diff --git a/Makefile b/Makefile index d3a4ccc25..c0d12931c 100644 --- a/Makefile +++ b/Makefile @@ -104,6 +104,9 @@ playwright-deploys: install-playwright install-rsconnect ## end-to-end tests on playwright-examples: install-playwright ## end-to-end tests on examples with playwright pytest tests/playwright/examples/$(SUB_FILE) +playwright-debug: install-playwright ## All end-to-end tests, chrome only, headed + pytest -c tests/playwright/playwright-pytest.ini tests/playwright/$(SUB_FILE) + testrail-junit: install-playwright install-trcli ## end-to-end tests with playwright and generate junit report pytest tests/playwright/shiny/$(SUB_FILE) --junitxml=report.xml diff --git a/pytest.ini b/pytest.ini index f7b1d8bad..e6879b461 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,6 +4,3 @@ testpaths=tests/pytest/ ; ; Debug version of options ; addopts = --strict-markers --durations=6 --durations-min=5.0 --browser chromium --numprocesses auto --video=retain-on-failure -vv addopts = --strict-markers --durations=6 --durations-min=5.0 --browser webkit --browser firefox --browser chromium --numprocesses auto -markers = - examples: Suite of tests to validate that examples do not produce errors (deselect with '-m "not examples"') - integrationtest: Suite of tests that check deploys and integration with other services (deselect with '-m "not integrationtest"') diff --git a/tests/playwright/README.md b/tests/playwright/README.md index 8258cc9fa..164857b94 100644 --- a/tests/playwright/README.md +++ b/tests/playwright/README.md @@ -2,7 +2,7 @@ This directory contains end-to-end (i.e. browser based) tests for Shiny for Python. -The Python files directly in this subdirectory are for Pytest fixtures and helper code +The Python files directly in this subdirectory (and `./utils`) are for Pytest fixtures and helper code to make test writing easier. (Eventually this logic may move to the `shiny` package itself or its own dedicated package, so that Shiny app authors can set up their own e2e tests against their apps.) @@ -17,14 +17,21 @@ optional, because the tests may also be for apps in the `../examples` or `../shi The following commands can be run from the repo root: ```sh -# Run all e2e tests +# Run tests related to shiny (in `./shiny` folder) make playwright-shiny +# Run tests related to examples (in `./examples` folder) +make playwright-examples +# Run tests on apps that should be deployed (in `./deploys` folder) +make playwright-deploys # Run just the tests in playwright/shiny/async/ make playwright-shiny SUB_FILE=e2e/async # Run just the tests in playwright/shiny/async/, in headed mode make playwright-shiny SUB_FILE="--headed e2e/async" + +# Run **all** tests on with pytest options best for debugging +make playwright-debug ``` ## Shiny app fixtures @@ -73,3 +80,5 @@ def test_airmass(page: Page, airmass_app: ShinyAppProc): plot = page.locator("#plot") expect(plot).to_have_class(re.compile(r"\bshiny-bound-output\b")) ``` + +However, it is suggested to alter the example app for testing and use a local app for testing. diff --git a/tests/playwright/deploys/express-accordion/test_accordion.py b/tests/playwright/deploys/express-accordion/test_deploys_express_accordion.py similarity index 100% rename from tests/playwright/deploys/express-accordion/test_accordion.py rename to tests/playwright/deploys/express-accordion/test_deploys_express_accordion.py diff --git a/tests/playwright/deploys/express-dataframe/test_dataframe.py b/tests/playwright/deploys/express-dataframe/test_deploys_express_dataframe.py similarity index 100% rename from tests/playwright/deploys/express-dataframe/test_dataframe.py rename to tests/playwright/deploys/express-dataframe/test_deploys_express_dataframe.py diff --git a/tests/playwright/deploys/express-folium/test_folium.py b/tests/playwright/deploys/express-folium/test_deploys_express_folium.py similarity index 100% rename from tests/playwright/deploys/express-folium/test_folium.py rename to tests/playwright/deploys/express-folium/test_deploys_express_folium.py diff --git a/tests/playwright/deploys/express-page_default/test_page_default.py b/tests/playwright/deploys/express-page_default/test_deploys_express_page_default.py similarity index 100% rename from tests/playwright/deploys/express-page_default/test_page_default.py rename to tests/playwright/deploys/express-page_default/test_deploys_express_page_default.py diff --git a/tests/playwright/deploys/express-page_fillable/test_page_fillable.py b/tests/playwright/deploys/express-page_fillable/test_deploys_express_page_fillable.py similarity index 100% rename from tests/playwright/deploys/express-page_fillable/test_page_fillable.py rename to tests/playwright/deploys/express-page_fillable/test_deploys_express_page_fillable.py diff --git a/tests/playwright/deploys/express-page_fluid/test_page_fluid.py b/tests/playwright/deploys/express-page_fluid/test_deploys_express_page_fluid.py similarity index 100% rename from tests/playwright/deploys/express-page_fluid/test_page_fluid.py rename to tests/playwright/deploys/express-page_fluid/test_deploys_express_page_fluid.py diff --git a/tests/playwright/deploys/express-page_sidebar/test_page_sidebar.py b/tests/playwright/deploys/express-page_sidebar/test_deploys_express_page_sidebar.py similarity index 100% rename from tests/playwright/deploys/express-page_sidebar/test_page_sidebar.py rename to tests/playwright/deploys/express-page_sidebar/test_deploys_express_page_sidebar.py diff --git a/tests/playwright/examples/example_apps.py b/tests/playwright/examples/example_apps.py index 3fa4c7ca2..296c64f31 100644 --- a/tests/playwright/examples/example_apps.py +++ b/tests/playwright/examples/example_apps.py @@ -12,6 +12,7 @@ is_interactive = hasattr(sys, "ps1") reruns = 1 if is_interactive else 3 +reruns_delay = 0 def get_apps(path: str) -> typing.List[str]: @@ -71,8 +72,6 @@ def get_apps(path: str) -> typing.List[str]: # TODO-garrick-future: Remove after fixing sidebar max_height_mobile warning "UserWarning: The `shiny.ui.sidebar(max_height_mobile=)`", "res = self.fn(*self.args, **self.kwargs)", - # if shiny express app detected - "Detected Shiny Express app", # pandas >= 2.2.0 # https://github.com/pandas-dev/pandas/blame/5740667a55aabffc660936079268cee2f2800225/pandas/core/groupby/groupby.py#L1129 "FutureWarning: When grouping with a length-1 list-like", @@ -161,6 +160,9 @@ def validate_example(page: Page, ex_app_path: str) -> None: def on_console_msg(msg: ConsoleMessage) -> None: if msg.type == "error": + # Do not report missing favicon errors + if msg.location["url"].endswith("favicon.ico"): + return console_errors.append(msg.text) page.on("console", on_console_msg) diff --git a/tests/playwright/examples/test_api_examples.py b/tests/playwright/examples/test_api_examples.py index 2ba9fac06..475dda70a 100644 --- a/tests/playwright/examples/test_api_examples.py +++ b/tests/playwright/examples/test_api_examples.py @@ -1,10 +1,9 @@ import pytest -from example_apps import get_apps, reruns, validate_example +from example_apps import get_apps, reruns, reruns_delay, validate_example from playwright.sync_api import Page -@pytest.mark.examples -@pytest.mark.flaky(reruns=reruns, reruns_delay=1) +@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay) @pytest.mark.parametrize("ex_app_path", get_apps("shiny/api-examples")) def test_api_examples(page: Page, ex_app_path: str) -> None: validate_example(page, ex_app_path) diff --git a/tests/playwright/examples/test_examples.py b/tests/playwright/examples/test_examples.py index 4c3942d36..60117d7df 100644 --- a/tests/playwright/examples/test_examples.py +++ b/tests/playwright/examples/test_examples.py @@ -1,9 +1,9 @@ import pytest -from example_apps import get_apps, reruns, validate_example +from example_apps import get_apps, reruns, reruns_delay, validate_example from playwright.sync_api import Page -@pytest.mark.flaky(reruns=reruns, reruns_delay=1) +@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay) @pytest.mark.parametrize("ex_app_path", get_apps("examples")) def test_examples(page: Page, ex_app_path: str) -> None: validate_example(page, ex_app_path) diff --git a/tests/playwright/examples/test_shiny_create.py b/tests/playwright/examples/test_shiny_create.py index 550cec0af..7571c0971 100644 --- a/tests/playwright/examples/test_shiny_create.py +++ b/tests/playwright/examples/test_shiny_create.py @@ -3,7 +3,7 @@ import tempfile import pytest -from example_apps import get_apps, reruns, validate_example +from example_apps import get_apps, reruns, reruns_delay, validate_example from playwright.sync_api import Page @@ -41,15 +41,13 @@ def subprocess_create( subprocess.run(cmd, check=True) -@pytest.mark.examples -@pytest.mark.flaky(reruns=reruns, reruns_delay=1) +@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay) @pytest.mark.parametrize("ex_app_path", get_apps("shiny/templates/app-templates")) def test_template_examples(page: Page, ex_app_path: str) -> None: validate_example(page, ex_app_path) -@pytest.mark.examples -@pytest.mark.flaky(reruns=reruns, reruns_delay=1) +@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay) @pytest.mark.parametrize("app_template", ["basic-app", "dashboard", "multi-page"]) def test_create_core(app_template: str, page: Page): with tempfile.TemporaryDirectory("example_apps") as tmpdir: @@ -57,8 +55,7 @@ def test_create_core(app_template: str, page: Page): validate_example(page, f"{tmpdir}/app.py") -@pytest.mark.examples -@pytest.mark.flaky(reruns=reruns, reruns_delay=1) +@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay) @pytest.mark.parametrize("app_template", ["basic-app"]) def test_create_express(app_template: str, page: Page): with tempfile.TemporaryDirectory("example_apps") as tmpdir: @@ -66,8 +63,7 @@ def test_create_express(app_template: str, page: Page): validate_example(page, f"{tmpdir}/app.py") -@pytest.mark.examples -@pytest.mark.flaky(reruns=reruns, reruns_delay=1) +@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay) @pytest.mark.parametrize("app_template", ["js-input", "js-output", "js-react"]) def test_create_js(app_template: str): with tempfile.TemporaryDirectory("example_apps") as tmpdir: diff --git a/tests/playwright/playwright-pytest.ini b/tests/playwright/playwright-pytest.ini new file mode 100644 index 000000000..74013315d --- /dev/null +++ b/tests/playwright/playwright-pytest.ini @@ -0,0 +1,13 @@ +[pytest] +asyncio_mode=strict +# --strict-markers: Markers not registered in the `markers` section of the configuration file raise errors +# --durations : show top `k` slowest durations +# --durations-min : Require that the top `k` slowest durations are longer than `n` seconds +# --browser : browser type to run on playwright +# --numprocesses auto: number of testing workers. auto is number of (virtual) cores +# --video=retain-on-failure: playwright saves recording of any failed test +# -vv: Extra extra verbose output +# # --headed: Headed browser testing +# # -r P: Show extra test summary info: (f)ailed, (E)rror, (s)kipped, (x)failed, (X)passed, (p)assed, (P)assed with output, (a)ll except passed (p/P), or (A)ll. (w)arnings... +# --maxfail=1: Stop after 1 failure has occurred +addopts = --strict-markers --durations=6 --durations-min=5.0 --browser chromium --numprocesses auto --video=retain-on-failure -vvv --maxfail=1 --headed diff --git a/tests/playwright/shiny/components/data_frame/test_data_frame.py b/tests/playwright/shiny/components/data_frame/test_data_frame.py index 9cf837b77..bdb1b5863 100644 --- a/tests/playwright/shiny/components/data_frame/test_data_frame.py +++ b/tests/playwright/shiny/components/data_frame/test_data_frame.py @@ -7,10 +7,9 @@ import pytest from conftest import ShinyAppProc, create_example_fixture, expect_to_change from controls import InputSelect, InputSwitch +from examples.example_apps import reruns, reruns_delay from playwright.sync_api import Locator, Page, expect -RERUNS = 3 - data_frame_app = create_example_fixture("dataframe") @@ -47,7 +46,7 @@ def do(): return do -@pytest.mark.flaky(reruns=RERUNS) +@pytest.mark.flaky(reruns=reruns, delay=reruns_delay) def test_grid_mode( page: Page, data_frame_app: ShinyAppProc, grid: Locator, grid_container: Locator ): @@ -61,7 +60,7 @@ def test_grid_mode( expect(grid_container).to_have_class(re.compile(r"\bshiny-data-grid-grid\b")) -@pytest.mark.flaky(reruns=RERUNS) +@pytest.mark.flaky(reruns=reruns, delay=reruns_delay) def test_summary_navigation( page: Page, data_frame_app: ShinyAppProc, @@ -81,7 +80,7 @@ def test_summary_navigation( expect(summary).to_have_text(re.compile("^Viewing rows \\d+ through 20 of 20$")) -@pytest.mark.flaky(reruns=RERUNS) +@pytest.mark.flaky(reruns=reruns, delay=reruns_delay) def test_full_width(page: Page, data_frame_app: ShinyAppProc, grid_container: Locator): page.goto(data_frame_app.url) @@ -101,7 +100,7 @@ def get_width() -> float: InputSwitch(page, "fullwidth").toggle() -@pytest.mark.flaky(reruns=RERUNS) +@pytest.mark.flaky(reruns=reruns, delay=reruns_delay) def test_table_switch( page: Page, data_frame_app: ShinyAppProc, @@ -134,7 +133,7 @@ def test_table_switch( ) -@pytest.mark.flaky(reruns=RERUNS) +@pytest.mark.flaky(reruns=reruns, delay=reruns_delay) def test_sort( page: Page, data_frame_app: ShinyAppProc, @@ -171,7 +170,7 @@ def test_sort( expect(first_cell_depth).to_have_text("67.6") -@pytest.mark.flaky(reruns=RERUNS) +@pytest.mark.flaky(reruns=reruns, delay=reruns_delay) def test_multi_selection( page: Page, data_frame_app: ShinyAppProc, grid_container: Locator, snapshot: Any ): @@ -201,7 +200,7 @@ def detail_text(): assert detail_text() == snapshot -@pytest.mark.flaky(reruns=RERUNS) +@pytest.mark.flaky(reruns=reruns, delay=reruns_delay) def test_single_selection( page: Page, data_frame_app: ShinyAppProc, grid_container: Locator, snapshot: Any ): diff --git a/tests/pytest/test_utils.py b/tests/pytest/test_utils.py index 6811a8d5c..ac99d4ab6 100644 --- a/tests/pytest/test_utils.py +++ b/tests/pytest/test_utils.py @@ -139,7 +139,7 @@ async def mutate_registrations(): # Timeout within 2 seconds @pytest.mark.timeout(2) -@pytest.mark.flaky(reruns=3, reruns_delay=1) +@pytest.mark.flaky(reruns=3) def test_random_port(): assert random_port(9000, 9000) == 9000