test: Test apps locally before deploying via pytest fixtures.

.github/workflows/pytest.yaml

@@ -67,7 +67,7 @@ jobs:
       - name: Run End-to-End tests
         timeout-minutes: 20
         run: |
-          make playwright-shiny
+          make playwright-shiny SUB_FILE=". -vv"
 
   playwright-examples:
     runs-on: ${{ matrix.os }}
@@ -99,7 +99,15 @@ jobs:
       - name: Run example app tests
         timeout-minutes: 20
         run: |
-          make playwright-examples
+          make playwright-examples SUB_FILE=". -vv"
+
+      - name: Test deploys example apps on python 3.10
+        if: ${{ matrix.python-version == '3.10' }}
+        timeout-minutes: 5
+        env:
+          DEPLOY_APPS: "false"
+        run: |
+          make playwright-deploys SUB_FILE=". -vv"
 
   playwright-deploys:
     # Only allow one `playwright-deploys` job to run at a time. (Independent of branch / PR)
@@ -121,15 +129,15 @@ jobs:
 
       - name: Run tests for deploys
         env:
+          DEPLOY_APPS: "true"
           DEPLOY_CONNECT_SERVER_URL: "https://rsc.radixu.com/"
           DEPLOY_CONNECT_SERVER_API_KEY: "${{ secrets.DEPLOY_CONNECT_SERVER_API_KEY }}"
           DEPLOY_SHINYAPPS_NAME: "${{ secrets.DEPLOY_SHINYAPPS_NAME }}"
           DEPLOY_SHINYAPPS_TOKEN: "${{ secrets.DEPLOY_SHINYAPPS_TOKEN }}"
           DEPLOY_SHINYAPPS_SECRET: "${{ secrets.DEPLOY_SHINYAPPS_SECRET }}"
         timeout-minutes: 30
         run: |
-          make playwright-deploys
-
+          make playwright-deploys SUB_FILE=". -vv"
   pypi:
     name: "Deploy to PyPI"
     runs-on: ubuntu-latest

.gitignore

@@ -113,4 +113,4 @@ docs/source/reference/
 
 # Developer scratch area
 _dev/
-tests/playwright/deploys/apps/*/requirements.txt
+tests/playwright/deploys/**/requirements.txt

Makefile

@@ -86,8 +86,6 @@ test: ## run tests quickly with the default Python
 # Default `SUB_FILE` to empty
 SUB_FILE:=
 
-DEPLOYS_FILE:=tests/playwright/deploys
-
 install-playwright:
 	playwright install --with-deps
 
@@ -100,12 +98,12 @@ install-rsconnect: ## install the main version of rsconnect till pypi version su
 playwright-shiny: install-playwright ## end-to-end tests with playwright
 	pytest tests/playwright/shiny/$(SUB_FILE)
 
+playwright-deploys: install-playwright install-rsconnect ## end-to-end tests on examples with playwright
+	pytest tests/playwright/deploys/$(SUB_FILE)
+
 playwright-examples: install-playwright ## end-to-end tests on examples with playwright
 	pytest tests/playwright/examples/$(SUB_FILE)
 
-playwright-deploys: install-playwright install-rsconnect ## end-to-end tests on deploys with playwright
-	pytest tests/playwright/deploys/$(SUB_FILE) -s
-
 testrail-junit: install-playwright install-trcli ## end-to-end tests with playwright and generate junit report
 	pytest tests/playwright/shiny/$(SUB_FILE) --junitxml=report.xml
 

pyrightconfig.json

@@ -8,12 +8,12 @@
     "sandbox",
     "_dev",
     "docs",
-    "tests/playwright/deploys/apps",
+    "tests/playwright/deploys/*/app.py",
     "shiny/templates"
   ],
   "typeCheckingMode": "strict",
   "reportImportCycles": "none",
   "reportUnusedFunction": "none",
   "reportPrivateUsage": "none",
-  "reportUnnecessaryIsInstance": "none",
+  "reportUnnecessaryIsInstance": "none"
 }

pytest.ini

@@ -1,6 +1,8 @@
 [pytest]
 asyncio_mode=strict
 testpaths=tests/pytest/
+; ; Debug version of options
+; addopts = --strict-markers --durations=6 --durations-min=5.0 --browser chromium --numprocesses auto --video=retain-on-failure -vv
 addopts = --strict-markers --durations=6 --durations-min=5.0 --browser webkit --browser firefox --browser chromium --numprocesses auto
 markers =
     examples: Suite of tests to validate that examples do not produce errors (deselect with '-m "not examples"')

tests/playwright/conftest.py

@@ -11,7 +11,18 @@
 from pathlib import PurePath
 from time import sleep
 from types import TracebackType
-from typing import IO, Any, Callable, Generator, List, Optional, TextIO, Type, Union
+from typing import (
+    IO,
+    Any,
+    Callable,
+    Generator,
+    List,
+    Literal,
+    Optional,
+    TextIO,
+    Type,
+    Union,
+)
 
 import pytest
 
@@ -176,25 +187,37 @@ def run_shiny_app(
     return sa
 
 
-def create_app_fixture(app: Union[PurePath, str], scope: str = "module"):
+def local_app_fixture_gen(app: PurePath | str):
+    sa = run_shiny_app(app, wait_for_start=False)
+    try:
+        with sa:
+            sa.wait_until_ready(30)
+            yield sa
+    finally:
+        logging.warning("Application output:\n" + str(sa.stderr))
+
+
+ScopeName = Literal["session", "package", "module", "class", "function"]
+
+
+def create_app_fixture(
+    app: Union[PurePath, str],
+    scope: ScopeName = "module",
+):
+    @pytest.fixture(scope=scope)
     def fixture_func():
-        sa = run_shiny_app(app, wait_for_start=False)
-        try:
-            with sa:
-                sa.wait_until_ready(30)
-                yield sa
-        finally:
-            logging.warning("Application output:\n" + str(sa.stderr))
+        # Pass through `yield` via `next(...)` call
+        # (`yield` must be on same line as `next`!)
+        app_gen = local_app_fixture_gen(app)
+        yield next(app_gen)
 
-    return pytest.fixture(
-        scope=scope,  # type: ignore
-    )(fixture_func)
+    return fixture_func
 
 
 def create_example_fixture(
     example_name: str,
     example_file: str = "app.py",
-    scope: str = "module",
+    scope: ScopeName = "module",
 ):
     """Used to create app fixtures from apps in py-shiny/examples"""
     return create_app_fixture(
@@ -205,7 +228,7 @@ def create_example_fixture(
 def create_doc_example_fixture(
     example_name: str,
     example_file: str = "app.py",
-    scope: str = "module",
+    scope: ScopeName = "module",
 ):
     """Used to create app fixtures from apps in py-shiny/shiny/api-examples"""
     return create_app_fixture(
@@ -215,28 +238,21 @@ def create_doc_example_fixture(
 
 def create_doc_example_core_fixture(
     example_name: str,
-    scope: str = "module",
+    scope: ScopeName = "module",
 ):
     """Used to create app fixtures from ``app-core.py`` example apps in py-shiny/shiny/api-examples"""
     return create_doc_example_fixture(example_name, "app-core.py", scope)
 
 
 def create_doc_example_express_fixture(
     example_name: str,
-    scope: str = "module",
+    scope: ScopeName = "module",
 ):
     """Used to create app fixtures from ``app-express.py`` example apps in py-shiny/shiny/api-examples"""
     return create_doc_example_fixture(example_name, "app-express.py", scope)
 
 
-def create_deploys_fixture(app: Union[PurePath, str], scope: str = "module"):
-    """Used to create app fixtures from apps in tests/playwright/deploys/apps"""
-    return create_app_fixture(
-        here_root / "tests/playwright/deploys/apps" / app / "app.py", scope
-    )
-
-
-def x_create_doc_example_fixture(example_name: str, scope: str = "module"):
+def x_create_doc_example_fixture(example_name: str, scope: ScopeName = "module"):
     """Used to create app fixtures from apps in py-shiny/shiny/examples"""
     return create_app_fixture(
         here_root / "shiny/experimental/api-examples" / example_name / "app.py", scope
@@ -245,13 +261,8 @@ def x_create_doc_example_fixture(example_name: str, scope: str = "module"):
 
 @pytest.fixture(scope="module")
 def local_app(request: pytest.FixtureRequest) -> Generator[ShinyAppProc, None, None]:
-    sa = run_shiny_app(PurePath(request.path).parent / "app.py", wait_for_start=False)
-    try:
-        with sa:
-            sa.wait_until_ready(30)
-            yield sa
-    finally:
-        logging.warning("Application output:\n" + str(sa.stderr))
+    app_gen = local_app_fixture_gen(PurePath(request.path).parent / "app.py")
+    yield next(app_gen)
 
 
 @contextmanager

tests/playwright/controls.py

@@ -2170,7 +2170,21 @@ def __init__(
         super().__init__(page, id=id, loc=f"#{id}.shiny-text-output")
 
 
-# TODO-Karan: Add OutputCode class
+class OutputCode(_OutputTextValue):
+    def __init__(self, page: Page, id: str) -> None:
+        super().__init__(page, id=id, loc=f"pre#{id}.shiny-text-output")
+
+    def expect_has_placeholder(
+        self, placeholder: bool = False, *, timeout: Timeout = None
+    ) -> None:
+        _expect_class_value(
+            self.loc,
+            cls="noplaceholder",
+            has_class=not placeholder,
+            timeout=timeout,
+        )
+
+
 class OutputTextVerbatim(_OutputTextValue):
     def __init__(self, page: Page, id: str) -> None:
         super().__init__(page, id=id, loc=f"pre#{id}.shiny-text-output")

tests/playwright/deploys/apps/plotly_app/app.py → tests/playwright/deploys/plotly/app.py

@@ -59,7 +59,7 @@ def summary_data():
             height="100%",
         )
 
-    @reactive.calc
+    @reactive.Calc
     def filtered_df():
         # input.summary_data_selected_rows() is a tuple, so we must convert it to list,
         # as that's what Pandas requires for indexing.
@@ -114,7 +114,7 @@ def synchronize_size(output_id):
     def wrapper(func):
         input = session.get_current_session().input
 
-        @reactive.effect
+        @reactive.Effect
         def size_updater():
             func(
                 input[f".clientdata_output_{output_id}_width"](),

tests/playwright/deploys/apps/plotly_app/app_requirements.txt → tests/playwright/deploys/plotly/app_requirements.txt

@@ -2,4 +2,4 @@ pandas
 plotly
 git+https://github.com/posit-dev/py-htmltools.git#egg=htmltools
 git+https://github.com/posit-dev/py-shinywidgets.git#egg=shinywidgets
-fastapi==0.108.0
+

tests/playwright/deploys/plotly/test_plotly_app.py

@@ -0,0 +1,22 @@
+from playwright.sync_api import Page, expect
+from utils.deploy_utils import (
+    create_deploys_app_url_fixture,
+    skip_if_not_python_310_or_chrome,
+)
+
+TIMEOUT = 2 * 60 * 1000
+app_url = create_deploys_app_url_fixture(__file__, "example_deploy_app_A")
+
+
+@skip_if_not_python_310_or_chrome
+def test_deploys(page: Page, app_url: str) -> None:
+    page.goto(app_url)
+
+    COUNTRY = "Afghanistan"
+    expect(page.get_by_text(COUNTRY)).to_have_count(1, timeout=TIMEOUT)
+    page.get_by_role("cell", name=COUNTRY).click()
+    expect(page.locator("#country_detail_pop")).to_contain_text(
+        COUNTRY, timeout=TIMEOUT
+    )
+    expect(page.locator("#country_detail_percap")).to_contain_text(COUNTRY)
+    expect(page.get_by_text(COUNTRY)).to_have_count(3)

tests/playwright/deploys/apps/shiny-express-page-fillable/app_requirements.txt → tests/playwright/deploys/shiny-express-accordion/app_requirements.txt

@@ -1,2 +1,2 @@
 git+https://github.com/posit-dev/py-htmltools.git#egg=htmltools
-fastapi==0.108.0
+

tests/playwright/deploys/shiny-express-accordion/test_accordion.py

@@ -0,0 +1,20 @@
+from controls import Accordion
+from playwright.sync_api import Page
+from utils.deploy_utils import (
+    create_deploys_app_url_fixture,
+    skip_if_not_python_310_or_chrome,
+)
+
+app_url = create_deploys_app_url_fixture(__file__, "shiny_express_accordion")
+
+
+@skip_if_not_python_310_or_chrome
+def test_express_accordion(page: Page, app_url: str) -> None:
+    page.goto(app_url)
+
+    acc = Accordion(page, "express_accordion")
+    acc_panel_2 = acc.accordion_panel("Panel 2")
+    acc_panel_2.expect_open(True)
+    acc_panel_2.expect_body("n = 50")
+    acc_panel_2.set(False)
+    acc_panel_2.expect_open(False)

tests/playwright/deploys/apps/shiny-express-dataframe/app_requirements.txt → tests/playwright/deploys/shiny-express-dataframe/app_requirements.txt

@@ -1,3 +1,3 @@
 git+https://github.com/posit-dev/py-htmltools.git#egg=htmltools
 pandas
-fastapi==0.108.0
+

tests/playwright/deploys/shiny-express-dataframe/test_dataframe.py

@@ -0,0 +1,16 @@
+from controls import OutputDataFrame
+from playwright.sync_api import Page
+from utils.deploy_utils import (
+    create_deploys_app_url_fixture,
+    skip_if_not_python_310_or_chrome,
+)
+
+app_url = create_deploys_app_url_fixture(__file__, "shiny-express-dataframe")
+
+
+@skip_if_not_python_310_or_chrome
+def test_express_dataframe_deploys(page: Page, app_url: str) -> None:
+    page.goto(app_url)
+
+    dataframe = OutputDataFrame(page, "sample_data_frame")
+    dataframe.expect_n_row(6)

tests/playwright/deploys/apps/shiny-express-folium/app_requirements.txt → tests/playwright/deploys/shiny-express-folium/app_requirements.txt

@@ -1,3 +1,3 @@
 git+https://github.com/posit-dev/py-htmltools.git#egg=htmltools
 folium
-fastapi==0.108.0
+

tests/playwright/deploys/shiny-express-folium/test_folium.py

@@ -0,0 +1,25 @@
+from playwright.sync_api import Page, expect
+from utils.deploy_utils import (
+    create_deploys_app_url_fixture,
+    skip_if_not_python_310_or_chrome,
+)
+
+app_url = create_deploys_app_url_fixture(__file__, "shiny-express-folium")
+
+
+@skip_if_not_python_310_or_chrome
+def test_folium_map(page: Page, app_url: str) -> None:
+    page.goto(app_url)
+
+    expect(page.get_by_text("Static Map")).to_have_count(1)
+    expect(page.get_by_text("Map inside of render express call")).to_have_count(1)
+    # map inside the @render.express
+    expect(
+        page.frame_locator("iframe").nth(1).get_by_role("link", name="OpenStreetMap")
+    ).to_have_count(1)
+    # map outside of the @render.express at the top level
+    expect(
+        page.frame_locator("iframe")
+        .nth(0)
+        .get_by_role("link", name="U.S. Geological Survey")
+    ).to_have_count(1)

tests/playwright/deploys/apps/shiny-express-page-default/app_requirements.txt → tests/playwright/deploys/shiny-express-page_default/app_requirements.txt

@@ -1,2 +1,2 @@
 git+https://github.com/posit-dev/py-htmltools.git#egg=htmltools
-fastapi==0.108.0
+

tests/playwright/deploys/apps/shiny-express-page-default/rsconnect-python/shiny-express-page-default.json → tests/playwright/deploys/shiny-express-page_default/rsconnect-python/shiny-express-page_default.json

@@ -4,7 +4,7 @@
         "app_url": "https://testing-apps.shinyapps.io/shiny_express_page_default/",
         "app_id": 10800233,
         "app_guid": null,
-        "title": "shiny_express_page_default",
+        "title": "shiny-express-folium",
         "app_mode": "python-shiny",
         "app_store_version": 1
     }