PageCoroutine checks

scrapy_playwright/handler.py

@@ -3,6 +3,7 @@
 import warnings
 from collections import defaultdict
 from contextlib import suppress
+from inspect import isawaitable
 from ipaddress import ip_address
 from time import time
 from typing import Callable, Dict, Optional, Type, TypeVar
@@ -251,9 +252,14 @@ async def _download_request_with_page(self, request: Request, page: Page) -> Res
             page_coroutines = page_coroutines.values()
         for pc in page_coroutines:
             if isinstance(pc, PageCoroutine):
-                method = getattr(page, pc.method)
-                pc.result = await method(*pc.args, **pc.kwargs)
-                await page.wait_for_load_state(timeout=self.default_navigation_timeout)
+                try:
+                    method = getattr(page, pc.method)
+                except AttributeError:
+                    logger.warning(f"Ignoring {repr(pc)}: could not find coroutine")
+                else:
+                    result = method(*pc.args, **pc.kwargs)
+                    pc.result = await result if isawaitable(result) else result
+                    await page.wait_for_load_state(timeout=self.default_navigation_timeout)
             else:
                 logger.warning(
                     f"Ignoring {repr(pc)}: expected PageCoroutine, got {repr(type(pc))}"

tests/test_page_coroutines.py

@@ -23,7 +23,7 @@ async def test_page_coroutines():
 
 class MixinPageCoroutineTestCase:
     @pytest.mark.asyncio
-    async def test_page_bad_page_coroutine(self, caplog):
+    async def test_page_non_page_coroutine(self, caplog):
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
             with StaticMockServer() as server:
                 req = Request(
@@ -52,6 +52,38 @@ async def test_page_bad_page_coroutine(self, caplog):
                     f"Ignoring {repr(obj)}: expected PageCoroutine, got {repr(type(obj))}",
                 ) in caplog.record_tuples
 
+    @pytest.mark.asyncio
+    async def test_page_mixed_page_coroutines(self, caplog):
+        async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
+            with StaticMockServer() as server:
+                req = Request(
+                    url=server.urljoin("/index.html"),
+                    meta={
+                        "playwright": True,
+                        "playwright_page_coroutines": {
+                            "does_not_exist": PageCoroutine("does_not_exist"),
+                            "is_closed": PageCoroutine("is_closed"),  # not awaitable
+                            "title": PageCoroutine("title"),  # awaitable
+                        },
+                    },
+                )
+                resp = await handler._download_request(req, Spider("foo"))
+
+            assert isinstance(resp, HtmlResponse)
+            assert resp.request is req
+            assert resp.url == server.urljoin("/index.html")
+            assert resp.status == 200
+            assert "playwright" in resp.flags
+
+            does_not_exist = req.meta["playwright_page_coroutines"]["does_not_exist"]
+            assert (
+                "scrapy-playwright",
+                logging.WARNING,
+                f"Ignoring {repr(does_not_exist)}: could not find coroutine",
+            ) in caplog.record_tuples
+            assert not req.meta["playwright_page_coroutines"]["is_closed"].result
+            assert req.meta["playwright_page_coroutines"]["title"].result == "Awesome site"
+
 
 class TestPageCoroutineChromium(MixinPageCoroutineTestCase):
     browser_type = "chromium"