Add cache, fetch, retry logic to tests (#829)

* Add cache, fetch, retry logic to tests

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* run in parallel

* add pytest-xdist

* undo parallelism. Need to remove http server to enable.

* woops a extra space

* Pass flake8

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* spell fulfill

* use decorator for fetch if not in cache

* Fix --headed and limit to PlaywrightRequestError

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* docs on cache

* CICD caching of conda on unstable builds

* fix config issues

* empty commit to trigger gh-actions

* restore build-unstable

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Remove http server, add parallel

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* temp: Bypass zip runtime test and point to v0.21.3 on CDN

* suport for files in zip under /pyodide

* remove test-one

* self.http_server and remove content_type

* domcontentloaded w no timeout on base url + longer timeout on wait_for_pyscript

* Fixed #678

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* set default timeout to 60000

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* seamless --headed support

* add test-integration-parallel and default for GHActions

* simplify the code. Use http://fakeserver instead of localhost:8080 so that it's clearer that the browser is NOT hitting a real server, and use urllib to parse the url. Moreover, the special case for pyodide is no longer needed, it's automatically handled by the normal 'fakeserver' logic

* The page-routing logic is becoming too much complicated to stay as an inner function. Move it to its own class, and add some logic to workaround a limitation of playwright which just hangs if a Python exception is raised inside it

* no need to use a hash, we can use the url as the key

* re-implement the retry logic. The old @Retry decorator was nice but a bit too over-engineered and most importantly failed silently in case of exceptions. This new approach is less powerful but since we want to retry only two times, simple is better than complex -- and in case of exception, the exception is actually raised

* improve logging

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Madhur Tandon <madhurtandon23@gmail.com>
Co-authored-by: Antonio Cuni <anto.cuni@gmail.com>

.github/workflows/build-unstable.yml

@@ -64,7 +64,7 @@ jobs:
             run: make test-py
 
         -   name: Integration Tests
-            run: make test-integration
+            run: make test-integration-parallel
 
         -   uses: actions/upload-artifact@v3
             with:

pyscriptjs/Makefile

@@ -86,6 +86,10 @@ test-integration:
 	make examples
 	$(PYTEST_EXE) -vv $(ARGS) tests/integration/ --log-cli-level=warning
 
+test-integration-parallel:
+	make examples
+	$(PYTEST_EXE) --numprocesses auto -vv $(ARGS) tests/integration/ --log-cli-level=warning
+
 test-py:
 	@echo "Tests from $(src_dir)"
 	$(PYTEST_EXE) -vv $(ARGS) tests/py-unit/ --log-cli-level=warning

pyscriptjs/environment.yml

@@ -17,3 +17,4 @@ dependencies:
 -   pip:
     -   playwright
     -   pytest-playwright
+    -   pytest-xdist

pyscriptjs/tests/integration/conftest.py

@@ -1,8 +1,4 @@
 """All data required for testing examples"""
-import threading
-from http.server import HTTPServer as SuperHTTPServer
-from http.server import SimpleHTTPRequestHandler
-
 import pytest
 
 from .support import Logger
@@ -11,41 +7,3 @@
 @pytest.fixture(scope="session")
 def logger():
     return Logger()
-
-
-class HTTPServer(SuperHTTPServer):
-    """
-    Class for wrapper to run SimpleHTTPServer on Thread.
-    Ctrl +Only Thread remains dead when terminated with C.
-    Keyboard Interrupt passes.
-    """
-
-    def run(self):
-        try:
-            self.serve_forever()
-        except KeyboardInterrupt:
-            pass
-        finally:
-            self.server_close()
-
-
-@pytest.fixture(scope="session")
-def http_server(logger):
-    class MyHTTPRequestHandler(SimpleHTTPRequestHandler):
-        def log_message(self, fmt, *args):
-            logger.log("http_server", fmt % args, color="blue")
-
-    host, port = "127.0.0.1", 8080
-    base_url = f"http://{host}:{port}"
-
-    # serve_Run forever under thread
-    server = HTTPServer((host, port), MyHTTPRequestHandler)
-
-    thread = threading.Thread(None, server.run)
-    thread.start()
-
-    yield base_url  # Transition to test here
-
-    # End thread
-    server.shutdown()
-    thread.join()

pyscriptjs/tests/integration/support.py

@@ -1,9 +1,14 @@
 import pdb
 import re
+import sys
 import time
+import traceback
+import urllib
+from dataclasses import dataclass
 
 import py
 import pytest
+from playwright.sync_api import Error as PlaywrightError
 
 ROOT = py.path.local(__file__).dirpath("..", "..", "..")
 BUILD = ROOT.join("pyscriptjs", "build")
@@ -43,18 +48,17 @@ class PyScriptTest:
     PY_COMPLETE = "Python initialization complete"
 
     @pytest.fixture()
-    def init(self, request, tmpdir, http_server, logger, page):
+    def init(self, request, tmpdir, logger, page):
         """
         Fixture to automatically initialize all the tests in this class and its
         subclasses.
 
-        The magic is done by the decorator @pyest.mark.usefixtures("init"),
+        The magic is done by the decorator @pytest.mark.usefixtures("init"),
         which tells pytest to automatically use this fixture for all the test
         method of this class.
 
         Using the standard pytest behavior, we can request more fixtures:
-        tmpdir, http_server and page; 'page' is a fixture provided by
-        pytest-playwright.
+        tmpdir, and page; 'page' is a fixture provided by pytest-playwright.
 
         Then, we save these fixtures on the self and proceed with more
         initialization. The end result is that the requested fixtures are
@@ -65,8 +69,12 @@ def init(self, request, tmpdir, http_server, logger, page):
         # create a symlink to BUILD inside tmpdir
         tmpdir.join("build").mksymlinkto(BUILD)
         self.tmpdir.chdir()
-        self.http_server = http_server
         self.logger = logger
+        self.fake_server = "http://fake_server"
+        self.router = SmartRouter(
+            "fake_server", logger=logger, usepdb=request.config.option.usepdb
+        )
+        self.router.install(page)
         self.init_page(page)
         #
         # this extra print is useful when using pytest -s, else we start printing
@@ -92,6 +100,10 @@ def init(self, request, tmpdir, http_server, logger, page):
 
     def init_page(self, page):
         self.page = page
+
+        # set default timeout to 60000 millliseconds from 30000
+        page.set_default_timeout(60000)
+
         self.console = ConsoleMessageCollection(self.logger)
         self._page_errors = []
         page.on("console", self.console.add_message)
@@ -145,8 +157,8 @@ def writefile(self, filename, content):
     def goto(self, path):
         self.logger.reset()
         self.logger.log("page.goto", path, color="yellow")
-        url = f"{self.http_server}/{path}"
-        self.page.goto(url)
+        url = f"{self.fake_server}/{path}"
+        self.page.goto(url, timeout=0)
 
     def wait_for_console(self, text, *, timeout=None, check_errors=True):
         """
@@ -212,8 +224,8 @@ def pyscript_run(self, snippet, *, extra_head="", wait_for_pyscript=True):
         doc = f"""
         <html>
           <head>
-              <link rel="stylesheet" href="{self.http_server}/build/pyscript.css" />
-              <script defer src="{self.http_server}/build/pyscript.js"></script>
+              <link rel="stylesheet" href="{self.fake_server}/build/pyscript.css" />
+              <script defer src="{self.fake_server}/build/pyscript.js"></script>
               {extra_head}
           </head>
           <body>
@@ -411,3 +423,125 @@ def escape_pair(cls, color):
         start = f"\x1b[{color}m"
         end = "\x1b[00m"
         return start, end
+
+
+class SmartRouter:
+    """
+    A smart router to be used in conjunction with playwright.Page.route.
+
+    Main features:
+
+      - it intercepts the requests to a local "fake server" and serve them
+        statically from disk
+
+      - it intercepts the requests to the network and cache the results
+        locally
+    """
+
+    @dataclass
+    class CachedResponse:
+        """
+        We cannot put playwright's APIResponse instances inside _cache, because
+        they are valid only in the context of the same page. As a workaround,
+        we manually save status, headers and body of each cached response.
+        """
+
+        status: int
+        headers: dict
+        body: str
+
+    # NOTE: this is a class attribute, which means that the cache is
+    # automatically shared between all instances of Fake_Server (and thus all
+    # tests of the pytest session)
+    _cache = {}
+
+    def __init__(self, fake_server, *, logger, usepdb=False):
+        """
+        fake_server: the domain name of the fake server
+        """
+        self.fake_server = fake_server
+        self.logger = logger
+        self.usepdb = usepdb
+        self.page = None
+
+    def install(self, page):
+        """
+        Install the smart router on a page
+        """
+        self.page = page
+        self.page.route("**", self.router)
+
+    def router(self, route):
+        """
+        Intercept and fulfill playwright requests.
+
+        NOTE!
+        If we raise an exception inside router, playwright just hangs and the
+        exception seems not to be propagated outside. It's very likely a
+        playwright bug.
+
+        This means that for example pytest doesn't have any chance to
+        intercept the exception and fail in a meaningful way.
+
+        As a workaround, we try to intercept exceptions by ourselves, print
+        something reasonable on the console and abort the request (hoping that
+        the test will fail cleaninly, that's the best we can do). We also try
+        to respect pytest --pdb, for what it's possible.
+        """
+        try:
+            return self._router(route)
+        except Exception:
+            print("***** Error inside Fake_Server.router *****")
+            info = sys.exc_info()
+            print(traceback.format_exc())
+            if self.usepdb:
+                pdb.post_mortem(info[2])
+            route.abort()
+
+    def log_request(self, status, kind, url):
+        color = "blue" if status == 200 else "red"
+        self.logger.log("request", f"{status} - {kind} - {url}", color=color)
+
+    def _router(self, route):
+        full_url = route.request.url
+        url = urllib.parse.urlparse(full_url)
+        assert url.scheme in ("http", "https")
+
+        # requests to http://fake_server/ are served from the current dir and
+        # never cached
+        if url.netloc == self.fake_server:
+            self.log_request(200, "fake_server", full_url)
+            assert url.path[0] == "/"
+            relative_path = url.path[1:]
+            route.fulfill(status=200, path=relative_path)
+            return
+
+        # network requests might be cached
+        if full_url in self._cache:
+            kind = "CACHED"
+            resp = self._cache[full_url]
+        else:
+            kind = "NETWORK"
+            resp = self.fetch_from_network(route.request)
+            self._cache[full_url] = resp
+
+        self.log_request(resp.status, kind, full_url)
+        route.fulfill(status=resp.status, headers=resp.headers, body=resp.body)
+
+    def fetch_from_network(self, request):
+        # sometimes the network is flaky and if the first request doesn't
+        # work, a subsequent one works. Instead of giving up immediately,
+        # let's try twice
+        try:
+            api_response = self.page.request.fetch(request)
+        except PlaywrightError:
+            # sleep a bit and try again
+            time.sleep(0.5)
+            api_response = self.page.request.fetch(request)
+
+        cached_response = self.CachedResponse(
+            status=api_response.status,
+            headers=api_response.headers,
+            body=api_response.body(),
+        )
+        return cached_response

pyscriptjs/tests/integration/test_00_support.py

@@ -91,7 +91,7 @@ def test_check_errors(self):
         # stack trace
         msg = str(exc.value)
         assert "Error: this is an error" in msg
-        assert f"at {self.http_server}/mytest.html" in msg
+        assert f"at {self.fake_server}/mytest.html" in msg
         #
         # after a call to check_errors, the errors are cleared
         self.check_errors()

pyscriptjs/tests/integration/test_zz_examples.py

@@ -241,7 +241,7 @@ def test_panel_deckgl(self):
     def test_panel_kmeans(self):
         # XXX improve this test
         self.goto("examples/panel_kmeans.html")
-        self.wait_for_pyscript(timeout=120 * 1000)
+        self.wait_for_pyscript()
         assert self.page.title() == "Pyscript/Panel KMeans Demo"
         wait_for_render(self.page, "*", "<div.*?class=['\"]bk-root['\"].*?>")
 
@@ -339,7 +339,6 @@ def test_toga_freedom(self):
         result = self.page.locator("#toga_c_input")
         assert "40.555" in result.input_value()
 
-    @pytest.mark.xfail(reason="it never finishes loading, issue #678")
     def test_webgl_raycaster_index(self):
         # XXX improve this test
         self.goto("examples/webgl/raycaster/index.html")