From bf91f64bfa15ddaafefbefdb680e0af3868c877b Mon Sep 17 00:00:00 2001 From: David Riccitelli Date: Thu, 6 Aug 2026 10:36:05 +0200 Subject: [PATCH] fix(render): block Google Analytics traffic --- AGENTS.md | 3 ++ README.md | 1 + docs/render.md | 9 ++++++ specs/INGESTION_PIPELINE.md | 4 +++ specs/validation.md | 6 ++++ tests/test_render_browser.py | 24 +++++++++++++++ tests/test_render_network_policy.py | 43 +++++++++++++++++++++++++++ tests/tools/run_slice_tests.py | 1 + wordlift_sdk/render/browser.py | 6 ++++ wordlift_sdk/render/network_policy.py | 18 +++++++++++ 10 files changed, 115 insertions(+) create mode 100644 tests/test_render_network_policy.py create mode 100644 wordlift_sdk/render/network_policy.py diff --git a/AGENTS.md b/AGENTS.md index 814703a..c698073 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -242,6 +242,9 @@ - Playwright ingestion is async-loop-safe in cloud workflows: when a caller thread already runs an asyncio event loop, rendering is offloaded away from that thread before Sync Playwright APIs are invoked. +- All Playwright rendering contexts disable service workers and apply the mandatory + context-wide direct Google Analytics measurement policy before creating pages; + Google Tag Manager and advertising endpoints remain available. - Playwright ingestion default `wait_until` is `domcontentloaded` (override with `PLAYWRIGHT_WAIT_UNTIL`), and navigation timeout now falls back to partial DOM extraction instead of immediate loader failure. diff --git a/README.md b/README.md index 1c20d89..7d24254 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Current release: see `CHANGELOG.md`. - Sitemap discovery requests use a browser-like header bundle aligned with Playwright defaults (including `User-Agent`, `Accept`, `Accept-Language`, and `Sec-CH-*` headers). - Change detection: skips URLs that are already imported unless `OVERWRITE` is enabled; re-imports when `lastmod` is newer. - Web page imports: sends URLs to WordLift with embedding requests, output types, retry logic, and pluggable callbacks. +- Playwright rendering blocks service workers and direct Google Analytics measurement traffic before network egress while keeping Google Tag Manager available. - Python 3.14 compatibility: retry filters use `pydantic_core.ValidationError` via the public API. - Search Console refresh: triggers analytics imports when top queries are stale. - GSC canonical clustering helper: builds `url,title,canonical` CSV outputs from Search Console impressions with exact-title clustering, interval parsing (`XX[d|w|m]`), optional URL regex filtering, and fixed/auto adaptive concurrency controls. diff --git a/docs/render.md b/docs/render.md index f7e31b4..48f0a3f 100644 --- a/docs/render.md +++ b/docs/render.md @@ -12,6 +12,15 @@ Renders a URL using `Browser` and converts HTML to XHTML with `HtmlConverter`. ### Browser Thin wrapper around Playwright that opens a page and returns the page, response, elapsed time, and resource list. +Each browser context blocks service workers and installs a mandatory context-wide +route that aborts direct traffic to the standard Google Analytics measurement +hosts: `*.google-analytics.com` and `*.analytics.google.com`. Google Tag Manager +and advertising endpoints remain available. Blocked URLs and payloads are not +logged or added to the response-resource list. Customer-specific first-party or +server-side tagging gateways are outside this hostname policy. +Disabling service workers intentionally trades PWA offline caching and +background-sync fidelity for complete context-route coverage. + ### RenderOptions Configuration for rendering: - `url`, `headless`, `timeout_ms`, `wait_until`, `locale`, `user_agent`, `viewport_width`, `viewport_height`, `ignore_https_errors` diff --git a/specs/INGESTION_PIPELINE.md b/specs/INGESTION_PIPELINE.md index d4d19b5..d208794 100644 --- a/specs/INGESTION_PIPELINE.md +++ b/specs/INGESTION_PIPELINE.md @@ -71,6 +71,10 @@ If item includes embedded HTML and `INGEST_PASSTHROUGH_WHEN_HTML=true`, orchestr - Playwright loader execution must be async-loop-safe: when called while an event loop is already running in the caller thread, rendering is offloaded away from that loop thread before invoking Sync Playwright APIs. +- Every Playwright browser context must block service workers and abort direct requests to the + standard Google Analytics measurement host families enumerated in the render documentation + before network egress. Google Tag Manager and advertising endpoints remain available; this + policy is mandatory and is not configurable. - Playwright default wait policy is `domcontentloaded`; explicit `PLAYWRIGHT_WAIT_UNTIL` still overrides. - On Playwright navigation timeout, loader should continue with available page DOM content instead of failing immediately, and only raise browser errors for non-timeout navigation failures. diff --git a/specs/validation.md b/specs/validation.md index 3e8267f..4324999 100644 --- a/specs/validation.md +++ b/specs/validation.md @@ -65,6 +65,12 @@ nodes, and pass them through the SHACL validation pipeline. Playwright is a required dependency for URL rendering. Install browser binaries with `playwright install` after the Python dependencies are installed. +All Playwright URL rendering disables service workers and aborts direct requests +to the standard Google Analytics measurement host families enumerated in the +render documentation before navigation traffic can leave the browser. Google +Tag Manager and advertising endpoints remain available. The policy also applies +to frames and popup pages. + ## Validation API composition contract Host tooling can validate one or more file/URL inputs via SDK APIs with diff --git a/tests/test_render_browser.py b/tests/test_render_browser.py index 17bbd2e..84624ee 100644 --- a/tests/test_render_browser.py +++ b/tests/test_render_browser.py @@ -37,11 +37,20 @@ def __init__(self): self.closed = False self.script = None self.page = _FakePage() + self.route_matcher = None + self.route_handler = None + self.events = [] + + def route(self, matcher, handler): + self.route_matcher = matcher + self.route_handler = handler + self.events.append("route") def add_init_script(self, script): self.script = script def new_page(self): + self.events.append("new_page") return self.page def close(self): @@ -112,6 +121,21 @@ def test_browser_enter_exit_and_open(monkeypatch: pytest.MonkeyPatch): assert pw.browser.kwargs["user_agent"] == "UA" assert pw.browser.kwargs["viewport"]["width"] == 1200 assert pw.browser.kwargs["ignore_https_errors"] is True + assert pw.browser.kwargs["service_workers"] == "block" + assert pw.browser.context.events == ["route", "new_page"] + assert pw.browser.context.route_matcher.search( + "https://region1.google-analytics.com/g/collect" + ) + + class _FakeRoute: + aborted_with = None + + def abort(self, error_code): + self.aborted_with = error_code + + route = _FakeRoute() + pw.browser.context.route_handler(route) + assert route.aborted_with == "blockedbyclient" assert pw.browser.context.closed is True assert pw.browser.closed is True diff --git a/tests/test_render_network_policy.py b/tests/test_render_network_policy.py new file mode 100644 index 0000000..6ef1900 --- /dev/null +++ b/tests/test_render_network_policy.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import pytest + +from wordlift_sdk.render.network_policy import ( + GOOGLE_ANALYTICS_URL_PATTERN, +) + + +@pytest.mark.parametrize( + "url", + [ + "https://www.google-analytics.com/g/collect", + "https://user:pass@www.google-analytics.com/g/collect", + "https://region1.google-analytics.com/mp/collect", + "https://ANALYTICS.GOOGLE.COM./g/collect", + ], +) +def test_google_analytics_urls_are_blocked(url: str) -> None: + assert GOOGLE_ANALYTICS_URL_PATTERN.search(url) + + +@pytest.mark.parametrize( + "url", + [ + "https://example.com/assets/analytics.js", + "https://evilgoogle-analytics.com/collect", + "https://google-analytics.com.example.org/collect", + "https://user@google-analytics.com.example.org/collect", + "https://www.googletagmanager.com/gtm.js?id=GTM-123", + "https://www.googletagmanager.com/gtag/js?id=G-123", + "https://tagmanager.google.com/", + "https://stats.g.doubleclick.net/g/collect", + "https://pagead2.googlesyndication.com/pagead/gen_204", + "https://example.com/g/collect", + "https://google.com/", + "https://" + "a." * 64 + "example.com/", + "not a url", + "https://[invalid", + ], +) +def test_unrelated_and_malformed_urls_are_allowed(url: str) -> None: + assert not GOOGLE_ANALYTICS_URL_PATTERN.search(url) diff --git a/tests/tools/run_slice_tests.py b/tests/tools/run_slice_tests.py index 39cacf0..6d08b34 100644 --- a/tests/tools/run_slice_tests.py +++ b/tests/tools/run_slice_tests.py @@ -21,6 +21,7 @@ "render": [ "tests/test_render_browser.py", "tests/test_render_html_renderer.py", + "tests/test_render_network_policy.py", "tests/test_xhtml_cleaner.py", ], "validation": [ diff --git a/wordlift_sdk/render/browser.py b/wordlift_sdk/render/browser.py index 2762ded..ffaa36c 100644 --- a/wordlift_sdk/render/browser.py +++ b/wordlift_sdk/render/browser.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from time import perf_counter +from .network_policy import GOOGLE_ANALYTICS_URL_PATTERN from .render_options import DEFAULT_BROWSER_REQUEST_HEADERS try: @@ -77,7 +78,12 @@ def __enter__(self) -> "Browser": context_kwargs["viewport"] = viewport context_kwargs["ignore_https_errors"] = self.ignore_https_errors context_kwargs["extra_http_headers"] = dict(DEFAULT_BROWSER_REQUEST_HEADERS) + context_kwargs["service_workers"] = "block" self._context = self._browser.new_context(**context_kwargs) + self._context.route( + GOOGLE_ANALYTICS_URL_PATTERN, + lambda route: route.abort("blockedbyclient"), + ) self._context.add_init_script( """ Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); diff --git a/wordlift_sdk/render/network_policy.py b/wordlift_sdk/render/network_policy.py new file mode 100644 index 0000000..33b3634 --- /dev/null +++ b/wordlift_sdk/render/network_policy.py @@ -0,0 +1,18 @@ +"""Mandatory browser network policy for analytics traffic.""" + +from __future__ import annotations + +import re + + +_BLOCKED_GOOGLE_ANALYTICS_HOST_SUFFIXES = ( + "google-analytics.com", + "analytics.google.com", +) + +GOOGLE_ANALYTICS_URL_PATTERN = re.compile( + r"^https?://(?:[^/?#@]*@)?(?:[^./?#:@]+\.)*(?:" + + "|".join(re.escape(host) for host in _BLOCKED_GOOGLE_ANALYTICS_HOST_SUFFIXES) + + r")\.?(?::\d+)?(?:[/?#]|$)", + re.IGNORECASE, +)