From a500ae04efac0db8c205504b22ef20d393f4038e Mon Sep 17 00:00:00 2001 From: Laszlo Takacs Date: Fri, 17 Apr 2026 09:48:33 +0200 Subject: [PATCH 1/3] Add prerender-django middleware for Django 5+ --- prerender-django/.env.example | 2 + prerender-django/.gitignore | 7 ++ prerender-django/README.md | 57 ++++++++++++ prerender-django/prerender_django/__init__.py | 0 .../prerender_django/middleware.py | 89 +++++++++++++++++++ prerender-django/pyproject.toml | 30 +++++++ prerender-django/tests/__init__.py | 0 prerender-django/tests/settings.py | 3 + prerender-django/tests/test_middleware.py | 69 ++++++++++++++ 9 files changed, 257 insertions(+) create mode 100644 prerender-django/.env.example create mode 100644 prerender-django/.gitignore create mode 100644 prerender-django/README.md create mode 100644 prerender-django/prerender_django/__init__.py create mode 100644 prerender-django/prerender_django/middleware.py create mode 100644 prerender-django/pyproject.toml create mode 100644 prerender-django/tests/__init__.py create mode 100644 prerender-django/tests/settings.py create mode 100644 prerender-django/tests/test_middleware.py diff --git a/prerender-django/.env.example b/prerender-django/.env.example new file mode 100644 index 0000000..07bced2 --- /dev/null +++ b/prerender-django/.env.example @@ -0,0 +1,2 @@ +PRERENDER_TOKEN= +PRERENDER_SERVICE_URL= diff --git a/prerender-django/.gitignore b/prerender-django/.gitignore new file mode 100644 index 0000000..3c9f148 --- /dev/null +++ b/prerender-django/.gitignore @@ -0,0 +1,7 @@ +.venv/ +__pycache__/ +*.pyc +*.egg-info/ +dist/ +build/ +.env diff --git a/prerender-django/README.md b/prerender-django/README.md new file mode 100644 index 0000000..9d4d1ea --- /dev/null +++ b/prerender-django/README.md @@ -0,0 +1,57 @@ +# prerender-django + +Django middleware for [Prerender.io](https://prerender.io). Intercepts requests from bots and crawlers and serves prerendered HTML, so your JavaScript-rendered app is fully indexable by search engines and social media scrapers. + +Compatible with **Django 5+** and **Python 3.10+**. + +## Installation + +```bash +pip install prerender-django +``` + +## Setup + +Add the middleware to your `settings.py`: + +```python +MIDDLEWARE = [ + 'prerender_django.middleware.PrerenderMiddleware', + # ... your other middleware +] + +PRERENDER_TOKEN = 'YOUR_PRERENDER_TOKEN' +``` + +The middleware must be placed **before** any session or authentication middleware to intercept bot requests early. + +## Settings + +| Setting | Default | Description | +|---------|---------|-------------| +| `PRERENDER_TOKEN` | `None` | Your Prerender.io token | +| `PRERENDER_SERVICE_URL` | `https://service.prerender.io/` | Prerender service URL (use this for self-hosted Prerender) | + +## Self-hosted Prerender + +```python +PRERENDER_SERVICE_URL = 'http://your-prerender-server:3000' +``` + +## How it works + +Requests are prerendered when **all** of the following are true: + +- The HTTP method is `GET` +- The `User-Agent` matches a known bot/crawler (Googlebot, Bingbot, Twitterbot, GPTBot, ClaudeBot, etc.) + — OR the URL contains `_escaped_fragment_` + — OR the `X-Bufferbot` header is present +- The URL does not end with a static asset extension (`.js`, `.css`, `.png`, etc.) + +Everything else passes through to your normal Django views. + +If the Prerender service is unreachable, the middleware falls back gracefully and serves the normal response. + +## License + +MIT diff --git a/prerender-django/prerender_django/__init__.py b/prerender-django/prerender_django/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prerender-django/prerender_django/middleware.py b/prerender-django/prerender_django/middleware.py new file mode 100644 index 0000000..a982f1c --- /dev/null +++ b/prerender-django/prerender_django/middleware.py @@ -0,0 +1,89 @@ +import logging +import urllib.error +import urllib.request + +from django.conf import settings +from django.http import HttpResponse + +logger = logging.getLogger(__name__) + +CRAWLER_USER_AGENTS = [ + 'googlebot', 'yahoo', 'bingbot', 'baiduspider', + 'facebookexternalhit', 'twitterbot', 'rogerbot', 'linkedinbot', + 'embedly', 'quora link preview', 'showyoubot', 'outbrain', + 'pinterest', 'slackbot', 'developers.google.com/+/web/snippet', + 'w3c_validator', 'perplexity', 'oai-searchbot', 'chatgpt-user', + 'gptbot', 'claudebot', 'amazonbot', +] + +EXTENSIONS_TO_IGNORE = frozenset([ + '.js', '.css', '.xml', '.less', '.png', '.jpg', '.jpeg', '.gif', + '.pdf', '.doc', '.txt', '.ico', '.rss', '.zip', '.mp3', '.rar', + '.exe', '.wmv', '.avi', '.ppt', '.mpg', '.mpeg', '.tif', '.wav', + '.mov', '.psd', '.ai', '.xls', '.mp4', '.m4a', '.swf', '.dat', + '.dmg', '.iso', '.flv', '.m4v', '.torrent', '.ttf', '.woff', '.svg', +]) + + +def _setting(name, default=None): + return getattr(settings, f'PRERENDER_{name}', default) + + +def _is_bot(user_agent): + ua = user_agent.lower() + return any(bot in ua for bot in CRAWLER_USER_AGENTS) + + +def _is_static_asset(path): + return any(path.endswith(ext) for ext in EXTENSIONS_TO_IGNORE) + + +def _should_prerender(request): + user_agent = request.META.get('HTTP_USER_AGENT', '') + if not user_agent or request.method != 'GET': + return False + if _is_static_asset(request.path): + return False + if '_escaped_fragment_' in request.GET: + return True + if request.META.get('HTTP_X_BUFFERBOT'): + return True + return _is_bot(user_agent) + + +def _build_api_url(request): + service_url = _setting('SERVICE_URL', 'https://service.prerender.io/') + if not service_url.endswith('/'): + service_url += '/' + return f'{service_url}{request.build_absolute_uri()}' + + +def _fetch_prerendered(api_url, user_agent): + token = _setting('TOKEN') + req = urllib.request.Request(api_url) + req.add_header('User-Agent', user_agent) + if token: + req.add_header('X-Prerender-Token', token) + try: + with urllib.request.urlopen(req) as resp: + return resp.status, resp.read().decode('utf-8') + except urllib.error.HTTPError as e: + return e.code, e.read().decode('utf-8') + + +class PrerenderMiddleware: + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + if not _should_prerender(request): + return self.get_response(request) + + try: + api_url = _build_api_url(request) + user_agent = request.META.get('HTTP_USER_AGENT', '') + status, body = _fetch_prerendered(api_url, user_agent) + return HttpResponse(body, status=status, content_type='text/html') + except urllib.error.URLError as e: + logger.error('Prerender error, falling back: %s', e) + return self.get_response(request) diff --git a/prerender-django/pyproject.toml b/prerender-django/pyproject.toml new file mode 100644 index 0000000..3bab1aa --- /dev/null +++ b/prerender-django/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "prerender-django" +version = "1.0.0" +description = "Django middleware for prerendering JavaScript-rendered pages for SEO via Prerender.io" +authors = [{ name = "Prerender.io" }] +license = { text = "MIT" } +requires-python = ">=3.10" +keywords = ["django", "prerender", "prerender.io", "seo", "middleware"] +dependencies = [] + +[project.urls] +Repository = "https://github.com/prerender/community-integrations" + +[project.optional-dependencies] +dev = [ + "django>=5.0", + "pytest>=8.0", + "pytest-django>=4.8", +] + +[tool.pytest.ini_options] +DJANGO_SETTINGS_MODULE = "tests.settings" +pythonpath = ["."] + +[tool.setuptools.packages.find] +include = ["prerender_django*"] diff --git a/prerender-django/tests/__init__.py b/prerender-django/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prerender-django/tests/settings.py b/prerender-django/tests/settings.py new file mode 100644 index 0000000..07608ba --- /dev/null +++ b/prerender-django/tests/settings.py @@ -0,0 +1,3 @@ +SECRET_KEY = 'test-secret-key' +DATABASES = {} +INSTALLED_APPS = [] diff --git a/prerender-django/tests/test_middleware.py b/prerender-django/tests/test_middleware.py new file mode 100644 index 0000000..d78b9f5 --- /dev/null +++ b/prerender-django/tests/test_middleware.py @@ -0,0 +1,69 @@ +import urllib.error +from unittest.mock import MagicMock, patch + +from django.http import HttpResponse +from django.test import RequestFactory + +from prerender_django.middleware import PrerenderMiddleware + +BOT_UA = 'Mozilla/5.0 (compatible; Googlebot/2.1)' +BROWSER_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36' +PRERENDERED_HTML = 'prerendered' + +factory = RequestFactory() + + +def normal_response(_request): + return HttpResponse('original') + + +def mock_urlopen(status=200, body=PRERENDERED_HTML): + cm = MagicMock() + cm.__enter__ = MagicMock(return_value=cm) + cm.__exit__ = MagicMock(return_value=False) + cm.status = status + cm.read.return_value = body.encode('utf-8') + return cm + + +def test_browser_passes_through(): + middleware = PrerenderMiddleware(normal_response) + request = factory.get('/', HTTP_USER_AGENT=BROWSER_UA) + response = middleware(request) + assert response.status_code == 200 + assert response.content == b'original' + + +def test_bot_receives_prerendered_response(): + middleware = PrerenderMiddleware(normal_response) + request = factory.get('/', HTTP_USER_AGENT=BOT_UA) + with patch('urllib.request.urlopen', return_value=mock_urlopen()): + response = middleware(request) + assert response.status_code == 200 + assert PRERENDERED_HTML in response.content.decode() + + +def test_static_asset_with_bot_ua_passes_through(): + middleware = PrerenderMiddleware(normal_response) + request = factory.get('/style.css', HTTP_USER_AGENT=BOT_UA) + response = middleware(request) + assert response.status_code == 200 + assert response.content == b'original' + + +def test_escaped_fragment_triggers_prerender(): + middleware = PrerenderMiddleware(normal_response) + request = factory.get('/', {'_escaped_fragment_': ''}, HTTP_USER_AGENT=BROWSER_UA) + with patch('urllib.request.urlopen', return_value=mock_urlopen()): + response = middleware(request) + assert response.status_code == 200 + assert PRERENDERED_HTML in response.content.decode() + + +def test_network_error_falls_back_to_normal_response(): + middleware = PrerenderMiddleware(normal_response) + request = factory.get('/', HTTP_USER_AGENT=BOT_UA) + with patch('urllib.request.urlopen', side_effect=urllib.error.URLError('network error')): + response = middleware(request) + assert response.status_code == 200 + assert response.content == b'original' From fcbe508c00265c2700482e4e539ef36bd5249a72 Mon Sep 17 00:00:00 2001 From: Laszlo Takacs Date: Fri, 17 Apr 2026 10:38:20 +0200 Subject: [PATCH 2/3] Fix repository URL to point to integrations repo --- prerender-django/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prerender-django/pyproject.toml b/prerender-django/pyproject.toml index 3bab1aa..1895f9d 100644 --- a/prerender-django/pyproject.toml +++ b/prerender-django/pyproject.toml @@ -13,7 +13,7 @@ keywords = ["django", "prerender", "prerender.io", "seo", "middleware"] dependencies = [] [project.urls] -Repository = "https://github.com/prerender/community-integrations" +Repository = "https://github.com/prerender/integrations" [project.optional-dependencies] dev = [ From 2aa545297e93e828f3f86a89b8114154a376a066 Mon Sep 17 00:00:00 2001 From: Laszlo Takacs Date: Fri, 17 Apr 2026 17:31:03 +0200 Subject: [PATCH 3/3] Bump prerender-django to v1.0.1, add readme field, normalize license format --- prerender-django/pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/prerender-django/pyproject.toml b/prerender-django/pyproject.toml index 1895f9d..30a979d 100644 --- a/prerender-django/pyproject.toml +++ b/prerender-django/pyproject.toml @@ -4,10 +4,11 @@ build-backend = "setuptools.build_meta" [project] name = "prerender-django" -version = "1.0.0" +version = "1.0.1" description = "Django middleware for prerendering JavaScript-rendered pages for SEO via Prerender.io" authors = [{ name = "Prerender.io" }] -license = { text = "MIT" } +license = "MIT" +readme = "README.md" requires-python = ">=3.10" keywords = ["django", "prerender", "prerender.io", "seo", "middleware"] dependencies = []