Add prerender-django middleware for Django 5+

prerender-django/.env.example

@@ -0,0 +1,2 @@
+PRERENDER_TOKEN=
+PRERENDER_SERVICE_URL=

prerender-django/.gitignore

@@ -0,0 +1,7 @@
+.venv/
+__pycache__/
+*.pyc
+*.egg-info/
+dist/
+build/
+.env

prerender-django/README.md

@@ -0,0 +1,57 @@
+# prerender-django
+
+Django middleware for [Prerender.io](https://prerender.io). Intercepts requests from bots and crawlers and serves prerendered HTML, so your JavaScript-rendered app is fully indexable by search engines and social media scrapers.
+
+Compatible with **Django 5+** and **Python 3.10+**.
+
+## Installation
+
+```bash
+pip install prerender-django
+```
+
+## Setup
+
+Add the middleware to your `settings.py`:
+
+```python
+MIDDLEWARE = [
+    'prerender_django.middleware.PrerenderMiddleware',
+    # ... your other middleware
+]
+
+PRERENDER_TOKEN = 'YOUR_PRERENDER_TOKEN'
+```
+
+The middleware must be placed **before** any session or authentication middleware to intercept bot requests early.
+
+## Settings
+
+| Setting | Default | Description |
+|---------|---------|-------------|
+| `PRERENDER_TOKEN` | `None` | Your Prerender.io token |
+| `PRERENDER_SERVICE_URL` | `https://service.prerender.io/` | Prerender service URL (use this for self-hosted Prerender) |
+
+## Self-hosted Prerender
+
+```python
+PRERENDER_SERVICE_URL = 'http://your-prerender-server:3000'
+```
+
+## How it works
+
+Requests are prerendered when **all** of the following are true:
+
+- The HTTP method is `GET`
+- The `User-Agent` matches a known bot/crawler (Googlebot, Bingbot, Twitterbot, GPTBot, ClaudeBot, etc.)  
+  — OR the URL contains `_escaped_fragment_`  
+  — OR the `X-Bufferbot` header is present
+- The URL does not end with a static asset extension (`.js`, `.css`, `.png`, etc.)
+
+Everything else passes through to your normal Django views.
+
+If the Prerender service is unreachable, the middleware falls back gracefully and serves the normal response.
+
+## License
+
+MIT

prerender-django/prerender_django/middleware.py

@@ -0,0 +1,89 @@
+import logging
+import urllib.error
+import urllib.request
+
+from django.conf import settings
+from django.http import HttpResponse
+
+logger = logging.getLogger(__name__)
+
+CRAWLER_USER_AGENTS = [
+    'googlebot', 'yahoo', 'bingbot', 'baiduspider',
+    'facebookexternalhit', 'twitterbot', 'rogerbot', 'linkedinbot',
+    'embedly', 'quora link preview', 'showyoubot', 'outbrain',
+    'pinterest', 'slackbot', 'developers.google.com/+/web/snippet',
+    'w3c_validator', 'perplexity', 'oai-searchbot', 'chatgpt-user',
+    'gptbot', 'claudebot', 'amazonbot',
+]
+
+EXTENSIONS_TO_IGNORE = frozenset([
+    '.js', '.css', '.xml', '.less', '.png', '.jpg', '.jpeg', '.gif',
+    '.pdf', '.doc', '.txt', '.ico', '.rss', '.zip', '.mp3', '.rar',
+    '.exe', '.wmv', '.avi', '.ppt', '.mpg', '.mpeg', '.tif', '.wav',
+    '.mov', '.psd', '.ai', '.xls', '.mp4', '.m4a', '.swf', '.dat',
+    '.dmg', '.iso', '.flv', '.m4v', '.torrent', '.ttf', '.woff', '.svg',
+])
+
+
+def _setting(name, default=None):
+    return getattr(settings, f'PRERENDER_{name}', default)
+
+
+def _is_bot(user_agent):
+    ua = user_agent.lower()
+    return any(bot in ua for bot in CRAWLER_USER_AGENTS)
+
+
+def _is_static_asset(path):
+    return any(path.endswith(ext) for ext in EXTENSIONS_TO_IGNORE)
+
+
+def _should_prerender(request):
+    user_agent = request.META.get('HTTP_USER_AGENT', '')
+    if not user_agent or request.method != 'GET':
+        return False
+    if _is_static_asset(request.path):
+        return False
+    if '_escaped_fragment_' in request.GET:
+        return True
+    if request.META.get('HTTP_X_BUFFERBOT'):
+        return True
+    return _is_bot(user_agent)
+
+
+def _build_api_url(request):
+    service_url = _setting('SERVICE_URL', 'https://service.prerender.io/')
+    if not service_url.endswith('/'):
+        service_url += '/'
+    return f'{service_url}{request.build_absolute_uri()}'
+
+
+def _fetch_prerendered(api_url, user_agent):
+    token = _setting('TOKEN')
+    req = urllib.request.Request(api_url)
+    req.add_header('User-Agent', user_agent)
+    if token:
+        req.add_header('X-Prerender-Token', token)
+    try:
+        with urllib.request.urlopen(req) as resp:
+            return resp.status, resp.read().decode('utf-8')
+    except urllib.error.HTTPError as e:
+        return e.code, e.read().decode('utf-8')
+
+
+class PrerenderMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        if not _should_prerender(request):
+            return self.get_response(request)
+
+        try:
+            api_url = _build_api_url(request)
+            user_agent = request.META.get('HTTP_USER_AGENT', '')
+            status, body = _fetch_prerendered(api_url, user_agent)
+            return HttpResponse(body, status=status, content_type='text/html')
+        except urllib.error.URLError as e:
+            logger.error('Prerender error, falling back: %s', e)
+            return self.get_response(request)

prerender-django/pyproject.toml

@@ -0,0 +1,31 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "prerender-django"
+version = "1.0.1"
+description = "Django middleware for prerendering JavaScript-rendered pages for SEO via Prerender.io"
+authors = [{ name = "Prerender.io" }]
+license = "MIT"
+readme = "README.md"
+requires-python = ">=3.10"
+keywords = ["django", "prerender", "prerender.io", "seo", "middleware"]
+dependencies = []
+
+[project.urls]
+Repository = "https://github.com/prerender/integrations"
+
+[project.optional-dependencies]
+dev = [
+    "django>=5.0",
+    "pytest>=8.0",
+    "pytest-django>=4.8",
+]
+
+[tool.pytest.ini_options]
+DJANGO_SETTINGS_MODULE = "tests.settings"
+pythonpath = ["."]
+
+[tool.setuptools.packages.find]
+include = ["prerender_django*"]

prerender-django/tests/settings.py

@@ -0,0 +1,3 @@
+SECRET_KEY = 'test-secret-key'
+DATABASES = {}
+INSTALLED_APPS = []

prerender-django/tests/test_middleware.py

@@ -0,0 +1,69 @@
+import urllib.error
+from unittest.mock import MagicMock, patch
+
+from django.http import HttpResponse
+from django.test import RequestFactory
+
+from prerender_django.middleware import PrerenderMiddleware
+
+BOT_UA = 'Mozilla/5.0 (compatible; Googlebot/2.1)'
+BROWSER_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
+PRERENDERED_HTML = '<html><body>prerendered</body></html>'
+
+factory = RequestFactory()
+
+
+def normal_response(_request):
+    return HttpResponse('original')
+
+
+def mock_urlopen(status=200, body=PRERENDERED_HTML):
+    cm = MagicMock()
+    cm.__enter__ = MagicMock(return_value=cm)
+    cm.__exit__ = MagicMock(return_value=False)
+    cm.status = status
+    cm.read.return_value = body.encode('utf-8')
+    return cm
+
+
+def test_browser_passes_through():
+    middleware = PrerenderMiddleware(normal_response)
+    request = factory.get('/', HTTP_USER_AGENT=BROWSER_UA)
+    response = middleware(request)
+    assert response.status_code == 200
+    assert response.content == b'original'
+
+
+def test_bot_receives_prerendered_response():
+    middleware = PrerenderMiddleware(normal_response)
+    request = factory.get('/', HTTP_USER_AGENT=BOT_UA)
+    with patch('urllib.request.urlopen', return_value=mock_urlopen()):
+        response = middleware(request)
+    assert response.status_code == 200
+    assert PRERENDERED_HTML in response.content.decode()
+
+
+def test_static_asset_with_bot_ua_passes_through():
+    middleware = PrerenderMiddleware(normal_response)
+    request = factory.get('/style.css', HTTP_USER_AGENT=BOT_UA)
+    response = middleware(request)
+    assert response.status_code == 200
+    assert response.content == b'original'
+
+
+def test_escaped_fragment_triggers_prerender():
+    middleware = PrerenderMiddleware(normal_response)
+    request = factory.get('/', {'_escaped_fragment_': ''}, HTTP_USER_AGENT=BROWSER_UA)
+    with patch('urllib.request.urlopen', return_value=mock_urlopen()):
+        response = middleware(request)
+    assert response.status_code == 200
+    assert PRERENDERED_HTML in response.content.decode()
+
+
+def test_network_error_falls_back_to_normal_response():
+    middleware = PrerenderMiddleware(normal_response)
+    request = factory.get('/', HTTP_USER_AGENT=BOT_UA)
+    with patch('urllib.request.urlopen', side_effect=urllib.error.URLError('network error')):
+        response = middleware(request)
+    assert response.status_code == 200
+    assert response.content == b'original'