Switch Bedrock to using Django's own i18n mechanism

bedrock/base/i18n.py

@@ -0,0 +1,165 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+import django.urls
+from django.conf import settings
+from django.utils.text import capfirst
+from django.utils.translation.trans_real import parse_accept_lang_header
+
+from lib.l10n_utils import translation
+
+
+class LocalePrefixPattern(django.urls.LocalePrefixPattern):
+    """
+    A sub-class of Django's "LocalePrefixPattern" that simply normalizes the language
+    prefix for Bedrock (e.g., upper-case country codes).
+
+    This is an essential piece, since it controls the resolution of the
+    language-code prefix of incoming URLs as well as the language code prefix
+    of the URLs generated by Django's "reverse".
+    """
+
+    @property
+    def language_prefix(self):
+        language_code = normalize_language(translation.get_language()) or settings.LANGUAGE_CODE
+        if language_code == settings.LANGUAGE_CODE and not self.prefix_default_language:
+            return ""
+        return f"{language_code}/"
+
+
+def path_needs_lang_code(path):
+    """Convenient way to spot if a path needs a prefix or not"""
+    if path in settings.SUPPORTED_LOCALE_IGNORE:
+        return False
+
+    if path.lstrip("/").split("/")[0] in settings.SUPPORTED_NONLOCALES:
+        return False
+
+    return True
+
+
+def bedrock_i18n_patterns(*urls, prefix_default_language=True):
+    """
+    Similar to Django's i18n_patterns but uses our "LocalePrefixPattern", defined above
+    """
+    if not settings.USE_I18N:
+        return list(urls)
+    return [
+        django.urls.URLResolver(
+            LocalePrefixPattern(prefix_default_language=prefix_default_language),
+            list(urls),
+        )
+    ]
+
+
+def normalize_language(language):
+    """
+    Given a language code, returns the language code supported by Bedrock
+    in the proper case, for example "eN-us" --> "en-US" - or None if
+    Bedrock doesn't support the language code.
+
+    We also convert it to a specified fallback language if the actual
+    presented code is not available _and_ we have a fallback specced in
+    settings.
+
+    """
+
+    if not language:
+        return None
+
+    if settings.IS_POCKET_MODE:
+        lang_code = language.lower()
+    else:
+        lang_code = language
+
+    if lang_code in settings.LANGUAGE_URL_MAP_WITH_FALLBACKS:
+        return settings.LANGUAGE_URL_MAP_WITH_FALLBACKS[lang_code]
+
+    # Reformat the lang code to be mixed-case, as we expect
+    # them to be for our lookup
+    lang, _partition, classifier = language.partition("-")
+    lang = lang.lower()  # this part is _always_ lowercase
+
+    _special_cases = ["hant"]
+
+    if classifier:
+        if "-" in classifier or classifier in _special_cases:
+            _classifier, _partition, rest = classifier.partition("-")
+            if _classifier.lower() in _special_cases:
+                # support zh-hant-tw -> zh-Hant-TW
+                _classifier = capfirst(_classifier)
+                if rest:
+                    classifier = f"{_classifier}-{rest.upper()}"
+                else:
+                    classifier = _classifier
+            else:
+                # Support patterns like ja-JP-mac
+                classifier = f"{_classifier.upper()}-{rest}"
+        else:
+            classifier = classifier.upper()
+
+        lang_code = f"{lang}-{classifier}"
+    else:
+        lang_code = lang
+
+    try:
+        return settings.LANGUAGE_URL_MAP_WITH_FALLBACKS[lang_code]
+    except KeyError:
+        pre = lang_code.split("-")[0]
+        return settings.LANGUAGE_URL_MAP_WITH_FALLBACKS.get(pre)
+
+
+def split_path_and_normalize_language(path_):
+    """
+    Split the requested path into (lang, path) and
+    switches to a supported lang if available
+
+    locale will be empty if it isn't found.
+
+    Returns:
+        - lang code (str, may be empty)
+        - remaining URL path (str)
+        - whether or not the lang code changed (bool)
+
+    """
+    path = path_.lstrip("/")
+
+    # Use partition instead of split since it always returns 3 parts
+    extracted_lang, _, rest = path.partition("/")
+
+    supported_lang = normalize_language(extracted_lang)
+    different = extracted_lang != supported_lang
+
+    if extracted_lang and supported_lang:
+        return supported_lang, rest, different
+    else:
+        return "", path, False
+
+
+def get_language_from_headers(request):
+    """
+    Return a locale code we support on the site using the
+    user's Accept-Language header to determine which is best. This
+    mostly follows the RFCs but read bug 439568 for details.
+    """
+    accept_lang = request.headers.get("Accept-Language")
+    if accept_lang:
+        best = get_best_language(accept_lang)
+        if best:
+            return best
+
+    return settings.LANGUAGE_CODE
+
+
+def get_best_language(accept_lang):
+    """Given an Accept-Language header, return the best-matching language."""
+    ranked = parse_accept_lang_header(accept_lang)
+    for lang, _ in ranked:
+        supported = normalize_language(lang)
+        if supported:
+            return supported
+
+
+def check_for_bedrock_language(lang_code):
+    return lang_code in [x[0] for x in settings.LANGUAGES]

bedrock/base/middleware.py

@@ -10,48 +10,188 @@
 """
 
 import base64
+import contextlib
 import inspect
 import time
+from functools import wraps
 
 from django.conf import settings
 from django.core.exceptions import MiddlewareNotUsed
-from django.http import Http404, HttpResponse
+from django.http import Http404, HttpResponse, HttpResponseRedirect
+from django.middleware.locale import LocaleMiddleware as DjangoLocaleMiddleware
+from django.utils import translation
 from django.utils.deprecation import MiddlewareMixin
+from django.utils.translation import trans_real
 
 from commonware.middleware import FrameOptionsHeader as OldFrameOptionsHeader
 
 from bedrock.base import metrics
-from lib.l10n_utils import translation
+from bedrock.base.i18n import (
+    check_for_bedrock_language,
+    get_language_from_headers,
+    normalize_language,
+    path_needs_lang_code,
+    split_path_and_normalize_language,
+)
+from lib.l10n_utils import is_root_path_with_no_language_clues
 
-from . import urlresolvers
 
+class BedrockLangCodeFixupMiddleware(MiddlewareMixin):
+    """Middleware focused on prepping a viable, Bedrock-compatible language code
+    in the URL, ready for the rest of the i18n logic.
+
+    It:
+
+    1) Redirects to a new language-coded path if we detect the lang=XX querystring.
+    This is basically a no-JS way for us to handle language switching.
+
+    2) Normalises language codes that are in the path - eg en-us -> en-US and also
+    goes to a prefix code if we don't have support (eg de-AT -> de)
+
+    3) If no redirect is needed, sets request.locale to be the normalized
+    lang code we've got from the URL
+
+    Querystrings are preserved in GET redirects.
 
-class LocaleURLMiddleware:
     """
-    This middleware adjusts the `path_info` for reverse URL resolving.
 
-    We split `request.path_info` into `locale` and `path`. The `path` portion
-    is saved back to `request.path_info` for reverse URL resolving, while the
-    `locale` will either be one we support or empty string.
+    def _redirect(self, request, lang_code, subpath):
+        dest = f"/{lang_code}/{subpath}"
+
+        # log the redirect without querystrings, in case there's a token or similar we don't want to end up in Markus
+        metrics.incr("bedrock.langfixup.redirect", tags=[f"from:{request.path}", f"to:{dest}"])
+
+        if request.GET:
+            dest += f"?{request.GET.urlencode()}"
+
+        return HttpResponseRedirect(dest)
 
+    def process_request(self, request):
+        # Initially see if we can separate a lang code from the rest of the URL
+        # path, along with a hint about whether the lang code changed, which
+        # suggests we will need to redirect using `lang_code` as a new prefix.
+
+        lang_code, subpath, lang_code_changed = split_path_and_normalize_language(request.path)
+
+        # Check for any no-JS language - doing this eagerly is good, as any further
+        # fixups due to the actual lang code in the lang=XX querystring will be
+        # handled by the rest of this function when we come back to it.
+        # Also fixes https://github.com/mozilla/bedrock/issues/5931
+        lang_via_querystring = request.GET.get("lang", None)
+        if lang_via_querystring is not None:
+            cleaned_lang_via_querystring = normalize_language(lang_via_querystring)
+            # Drop the lang querystring to avoid a redirect loop;
+            # request.GET is immutable so we have to edit a copy
+            if not cleaned_lang_via_querystring:
+                cleaned_lang_via_querystring = settings.LANGUAGE_CODE
+            qs = request.GET.copy()
+            del qs["lang"]
+            request.GET = qs
+            return self._redirect(request, cleaned_lang_via_querystring, subpath)
+
+        # If we're not redirecting based on a querystring, do we need to find
+        # a language code? If so, find from the headers and redirect.
+        if not lang_code and path_needs_lang_code(request.path) and not is_root_path_with_no_language_clues(request):
+            lang_code = get_language_from_headers(request)
+            return self._redirect(request, lang_code, subpath)
+
+        # Or if we have a lang code, but it's not the one that was in the
+        # original URL path – e.g. fixing es-mx to es-MX - we also need to redirect
+        if lang_code and lang_code_changed:
+            return self._redirect(request, lang_code, subpath)
+
+        # If we get this far, the path contains a validly formatted lang code
+        # OR the path does not need a lang code. We annotate the request appropriately
+        request.locale = lang_code if lang_code else ""
+
+
+class BedrockLocaleMiddleware(DjangoLocaleMiddleware):
+    """Middleware that usually* wraps Django's own i18n middleware in order to
+    ensure we normalize language codes - i..e. we ensure they are in
+    mixed case we use, rather than Django's internal all-lowercase codes.
+
+    *for one specific situation, though, we skip Django's LocaleMiddleware entirely:
+    we have a special SEO-helping page that gets returned to robots/spiders that
+    don't declare an accept-language header, showing a list of locales to pick.
+
+    It needs to be kept super-light so that it doesn't diverge too far from
+    the stock LocaleMiddleware, lest we wake up dragons when we use
+    wagtail-localize, which squarely depends on django's LocaleMiddleware.
+
+    Note: this is not SUMO's LocaleMiddleware, this just a tribute.
+    (https://github.com/escattone/kitsune/blob/main/kitsune/sumo/middleware.py#L128)
     """
 
-    def __init__(self, get_response):
-        self.get_response = get_response
+    def process_request(self, request):
+        if is_root_path_with_no_language_clues(request):
+            # Skip using Django's LocaleMiddleware
+            metrics.incr(
+                "bedrock.localemiddleware.skipdjangolocalemiddleware",
+                tags=[f"path:{request.path}"],
+            )
+        else:
+            with normalized_get_language():
+                with simplified_check_for_language():
+                    return super().process_request(request)
 
-    def __call__(self, request):
-        response = self.process_request(request)
-        if response:
+    def process_response(self, request, response):
+        if is_root_path_with_no_language_clues(request):
+            # Skip using Django's LocaleMiddleware on the response cycle too
             return response
-        return self.get_response(request)
 
-    def process_request(self, request):
-        prefixer = urlresolvers.Prefixer(request)
-        urlresolvers.set_url_prefix(prefixer)
+        with normalized_get_language():
+            with simplified_check_for_language():
+                return super().process_response(request, response)
+
+
+@contextlib.contextmanager
+def normalized_get_language():
+    """
+    Ensures that any use of django.utils.translation.get_language()
+    within its context will return a normalized language code. This
+    context manager only works when the "get_language" function is
+    acquired from the "django.utils.translation" module at call time,
+    so for example, if it's called like "translation.get_language()".
+
+    Note that this does not cover every call to translation.get_language()
+    as there are calls to it on Django startup and more, but this gives us an
+    extra layer of (idempotent) fixing up within the request/response cycle.
+    """
+    get_language = translation.get_language
+
+    @wraps(get_language)
+    def get_normalized_language():
+        return normalize_language(get_language())
+
+    translation.get_language = get_normalized_language
+
+    try:
+        yield
+    finally:
+        translation.get_language = get_language
+
+
+@contextlib.contextmanager
+def simplified_check_for_language():
+    """Ensures that calls to trans_real.check_for_language within
+    its context will not check for the existence of files in
+    `appname/LANG_CODE/locale` and therefore avoid a false negative about
+    the langs we support (our lang files are in ./data/l10n-team/LANG_CODE)
+    but check_for_language is opinionated and expects only the Django directory
+    pattern for lang files."""
+
+    check_for_language = trans_real.check_for_language
+
+    @wraps(check_for_language)
+    def simpler_check_for_language(lang_code):
+        return check_for_bedrock_language(lang_code)
+
+    trans_real.check_for_language = simpler_check_for_language
 
-        request.path_info = f"/{prefixer.shortened_path}"
-        request.locale = prefixer.locale
-        translation.activate(prefixer.locale or settings.LANGUAGE_CODE)
+    try:
+        yield
+    finally:
+        trans_real.check_for_language = check_for_language
 
 
 class BasicAuthMiddleware: