Merge pull request #6406 from freedomofpress/6366-check-usable-languages

feat: validate `SDConfig.SUPPORTED_LANGUAGES` for usable locales

securedrop/i18n.py

@@ -17,7 +17,7 @@
 #
 import collections
 
-from typing import Dict, List
+from typing import List, Set
 
 from babel.core import (
     Locale,
@@ -29,7 +29,7 @@
 from flask import Flask, g, request, session
 from flask_babel import Babel
 
-from sdconfig import SDConfig
+from sdconfig import SDConfig, FALLBACK_LOCALE
 
 
 class RequestLocaleInfo:
@@ -126,32 +126,46 @@ def configure_babel(config: SDConfig, app: Flask) -> Babel:
     return babel
 
 
+def parse_locale_set(codes: List[str]) -> Set[Locale]:
+    return {Locale.parse(code) for code in codes}
+
+
 def validate_locale_configuration(config: SDConfig, babel: Babel) -> None:
     """
-    Ensure that the configured locales are valid and translated.
+    Check that configured locales are available in the filesystem and therefore usable by
+    Babel.  Warn about configured locales that are not usable, unless we're left with
+    no usable default or fallback locale, in which case raise an exception.
     """
-    if config.DEFAULT_LOCALE not in config.SUPPORTED_LOCALES:
-        raise ValueError(
-            'The default locale "{}" is not included in the set of supported locales "{}"'.format(
-                config.DEFAULT_LOCALE, config.SUPPORTED_LOCALES
-            )
+    # These locales are available and loadable from the filesystem.
+    available = set(babel.list_translations())
+    available.add(Locale.parse(FALLBACK_LOCALE))
+
+    # These locales were configured via "securedrop-admin sdconfig", meaning
+    # they were present on the Admin Workstation at "securedrop-admin" runtime.
+    configured = parse_locale_set(config.SUPPORTED_LOCALES)
+
+    # The intersection of these sets is the set of locales usable by Babel.
+    usable = available & configured
+
+    missing = configured - usable
+    if missing:
+        babel.app.logger.error(
+            f'Configured locales {missing} are not in the set of usable locales {usable}'
         )
 
-    translations = babel.list_translations()
-    for locale in config.SUPPORTED_LOCALES:
-        if locale == "en_US":
-            continue
+    defaults = parse_locale_set([config.DEFAULT_LOCALE, FALLBACK_LOCALE])
+    if not defaults & usable:
+        raise ValueError(
+            f'None of the default locales {defaults} are in the set of usable locales {usable}'
+        )
 
-        parsed = Locale.parse(locale)
-        if parsed not in translations:
-            raise ValueError(
-                'Configured locale "{}" is not in the set of translated locales "{}"'.format(
-                    parsed, translations
-                )
-            )
+    global USABLE_LOCALES
+    USABLE_LOCALES = usable
 
 
+# TODO(#6420): avoid relying on and manipulating on this global state
 LOCALES = collections.OrderedDict()  # type: collections.OrderedDict[str, RequestLocaleInfo]
+USABLE_LOCALES = set()  # type: Set[Locale]
 
 
 def map_locale_display_names(config: SDConfig) -> None:
@@ -163,16 +177,19 @@ def map_locale_display_names(config: SDConfig) -> None:
     to distinguish them. For languages with more than one translation,
     like Chinese, we do need the additional detail.
     """
-    language_locale_counts = collections.defaultdict(int)  # type: Dict[str, int]
-    for l in sorted(config.SUPPORTED_LOCALES):
-        locale = RequestLocaleInfo(l)
-        language_locale_counts[locale.language] += 1
-
+    seen: Set[str] = set()
     locale_map = collections.OrderedDict()
     for l in sorted(config.SUPPORTED_LOCALES):
+        if Locale.parse(l) not in USABLE_LOCALES:
+            continue
+
         locale = RequestLocaleInfo(l)
-        if language_locale_counts[locale.language] > 1:
+        if locale.language in seen:
+            # Disambiguate translations for this language.
             locale.use_display_name = True
+        else:
+            seen.add(locale.language)
+
         locale_map[str(locale)] = locale
 
     global LOCALES
@@ -193,23 +210,24 @@ def get_locale(config: SDConfig) -> str:
     - l request argument or session['locale']
     - browser suggested locale, from the Accept-Languages header
     - config.DEFAULT_LOCALE
+    - config.FALLBACK_LOCALE
     """
-    # Default to any locale set in the session.
-    locale = session.get("locale")
-
-    # A valid locale specified in request.args takes precedence.
+    preferences = []
+    if session.get("locale"):
+        preferences.append(session.get("locale"))
     if request.args.get("l"):
-        negotiated = negotiate_locale([request.args["l"]], LOCALES.keys())
-        if negotiated:
-            locale = negotiated
+        preferences.insert(0, request.args.get("l"))
+    if not preferences:
+        preferences.extend(get_accepted_languages())
+    preferences.append(config.DEFAULT_LOCALE)
+    preferences.append(FALLBACK_LOCALE)
+
+    negotiated = negotiate_locale(preferences, LOCALES.keys())
 
-    # If the locale is not in the session or request.args, negotiate
-    # the best supported option from the browser's accepted languages.
-    if not locale:
-        locale = negotiate_locale(get_accepted_languages(), LOCALES.keys())
+    if not negotiated:
+        raise ValueError("No usable locale")
 
-    # Finally, fall back to the default locale if necessary.
-    return locale or config.DEFAULT_LOCALE
+    return negotiated
 
 
 def get_accepted_languages() -> List[str]:

securedrop/sdconfig.py

@@ -8,6 +8,9 @@
 from typing import Set
 
 
+FALLBACK_LOCALE = "en_US"
+
+
 class SDConfig:
     def __init__(self) -> None:
         try:
@@ -120,7 +123,7 @@ def __init__(self) -> None:
         # Config entries used by i18n.py
         # Use en_US as the default locale if the key is not defined in _config
         self.DEFAULT_LOCALE = getattr(
-            _config, "DEFAULT_LOCALE", "en_US"
+            _config, "DEFAULT_LOCALE", FALLBACK_LOCALE,
         )  # type: str
         supported_locales = set(getattr(
             _config, "SUPPORTED_LOCALES", [self.DEFAULT_LOCALE]

securedrop/tests/test_i18n.py

@@ -32,13 +32,17 @@
 from flask import request
 from flask import session
 from flask_babel import gettext
-from sdconfig import SDConfig
+from i18n import parse_locale_set
+from sdconfig import FALLBACK_LOCALE, SDConfig
 from sh import pybabel
 from sh import sed
 from .utils.env import TESTS_DIR
 from werkzeug.datastructures import Headers
 
 
+NEVER_LOCALE = 'eo'  # Esperanto
+
+
 def verify_i18n(app):
     not_translated = 'code hello i18n'
     translated_fr = 'code bonjour'
@@ -225,32 +229,77 @@ def test_i18n(journalist_app, config):
         verify_i18n(app)
 
 
-def test_supported_locales(config):
-    fake_config = SDConfig()
+def test_parse_locale_set():
+    assert parse_locale_set([FALLBACK_LOCALE]) == set([Locale.parse(FALLBACK_LOCALE)])
+
 
-    # Check that an invalid locale raises an error during app
-    # configuration.
-    fake_config.SUPPORTED_LOCALES = ['en_US', 'yy_ZZ']
+def test_no_usable_fallback_locale(journalist_app, config):
+    """
+    The apps fail if neither the default nor the fallback locale is usable.
+    """
+    fake_config = SDConfig()
+    fake_config.DEFAULT_LOCALE = NEVER_LOCALE
+    fake_config.SUPPORTED_LOCALES = [NEVER_LOCALE]
     fake_config.TRANSLATION_DIRS = Path(config.TEMP_DIR)
 
-    with pytest.raises(UnknownLocaleError):
+    i18n.USABLE_LOCALES = set()
+
+    with pytest.raises(ValueError, match='in the set of usable locales'):
         journalist_app_module.create_app(fake_config)
 
-    with pytest.raises(UnknownLocaleError):
+    with pytest.raises(ValueError, match='in the set of usable locales'):
         source_app.create_app(fake_config)
 
-    # Check that a valid but unsupported locale raises an error during
-    # app configuration.
-    fake_config.SUPPORTED_LOCALES = ['en_US', 'wae_CH']
+
+def test_unusable_default_but_usable_fallback_locale(config, caplog):
+    """
+    The apps start even if the default locale is unusable, as along as the fallback locale is
+    usable, but log an error for OSSEC to pick up.
+    """
+    fake_config = SDConfig()
+    fake_config.DEFAULT_LOCALE = NEVER_LOCALE
+    fake_config.SUPPORTED_LOCALES = [NEVER_LOCALE, FALLBACK_LOCALE]
     fake_config.TRANSLATION_DIRS = Path(config.TEMP_DIR)
 
-    with pytest.raises(ValueError, match="not in the set of translated locales"):
+    for app in (journalist_app_module.create_app(fake_config),
+                source_app.create_app(fake_config)):
+        with app.app_context():
+            assert NEVER_LOCALE in caplog.text
+            assert 'not in the set of usable locales' in caplog.text
+
+
+def test_invalid_locales(config):
+    """
+    An invalid locale raises an error during app configuration.
+    """
+    fake_config = SDConfig()
+    fake_config.SUPPORTED_LOCALES = [FALLBACK_LOCALE, 'yy_ZZ']
+    fake_config.TRANSLATION_DIRS = Path(config.TEMP_DIR)
+
+    with pytest.raises(UnknownLocaleError):
         journalist_app_module.create_app(fake_config)
 
-    with pytest.raises(ValueError, match="not in the set of translated locales"):
+    with pytest.raises(UnknownLocaleError):
         source_app.create_app(fake_config)
 
 
+def test_valid_but_unusable_locales(config, caplog):
+    """
+    The apps start with one or more unusable, but still valid, locales, but log an error for
+    OSSEC to pick up.
+    """
+    fake_config = SDConfig()
+
+    fake_config.SUPPORTED_LOCALES = [FALLBACK_LOCALE, 'wae_CH']
+    fake_config.TRANSLATION_DIRS = Path(config.TEMP_DIR)
+
+    for app in (journalist_app_module.create_app(fake_config),
+                source_app.create_app(fake_config)):
+        with app.app_context():
+            assert 'wae' in caplog.text
+            assert 'not in the set of usable locales' in caplog.text
+
+
 def test_language_tags():
     assert i18n.RequestLocaleInfo(Locale.parse('en')).language_tag == 'en'
     assert i18n.RequestLocaleInfo(Locale.parse('en-US', sep="-")).language_tag == 'en-US'