Merge pull request #1 from django-ftl/fluent-compiler-next

Use fluent_compiler 0.2

CONTRIBUTING.rst

@@ -86,6 +86,11 @@ Ready to contribute? Here's how to set up ``django-ftl`` for local development.
         $ pip install tox
         $ tox
 
+    You can also run tests with py.test (although it is much slower for some reason)::
+
+        $ pip install pytest
+        $ py.test
+
 6. Commit your changes and push your branch to GitHub::
 
     $ git add .

HISTORY.rst

@@ -3,6 +3,17 @@
 History
 -------
 
+0.12 (under development)
+++++++++++++++++++++++++
+
+* Switch to the new APIs available in ``fluent_compiler`` 0.2.
+* Performance improvements - large reduction in the percentage overhead
+  introduced by django-ftl (compared to raw ``fluent_compiler`` performance).
+* Undocumented ``MessageFinderBase`` class has changed slightly: its ``load``
+  method now returns a ``fluent_compiler.resource.FtlResource`` object instead
+  of a string. If you used a custom ``finder`` for ``Bundle`` you may need to
+  update it for this change.
+
 0.11 (2020-02-24)
 +++++++++++++++++
 

MANIFEST.in

@@ -1,7 +1,7 @@
 include *.rst
 include *.py
 include LICENSE
-include tox.ini
+include *.ini
 include release.sh
 include requirements*.txt
 include .coveragerc

docs/usage.rst

@@ -303,7 +303,7 @@ could be done this way:
            help_text=ftl_bundle.format_lazy('kitten-name.help-text'))
 
 
-.. code-block:: ftl
+::
 
    # kittens.ftl
 

pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+DJANGO_SETTINGS_MODULE = tests.settings

requirements-test.txt

@@ -6,3 +6,4 @@ flake8-future-import>=0.4.5
 django-functest==1.0.4
 check-manifest
 isort
+testfixtures

setup.cfg

@@ -25,6 +25,6 @@ max-line-length = 119
 
 [isort]
 line_length=119
-known_third_party=fluent,six,django_functest,django,pyinotify
+known_third_party=fluent,six,django_functest,django,pyinotify,fluent_compiler
 known_first_party=django_ftl
 skip = docs,.tox,.eggs

src/django_ftl/bundles.py

@@ -6,11 +6,13 @@
 from threading import Lock, local
 
 import six
+from babel.core import UnknownLocaleError
 from django.conf import settings
 from django.utils.functional import cached_property, lazy
 from django.utils.html import conditional_escape as conditional_html_escape
 from django.utils.html import mark_safe as mark_html_escaped
-from fluent_compiler import FluentBundle
+from fluent_compiler.compiler import compile_messages
+from fluent_compiler.resource import FtlResource
 
 from .conf import get_setting
 from .utils import make_namespace
@@ -72,13 +74,11 @@ def load(self, locale, path, reloader=None):
                 continue
 
             full_path = os.path.join(locale_dir, path)
-            try:
-                with open(full_path, "rb") as f:
-                    if reloader is not None:
-                        reloader.add_watched_path(full_path)
-                    return f.read().decode('utf-8')
-
-            except (IOError, OSError):
+            if os.path.exists(full_path):
+                if reloader is not None:
+                    reloader.add_watched_path(full_path)
+                return FtlResource.from_file(full_path)
+            else:
                 tried.append(full_path)
 
         raise FileNotFoundError("Could not find locate FTL file {0}/{1}. Tried: {2}"
@@ -175,14 +175,10 @@ def __init__(self, paths,
             self._reloader = None
         self.reload()
 
-    def _make_fluent_bundle(self, locale):
-        return FluentBundle([locale], use_isolating=self._use_isolating,
-                            escapers=[html_escaper])
-
     def reload(self):
         with self._lock:
-            self._all_fluent_bundles = {}  # dict from available locale to FluentBundle
-            self._fluent_bundles_for_current_locale = {}  # dict from chosen locale to list of FluentBundle
+            self._message_function_cache = {}
+            self._compiled_unit_for_locale = {}
 
     def _get_default_locale(self):
         default_locale = self._default_locale
@@ -191,86 +187,112 @@ def _get_default_locale(self):
 
         return get_setting('LANGUAGE_CODE')
 
-    def get_fluent_bundles_for_current_locale(self):
-        current_locale = activator.get_current_value()
-        if current_locale is None:
-            if self._require_activate:
-                raise NoLocaleSet("activate() must be used before using Bundle.format "
-                                  "- or, use Bundle(require_activate=False)")
+    def get_compiled_unit_for_locale_list(self, locales):
+        for locale in locales:
+            try:
+                unit = self.get_compiled_unit_for_locale(locale)
+            except UnknownLocaleError:
+                continue
+            yield unit
+
+    def get_compiled_unit_for_locale(self, locale):
         try:
-            return self._fluent_bundles_for_current_locale[current_locale]
+            return self._compiled_unit_for_locale[locale]
         except KeyError:
             pass
 
-        # Fill out _fluent_bundles_for_current_locale and _all_fluent_bundles as
+        # Fill out _compiled_unit_for_locale
         # necessary, but do this synchronized for all threads.
         with self._lock:
             # Double checked locking pattern.
             try:
-                return self._fluent_bundles_for_current_locale[current_locale]
+                return self._compiled_unit_for_locale[locale]
             except KeyError:
                 pass
 
-            to_try = list(locale_lookups(current_locale)) if current_locale is not None else []
+            # Do the compilation:
+            resources = []
+            for path in self._paths:
+                try:
+                    resource = self._finder.load(locale, path, reloader=self._reloader)
+                except FileNotFoundError:
+                    pass
+                    if locale == self._get_default_locale():
+                        # Can't find any FTL with the specified filename, we
+                        # want to bail early and alert developer.
+                        raise
+                    # Allow missing files otherwise
+                else:
+                    resources.append(resource)
+
+            unit = compile_messages(
+                locale,
+                resources,
+                use_isolating=self._use_isolating,
+                escapers=[html_escaper]
+            )
+            errors = unit.errors
+            for msg_id, error in errors:
+                self._log_error(locale, msg_id, {}, error)
+            self._compiled_unit_for_locale[locale] = unit
+            return unit
+
+    def format(self, message_id, args=None):
+        # This is the hot path for performance, so we try to optimise,
+        # especially the 'happy path' which will hit caches.
+
+        # FAST PATH:
+        # Avoid Activator.get_current_value() here because it adds measurable
+        # overhead.
+        current_locale = getattr(_active_locale, 'value', None)
+        errors = []
+        try:
+            func = self._message_function_cache[current_locale, message_id]
+        except KeyError:
+            # SLOW PATH:
+            if current_locale is None:
+                if self._require_activate:
+                    raise NoLocaleSet("activate() must be used before using Bundle.format "
+                                      "- or, use Bundle(require_activate=False)")
+
+            # current_locale can be `None`, and we will create cache entries
+            # against (None, message_id). This gives us small performance
+            # improvement by moving `if current_locale is None` check out of the
+            # hot path.
+            locale_to_use = current_locale or self._get_default_locale()
+            to_try = list(locale_lookups(locale_to_use))
             default_locale = self._get_default_locale()
             if default_locale is not None:
                 to_try = uniquify(to_try + [default_locale])
 
-            bundles = []
-            for i, locale in enumerate(to_try):
+            func = None
+            for unit in self.get_compiled_unit_for_locale_list(to_try):
                 try:
-                    bundle = self._all_fluent_bundles[locale]
-                    if bundle is not None:
-                        bundles.append(bundle)
-                except KeyError:
-                    # Create the FluentBundle on demand
-                    bundle = self._make_fluent_bundle(locale)
-
-                    for path in self._paths:
-                        try:
-                            contents = self._finder.load(locale, path, reloader=self._reloader)
-                        except FileNotFoundError:
-                            if locale == default_locale:
-                                # Can't find any FTL with the specified filename, we
-                                # want to bail early and alert developer.
-                                raise
-                            # Allow missing files otherwise
-                        else:
-                            bundle.add_messages(contents)
-                    errors = bundle.check_messages()
-                    for msg_id, error in errors:
-                        self._log_error(bundle, msg_id, {}, error)
-                    bundles.append(bundle)
-                    self._all_fluent_bundles[locale] = bundle
-
-            # Shortcut next time
-            self._fluent_bundles_for_current_locale[current_locale] = bundles
-            return bundles
-
-    def format(self, message_id, args=None):
-        fluent_bundles = self.get_fluent_bundles_for_current_locale()
-        for i, bundle in enumerate(fluent_bundles):
-            try:
-                value, errors = bundle.format(message_id, args=args)
-                for e in errors:
-                    self._log_error(bundle, message_id, args, e)
-                return value
-            except LookupError as e:
-                self._log_error(bundle, message_id, args, e)
-
-        # None were successful, return default
-        return '???'
+                    func = unit.message_functions[message_id]
+                    break
+                except LookupError as e:
+                    self._log_error(unit.locale, message_id, args, e)
+                    continue
+            if func is None:
+                func = _missing_message
+            self._message_function_cache[current_locale, message_id] = func
+            # SLOW PATH END
+
+        value = func(args, errors)
+        if errors:
+            for e in errors:
+                self._log_error(current_locale, message_id, args, e)
+        return value
 
     format_lazy = lazy(format, text_type)
 
     def _log_error(self,
-                   bundle,
+                   locale,
                    message_id,
                    args,
                    exception):
-        # TODO - nicer error that includes path and source line
         ftl_logger.error("FTL exception for locale [%s], message '%s', args %r: %s",
-                         ", ".join(bundle.locales),
+                         locale,
                          message_id,
                          args,
                          repr(exception))
@@ -287,6 +309,10 @@ def check_all(self, locales):
         return errors
 
 
+def _missing_message(args, errors):
+    return '???'
+
+
 def locale_lookups(locale):
     """
     Utility for implementing RFC 4647 lookup algorithm

tests/benchmarks/README.md

@@ -0,0 +1,11 @@
+To run the benchmarks, do the following (assuming cwd at top level):
+
+    $ pip install -r tests/benchmarks/requirements.txt
+
+Then, run the benchmarks as a script:
+
+    $ ./tests/benchmarks/benchmarks.py
+
+You can also run them using py.test:
+
+    $ py.test --benchmark-warmup=on tests/benchmarks/benchmarks.py

tests/benchmarks/benchmarks.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+
+# A set of benchmarks, generally used to test out different implementation
+# choices against each other.
+
+# This should be run using pytest, see end of file
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import subprocess
+import sys
+
+import pytest
+
+from django_ftl import activate
+from django_ftl.bundles import Bundle
+
+this_file = os.path.abspath(__file__)
+
+
+# For testing changes, can use multiple runs and compare mean or OPS figures on
+# subsequent runs, or test at the same time by creating multiple `Bundle`
+# implementations and add them to `params` below.
+
+@pytest.fixture(params=[Bundle])
+def bundle(request):
+    return request.param(['benchmarks/benchmarks.ftl'], default_locale='en')
+
+
+def test_simple_string_default_locale_present(bundle, benchmark):
+    activate('en')
+    result = benchmark(lambda: bundle.format('simple-string'))
+    assert result == "Hello I am a simple string"
+
+
+def test_simple_string_other_locale_present(bundle, benchmark):
+    activate('tr')
+    result = benchmark(lambda: bundle.format('simple-string'))
+    assert result == "Merhaba ben basit bir metinim"
+
+
+def test_simple_string_present_in_fallback(bundle, benchmark):
+    activate('tr')
+    result = benchmark(lambda: bundle.format('simple-string-present-in-fallback'))
+    assert result == "Hello I am a simple string present in fallback"
+
+
+if __name__ == '__main__':
+    # You can execute this file directly, and optionally add more py.test args
+    # to the command line (e.g. -k for keyword matching certain tests).
+    subprocess.check_call(["py.test", "--benchmark-warmup=on", "--benchmark-sort=name", this_file] + sys.argv[1:])

tests/benchmarks/requirements.txt

@@ -0,0 +1,3 @@
+pytest
+pytest-benchmark
+pytest-django

tests/locales/en/benchmarks/benchmarks.ftl

@@ -0,0 +1,3 @@
+simple-string = Hello I am a simple string
+
+simple-string-present-in-fallback = Hello I am a simple string present in fallback

tests/locales/tr/benchmarks/benchmarks.ftl

@@ -0,0 +1 @@
+simple-string = Merhaba ben basit bir metinim