feat(i18n): add locale override option when using gettext

spoonbill/__init__.py

@@ -4,6 +4,7 @@
 
 from spoonbill.common import COMBINED_TABLES, ROOT_TABLES
 from spoonbill.flatten import Flattener
+from spoonbill.i18n import LOCALE, _
 from spoonbill.stats import DataPreprocessor
 from spoonbill.utils import iter_file
 from spoonbill.writers import CSVWriter, XlsxWriter
@@ -29,14 +30,15 @@ def __init__(
         root_tables=ROOT_TABLES,
         combined_tables=COMBINED_TABLES,
         root_key="releases",
+        language=LOCALE,
     ):
         self.workdir = Path(workdir)
         if state_file:
             with open(state_file) as fd:
                 data = json.load(fd)
             self.spec = DataPreprocessor.restore(data)
         else:
-            self.spec = DataPreprocessor(schema, root_tables, combined_tables=combined_tables)
+            self.spec = DataPreprocessor(schema, root_tables, combined_tables=combined_tables, language=language)
         self.root_key = root_key
 
     def analyze_file(self, filename, with_preview=True):
@@ -71,8 +73,8 @@ class FileFlattener:
     :param xlsx: Generate combined xlsx table
     """
 
-    def __init__(self, workdir, options, tables, root_key="releases", csv=True, xlsx=True):
-        self.flattener = Flattener(options, tables)
+    def __init__(self, workdir, options, tables, root_key="releases", csv=True, xlsx=True, language=LOCALE):
+        self.flattener = Flattener(options, tables, language=language)
         self.workdir = Path(workdir)
         # TODO: detect package, where?
         self.root_key = root_key

spoonbill/cli.py

@@ -11,7 +11,7 @@
 from spoonbill import FileAnalyzer, FileFlattener
 from spoonbill.common import COMBINED_TABLES, ROOT_TABLES, TABLE_THRESHOLD
 from spoonbill.flatten import FlattenOptions
-from spoonbill.i18n import _, set_locale
+from spoonbill.i18n import LOCALE, _
 from spoonbill.utils import read_lines, resolve_file_uri
 
 LOGGER = logging.getLogger("spoonbill")
@@ -122,7 +122,7 @@ def get_selected_tables(base, selection):
 @click.option(
     "--language",
     help=_("Language for headings"),
-    default="en",
+    default=LOCALE.split("_")[0],
     type=click.Choice(["en", "es"]),
 )
 @click_logging.simple_verbosity_option(LOGGER)
@@ -148,7 +148,6 @@ def cli(
     language,
 ):
     """Spoonbill cli entry point"""
-    set_locale(language)
     click.echo(_("Detecting input file format"))
     # TODO: handle line separated json
     # TODO: handle single release/record
@@ -203,6 +202,7 @@ def cli(
             root_key=root_key,
             root_tables=root_tables,
             combined_tables=combined_tables,
+            language=language,
         )
         click.echo(_("Processing file: {}").format(click.style(str(path), fg="cyan")))
         total = path.stat().st_size
@@ -276,7 +276,15 @@ def cli(
     options = FlattenOptions(**options)
     all_tables = chain(options.selection.keys(), combined_tables.keys())
     click.echo(_("Going to export tables: {}").format(click.style(",".join(all_tables), fg="magenta")))
-    flattener = FileFlattener(workdir, options, analyzer.spec.tables, root_key=root_key, csv=csv, xlsx=xlsx)
+    flattener = FileFlattener(
+        workdir,
+        options,
+        analyzer.spec.tables,
+        root_key=root_key,
+        csv=csv,
+        xlsx=xlsx,
+        language=language,
+    )
     click.echo(_("Flattening input file"))
     with click.progressbar(
         flattener.flatten_file(filename),

spoonbill/flatten.py

@@ -4,7 +4,7 @@
 from typing import List, Mapping, Sequence
 
 from spoonbill.common import DEFAULT_FIELDS, JOINABLE
-from spoonbill.i18n import _
+from spoonbill.i18n import LOCALE, _
 from spoonbill.spec import Table
 from spoonbill.utils import generate_row_id, get_matching_tables, get_pointer, get_root
 
@@ -68,11 +68,12 @@ class Flattener:
     :param tables: Analyzed tables data
     """
 
-    def __init__(self, options: FlattenOptions, tables: Mapping[str, Table]):
+    def __init__(self, options: FlattenOptions, tables: Mapping[str, Table], language=LOCALE):
         if not is_dataclass(options):
             options = FlattenOptions(**options)
         self.options = options
         self.tables = tables
+        self.language = language
 
         self._lookup_cache = {}
         self._types_cache = {}
@@ -170,6 +171,7 @@ def _init_options(self, tables):
                         target.add_column(
                             path,
                             "integer",
+                            _(path, self.language),
                             additional=True,
                             combined_only=not combined,
                             propagate=False,

spoonbill/i18n.py

@@ -1,31 +1,38 @@
 import gettext
 import json
+import locale
+import os
 from collections import deque
+from functools import lru_cache
 from pathlib import Path
 
 import jsonref
 
 from spoonbill.common import COMBINED_TABLES
 from spoonbill.utils import common_prefix, extract_type
 
-domain = "spoonbill"
-locale_dir = str(Path(__file__).parent / "locales")
-LOCALE = "en"
-LANG = None
-_ = lambda x: x  # noqa
+DOMAIN = "spoonbill"
+LOCALEDIR = str(Path(__file__).parent / "locales")
+LOCALE = "en_US"
+system_locale = locale.getdefaultlocale()
+if system_locale and system_locale[0]:
+    LOCALE = system_locale[0]
 
 
-def set_locale(locale):
-    global LOCALE
-    global LANG
-    global _
-    LOCALE = locale
-    LANG = gettext.translation(domain, locale_dir, languages=[LOCALE], fallback=True)
-    LANG.install()
-    _ = LANG.gettext
+def translate(msg_id, lang=LOCALE):
+    """Simple wrapper of python's gettext with ability to override desired language"""
+    translator = _translation(lang)
+    if translator:
+        return translator.gettext(msg_id)
+    return msg_id
 
 
-set_locale(LOCALE)
+@lru_cache(maxsize=None)
+def _translation(lang):
+    return gettext.translation(DOMAIN, LOCALEDIR, languages=[lang], fallback=None)
+
+
+_ = translate
 
 
 # slightly modified version of ocds-babel's extract_schema

spoonbill/spec.py

@@ -114,6 +114,7 @@ def add_column(
         self,
         path,
         item_type,
+        title,
         *,
         combined_only=False,
         propagate=True,
@@ -129,7 +130,6 @@ def add_column(
         :param combined_only: Make this column available only in combined version of table
         :param additional: Mark this column as missing in schema
         """
-        title = _(path)
         is_array = self.is_array(path)
         combined_path = combine_path(self, path)
         if not combined_only:
@@ -150,6 +150,7 @@ def add_column(
             self.parent.add_column(
                 path,
                 item_type,
+                title,
                 combined_only=combined_only,
                 additional=additional,
                 abs_path=abs_path,

spoonbill/stats.py

@@ -1,11 +1,12 @@
+import locale
 import logging
 from collections import deque
 from typing import List, Mapping
 
 import jsonref
 
 from spoonbill.common import ARRAY, DEFAULT_FIELDS, JOINABLE, JOINABLE_SEPARATOR, TABLE_THRESHOLD
-from spoonbill.i18n import _
+from spoonbill.i18n import DOMAIN, LOCALE, LOCALEDIR, _
 from spoonbill.spec import Column, Table, add_child_table
 from spoonbill.utils import (
     PYTHON_TO_JSON_TYPE,
@@ -45,6 +46,7 @@ def __init__(
         table_threshold=TABLE_THRESHOLD,
         total_items=0,
         header_separator="/",
+        language=LOCALE,
     ):
         self.schema = schema
         self.root_tables = root_tables
@@ -58,6 +60,7 @@ def __init__(
 
         self._lookup_cache = {}
         self._table_by_path = {}
+        self.language = language
         if not self.tables:
             self.parse_schema()
 
@@ -118,11 +121,11 @@ def parse_schema(self):
                             # This means we in array of strings, so this becomes a single joinable column
                             typeset = ARRAY.format(items_type)
                             self.current_table.types[pointer] = JOINABLE
-                            self.current_table.add_column(pointer, typeset)
+                            self.current_table.add_column(pointer, typeset, _(pointer, self.language))
                     else:
                         if self.current_table.is_combined:
                             pointer = separator + separator.join((parent_key, key))
-                        self.current_table.add_column(pointer, typeset)
+                        self.current_table.add_column(pointer, typeset, _(pointer, self.language))
             else:
                 # TODO: not sure what to do here
                 continue
@@ -291,6 +294,7 @@ def process_items(self, releases, with_preview=True):
                             self.current_table.add_column(
                                 pointer,
                                 PYTHON_TO_JSON_TYPE.get(type(item).__name__, "N/A"),
+                                _(pointer, self.language),
                                 additional=True,
                                 abs_path=abs_pointer,
                             )

spoonbill/writers/csv.py

@@ -1,6 +1,7 @@
 import csv
 import logging
 
+from spoonbill.i18n import _
 from spoonbill.utils import get_headers
 
 LOGGER = logging.getLogger("spoonbill")

tests/test_spec.py

@@ -22,7 +22,7 @@ def test_combine_path(root_table):
 
 def test_inc_column(root_table):
     child = add_child_table(root_table, "/tender/items", "tender", "items")
-    child.add_column("/tender/items/id", ["string"], additional=True, abs_path="/tender/items/0/test")
+    child.add_column("/tender/items/id", ["string"], "Tender Id", additional=True, abs_path="/tender/items/0/test")
     child.arrays["/tender/items/additionalClassifications"] = 0
     root_table.inc_column("ocid", "ocid")
     assert root_table["ocid"].hits == 1
@@ -44,6 +44,7 @@ def test_inc_column(root_table):
     child_child.add_column(
         "/tender/items/additionalClassifications/id",
         ["string"],
+        "Classification Id",
         additional=True,
     )
     child_child.inc_column(
@@ -56,17 +57,16 @@ def test_inc_column(root_table):
 
 
 def test_add_column(root_table):
-    root_table.add_column("/tender/id", ["string", "integer"])
+    root_table.add_column("/tender/id", ["string", "integer"], "Tender Id")
     assert "/tender/id" in root_table
     assert "/tender/id" in root_table.combined_columns
 
-    root_table.add_column("/tender/itemsCount", ["string", "integer"])
+    root_table.add_column("/tender/itemsCount", ["string", "integer"], "Items Count")
     assert "/tender/itemsCount" in root_table
     assert "/tender/itemsCount" in root_table.combined_columns
 
     root_table.add_column(
-        "/tender/items/additionalClassificationsCount",
-        ["string", "integer"],
+        "/tender/items/additionalClassificationsCount", ["string", "integer"], "Classifications Count"
     )
     assert "/tender/items/0/additionalClassificationsCount" in root_table
     assert "/tender/items/0/additionalClassificationsCount" in root_table.combined_columns
@@ -75,17 +75,18 @@ def test_add_column(root_table):
     child.add_column(
         "/tender/items/test",
         ["string", "integer"],
+        "/tender/items/test",
         additional=True,
         abs_path="/tender/items/0/test",
     )
     assert "/tender/items/test" in child
     assert "/tender/items/test" in child.combined_columns
     assert "/tender/items/0/test" in root_table
     assert "/tender/items/0/test" in root_table.combined_columns
-    child.add_column("/tender/items/id", ["string", "integer"])
+    child.add_column("/tender/items/id", ["string", "integer"], "Items Id")
 
     child.arrays["/tender/items/additionalClassifications"] = 0
-    child.add_column("/tender/items/additionalClassifications/id", ["string", "integer"])
+    child.add_column("/tender/items/additionalClassifications/id", ["string", "integer"], "Classification ID")
     assert "/tender/items/additionalClassifications/0/id" in child
     assert "/tender/items/id" in child
     assert "/tender/items/0/id" in root_table
@@ -94,6 +95,7 @@ def test_add_column(root_table):
     child.add_column(
         "/tender/items/additionalClassificationsCount",
         ["string", "integer"],
+        "Classification Count",
         propagate=False,
     )
     assert "/tender/items/additionalClassificationsCount" in child