feat: gettext translations (v15 port) (#25982)

* feat: gettext translations

+ many misc translation fixes

Co-Authored-By: Raffael Meyer <raffael@alyf.de>

* chore: add crowdin conf

* fix: LT fixes

* chore: port babel extractors

* fix: port get translated country info

* fix: dont supress exception in tests

* fix!: remove redundant sending of translations

* chore: drop docs checker from stable versions

* fix: bad usage of _

* fix: translations in webform

---------

Co-authored-by: Raffael Meyer <raffael@alyf.de>

.github/workflows/linters.yml

@@ -31,25 +31,6 @@ jobs:
           npm install @commitlint/cli @commitlint/config-conventional
           npx commitlint --verbose --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }}
 
-  docs-required:
-    name: 'Documentation Required'
-    runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request'
-
-    steps:
-      - name: 'Setup Environment'
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-      - uses: actions/checkout@v4
-
-      - name: Validate Docs
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          pip install requests --quiet
-          python $GITHUB_WORKSPACE/.github/helper/documentation.py $PR_NUMBER
-
   linter:
     name: 'Semgrep Rules'
     runs-on: ubuntu-latest

babel_extractors.csv

@@ -0,0 +1,9 @@
+hooks.py,frappe.gettext.extractors.navbar.extract
+**/doctype/*/*.json,frappe.gettext.extractors.doctype.extract
+**/workspace/*/*.json,frappe.gettext.extractors.workspace.extract
+**/onboarding_step/*/*.json,frappe.gettext.extractors.onboarding_step.extract
+**/module_onboarding/*/*.json,frappe.gettext.extractors.module_onboarding.extract
+**/report/*/*.json,frappe.gettext.extractors.report.extract
+**.py,frappe.gettext.extractors.python.extract
+**.js,frappe.gettext.extractors.javascript.extract
+**.html,frappe.gettext.extractors.html_template.extract

crowdin.yml

@@ -0,0 +1,8 @@
+files:
+  - source: /frappe/locale/main.pot
+    translation: /frappe/locale/%two_letters_code%.po
+pull_request_title: "fix: sync translations from crowdin"
+pull_request_labels:
+  - translation
+commit_message: "fix: %language% translations"
+append_commit_message: false

frappe/__init__.py

@@ -90,7 +90,7 @@ def copy(self):
 
 
 def _(msg: str, lang: str | None = None, context: str | None = None) -> str:
-	"""Returns translated string in current lang, if exists.
+	"""Return translated string in current lang, if exists.
 	Usage:
 	        _('Change')
 	        _('Change', context='Coins')
@@ -125,8 +125,61 @@ def _(msg: str, lang: str | None = None, context: str | None = None) -> str:
 	return translated_string or non_translated_string
 
 
-def as_unicode(text: str, encoding: str = "utf-8") -> str:
-	"""Convert to unicode if required"""
+def _lt(msg: str, lang: str | None = None, context: str | None = None):
+	"""Lazily translate a string.
+
+
+	This function returns a "lazy string" which when casted to string via some operation applies
+	translation first before casting.
+
+	This is only useful for translating strings in global scope or anything that potentially runs
+	before `frappe.init()`
+
+	Note: Result is not guaranteed to equivalent to pure strings for all operations.
+	"""
+	return _LazyTranslate(msg, lang, context)
+
+
+@functools.total_ordering
+class _LazyTranslate:
+	__slots__ = ("msg", "lang", "context")
+
+	def __init__(self, msg: str, lang: str | None = None, context: str | None = None) -> None:
+		self.msg = msg
+		self.lang = lang
+		self.context = context
+
+	@property
+	def value(self) -> str:
+		return _(str(self.msg), self.lang, self.context)
+
+	def __str__(self):
+		return self.value
+
+	def __add__(self, other):
+		if isinstance(other, str | _LazyTranslate):
+			return self.value + str(other)
+		raise NotImplementedError
+
+	def __radd__(self, other):
+		if isinstance(other, str | _LazyTranslate):
+			return str(other) + self.value
+		return NotImplementedError
+
+	def __repr__(self) -> str:
+		return f"'{self.value}'"
+
+	# NOTE: it's required to override these methods and raise error as default behaviour will
+	# return `False` in all cases.
+	def __eq__(self, other):
+		raise NotImplementedError
+
+	def __lt__(self, other):
+		raise NotImplementedError
+
+
+def as_unicode(text, encoding: str = "utf-8") -> str:
+	"""Convert to unicode if required."""
 	if isinstance(text, str):
 		return text
 	elif text is None:
@@ -137,16 +190,6 @@ def as_unicode(text: str, encoding: str = "utf-8") -> str:
 		return str(text)
 
 
-def get_lang_dict(fortype: str, name: str | None = None) -> dict[str, str]:
-	"""Returns the translated language dict for the given type and name.
-
-	:param fortype: must be one of `doctype`, `page`, `report`, `include`, `jsfile`, `boot`
-	:param name: name of the document for which assets are to be returned."""
-	from frappe.translate import get_dict
-
-	return get_dict(fortype, name)
-
-
 def set_user_lang(user: str, user_language: str | None = None) -> None:
 	"""Guess and set user language for the session. `frappe.local.lang`"""
 	from frappe.translate import get_user_lang

frappe/commands/__init__.py

@@ -105,14 +105,22 @@ def call_command(cmd, context):
 
 def get_commands():
 	# prevent circular imports
+	from .gettext import commands as gettext_commands
 	from .redis_utils import commands as redis_commands
 	from .scheduler import commands as scheduler_commands
 	from .site import commands as site_commands
 	from .translate import commands as translate_commands
 	from .utils import commands as utils_commands
 
 	clickable_link = "https://frappeframework.com/docs"
-	all_commands = scheduler_commands + site_commands + translate_commands + utils_commands + redis_commands
+	all_commands = (
+		scheduler_commands
+		+ site_commands
+		+ translate_commands
+		+ gettext_commands
+		+ utils_commands
+		+ redis_commands
+	)
 
 	for command in all_commands:
 		if not command.help:

frappe/commands/gettext.py

@@ -0,0 +1,97 @@
+import click
+
+from frappe.commands import pass_context
+from frappe.exceptions import SiteNotSpecifiedError
+
+
+@click.command("generate-pot-file", help="Translation: generate POT file")
+@click.option("--app", help="Only generate for this app. eg: frappe")
+@pass_context
+def generate_pot_file(context, app: str | None = None):
+	from frappe.gettext.translate import generate_pot
+
+	if not app:
+		connect_to_site(context.sites[0] if context.sites else None)
+
+	generate_pot(app)
+
+
+@click.command("compile-po-to-mo", help="Translation: compile PO files to MO files")
+@click.option("--app", help="Only compile for this app. eg: frappe")
+@click.option(
+	"--force",
+	is_flag=True,
+	default=False,
+	help="Force compile even if there are no changes to PO files",
+)
+@click.option("--locale", help="Compile transaltions only for this locale. eg: de")
+@pass_context
+def compile_translations(context, app: str | None = None, locale: str | None = None, force=False):
+	from frappe.gettext.translate import compile_translations as _compile_translations
+
+	if not app:
+		connect_to_site(context.sites[0] if context.sites else None)
+
+	_compile_translations(app, locale, force=force)
+
+
+@click.command("migrate-csv-to-po", help="Translation: migrate from CSV files (old) to PO files (new)")
+@click.option("--app", help="Only migrate for this app. eg: frappe")
+@click.option("--locale", help="Compile translations only for this locale. eg: de")
+@pass_context
+def csv_to_po(context, app: str | None = None, locale: str | None = None):
+	from frappe.gettext.translate import migrate
+
+	if not app:
+		connect_to_site(context.sites[0] if context.sites else None)
+
+	migrate(app, locale)
+
+
+@click.command(
+	"update-po-files",
+	help="""Translation: sync PO files with POT file.
+You might want to run generate-pot-file first.""",
+)
+@click.option("--app", help="Only update for this app. eg: frappe")
+@click.option("--locale", help="Update PO files only for this locale. eg: de")
+@pass_context
+def update_po_files(context, app: str | None = None, locale: str | None = None):
+	from frappe.gettext.translate import update_po
+
+	if not app:
+		connect_to_site(context.sites[0] if context.sites else None)
+
+	update_po(app, locale=locale)
+
+
+@click.command("create-po-file", help="Translation: create a new PO file for a locale")
+@click.argument("locale", nargs=1)
+@click.option("--app", help="Only create for this app. eg: frappe")
+@pass_context
+def create_po_file(context, locale: str, app: str | None = None):
+	"""Create PO file for lang code"""
+	from frappe.gettext.translate import new_po
+
+	if not app:
+		connect_to_site(context.sites[0] if context.sites else None)
+
+	new_po(locale, app)
+
+
+def connect_to_site(site):
+	from frappe import connect
+
+	if not site:
+		raise SiteNotSpecifiedError
+
+	connect(site=site)
+
+
+commands = [
+	generate_pot_file,
+	compile_translations,
+	csv_to_po,
+	update_po_files,
+	create_po_file,
+]

frappe/commands/utils.py

@@ -52,6 +52,7 @@ def build(
 ):
 	"Compile JS and CSS source files"
 	from frappe.build import bundle, download_frappe_assets
+	from frappe.gettext.translate import compile_translations
 	from frappe.utils.synchronization import filelock
 
 	frappe.init("")
@@ -82,6 +83,16 @@ def build(
 			save_metafiles=save_metafiles,
 		)
 
+		if apps and isinstance(apps, str):
+			apps = apps.split(",")
+
+		if not apps:
+			apps = frappe.get_all_apps()
+
+		for app in apps:
+			print("Compiling translations for", app)
+			compile_translations(app, force=force)
+
 
 @click.command("watch")
 @click.option("--apps", help="Watch assets for specific apps")
@@ -98,14 +109,13 @@ def watch(apps=None):
 def clear_cache(context):
 	"Clear cache, doctype cache and defaults"
 	import frappe.sessions
-	from frappe.desk.notifications import clear_notifications
 	from frappe.website.utils import clear_website_cache
 
 	for site in context.sites:
 		try:
-			frappe.connect(site)
+			frappe.init(site=site)
+			frappe.connect()
 			frappe.clear_cache()
-			clear_notifications()
 			clear_website_cache()
 		finally:
 			frappe.destroy()

frappe/core/doctype/page/page.py

@@ -173,11 +173,6 @@ def load_assets(self):
 					# flag for not caching this page
 					self._dynamic_page = True
 
-		if frappe.lang != "en":
-			from frappe.translate import get_lang_js
-
-			self.script += get_lang_js("page", self.name)
-
 		for path in get_code_files_via_hooks("page_js", self.name):
 			js = get_js(path)
 			if js:

frappe/core/doctype/translation/test_translation.py

@@ -3,7 +3,6 @@
 import frappe
 from frappe import _
 from frappe.tests.utils import FrappeTestCase
-from frappe.translate import clear_cache
 
 
 class TestTranslation(FrappeTestCase):
@@ -12,6 +11,8 @@ def setUp(self):
 
 	def tearDown(self):
 		frappe.local.lang = "en"
+		from frappe.translate import clear_cache
+
 		clear_cache()
 
 	def test_doctype(self):