update deprecated function

i18n.py

@@ -4,16 +4,18 @@
 # Script to generate the template file and update the translation files.
 # Copy the script into the mod or modpack root folder and run it there.
 #
-# Copyright (C) 2019 Joachim Stolberg
+# Copyright (C) 2019 Joachim Stolberg, 2020 FaceDeer
 # LGPLv2.1+
 
 from __future__ import print_function
 import os, fnmatch, re, shutil, errno
 
+verbose = False
+
 #group 2 will be the string, groups 1 and 3 will be the delimiters (" or ')
 #See https://stackoverflow.com/questions/46967465/regex-match-text-in-either-single-or-double-quote
-#TODO: support [[]] delimiters
 pattern_lua = re.compile(r'[\.=^\t,{\(\s]N?S\(\s*(["\'])((?:\\\1|(?:(?!\1)).)*)(\1)[\s,\)]', re.DOTALL)
+pattern_lua_bracketed = re.compile(r'[\.=^\t,{\(\s]N?S\(\s*\[\[(.*?)\]\][\s,\)]', re.DOTALL)
 
 # Handles "concatenation" .. " of strings"
 pattern_concat = re.compile(r'["\'][\s]*\.\.[\s]*["\']', re.DOTALL)
@@ -83,11 +85,13 @@ def process_po_files(folder, modname):
             tr_name = modname + "." + language_code + ".tr"
             tr_file = os.path.join(root, tr_name)
             if os.path.exists(tr_file):
-                print(tr_name + " already exists, ignoring " + name)
+                if verbose:
+                    print(tr_name + " already exists, ignoring " + name)
                 continue
             fname = os.path.join(root, name)
             with open(fname, "r", encoding='utf-8') as po_file:
-                print("Importing translations from " + name)
+                if verbose:
+                    print("Importing translations from " + name)
                 text = process_po_file(po_file.read())
                 with open(tr_file, "wt", encoding='utf-8') as tr_out:
                     tr_out.write(text)
@@ -103,24 +107,68 @@ def mkdir_p(path):
             pass
         else: raise
 
+# Converts the template dictionary to a text to be written as a file
+# dKeyStrings is a dictionary of localized string to source file sets
+# dOld is a dictionary of existing translations, for use when updating
+# existing .tr files
+def strings_to_text(dkeyStrings, dOld, mod_name):
+    lOut = ["# textdomain: %s\n" % mod_name]
+
+    dGroupedBySource = {}
+
+    for key in dkeyStrings:
+        sourceList = list(dkeyStrings[key])
+        sourceList.sort()
+        sourceString = "\n".join(sourceList)
+        listForSource = dGroupedBySource.get(sourceString, [])
+        listForSource.append(key)
+        dGroupedBySource[sourceString] = listForSource
+
+    lSourceKeys = list(dGroupedBySource.keys())
+    lSourceKeys.sort()
+    for source in lSourceKeys:
+        lOut.append("")
+        localizedStrings = dGroupedBySource[source]
+        localizedStrings.sort()
+        lOut.append(source)
+        for localizedString in localizedStrings:
+            val = dOld.get(localizedString, "")
+            lOut.append("%s=%s" % (localizedString, val))
+
+    unusedExist = False
+    for key in dOld:
+        if key not in dkeyStrings:
+            if not unusedExist:
+                unusedExist = True
+                lOut.append("\n##### not used anymore #####")
+            lOut.append("%s=%s" % (key, dOld[key]))
+    return "\n".join(lOut)
+
 # Writes a template.txt file
-def write_template(templ_file, lkeyStrings):
-    lOut = []
-    lkeyStrings.sort()
-    for s in lkeyStrings:
-        lOut.append("%s=" % s)
+# dkeyStrings is the dictionary returned by generate_template
+def write_template(templ_file, dkeyStrings, mod_name):
+    text = strings_to_text(dkeyStrings, {}, mod_name)
     mkdir_p(os.path.dirname(templ_file))
     with open(templ_file, "wt", encoding='utf-8') as template_file:
-        template_file.write("\n".join(lOut))
+        template_file.write(text)
+
 
 # Gets all translatable strings from a lua file
 def read_lua_file_strings(lua_file):
     lOut = []
     with open(lua_file, encoding='utf-8') as text_file:
         text = text_file.read()
-        text = re.sub(pattern_concat, "", text)        
+        #TODO remove comments here
+
+        text = re.sub(pattern_concat, "", text)
+
+        strings = []
         for s in pattern_lua.findall(text):
-            s = s[1]
+            strings.append(s[1])
+        for s in pattern_lua_bracketed.findall(text):
+            strings.append(s)
+
+        for s in strings:
             s = re.sub(r'"\.\.\s+"', "", s)
             s = re.sub("@[^@=0-9]", "@@", s)
             s = s.replace('\\"', '"')
@@ -132,65 +180,78 @@ def read_lua_file_strings(lua_file):
     return lOut
 
 # Gets strings from an existing translation file
+# returns both a dictionary of translations
+# and the full original source text so that the new text
+# can be compared to it for changes.
 def import_tr_file(tr_file):
     dOut = {}
+    text = None
     if os.path.exists(tr_file):
         with open(tr_file, "r", encoding='utf-8') as existing_file :
+            text = existing_file.read()
+            existing_file.seek(0)
             for line in existing_file.readlines():
                 s = line.strip()
                 if s == "" or s[0] == "#":
                      continue
                 match = pattern_tr.match(s)
                 if match:
                     dOut[match.group(1)] = match.group(2)
-    return dOut
+    return (dOut, text)
 
 # Walks all lua files in the mod folder, collects translatable strings,
 # and writes it to a template.txt file
-def generate_template(folder):
-    lOut = []
+# Returns a dictionary of localized strings to source file sets
+# that can be used with the strings_to_text function.
+def generate_template(folder, mod_name):
+    dOut = {}
     for root, dirs, files in os.walk(folder):
         for name in files:
             if fnmatch.fnmatch(name, "*.lua"):
                 fname = os.path.join(root, name)
                 found = read_lua_file_strings(fname)
-                print(fname + ": " + str(len(found)) + " translatable strings")
-                lOut.extend(found)
-    lOut = list(set(lOut))
-    lOut.sort()
-    if len(lOut) == 0:
+                if verbose:
+                    print(fname + ": " + str(len(found)) + " translatable strings")
+
+                for s in found:
+                    sources = dOut.get(s, set())
+                    sources.add("# " + fname)
+                    dOut[s] = sources
+
+    if len(dOut) == 0:
         return None
     templ_file = folder + "locale/template.txt"
-    write_template(templ_file, lOut)
-    return lOut
+    write_template(templ_file, dOut, mod_name)
+    return dOut
 
 # Updates an existing .tr file, copying the old one to a ".old" file
-def update_tr_file(lNew, mod_name, tr_file):
-    print("updating " + tr_file)
-    lOut = ["# textdomain: %s\n" % mod_name]
+# if any changes have happened
+# dNew is the data used to generate the template, it has all the
+# currently-existing localized strings
+def update_tr_file(dNew, mod_name, tr_file):
+    if verbose:
+        print("updating " + tr_file)
 
-    #TODO only make a .old if there are actual changes from the old file
-    if os.path.exists(tr_file):
+    tr_import = import_tr_file(tr_file)
+    dOld = tr_import[0]
+    textOld = tr_import[1]
+
+    textNew = strings_to_text(dNew, dOld, mod_name)
+
+    if textOld and textOld != textNew:
+        print(tr_file + " has changed.")
         shutil.copyfile(tr_file, tr_file+".old")
 
-    dOld = import_tr_file(tr_file)
-    for key in lNew:
-        val = dOld.get(key, "")
-        lOut.append("%s=%s" % (key, val))
-    lOut.append("##### not used anymore #####")
-    for key in dOld:
-        if key not in lNew:
-            lOut.append("%s=%s" % (key, dOld[key]))
     with open(tr_file, "w", encoding='utf-8') as new_tr_file:
-        new_tr_file.write("\n".join(lOut))
+        new_tr_file.write(textNew)
 
 # Updates translation files for the mod in the given folder
 def update_mod(folder):
     modname = get_modname(folder)
     if modname is not None:
         process_po_files(folder, modname)
         print("Updating translations for " + modname)
-        data = generate_template(folder)
+        data = generate_template(folder, modname)
         if data == None:
             print("No translatable strings found in " + modname)
         else:
@@ -199,6 +260,8 @@ def update_mod(folder):
     else:
         print("Unable to find modname in folder " + folder)
 
+# Determines if the folder being pointed to is a mod or a mod pack
+# and then runs update_mod accordingly
 def update_folder(folder):
     is_modpack = os.path.exists(folder+"modpack.txt") or os.path.exists(folder+"modpack.conf")
     if is_modpack:

init.lua

@@ -2,7 +2,7 @@
 -- simple anvil that can be used to repair tools
 ---------------------------------------------------------------------------------------
 -- * can be used to repair tools
--- * the hammer gets dammaged a bit at each repair step
+-- * the hammer gets damaged a bit at each repair step
 ---------------------------------------------------------------------------------------
 
 anvil = {
@@ -13,8 +13,7 @@ anvil = {
 
 minetest.register_alias("castle:anvil", "anvil:anvil")
 
-local hammer_repairable = minetest.setting_getbool("anvil_hammer_is_repairable")
-if hammer_repairable == nil then hammer_repairable = true end
+local hammer_repairable = minetest.settings:get_bool("anvil_hammer_is_repairable", true)
 
 anvil.make_unrepairable = function(item_name)
 	local item_def = minetest.registered_items[item_name]

locale/anvil.de.tr

@@ -1,5 +1,7 @@
 # textdomain: anvil
 
+
+# ./init.lua
 @1 cannot be repaired with an anvil.=
 @1's anvil=
 A tool for repairing other tools at a blacksmith's anvil.=Stahlhammer um Werkzeuge auf dem Amboss zu reparieren

locale/anvil.es.tr

@@ -1,5 +1,7 @@
 # textdomain: anvil
 
+
+# ./init.lua
 @1 cannot be repaired with an anvil.=
 @1's anvil=
 A tool for repairing other tools at a blacksmith's anvil.=Es una herramienta para reparar otras herramientas en el yunque del herrero
@@ -9,5 +11,4 @@ Right-click on this anvil with a damaged tool to place the damaged tool upon it.
 Steel blacksmithing hammer=Martillo de acero para la herrería
 This anvil is for damaged tools only.=Este yunque es sólo para herramientas dañadas
 Use this hammer to strike blows upon an anvil bearing a damaged tool and you can repair it. It can also be used for smashing stone, but it is not well suited to this task.=Use este martillo para dar golpes sobre el yunque donde puso la herramienta dañada Tambien puede ser usado para romper piedra pero no es muy adecuado para esa tarea.
-Your @1 has been repaired successfully.=Su @1 ha sido reparado correctamente.
-##### not used anymore #####
+Your @1 has been repaired successfully.=Su @1 ha sido reparado correctamente.

locale/anvil.fr.tr

@@ -1,5 +1,7 @@
 # textdomain: anvil
 
+
+# ./init.lua
 @1 cannot be repaired with an anvil.=
 @1's anvil=
 A tool for repairing other tools at a blacksmith's anvil.=Un outil pour réparer les autres outils avec une enclume de forgeron.
@@ -9,5 +11,4 @@ Right-click on this anvil with a damaged tool to place the damaged tool upon it.
 Steel blacksmithing hammer=Marteau de forgeron en acier
 This anvil is for damaged tools only.=L'enclume s'utilise sur les outils endommagés.
 Use this hammer to strike blows upon an anvil bearing a damaged tool and you can repair it. It can also be used for smashing stone, but it is not well suited to this task.=Utilisez ce marteau pour frapper une enclume contenant un outil endommagé, ainsi vous pourrez le réparer. Il peut être aussi utilisé pour casser de la pierre, mais il n'est pas adapté à cette tâche.
-Your @1 has been repaired successfully.=Votre @1 a été réparé avec succès.
-##### not used anymore #####
+Your @1 has been repaired successfully.=Votre @1 a été réparé avec succès.

locale/anvil.it.tr

@@ -1,5 +1,7 @@
 # textdomain: anvil
 
+
+# ./init.lua
 @1 cannot be repaired with an anvil.=
 @1's anvil=
 A tool for repairing other tools at a blacksmith's anvil.=Un attrezzo per riparare altri attrezzi su di una incudine da fabbro.
@@ -9,5 +11,4 @@ Right-click on this anvil with a damaged tool to place the damaged tool upon it.
 Steel blacksmithing hammer=Martello da fabbro di acciaio
 This anvil is for damaged tools only.=Questa incudine è solo per attrezzi danneggiati.
 Use this hammer to strike blows upon an anvil bearing a damaged tool and you can repair it. It can also be used for smashing stone, but it is not well suited to this task.=Usate questo martello per colpire una incudine su cui è posto un attrezzo danneggiato e potrete ripararlo. Può anche essere usato per colpire la pietra, ma non è molto adatto a questo compito.
-Your @1 has been repaired successfully.=La/il vostr* @1 è stat* riparat* con successo.
-##### not used anymore #####
+Your @1 has been repaired successfully.=La/il vostr* @1 è stat* riparat* con successo.

locale/template.txt

@@ -1,3 +1,7 @@
+# textdomain: anvil
+
+
+# ./init.lua
 @1 cannot be repaired with an anvil.=
 @1's anvil=
 A tool for repairing other tools at a blacksmith's anvil.=