wmlxgettext: added support for _ << translatable string >> on WML

utils/pywmlx/state/wml_states.py

@@ -102,17 +102,67 @@ def run(self, xline, lineno, match):
 class WmlCommentState:
     def __init__(self):
         self.regex = re.compile(r'\s*#.+')
-        self.iffail = 'wml_tag'
+        self.iffail = 'wml_str02'
 
     def run(self, xline, lineno, match):
         xline = None
         return (xline, 'wml_idle')
 
 
 
+# On WML you can also have _ << translatable string >>, even if quite rare
+# This is considered here as "WML string 02" since it is a rare case.
+# However, for code safety, it is evaluated here before evaluating tags, and
+# before string 1. Unlike all other strings, this will be evaluated ONLY if
+# translatable, to avoid possible conflicts with WmlGoLuaState (and to prevent
+# to make that state unreachable).
+# In order to ensure that the order of sentences will be respected, the regexp
+# does not match if " is found before _ <<
+# In this way the WmlStr01 state (wich is placed after) can be reached and the
+# sentence will not be lost.
+# WARNING: This also means that it is impossible to capture any wmlinfo which
+#          uses this kind of translatable string
+#          example: name = _ <<Name>>
+#          in that case the string "name" will be captured, but the wmlinfo
+#          name = Name will be NOT added to automatic informations.
+#          This solution is necessary, since extending the workaround
+#          done for _ "standard translatable strings" to _ << wmlstr02 >>
+#          can introduce serious bugs
+class WmlStr02:
+    def __init__(self):
+        rx = r'[^"]*_\s*<<(?:(.*?)>>|(.*))'
+        self.regex = re.compile(rx)
+        self.iffail = 'wml_tag'
+
+    def run(self, xline, lineno, match):
+        _nextstate = 'wml_idle'
+        loc_translatable = True
+        loc_multiline = False
+        loc_string = None
+        if match.group(1):
+            loc_multiline = False
+            loc_string = match.group(1)
+            xline = xline [ match.end(): ]
+        elif match.group(2):
+            loc_multiline = True
+            loc_string = match.group(2)
+            _nextstate = 'wml_str20'
+            xline = None
+        else:
+            wmlerr('wmlxgettext python sources', 'wmlstr02 assertion error\n'
+                   'please report a bug if you encounter this error message')    
+        pywmlx.state.machine._pending_wmlstring = (
+            pywmlx.state.machine.PendingWmlString( 
+                lineno, loc_string, loc_multiline, loc_translatable
+            )
+        ) 
+        return (xline, _nextstate)
+
+
+
 class WmlTagState:
     def __init__(self):
-        # this regexp is deeply discussed in Source Documentation, chapter 6
+        # this regexp is discussed in depth in Source Documentation, chapter 6
         rx = r'\s*(?:[^"]+\(\s*)?\[\s*([\/+-]?)\s*([A-Za-z0-9_]+)\s*\]'
         self.regex = re.compile(rx)
         self.iffail = 'wml_getinf'
@@ -210,9 +260,30 @@ def run(self, xline, lineno, match):
 
 
 
-# Only if the symbol '<<' is found inside a [lua] tag, than it means we are
+class WmlStr20:
+    def __init__(self):
+        self.regex = None
+        self.iffail = None
+
+    def run(self, xline, lineno, match):
+        realmatch = re.match(r'(.*?)>>', xline)
+        _nextstate = 'wml_str20'
+        if realmatch:
+            pywmlx.state.machine._pending_wmlstring.addline( 
+                realmatch.group(1) )
+            xline = xline [ realmatch.end(): ]
+            _nextstate = 'wml_idle'
+        else:
+            pywmlx.state.machine._pending_wmlstring.addline(xline)
+            xline = None
+            _nextstate = 'wml_str20'
+        return (xline, _nextstate)
+
+
+
+# Only if the symbol '<<' is found inside a [lua] tag, then it means we are
 # actually starting a lua code.
-# It can happen that WML use the '<<' symbol in a very different context
+# It can happen that WML uses the '<<' symbol in a very different context
 # wich has nothing to do with lua, so switching to the lua states in that
 # case can lead to problems.
 # This happened on the file data/gui/default/widget/toggle_button_orb.cfg
@@ -222,18 +293,18 @@ def run(self, xline, lineno, match):
 #
 # In that case, after 'name' there is a WML string 
 # "('buttons/misc/orb{STATE}.png"
-# And after that you find a cuncatenation with a literal string 
+# And after that you find a concatenation with a literal string 
 # <<~RC(magenta>{icon})')>>
 #
 # That second string has nothing to do with lua, and, most importantly, if 
-# it is parsed with lua states, it return an error... why?
-# Semply becouse of the final ' symbol, wich is valid symbol, in lua, for
+# it is parsed with lua states, it returns an error... why?
+# Simply because of the final ' symbol, wich is a valid symbol, in lua, for
 # opening a new string; but, in that case, there is not an opening string,
 # but a ' symbol that must be used literally.
 #
 # This is why we use a global var _on_luatag in state.py wich is usually False.
-# it will be setted True only when opening a lua tag (see WmlTagState)
-# it will be setted to False again when the lua tag is closed (see WmlTagState)
+# it will be set to True only when opening a lua tag (see WmlTagState)
+# it will be set to False again when the lua tag is closed (see WmlTagState)
 class WmlGoluaState:
     def __init__(self):
         self.regex = re.compile(r'.*?<<\s*')
@@ -268,13 +339,14 @@ def setup_wmlstates():
                                    ('wml_checkdom', WmlCheckdomState),
                                    ('wml_checkpo', WmlCheckpoState),
                                    ('wml_comment', WmlCommentState),
+                                   ('wml_str02', WmlStr02),
                                    ('wml_tag', WmlTagState),                    
                                    ('wml_getinf', WmlGetinfState),
                                    ('wml_str01', WmlStr01),
                                    ('wml_str10', WmlStr10),
+                                   ('wml_str20', WmlStr20),
                                    ('wml_golua', WmlGoluaState),
                                    ('wml_final', WmlFinalState)]:
         st = stateclass()
         pywmlx.state.machine.addstate(statename, 
             State(st.regex, st.run, st.iffail) )
-

utils/wmlxgettext

@@ -3,9 +3,8 @@
 
 # encoding: utf8
 #
-# wmlxgettext -- generate a blank .po file for official campaigns translations 
+# wmlxgettext -- generate a blank .pot file for official campaigns translations
 #                    (build tool for wesnoth core)
-#             -- if you are a UMC developer, you could use umcpo, instead
 #                
 #
 # By Nobun, october 2015
@@ -19,22 +18,17 @@
 # wmlxgettext is a tool that is directly used during wesnoth build process 
 # to generate the pot files for the core campaigns.
 #
-# If you are an UMC developer you might want to use umcpo instead of using
-# wmlxgettext directly (but using wmlxgettext directly is however possible).
+#                              USAGE
 #
-#                              BASIC PROCEDURE
+# If you want to learn how to use wmlxgettext, read the online End-User
+# documentation at:
+# http://wmlxgettext-unoff.readthedocs.org/en/latest/enduser/index.html
 #
-# todo
+#                   SOURCE CODE DOCUMENTATION
 #
-#                              MAGIC COMMENTS
-#
-# todo
-#
-#                              DEVELOPER INFORMATION
-#
-# todo.
-#
-
+# While the source code contains some comments that explain what it does at 
+# that point, the source code is mainly explained on source documentation at:
+# http://wmlxgettext-unoff.readthedocs.org/en/latest/srcdoc/index.html
 
 import os
 import re
@@ -142,9 +136,6 @@ def commandline(args):
 
 
 def main():
-    # nota: rimane scoperto il controllo dell' eventualità che l'ultimo #define
-    # di un file WML non sia stato correttamente chiuso da un #enddef entro
-    # la fine del file
     args = commandline(sys.argv[1:])
     pywmlx.ansi_setEnabled(args.ansi_col)
     pywmlx.set_warnall(args.warnall)
@@ -170,22 +161,22 @@ def main():
                "in FILELIST or use the --recursive option.")
     elif args.recursive is False:
         filelist = args.filelist
-    # the following elif cases implicitly expexcts that args.recursive is True
+    # the following elif case implicitly expects that args.recursive is True
     elif args.filelist is not None:
         if len(args.filelist) > 0:
             pywmlx.wmlwarn("command line warning", "Option --recursive was "
                 "used, but FILELIST is not empty. All extra file listed in "
                 "FILELIST will be ignored.")
-        # if we use the --recursive option we recursively scan the add-on
-        # directory. 
+        # If we use the --recursive option we recursively scan the add-on
+        # directory.
         #    But we want that the file reference informations placed 
-        # into the .po file will remember the (relative) root name of the
+        # in the .po file will remember the (relative) root name of the
         # addon. 
         #    This is why the autof.autoscan function returns a tuple of 
         # values: 
         #   the first one is the parent directory of the original startPath
         #   the second one is the filelist (with the "fixed" file references)
-        # In this way we will override startPath to the parent directory
+        # This way, we can override startPath with its parent directory
         # containing the main directory of the wesnoth add-on, without 
         # introducing bugs.
         startPath, filelist = pywmlx.autof.autoscan(startPath)
@@ -248,14 +239,3 @@ def main():
 
 if __name__ == "__main__":
     main()
-
-
-
-# def test_wmlerr_internal():
-#     wmlwarn("file1:5", "testing warning...")
-#     wmlerr("file2:7", "testing error...")
-#     print("this string should not be printed")
-
-
-# if __name__ == "__main__":   test_wmlerr_internal()
-