diff --git a/utils/pywmlx/state/machine.py b/utils/pywmlx/state/machine.py index ff8e05422ed8..9c090330dfba 100644 --- a/utils/pywmlx/state/machine.py +++ b/utils/pywmlx/state/machine.py @@ -11,6 +11,7 @@ from pywmlx.state.wml_states import setup_wmlstates import pywmlx.nodemanip +import pdb @@ -335,53 +336,87 @@ def run(*, filebuf, fileref, fileno, startstate, waitwml=True): _currentdomain = _initialdomain pywmlx.nodemanip.newfile(fileref, fileno) # debug_cs = startstate - for xline in filebuf: - xline = xline.strip('\n\r') - _current_lineno += 1 - # on new line, debug file will write another marker - if _debugmode: - print('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@', - file=_fdebug) - while xline is not None: - # print debug infos (if debugmode is on) + try: + for xline in filebuf: + xline = xline.strip('\n\r') + _current_lineno += 1 + # on new line, debug file will write another marker if _debugmode: - lno = '%05d' % _current_lineno - print('------------------------------------------------------', + print('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@', file=_fdebug) - print('LINE', lno, '|', xline, file=_fdebug) - # action number is used to know what function we should run - action = 0 - v = None - m = None - if cs.regex is None: - # action = 1 --> execute state.run - action = 1 + while xline is not None: + # print debug infos (if debugmode is on) if _debugmode: - print('ALWAYS-RUN x', cs_debug, file=_fdebug) - else: - # m is match - m = re.match(cs.regex, xline) - if m: + lno = '%05d' % _current_lineno + print('---------------------------------------------------', + file=_fdebug) + print('LINE', lno, '|', xline, file=_fdebug) + # action number is used to know what function we should run + action = 0 + v = None + m = None + if cs.regex is None: # action = 1 --> execute state.run action = 1 if _debugmode: - print('RUN state \\', cs_debug, file=_fdebug) + print('ALWAYS-RUN x', cs_debug, file=_fdebug) else: - # action = 2 --> change to the state pointed by - # state.iffail - action = 2 - if _debugmode: - print('FAIL state |', cs_debug, file=_fdebug) - if action == 1: - # xline, ns: xline --> override xline with new value - # ns --> value of next state - xline, ns = cs.run(xline, _current_lineno, m) - cs_debug = ns - cs = _states.get(ns) - else: - cs_debug = cs.iffail - cs = _states.get(cs.iffail) - # end while xline - # end for xline + # m is match + m = re.match(cs.regex, xline) + if m: + # action = 1 --> execute state.run + action = 1 + if _debugmode: + print('RUN state \\', cs_debug, file=_fdebug) + else: + # action = 2 --> change to the state pointed by + # state.iffail + action = 2 + if _debugmode: + print('FAIL state |', cs_debug, file=_fdebug) + if action == 1: + # xline, ns: xline --> override xline with new value + # ns --> value of next state + xline, ns = cs.run(xline, _current_lineno, m) + cs_debug = ns + cs = _states.get(ns) + else: + cs_debug = cs.iffail + cs = _states.get(cs.iffail) + # end while xline + # end for xline + except UnicodeDecodeError as e: + errpos = int(e.start) # error position on file object with UTF-8 error + errbval = hex(e.object[errpos]) # value of byte wich causes UTF-8 error + # well... when exception occurred, the _current_lineno value + # was not updated at all due to the failure of the try block. + # (it is = 0) + # this means we need to make a workaround to obtain in what line of the + # file the problem happened. + # In order to perform this task (and not only) we create a temporary + # string wich contains all the file text UNTIL the UTF-8 + untilerr_buf = e.object[0:errpos] # buffer containing file text + untilerr = "".join(map(chr, untilerr_buf)) + # splituntil will be a array of strings (each item is a line of text). + # the last item will show the point where the invalid UTF-8 character + # was found. + splituntil = untilerr.split('\n') + # error line is equal of lines of text until error occurs (number of + # items on splituntil string array) + errlineno = len(splituntil) + # finally we can know the actual file info + finfo = pywmlx.nodemanip.fileref + ":" + str(errlineno) + errmsg = ( + "UTF-8 Format error.\nCan't decode byte " + str(errbval) + ' (' + + e.reason + ').\n' + + 'Probably your file is not encoded with UTF-8 encoding: you ' + + 'should open the file with an advanced text editor, and re-save ' + + 'it with UTF-8 encoding.\n' + + 'To avoid this problem in the future, you might want to set ' + + 'the default encoding of your editor to UTF-8.\n\n' + + 'Text preceding the invalid byte (source file, line ' + + str(errlineno) + '):\n' + splituntil[-1] + '\n' + ) + wmlerr(finfo, errmsg) pywmlx.nodemanip.closefile(_dictionary, _current_lineno) diff --git a/utils/wmlxgettext b/utils/wmlxgettext index 326cff9df357..8f4f0a543c17 100755 --- a/utils/wmlxgettext +++ b/utils/wmlxgettext @@ -44,23 +44,29 @@ import pywmlx def commandline(args): parser = argparse.ArgumentParser( description='Generate .po from WML/lua file list.', - usage='''wmlxgettext --domain=DOMAIN [--directory=START_PATH] + usage='''wmlxgettext --domain=DOMAIN -o OUTPUT_FILE + [--directory=START_PATH] [--recursive] [--initialdomain=INITIALDOMAIN] [--package-version=PACKAGE_VERSION] - [--no-text-colors] [--fuzzy] [--warnall] [-o OUTPUT_FILE] + [--no-text-colors] [--fuzzy] [--warnall] FILE1 FILE2 ... FILEN''' ) parser.add_argument( '--version', action='version', - version='wmlxgettext 2016.10.03.py3' + version='wmlxgettext 2017.06.25.py3' ) parser.add_argument( '-o', + required=True, default=None, dest='outfile', - help= ('Destination file. By default the output ' - 'will be printed on stdout') + help= ('Destination file. In some special situations you might want ' + 'to write the output to STDOUT instead of writing ' + 'an actual file (using "-o -"). On a standard usage, however, ' + 'you should avoid to write the output to STDOUT (or you can' + 'face some issues related to text encoding ' + '[**REQUIRED ARGUMENT**]') ) parser.add_argument( '--domain', @@ -156,16 +162,12 @@ def main(): sentlist = dict() fileno = 0 fdebug = None + if args.outfile == '-': + args.outfile = None if args.debugmode: fdebug = open('debug.txt', 'w', encoding='utf-8') pywmlx.statemachine.setup(sentlist, args.initdom, args.domain, args.warnall, fdebug) - if args.warnall is True and args.outfile is None: - pywmlx.wmlwarn('command line warning', 'Writing the output to stdout ' - '(and then eventually redirect that output to a file) ' - 'is a deprecated usage. Please, consider to use the ' - '"-o " option, instead of using the ' - 'output redirection') filelist = None if args.recursive is False and args.filelist is None: pywmlx.wmlerr("bad command line", "FILELIST must not be empty. "