Don't use docstring when @TOKEN decorator is being used

[kristofferkoch]

I think it's an elegant solution to enter the lexer regexes in the docstrings of function, but as stated in the documentation, this has some drawbacks. When using the @token decorator, there should be no reason to set doc over any other atribute (such as e.g. 'regex') on the function. I'll attach a patch to show how it might be solved differently. Setting another attribute wil allow users of ply (me :) to write a lexer that works with "python -O", if they want to.

Thanks for a very nice framework, I'll let you know if I make something cool with it :)

[kristofferkoch]

Sorry for not submitting a proper commit, but this is more of a RFC. Please let me know what you think of this:

diff --git a/ply/lex.py b/ply/lex.py
index bd32da9..2350440 100644
--- a/ply/lex.py
+++ b/ply/lex.py
@@ -747,20 +747,20 @@ class LexerReflect(object):
                     self.error = 1
                     continue

-                if not f.__doc__:
+                if not _get_regex(f):
                     self.log.error("%s:%d: No regular expression defined for rule '%s'",file,line,f.__name__)
                     self.error = 1
                     continue

                 try:
-                    c = re.compile("(?P<%s>%s)" % (fname,f.__doc__), re.VERBOSE | self.reflags)
+                    c = re.compile("(?P<%s>%s)" % (fname,_get_regex(f)), re.VERBOSE | self.reflags)
                     if c.match(""):
                         self.log.error("%s:%d: Regular expression for rule '%s' matches empty string", file,line,f.__name__)
                         self.error = 1
                 except re.error:
                     _etype, e, _etrace = sys.exc_info()
                     self.log.error("%s:%d: Invalid regular expression for rule '%s'. %s", file,line,f.__name__,e)
-                    if '#' in f.__doc__:
+                    if '#' in _get_regex(f):
                         self.log.error("%s:%d. Make sure '#' in rule '%s' is escaped with '\\#'",file,line, f.__name__)
                     self.error = 1

@@ -933,9 +933,9 @@ def lex(module=None,object=None,debug=0,optimize=0,lextab="lextab",reflags=0,now
         for fname, f in linfo.funcsym[state]:
             line = func_code(f).co_firstlineno
             file = func_code(f).co_filename
-            regex_list.append("(?P<%s>%s)" % (fname,f.__doc__))
+            regex_list.append("(?P<%s>%s)" % (fname,_get_regex(f)))
             if debug:
-                debuglog.info("lex: Adding rule %s -> '%s' (state '%s')",fname,f.__doc__, state)
+                debuglog.info("lex: Adding rule %s -> '%s' (state '%s')",fname,_get_regex(f), state)

         # Now add all of the simple rules
         for name,r in linfo.strsym[state]:
@@ -1037,6 +1037,12 @@ def runmain(lexer=None,data=None):
         if not tok: break
         sys.stdout.write("(%s,%r,%d,%d)\n" % (tok.type, tok.value, tok.lineno,tok.lexpos))

+def _get_regex(func):
+    if hasattr(func, 'regex'):
+        return func.regex
+    # Fall back to docstring if @TOKEN has not been used
+    return func.__doc__
+
 # -----------------------------------------------------------------------------
 # @TOKEN(regex)
 #
@@ -1047,9 +1053,9 @@ def runmain(lexer=None,data=None):
 def TOKEN(r):
     def set_doc(f):
         if hasattr(r,"__call__"):
-            f.__doc__ = r.__doc__
+            f.regex = _get_regex(r)
         else:
-            f.__doc__ = r
+            f.regex = r
         return f
     return set_doc

[kristofferkoch]

closing in favor of real pull request.