Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Make the tokenizer easier to debug

Signed-off-by: Nicolai Hähnle <nhaehnle@gmail.com>
  • Loading branch information...
commit 6819f36c7ae54d901c183bd261a0376344f850a9 1 parent 844ae91
@nhaehnle authored
Showing with 7 additions and 1 deletion.
  1. +1 −1  patre/cpp.py
  2. +6 −0 patre/parse.py
View
2  patre/cpp.py
@@ -61,7 +61,7 @@ def cppliteral(text, pos):
return None, None
tokenizer = Tokenizer()
-tokenizer.addfn(tok_regex(re.compile("[a-zA-z_][a-zA-Z_0-9]*", re.VERBOSE), "id"))
+tokenizer.addfn(tok_regex(re.compile("[a-zA-Z_][a-zA-Z_0-9]*", re.VERBOSE), "id"))
tokenizer.addfn(cppcomment)
tokenizer.addfn(cppliteral)
tokenizer.addfn(tok_whitespace([' ', '\t', '\n']), -100)
View
6 patre/parse.py
@@ -39,6 +39,7 @@ def inner(text, pos):
while pos < len(text) and text[pos] in white:
pos += 1
return None, pos
+ inner.func_name = "whitespace"
return inner
def tok_fallback():
@@ -48,6 +49,7 @@ def tok_fallback():
"""
def inner(text, pos):
return TextRange(text, pos, pos + 1), pos + 1
+ inner.func_name = "fallback"
return inner
def tok_regex(regex, tag=None):
@@ -64,6 +66,10 @@ def inner(text, pos):
return TextRange(text, pos, m.end(), tag), m.end()
else:
return None, None
+ if tag != None:
+ inner.func_name = "regex(tag=%s)" % (tag)
+ else:
+ inner.func_name = "regex"
return inner
class Tokenizer(object):
Please sign in to comment.
Something went wrong with that request. Please try again.