Skip to content
Browse files

Fixed whitespace handling in comments

  • Loading branch information...
1 parent d761e0a commit 6437a744b6c5cf885cb7e4a01efc9c1b17582675 @dabeaz committed Oct 15, 2012
Showing with 15 additions and 5 deletions.
  1. +15 −5 ply/cpp.py
View
20 ply/cpp.py
@@ -15,7 +15,7 @@
# -----------------------------------------------------------------------------
tokens = (
- 'CPP_ID','CPP_INTEGER', 'CPP_FLOAT', 'CPP_STRING', 'CPP_CHAR', 'CPP_WS', 'CPP_COMMENT', 'CPP_POUND','CPP_DPOUND'
+ 'CPP_ID','CPP_INTEGER', 'CPP_FLOAT', 'CPP_STRING', 'CPP_CHAR', 'CPP_WS', 'CPP_COMMENT1', 'CPP_COMMENT2', 'CPP_POUND','CPP_DPOUND'
)
literals = "+-*/%|&~^<>=!?()[]{}.,;:\\\'\""
@@ -55,10 +55,19 @@ def t_CPP_CHAR(t):
return t
# Comment
-def t_CPP_COMMENT(t):
- r'(/\*(.|\n)*?\*/)|(//.*?\n)'
- t.lexer.lineno += t.value.count("\n")
+def t_CPP_COMMENT1(t):
+ r'(/\*(.|\n)*?\*/)'
+ ncr = t.value.count("\n")
+ t.lexer.lineno += ncr
+ # replace with one space or a number of '\n'
+ t.type = 'CPP_WS'; t.value = '\n' * ncr if ncr else ' '
return t
+
+# Line comment
+def t_CPP_COMMENT2(t):
+ r'(//.*?(\n|$))'
+ # replace with '/n'
+ t.type = 'CPP_WS'; t.value = '\n'
def t_error(t):
t.type = t.value[0]
@@ -614,8 +623,9 @@ def parsegen(self,input,source=None):
if tok.value == '#':
# Preprocessor directive
+ # insert necessary whitespace instead of eaten tokens
for tok in x:
- if tok in self.t_WS and '\n' in tok.value:
+ if tok.type in self.t_WS and '\n' in tok.value:
chunk.append(tok)
dirtokens = self.tokenstrip(x[i+1:])

0 comments on commit 6437a74

Please sign in to comment.
Something went wrong with that request. Please try again.