Index: test/lexer.py =================================================================== --- test/lexer.py (revision 302) +++ test/lexer.py (working copy) @@ -176,6 +176,47 @@ nodes = Lexer(template).parse() #print nodes assert repr(nodes) == r"""TemplateNode({}, [Text(u'\n some text\n \n ', (1, 1)), Code(u'\nprint "hi"\nfor x in range(1,5):\n print x\n \n', False, (4, 9)), Text(u'\n \n more text\n \n ', (8, 11)), Code(u'\nimport foo\n \n', True, (12, 9)), Text(u'\n ', (14, 11))])""" + + def test_bad_code(self): + template = """ + some text + + <% + print "hi" + print "error + for x in range(1,5): + print x + %> +""" + try: + nodes = Lexer(template).parse() + assert False + except exceptions.SyntaxException, e: + assert str(e) == "Syntax error: EOL while scanning single-quoted string at line: 6 char: 24" + + template = """ + some text + + <% + print "hi" + except: + for x in range(1,5): + print x + %> +""" + try: + nodes = Lexer(template).parse() + assert False + except exceptions.SyntaxException, e: + assert str(e) == "Syntax error: invalid syntax at line: 6 char: 18" + + template = """ + some text <% print }%>""" + try: + nodes = Lexer(template).parse() + assert False + except exceptions.SyntaxException, e: + assert str(e) == "Syntax error: invalid syntax at line: 2 char: 28" def test_code_and_tags(self): template = """ @@ -226,6 +267,22 @@ """ nodes = Lexer(template).parse() assert repr(nodes) == r"""TemplateNode({}, [Text(u'\n \n ', (1, 1)), Expression(u"hello + '''heres '{|}' text | | }''' ", ['escape1'], (3, 13)), Text(u'\n ', (3, 62))])""" + + def test_embedded_braces(self): + template = """ + + ${{1:1}[0]} + """ + nodes = Lexer(template).parse() + assert repr(nodes) == r"""TemplateNode({}, [Text(u'\n \n ', (1, 1)), Expression(u'{1:1}[0]', [], (3, 13)), Text(u'\n ', (3, 24))])""" + + template = """ + + ${{'a':'}', + 'b':'{'}[0]} + """ + nodes = Lexer(template).parse() + assert repr(nodes) == r"""TemplateNode({}, [Text(u'\n \n ', (1, 1)), Expression(u"{'a':'}',\n 'b':'{'}[0]", [], (3, 13)), Text(u'\n ', (4, 29))])""" def test_tricky_code(self): template = """<% print 'hi %>' %>""" @@ -247,6 +304,7 @@ nodes = Lexer(template).parse() assert repr(nodes) == r"""TemplateNode({}, [Code(u" \n # someone's comment\n \n", False, (1, 1)), Text(u'\n ', (3, 11))])""" + def test_tricky_code_3(self): template= """<% print 'hi' # this is a comment Index: lib/mako/lexer.py =================================================================== --- lib/mako/lexer.py (revision 302) +++ lib/mako/lexer.py (working copy) @@ -9,6 +9,7 @@ import re, codecs from mako import parsetree, exceptions from mako.pygen import adjust_whitespace +import parser _regexp_cache = {} @@ -65,25 +66,70 @@ #print "MATCH:", regexp, "\n", self.text[mp : mp + 15], (match and "TRUE" or "FALSE") return match - def parse_until_text(self, *text): + def parse_python_code_until(self, symbol, *text): startpos = self.match_position while True: - match = self.match(r'#.*\n') + match = self.match(r'.*?(%s)' % r'|'.join(text), re.S) if match: - continue - match = self.match(r'(\"\"\"|\'\'\'|\"|\')') - if match: - m = self.match(r'.*?%s' % match.group(1), re.S) - if not m: - raise exceptions.SyntaxException("Unmatched '%s'" % match.group(1), **self.exception_kwargs) + code = self.text[startpos:self.match_position-len(match.group(1))] + if not code: + raise exceptions.SyntaxException("Empty python block", **self.exception_kwargs) + indent = False + newline = False + strip = False + lines = code.split("\n") + for line in lines: + stripped = line.lstrip() + if stripped and stripped[0] != '#': + # Here code starts + if len(stripped) < len(line): + indent = True # Need normalize indentation + stripped = lines[-1].strip() + if not stripped or stripped[0] == '#': + # Compiler doesn't like if there is no newline at end of comment + # or if last line consists of only whitespace + newline = True + break + else: + if symbol != "eval": + return (code, match.group(1)) + last_lineno = len(lines) + last_offset = len(lines[-1]) + nc = code + if newline: + nc = nc + '\n' + else: + # Adding a space to check + # if error ocurs before it + nc = nc + ' ' + last_offset += 1 + if indent: + if symbol == 'eval': + nc = '(\n%s)' % nc + last_lineno += 1 + last_offset += 1 + else: + nc = 'if 1:\n' + nc + last_lineno += 1 + try: + if symbol == 'exec': + parser.suite(nc) + else: + parser.expr(nc) + except SyntaxError, err: + # If code fails to compile at last position - + # it probably needs more text, if before - + # it is definetly broken + exc = self.exception_kwargs + if err.lineno < last_lineno or err.offset < last_offset: + exc['lineno'] += err.lineno-2 + exc['pos'] = len(lines)==1 and err.offset + exc['pos']-1 or err.offset + raise exceptions.SyntaxException("Syntax error: %s" % err.msg, **exc) + else: + continue + return (code, match.group(1)) else: - match = self.match(r'(%s)' % r'|'.join(text)) - if match: - return (self.text[startpos:self.match_position-len(match.group(1))], match.group(1)) - else: - match = self.match(r".*?(?=\"|\'|#|%s)" % r'|'.join(text), re.S) - if not match: - raise exceptions.SyntaxException("Expected: %s" % ','.join(text), **self.exception_kwargs) + raise exceptions.SyntaxException("Expected: %s" % ','.join(text), **self.exception_kwargs) def append_node(self, nodecls, *args, **kwargs): kwargs.setdefault('source', self.text) @@ -269,7 +315,7 @@ match = self.match(r"<%(!)?") if match: (line, pos) = (self.matched_lineno, self.matched_charpos) - (text, end) = self.parse_until_text(r'%>') + (text, end) = self.parse_python_code_until('exec', r'%>') text = adjust_whitespace(text) self.append_node(parsetree.Code, self.escape_code(text), match.group(1)=='!', lineno=line, pos=pos) return True @@ -280,9 +326,9 @@ match = self.match(r"\${") if match: (line, pos) = (self.matched_lineno, self.matched_charpos) - (text, end) = self.parse_until_text(r'\|', r'}') + (text, end) = self.parse_python_code_until('eval', r'\|', r'}') if end == '|': - (escapes, end) = self.parse_until_text(r'}') + (escapes, end) = self.parse_python_code_until('eval', r'}') else: escapes = "" self.append_node(parsetree.Expression, self.escape_code(text), escapes.strip(), lineno=line, pos=pos)