Permalink
Browse files

add whitespace control

  • Loading branch information...
1 parent 143afb6 commit cbfb747f97815d871c9b70a161e4394f37d15ee1 @jlongster committed Jan 31, 2013
Showing with 123 additions and 16 deletions.
  1. +7 −5 bench/jinja.py
  2. +4 −2 src/lexer.js
  3. +45 −9 src/parser.js
  4. +67 −0 tests/parser.js
View
@@ -2,12 +2,14 @@
import time
from jinja2 import Template, Environment, FileSystemLoader
-# env = Environment(loader=FileSystemLoader('.'))
-# src = open('index.html').read()
+env = Environment(loader=FileSystemLoader('.'))
+src = open('index.html').read()
-# print env._generate(env._parse(src, 'poop', 'hello.html'),
+# print(env._generate(env._parse(src, 'poop', 'hello.html'),
# 'poop',
-# 'hello.html')
+# 'hello.html'))
+
+# print([x for x in env._tokenize(src, 'poop', 'hello.html')])
env = Environment(loader=FileSystemLoader('.'))
times = []
@@ -23,4 +25,4 @@
times.append(t2-t1)
-print reduce(lambda x, y: x+y, times) / len(times)
+print( reduce(lambda x, y: x+y, times) / len(times))
View
@@ -74,7 +74,8 @@ Tokenizer.prototype.nextToken = function() {
// We hit some whitespace
return token(TOKEN_WHITESPACE, tok, lineno, colno);
}
- else if((tok = this._extractString(BLOCK_END))) {
+ else if((tok = this._extractString(BLOCK_END)) ||
+ (tok = this._extractString('-' + BLOCK_END))) {
// Special check for the block end tag
//
// It is a requirement that start and end tags are composed of
@@ -155,7 +156,8 @@ Tokenizer.prototype.nextToken = function() {
if(this.is_finished()) {
return null;
}
- else if((tok = this._extractString(BLOCK_START))) {
+ else if((tok = this._extractString(BLOCK_START + '-')) ||
+ (tok = this._extractString(BLOCK_START))) {
this.in_code = true;
return token(TOKEN_BLOCK_START, tok, lineno, colno);
}
View
@@ -9,6 +9,7 @@ var Parser = Object.extend({
this.tokens = tokens;
this.peeked = null;
this.breakOnBlocks = null;
+ this.dropLeadingWhitespace = false;
},
nextToken: function (withWhitespace) {
@@ -112,7 +113,14 @@ var Parser = Object.extend({
name = this.nextToken().value;
}
- if(!this.skip(lexer.TOKEN_BLOCK_END)) {
+ var tok = this.nextToken();
+
+ if(tok.type == lexer.TOKEN_BLOCK_END) {
+ if(tok.value.charAt(0) === '-') {
+ this.dropLeadingWhitespace = true;
+ }
+ }
+ else {
this.fail("expected block end in " + name + " statement");
}
},
@@ -232,14 +240,21 @@ var Parser = Object.extend({
var names = node.names;
while(1) {
- var type = this.peekToken().type;
- if(type == lexer.TOKEN_BLOCK_END) {
+ var nextTok = this.peekToken();
+ if(nextTok.type == lexer.TOKEN_BLOCK_END) {
if(!names.children.length) {
this.fail('parseFrom: Expected at least one import name',
fromTok.lineno,
fromTok.colno);
}
+ // Since we are manually advancing past the block end,
+ // need to keep track of whitespace control (normally
+ // this is done in `advanceAfterBlockEnd`
+ if(nextTok.value.charAt(0) == '-') {
+ this.dropLeadingWhitespace = true;
+ }
+
this.nextToken();
break;
}
@@ -930,11 +945,32 @@ var Parser = Object.extend({
while((tok = this.nextToken())) {
if(tok.type == lexer.TOKEN_DATA) {
+ var data = tok.value;
+ var nextToken = this.peekToken();
+ var nextVal = nextToken && nextToken.value;
+
+ // If the last token has "-" we need to trim the
+ // leading whitespace of the data. This is marked with
+ // the `dropLeadingWhitespace` variable.
+ if(this.dropLeadingWhitespace) {
+ // TODO: this could be optimized (don't use regex)
+ data = data.replace(/^\s*/, '');
+ this.dropLeadingWhitespace = false;
+ }
+
+ // Same for the succeding block start token
+ if(nextToken &&
+ nextToken.type == lexer.TOKEN_BLOCK_START &&
+ nextVal.charAt(nextVal.length - 1) == '-') {
+ // TODO: this could be optimized (don't use regex)
+ data = data.replace(/\s*$/, '');
+ }
+
buf.push(new nodes.Output(tok.lineno,
tok.colno,
[new nodes.TemplateData(tok.lineno,
tok.colno,
- tok.value)]));
+ data)]));
}
else if(tok.type == lexer.TOKEN_BLOCK_START) {
var n = this.parseStatement();
@@ -967,17 +1003,17 @@ var Parser = Object.extend({
}
});
-var util = require('util');
+// var util = require('util');
-// var l = lexer.lex('{% set x = 3 %}');
+// var l = lexer.lex('{%- if x -%}\n hello {% endif %}');
// var t;
// while((t = l.nextToken())) {
// console.log(util.inspect(t));
// }
-// var p = new Parser(lexer.lex('{% raw %}hello{% endraw %}'));
-// var n = p.parse();
-// nodes.printNodes(n);
+var p = new Parser(lexer.lex('{% from x import y -%}\n hi \n'));
+var n = p.parse();
+nodes.printNodes(n);
module.exports = {
parse: function(src) {
View
@@ -264,6 +264,73 @@ describe('parser', function() {
[nodes.Symbol, 'foobarbaz']]]]]);
});
+ it('should parse whitespace control', function() {
+ // Every start/end tag with "-" should trim the whitespace
+ // before or after it
+
+ isAST(parser.parse('{% if x %}\n hi \n{% endif %}'),
+ [nodes.Root,
+ [nodes.If,
+ [nodes.Symbol, 'x'],
+ [nodes.NodeList,
+ [nodes.Output,
+ [nodes.TemplateData, '\n hi \n']]]]]);
+
+ isAST(parser.parse('{% if x -%}\n hi \n{% endif %}'),
+ [nodes.Root,
+ [nodes.If,
+ [nodes.Symbol, 'x'],
+ [nodes.NodeList,
+ [nodes.Output,
+ [nodes.TemplateData, 'hi \n']]]]]);
+
+ isAST(parser.parse('{% if x %}\n hi \n{%- endif %}'),
+ [nodes.Root,
+ [nodes.If,
+ [nodes.Symbol, 'x'],
+ [nodes.NodeList,
+ [nodes.Output,
+ [nodes.TemplateData, '\n hi']]]]]);
+
+ isAST(parser.parse('{% if x -%}\n hi \n{%- endif %}'),
+ [nodes.Root,
+ [nodes.If,
+ [nodes.Symbol, 'x'],
+ [nodes.NodeList,
+ [nodes.Output,
+ [nodes.TemplateData, 'hi']]]]]);
+
+ isAST(parser.parse('poop \n{%- if x -%}\n hi \n{%- endif %}'),
+ [nodes.Root,
+ [nodes.Output,
+ [nodes.TemplateData, 'poop']],
+ [nodes.If,
+ [nodes.Symbol, 'x'],
+ [nodes.NodeList,
+ [nodes.Output,
+ [nodes.TemplateData, 'hi']]]]]);
+
+ // The from statement required a special case so make sure to
+ // test it
+ isAST(parser.parse('{% from x import y %}\n hi \n'),
+ [nodes.Root,
+ [nodes.FromImport,
+ [nodes.Symbol, 'x'],
+ [nodes.NodeList,
+ [nodes.Symbol, 'y']]],
+ [nodes.Output,
+ [nodes.TemplateData, '\n hi \n']]]);
+
+ isAST(parser.parse('{% from x import y -%}\n hi \n'),
+ [nodes.Root,
+ [nodes.FromImport,
+ [nodes.Symbol, 'x'],
+ [nodes.NodeList,
+ [nodes.Symbol, 'y']]],
+ [nodes.Output,
+ [nodes.TemplateData, 'hi \n']]]);
+ });
+
it('should throw errors', function() {
(function() {
parser.parse('hello {{ foo');

0 comments on commit cbfb747

Please sign in to comment.