Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Implement lstripBlocks and trimBlocks from Jinja2 #355

Merged
merged 2 commits into from
Feb 4, 2015
Merged

Implement lstripBlocks and trimBlocks from Jinja2 #355

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fecfa42
Implement lstripBlocks and trimBlocks from Jinja2
radev Jan 17, 2015
9a77099
Moved environment options to opts
radev Jan 24, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions src/compiler.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ var Compiler = Object.extend({
_bufferAppend: function(func) {
this.emit(this.buffer + ' += runtime.suppressValue(');
func.call(this);
this.emit(', env.autoesc);\n');
this.emit(', env.opts.autoescape);\n');
},

_compileChildren: function(node, frame) {
Expand Down Expand Up @@ -277,12 +277,12 @@ var Compiler = Object.extend({
if(async) {
var res = this.tmpid();
this.emitLine(', ' + this.makeCallback(res));
this.emitLine(this.buffer + ' += runtime.suppressValue(' + res + ', ' + autoescape + ' && env.autoesc);');
this.emitLine(this.buffer + ' += runtime.suppressValue(' + res + ', ' + autoescape + ' && env.opts.autoescape);');
this.addScopeLevel();
}
else {
this.emit(')');
this.emit(', ' + autoescape + ' && env.autoesc);\n');
this.emit(', ' + autoescape + ' && env.opts.autoescape);\n');
}
},

Expand Down Expand Up @@ -427,7 +427,7 @@ var Compiler = Object.extend({
this._compileExpression(node.target, frame);
this.emit('),');
this._compileExpression(node.val, frame);
this.emit(', env.autoesc)');
this.emit(', env.opts.autoescape)');
},

_getNodeName: function(node) {
Expand Down Expand Up @@ -1034,7 +1034,7 @@ var Compiler = Object.extend({
else {
this.emit(this.buffer + ' += runtime.suppressValue(');
this.compile(children[i], frame);
this.emit(', env.autoesc);\n');
this.emit(', env.opts.autoescape);\n');
}
}
},
Expand Down Expand Up @@ -1103,7 +1103,7 @@ var Compiler = Object.extend({
// console.log(tmpl);

module.exports = {
compile: function(src, asyncFilters, extensions, name, lexerTags) {
compile: function(src, asyncFilters, extensions, name, opts) {
var c = new Compiler();

// Run the extension preprocessors against the source.
Expand All @@ -1115,7 +1115,9 @@ module.exports = {
}
}

c.compile(transformer.transform(parser.parse(src, extensions, lexerTags),
c.compile(transformer.transform(parser.parse(src,
extensions,
opts),
asyncFilters,
name));
return c.getCode();
Expand Down
13 changes: 8 additions & 5 deletions src/environment.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,18 @@ var Environment = Obj.extend({
// (the full trace from within nunjucks may confuse developers using
// the library)
// defaults to false
opts = opts || {};
this.dev = !!opts.dev;
this.lexerTags = opts.tags;
var opts = this.opts = opts || {};
this.opts.dev = !!opts.dev;

// The autoescape flag sets global autoescaping. If true,
// every string variable will be escaped by default.
// If false, strings can be manually escaped using the `escape` filter.
// defaults to false
this.autoesc = !!opts.autoescape;
this.opts.autoescape = !!opts.autoescape;

this.opts.trimBlocks = !!opts.trimBlocks;

this.opts.lstripBlocks = !!opts.lstripBlocks;

if(!loaders) {
// The filesystem loader is only available client-side
Expand Down Expand Up @@ -415,7 +418,7 @@ var Template = Obj.extend({
this.env.asyncFilters,
this.env.extensionsList,
this.path,
this.env.lexerTags);
this.env.opts);

var func = new Function(source);
props = func();
Expand Down
38 changes: 32 additions & 6 deletions src/lexer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ function token(type, value, lineno, colno) {
};
}

function Tokenizer(str, tags) {
function Tokenizer(str, opts) {
this.str = str;
this.index = 0;
this.len = str.length;
Expand All @@ -54,7 +54,9 @@ function Tokenizer(str, tags) {

this.in_code = false;

tags = tags || {};
opts = opts || {};

tags = opts.tags || {};
this.tags = {
BLOCK_START: tags.blockStart || BLOCK_START,
BLOCK_END: tags.blockEnd || BLOCK_END,
Expand All @@ -63,16 +65,19 @@ function Tokenizer(str, tags) {
COMMENT_START: tags.commentStart || COMMENT_START,
COMMENT_END: tags.commentEnd || COMMENT_END
};

this.trimBlocks = !!opts.trimBlocks;
this.lstripBlocks = !!opts.lstripBlocks;
}

Tokenizer.prototype.nextToken = function() {
var lineno = this.lineno;
var colno = this.colno;
var tok;

if(this.in_code) {
// Otherwise, if we are in a block parse it as code
var cur = this.current();
var tok;

if(this.is_finished()) {
// We have nothing else to parse
Expand All @@ -95,6 +100,13 @@ Tokenizer.prototype.nextToken = function() {
// breaks on delimiters so we can assume the token parsing
// doesn't consume these elsewhere
this.in_code = false;
if(this.trimBlocks) {
cur = this.current();
if(cur === '\n') {
// Skip newline
this.forward();
}
}
return token(TOKEN_BLOCK_END, tok, lineno, colno);
}
else if((tok = this._extractString(this.tags.VARIABLE_END))) {
Expand Down Expand Up @@ -195,7 +207,6 @@ Tokenizer.prototype.nextToken = function() {
this.tags.VARIABLE_START.charAt(0) +
this.tags.COMMENT_START.charAt(0) +
this.tags.COMMENT_END.charAt(0));
var tok;

if(this.is_finished()) {
return null;
Expand Down Expand Up @@ -232,6 +243,21 @@ Tokenizer.prototype.nextToken = function() {
this._matches(this.tags.VARIABLE_START) ||
this._matches(this.tags.COMMENT_START)) &&
!in_comment) {
if(this.lstripBlocks &&
this._matches(this.tags.BLOCK_START) &&
this.colno > 0 &&
this.colno <= tok.length) {
var lastLine = tok.slice(-this.colno);
if(/^\s+$/.test(lastLine)) {
// Remove block leading whitespace from beginning of the string
tok = tok.slice(0, -this.colno);
if(!tok.length) {
// All data removed, collapse to avoid unnecessary nodes
// by returning next token (block start)
return this.nextToken();
}
}
}
// If it is a start tag, stop looping
break;
}
Expand Down Expand Up @@ -412,8 +438,8 @@ Tokenizer.prototype.previous = function() {
};

module.exports = {
lex: function(src, tags) {
return new Tokenizer(src, tags);
lex: function(src, opts) {
return new Tokenizer(src, opts);
},

TOKEN_STRING: TOKEN_STRING,
Expand Down
4 changes: 2 additions & 2 deletions src/parser.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1189,8 +1189,8 @@ var Parser = Object.extend({
// nodes.printNodes(n);

module.exports = {
parse: function(src, extensions, lexerTags) {
var p = new Parser(lexer.lex(src, lexerTags));
parse: function(src, extensions, opts) {
var p = new Parser(lexer.lex(src, opts));
if (extensions !== undefined) {
p.extensions = extensions;
}
Expand Down
57 changes: 51 additions & 6 deletions tests/lexer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,51 @@
lexer.TOKEN_DATA);
});

it('should trim blocks', function () {
tokens = lexer.lex(' {% if true %}\n foo\n {% endif %}\n', {trimBlocks: true});
hasTokens(tokens,
[lexer.TOKEN_DATA, ' '],
lexer.TOKEN_BLOCK_START,
lexer.TOKEN_SYMBOL,
lexer.TOKEN_BOOLEAN,
lexer.TOKEN_BLOCK_END,
[lexer.TOKEN_DATA, ' foo\n '],
lexer.TOKEN_BLOCK_START,
lexer.TOKEN_SYMBOL,
lexer.TOKEN_BLOCK_END);
});

it('should lstrip and trim blocks', function () {
tokens = lexer.lex('test\n {% if true %}\n foo\n {% endif %}\n</div>', {
lstripBlocks: true,
trimBlocks: true
});
hasTokens(tokens,
[lexer.TOKEN_DATA, 'test\n'],
lexer.TOKEN_BLOCK_START,
lexer.TOKEN_SYMBOL,
lexer.TOKEN_BOOLEAN,
lexer.TOKEN_BLOCK_END,
[lexer.TOKEN_DATA, ' foo\n'],
lexer.TOKEN_BLOCK_START,
lexer.TOKEN_SYMBOL,
lexer.TOKEN_BLOCK_END,
[lexer.TOKEN_DATA, '</div>']);
});

it('should lstrip and not collapse whitespace between blocks', function () {
tokens = lexer.lex(' {% t %} {% t %}', {lstripBlocks: true});
hasTokens(tokens,
lexer.TOKEN_BLOCK_START,
lexer.TOKEN_SYMBOL,
lexer.TOKEN_BLOCK_END,
[lexer.TOKEN_DATA, ' '],
lexer.TOKEN_BLOCK_START,
lexer.TOKEN_SYMBOL,
lexer.TOKEN_BLOCK_END);
});


it('should parse variable start and end', function() {
tokens = lexer.lex('data {{ foo }} bar bizzle');
hasTokens(tokens,
Expand Down Expand Up @@ -242,7 +287,7 @@
}),

it('should allow changing the variable start and end', function() {
tokens = lexer.lex('data {= var =}', {variableStart: '{=', variableEnd: '=}'});
tokens = lexer.lex('data {= var =}', {tags: {variableStart: '{=', variableEnd: '=}'}});
hasTokens(tokens,
lexer.TOKEN_DATA,
lexer.TOKEN_VARIABLE_START,
Expand All @@ -251,14 +296,14 @@
}),

it('should allow changing the block start and end', function() {
tokens = lexer.lex('{= =}', {blockStart: '{=', blockEnd: '=}'});
tokens = lexer.lex('{= =}', {tags: {blockStart: '{=', blockEnd: '=}'}});
hasTokens(tokens,
lexer.TOKEN_BLOCK_START,
lexer.TOKEN_BLOCK_END);
}),

it('should allow changing the variable start and end', function() {
tokens = lexer.lex('data {= var =}', {variableStart: '{=', variableEnd: '=}'});
tokens = lexer.lex('data {= var =}', {tags: {variableStart: '{=', variableEnd: '=}'}});
hasTokens(tokens,
lexer.TOKEN_DATA,
lexer.TOKEN_VARIABLE_START,
Expand All @@ -267,7 +312,7 @@
}),

it('should allow changing the comment start and end', function() {
tokens = lexer.lex('<!-- A comment! -->', {commentStart: '<!--', commentEnd: '-->'});
tokens = lexer.lex('<!-- A comment! -->', {tags: {commentStart: '<!--', commentEnd: '-->'}});
hasTokens(tokens,
lexer.TOKEN_COMMENT);
}),
Expand All @@ -276,13 +321,13 @@
* Test that this bug is fixed: https://github.com/mozilla/nunjucks/issues/235
*/
it('should have individual lexer tag settings for each environment', function() {
tokens = lexer.lex('{=', {variableStart: '{='});
tokens = lexer.lex('{=', {tags: {variableStart: '{='}});
hasTokens(tokens, lexer.TOKEN_VARIABLE_START);

tokens = lexer.lex('{{');
hasTokens(tokens, lexer.TOKEN_VARIABLE_START);

tokens = lexer.lex('{{', {variableStart: '<<<'});
tokens = lexer.lex('{{', {tags: {variableStart: '<<<'}});
hasTokens(tokens, lexer.TOKEN_DATA);

tokens = lexer.lex('{{');
Expand Down