Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@ericmorand
ericmorand committed Jun 4, 2019
1 parent d954a76 commit 69fc2f8
Show file tree
Hide file tree
Showing 12 changed files with 129 additions and 78 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
"fastest:all": "tape 'test/tests/**/test.js' | tap-bail | tap-spec",
"fastest:integration": "tape test/tests/integration/**/test.js | tap-bail | tap-spec",
"fastest:integration:browser": "node test/tests/integration/browser.js | browserify - --basedir ./test -t [ stringify --extensions [.html .twig] ] | tape-run --render='tap-spec'",
"fastest:unit": "tape 'test/tests/unit/**/token/test.js' | tap-bail | tap-spec",
"fastest:unit": "tape 'test/tests/unit/**/test.js' | tap-bail | tap-spec",
"cover": "nyc npm t",
"coverage": "nyc report --reporter=text-lcov | coveralls",
"build": "tsc --project .",
Expand Down
22 changes: 2 additions & 20 deletions src/lexer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -484,24 +484,6 @@ export class TwingLexer {
}
}

private handleLeadingLineFeeds(text: string) {
let match: RegExpExecArray = /^\n+/.exec(text);

if (match) {
let lineno = this.lineno;

this.moveCoordinates(match[0]);

let lineCount = this.lineno - lineno;

this.cursor += lineCount;

text = text.substring(lineCount);
}

return text;
}

private moveCursor(text: string) {
this.cursor += text.length;
}
Expand Down Expand Up @@ -571,9 +553,9 @@ export class TwingLexer {

if (modifier === this.options.whitespace_trim) {
type = TwingTokenType.WHITESPACE_CONTROL_MODIFIER_TRIMMING;
} else {
} /** else {
type = TwingTokenType.WHITESPACE_CONTROL_MODIFIER_LINE_TRIMMING;
}
} **/

this.tokens.push(new TwingToken(type, modifier, this.lineno, this.columnno));
this.moveCoordinates(modifier);
Expand Down
2 changes: 1 addition & 1 deletion src/node/verbatim.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import {TwingNodeText} from "./text";
* @author Eric Morand <eric.morand@gmail.com>
*/
export class TwingNodeVerbatim extends TwingNodeText {
constructor(data: string, lineno: number, columnno: number, tag: string = null) {
constructor(data: string, lineno: number, columnno: number, tag: string) {
super(data, lineno, columnno);

this.type = TwingNodeType.VERBATIM;
Expand Down
6 changes: 0 additions & 6 deletions src/parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import {TwingTokenParser} from "./token-parser";
import {first} from "./helper/first";
import {push} from "./helper/push";
import {TwingNodeComment} from "./node/comment";
import {TwingNodeVerbatim} from "./node/verbatim";

const ctype_space = require('locutus/php/ctype/ctype_space');
const sha256 = require('crypto-js/sha256');
Expand Down Expand Up @@ -176,11 +175,6 @@ export class TwingParser {

while (!this.stream.isEOF()) {
switch (this.getCurrentToken().getType()) {
// case TwingTokenType.VERBATIM_START:
// token = this.stream.next();
// rv.set(i++, new TwingNodeVerbatim(token.getContent(), token.getLine(), token.getColumn()));
//
// break;
case TwingTokenType.TEXT:
token = this.stream.next();
rv.set(i++, new TwingNodeText(token.getContent(), token.getLine(), token.getColumn()));
Expand Down
12 changes: 12 additions & 0 deletions src/token-stream.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,11 @@ export class TwingTokenStream {
return new TwingTokenStream(tokens, this.source);
}

/**
* Inject tokens after the current one.
*
* @param tokens
*/
injectTokens(tokens: Array<TwingToken>) {
this.tokens = array_merge(this.tokens.slice(0, this.current), tokens, this.tokens.slice(this.current));
}
Expand Down Expand Up @@ -234,4 +239,11 @@ export class TwingTokenStream {
getSourceContext() {
return this.source;
}

/**
* @return TwingToken[]
*/
getTokens(): TwingToken[] {
return this.tokens;
}
}
16 changes: 11 additions & 5 deletions test/tests/unit/expression-parser/test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@ tap.test('expression-parser', function (test) {
['{% set NULL = "foo" %}', 'You cannot assign a value to "NULL".'],
['{% set 3 = "foo" %}', 'Only variables can be assigned to. Unexpected token "number" of value "3" ("name" expected).'],
['{% set 1 + 2 = "foo" %}', 'Only variables can be assigned to. Unexpected token "number" of value "1" ("name" expected).'],
['{% set "bar" = "foo" %}', 'Only variables can be assigned to. Unexpected token "string" of value "bar" ("name" expected).'],
['{% set %}{% endset %})', 'Only variables can be assigned to. Unexpected token "end of statement block" of value "null" ("name" expected).']
['{% set "bar" = "foo" %}', 'Only variables can be assigned to. Unexpected token "opening quote" of value """ ("name" expected).'],
['{% set %}{% endset %})', 'Only variables can be assigned to. Unexpected token "end of statement block" of value "%}" ("name" expected).']
];

let loader = new TwingTestMockLoader();
Expand All @@ -120,9 +120,15 @@ tap.test('expression-parser', function (test) {
for (let templateAndMessage of templatesAndMessages) {
let source = new TwingSource(templateAndMessage[0], 'index');

test.throws(function () {
parser.parse(env.tokenize(source));
}, new TwingErrorSyntax(templateAndMessage[1], 1, source), 'should throw a TwingErrorSyntax')
try {
parser.parse(env.tokenize(source).toAst());

test.fail('Should throw a TwingErrorSyntax "' + templateAndMessage[1] + '"')
}
catch (e) {
test.same(e.name, 'TwingErrorSyntax', 'name is TwingErrorSyntax');
test.same(e.getRawMessage(), templateAndMessage[1], 'message is "' + templateAndMessage[1] + '"');
}
}

test.end();
Expand Down
24 changes: 24 additions & 0 deletions test/tests/unit/lexer/test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -784,6 +784,26 @@ bla
test.end();
});

test.test('macro', function(test) {
let template = '{% macro a() %}';

let lexer = createLexer();
let stream = lexer.tokenize(new TwingSource(template, 'index'));

testToken(test, stream.expect(TwingTokenType.BLOCK_START), '{%', 1, 1);
testToken(test, stream.expect(TwingTokenType.WHITESPACE), ' ', 1, 3);
testToken(test, stream.expect(TwingTokenType.NAME), 'macro', 1, 4);
testToken(test, stream.expect(TwingTokenType.WHITESPACE), ' ', 1, 9);
testToken(test, stream.expect(TwingTokenType.NAME), 'a', 1, 10);
testToken(test, stream.expect(TwingTokenType.PUNCTUATION), '(', 1, 11);
testToken(test, stream.expect(TwingTokenType.PUNCTUATION), ')', 1, 12);
testToken(test, stream.expect(TwingTokenType.WHITESPACE), ' ', 1, 13);
testToken(test, stream.expect(TwingTokenType.BLOCK_END), '%}', 1, 14);
testToken(test, stream.getCurrent(), null, 1, 16, TwingTokenType.EOF);

test.end();
});

test.test('CST is lossless', function(test) {
let template = `{{ [1, 2] }}
{% embed __foo %}
Expand All @@ -792,6 +812,10 @@ bla
{{ a_function ( "a": 5)
}}
{% verbatim %}
{{ foo }}
{% bar %}{% endbar %}
{% endverbatim %}
{%endembed%}
`;
Expand Down
35 changes: 19 additions & 16 deletions test/tests/unit/token-parser/block/test.js
View file
Original file line number Diff line number Diff line change
@@ -1,41 +1,44 @@
const {TwingTokenParserBlock} = require('../../../../../build/token-parser/block');
const {TwingTokenStream} = require('../../../../../build/token-stream');
const {TwingToken} = require('../../../../../build/token');
const {TwingErrorSyntax} = require('../../../../../build/error/syntax');
const {TwingSource} = require('../../../../../build/source');
const {TwingToken, TwingTokenType} = require('../../../../../build/token');

const TwingTestMockBuilderParser = require('../../../../mock-builder/parser');

const tap = require('tape');

class ExpressionParser {
parseExpression() {
return new TwingToken(TwingToken.NAME_TYPE, 'foo', 1);
return new TwingToken(TwingTokenType.NAME, 'foo', 1, 1);
}
}

tap.test('token-parser/block', function (test) {
test.test('parse', function (test) {
test.test('when endblock name doesn\'t match', function(test) {
let stream = new TwingTokenStream([
new TwingToken(TwingToken.NAME_TYPE, 'foo', 1),
new TwingToken(TwingToken.BLOCK_END_TYPE, null, 1),
new TwingToken(TwingToken.TEXT_TYPE, 'FOO', 1),
new TwingToken(TwingToken.BLOCK_START_TYPE, null, 1),
new TwingToken(TwingToken.NAME_TYPE, 'endblock', 1),
new TwingToken(TwingToken.NAME_TYPE, 'bar', 1),
new TwingToken(TwingToken.BLOCK_END_TYPE, null, 1)
new TwingToken(TwingTokenType.NAME, 'foo', 1, 1),
new TwingToken(TwingTokenType.BLOCK_END, null, 1, 1),
new TwingToken(TwingTokenType.TEXT, 'FOO', 1, 1),
new TwingToken(TwingTokenType.BLOCK_START, null, 1, 1),
new TwingToken(TwingTokenType.NAME, 'endblock', 1, 1),
new TwingToken(TwingTokenType.NAME, 'bar', 1, 1),
new TwingToken(TwingTokenType.BLOCK_END, null, 1, 1)
]);

let tokenParser = new TwingTokenParserBlock();
let parser = TwingTestMockBuilderParser.getParser(stream, new ExpressionParser());

tokenParser.setParser(parser);

test.throws(function () {
tokenParser.parse(new TwingToken(TwingToken.NAME_TYPE, 'block', 1));
}, new TwingErrorSyntax('Expected endblock for block "foo" (but "bar" given).', 1, new TwingSource('', '')));


try {
tokenParser.parse(new TwingToken(TwingTokenType.NAME, 'block', 1, 1));

test.fail('should throw an error');
}
catch (e) {
test.same(e.getRawMessage(), 'Expected endblock for block "foo" (but "bar" given).')
}

test.end();
});

Expand Down
31 changes: 17 additions & 14 deletions test/tests/unit/token-parser/macro/test.js
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,15 @@
const {TwingTokenParserMacro} = require('../../../../../build/token-parser/macro');
const {TwingTokenStream} = require('../../../../../build/token-stream');
const {TwingToken} = require('../../../../../build/token');
const {TwingToken, TwingTokenType} = require('../../../../../build/token');
const {TwingNode} = require('../../../../../build/node');
const {TwingErrorSyntax} = require('../../../../../build/error/syntax');
const {TwingSource} = require('../../../../../build/source');

const TwingTestMockBuilderParser = require('../../../../mock-builder/parser');

const tap = require('tape');

class ExpressionParser {
parseExpression() {
return new TwingToken(TwingToken.NAME_TYPE, 'foo', 1, 1);
return new TwingToken(TwingTokenType.NAME, 'foo', 1, 1);
}

parseArguments() {
Expand All @@ -23,23 +21,28 @@ tap.test('token-parser/macro', function (test) {
test.test('parse', function (test) {
test.test('when endmacro name doesn\'t match', function(test) {
let stream = new TwingTokenStream([
new TwingToken(TwingToken.NAME_TYPE, 'foo', 1, 1),
new TwingToken(TwingToken.BLOCK_END_TYPE, null, 1, 1),
new TwingToken(TwingToken.TEXT_TYPE, 'FOO', 1, 1),
new TwingToken(TwingToken.BLOCK_START_TYPE, null, 1, 1),
new TwingToken(TwingToken.NAME_TYPE, 'endmacro', 1, 1),
new TwingToken(TwingToken.NAME_TYPE, 'bar', 1, 1),
new TwingToken(TwingToken.BLOCK_END_TYPE, null, 1, 1)
new TwingToken(TwingTokenType.NAME, 'foo', 1, 1),
new TwingToken(TwingTokenType.BLOCK_END, '%}', 1, 1),
new TwingToken(TwingTokenType.TEXT, 'FOO', 1, 1),
new TwingToken(TwingTokenType.BLOCK_START, '{%', 1, 1),
new TwingToken(TwingTokenType.NAME, 'endmacro', 1, 1),
new TwingToken(TwingTokenType.NAME, 'bar', 1, 1),
new TwingToken(TwingTokenType.BLOCK_END, '%}', 1, 1)
]);

let tokenParser = new TwingTokenParserMacro();
let parser = TwingTestMockBuilderParser.getParser(stream, new ExpressionParser());

tokenParser.setParser(parser);

test.throws(function () {
tokenParser.parse(new TwingToken(TwingToken.NAME_TYPE, 'block', 1, 1));
}, new TwingErrorSyntax('Expected endmacro for macro "foo" (but "bar" given).', 1, new TwingSource('', '')));
try {
tokenParser.parse(new TwingToken(TwingTokenType.NAME, 'macro', 1, 1));

test.fail('should throw an error');
}
catch (e) {
test.same(e.getRawMessage(), 'Expected endmacro for macro "foo" (but "bar" given).')
}

test.end();
});
Expand Down
36 changes: 22 additions & 14 deletions test/tests/unit/token-parser/set/test.js
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
const {TwingTokenParserSet} = require('../../../../../build/token-parser/set');
const {TwingTokenStream} = require('../../../../../build/token-stream');
const {TwingToken} = require('../../../../../build/token');
const {TwingToken, TwingTokenType} = require('../../../../../build/token');
const {TwingNode} = require('../../../../../build/node');
const {TwingErrorSyntax} = require('../../../../../build/error/syntax');
const {TwingSource} = require('../../../../../build/source');

const TwingTestMockBuilderParser = require('../../../../mock-builder/parser');

Expand All @@ -15,9 +13,9 @@ tap.test('token-parser/set', function (test) {
test.test('when direct assignment', function (test) {
test.test('when different number of variables and assignments', function (test) {
let stream = new TwingTokenStream([
new TwingToken(TwingToken.OPERATOR_TYPE, '=', 1, 1),
new TwingToken(TwingToken.BLOCK_END_TYPE, null, 1, 1),
new TwingToken(TwingToken.EOF_TYPE, null, 1, 1)
new TwingToken(TwingTokenType.OPERATOR, '=', 1, 1),
new TwingToken(TwingTokenType.BLOCK_END, null, 1, 1),
new TwingToken(TwingTokenType.EOF, null, 1, 1)
]);

let tokenParser = new TwingTokenParserSet();
Expand All @@ -29,9 +27,14 @@ tap.test('token-parser/set', function (test) {
sinon.stub(expressionParser, 'parseAssignmentExpression').returns(new TwingNode(new Map([[0, 'foo']])));
sinon.stub(expressionParser, 'parseMultitargetExpression').returns(new TwingNode(new Map([[0, 'oof'], [1, 'bar']])));

test.throws(function () {
tokenParser.parse(new TwingToken(TwingToken.NAME_TYPE, 'set', 1, 1))
}, new TwingErrorSyntax('When using set, you must have the same number of variables and assignments.', 1, new TwingSource('', '')));
try {
tokenParser.parse(new TwingToken(TwingTokenType.NAME, 'set', 1, 1));

test.fail('should throw an error');
}
catch (e) {
test.same(e.getRawMessage(), 'When using set, you must have the same number of variables and assignments.')
}

test.end();
});
Expand All @@ -42,8 +45,8 @@ tap.test('token-parser/set', function (test) {
test.test('when capture', function (test) {
test.test('when multiple targets', function (test) {
let stream = new TwingTokenStream([
new TwingToken(TwingToken.BLOCK_END_TYPE, null, 1, 1),
new TwingToken(TwingToken.EOF_TYPE, null, 1, 1)
new TwingToken(TwingTokenType.BLOCK_END, null, 1, 1),
new TwingToken(TwingTokenType.EOF, null, 1, 1)
]);

let tokenParser = new TwingTokenParserSet();
Expand All @@ -54,9 +57,14 @@ tap.test('token-parser/set', function (test) {

sinon.stub(expressionParser, 'parseAssignmentExpression').returns(new TwingNode(new Map([[0, 'foo'], [1, 'bar']])));

test.throws(function () {
tokenParser.parse(new TwingToken(TwingToken.NAME_TYPE, 'set', 1, 1))
}, new TwingErrorSyntax('When using set with a block, you cannot have a multi-target.', 1, new TwingSource('', '')));
try {
tokenParser.parse(new TwingToken(TwingTokenType.NAME, 'set', 1, 1));

test.fail('should throw an error');
}
catch (e) {
test.same(e.getRawMessage(), 'When using set with a block, you cannot have a multi-target.')
}

test.end();
});
Expand Down
15 changes: 15 additions & 0 deletions test/tests/unit/token-stream/test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,21 @@ const {TwingEnvironmentNode: TwingEnvironment} = require('../../../../build/envi
const tap = require('tape');

tap.test('token-stream', function (test) {
test.test('constructor', function (test) {
let tokens = [
new TwingToken(TwingTokenType.PUNCTUATION, '{', 1, 1),
new TwingToken(TwingTokenType.WHITESPACE, '\n ', 1, 1),
new TwingToken(TwingTokenType.NAME, 'foo', 1, 1),
new TwingToken(TwingTokenType.EOF, null, 1, 1)
];

let stream = new TwingTokenStream(tokens);

test.same(stream.getTokens(), tokens);

test.end();
});

test.test('should provide textual representation', function (test) {
let loader = new TwingLoader({
index: ''
Expand Down
6 changes: 5 additions & 1 deletion test/tests/unit/token/test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,13 @@ tap.test('token', function (test) {
test.same(TwingToken.typeToEnglish(TwingTokenType.COMMENT_START), 'begin of comment statement');
test.same(TwingToken.typeToEnglish(TwingTokenType.COMMENT_END), 'end of comment statement');
test.same(TwingToken.typeToEnglish(TwingTokenType.WHITESPACE), 'whitespace');
test.same(TwingToken.typeToEnglish(TwingTokenType.CLOSING_QUOTE), 'closing quote');
test.same(TwingToken.typeToEnglish(TwingTokenType.OPENING_QUOTE), 'opening quote');
test.same(TwingToken.typeToEnglish(TwingTokenType.WHITESPACE_CONTROL_MODIFIER_TRIMMING), 'trimming whitespace control modifier');
test.same(TwingToken.typeToEnglish(TwingTokenType.WHITESPACE_CONTROL_MODIFIER_LINE_TRIMMING), 'line trimming whitespace control modifier');

test.throws(function() {
TwingToken.typeToEnglish(-999);
TwingToken.typeToEnglish('999');
}, new Error('Token of type "-999" does not exist.'));

test.end();
Expand Down

0 comments on commit 69fc2f8

Please sign in to comment.