Merge 66970a0 into e8d942c

docs/internals.md

@@ -19,19 +19,7 @@ The rendering of a Twig template can be summarized into four key steps:
 
 ## The Lexer
 
-The lexer tokenizes a template source code into a token stream (each token is an instance of `TwingToken`, and the stream is an instance of `TwingTokenStream`). The default lexer recognizes 15 different token types:
-
-* `TwingToken.BLOCK_START_TYPE`, `TwingToken.BLOCK_END_TYPE`: Delimiters for blocks (`{% %}`)
-* `TwingToken.VAR_START_TYPE`, `TwingToken.VAR_END_TYPE`: Delimiters for variables (`{{ }}`)
-* `TwingToken.COMMENT_START_TYPE`, `TwingToken.COMMENT_END_TYPE`: Delimiters for comments (`{# #}`)
-* `TwingToken.TEXT_TYPE`: A text outside an expression;
-* `TwingToken.NAME_TYPE`: A name in an expression;
-* `TwingToken.NUMBER_TYPE`: A number in an expression;
-* `TwingToken.STRING_TYPE`: A string in an expression;
-* `TwingToken.OPERATOR_TYPE`: An operator;
-* `TwingToken.PUNCTUATION_TYPE`: A punctuation sign;
-* `TwingToken.INTERPOLATION_START_TYPE`, `TwingToken.INTERPOLATION_END_TYPE`: Delimiters for string interpolation;
-* `TwingToken.EOF_TYPE`: Ends of template.
+The lexer tokenizes a template source code into a token stream. Each token is an instance of  [twig-lexer][twig-lexer-url] `Token` and the stream is an instance of `TwingTokenStream`.
 
 You can manually convert a source code into a token stream by calling the `tokenize()` method of an environment:
 
@@ -48,11 +36,11 @@ console.log(stream.toString());
 Here is the output for the `Hello {{ name }}` template:
 
 ````
-TEXT_TYPE(Hello )
-VAR_START_TYPE()
-NAME_TYPE(name)
-VAR_END_TYPE()
-EOF_TYPE()
+TEXT(Hello )
+VARIABLE_START({{)
+NAME(name)
+VARIABLE_END(}})
+EOF()
 ```` 
 
 > The default lexer (`TwingLexer`) can be changed by calling the `setLexer()` method:
@@ -178,3 +166,4 @@ twing.setCompiler(compiler);
 [back][back-url]
 
 [back-url]: {{ site.baseurl }}{% link index.md %}
+[twig-lexer-url]: https://www.npmjs.com/package/twig-lexer

package.json

@@ -62,7 +62,7 @@
     "isobject": "^3.0.1",
     "levenshtein": "^1.0.5",
     "locutus": "^2.0.9",
-    "luxon": "^1.4.1",
+    "luxon": "^1.19.3",
     "merge": "^1.2.1",
     "object-hash": "^1.2.0",
     "pad": "^2.0.3",
@@ -71,6 +71,7 @@
     "snake-case": "^2.1.0",
     "source-map": "^0.6.1",
     "tmp": "0.0.33",
+    "twig-lexer": "^0.6.3",
     "utf8-binary-cutter": "^0.9.2",
     "var-validator": "0.0.3"
   },

src/base.ts

@@ -121,7 +121,6 @@ export {TwingSourceMapNodeFactorySpaceless} from "./lib/source-map/node-factory/
 export {TwingSourceMapNodeSpaceless} from "./lib/source-map/node/spaceless";
 export {TwingTemplate} from "./lib/template";
 export {TwingTest} from "./lib/test";
-export {TwingToken} from "./lib/token";
 export {TwingTokenParserAutoEscape} from "./lib/token-parser/auto-escape";
 export {TwingTokenParserBlock} from "./lib/token-parser/block";
 export {TwingTokenParserDo} from "./lib/token-parser/do";

src/lib/cache/null.ts

@@ -4,7 +4,7 @@ import {TwingTemplatesModule} from "../environment";
 /**
  * Implements a no-cache strategy.
  *
- * @author Fabien Potencier <fabien@symfony.com>
+ * @author Eric MORAND <eric.morand@gmail.com>
  */
 export class TwingCacheNull implements TwingCacheInterface {
     TwingCacheInterfaceImpl: TwingCacheInterface;

src/lib/environment.ts

@@ -30,6 +30,7 @@ import {TwingSandboxSecurityPolicyInterface} from "./sandbox/security-policy-int
 import {TwingEnvironmentOptions} from "./environment-options";
 import {TwingNodeType} from "./node";
 import {TwingSourceMapNodeFactory} from "./source-map/node-factory";
+import {Token, TokenStream} from "twig-lexer";
 
 const path = require('path');
 const sha256 = require('crypto-js/sha256');
@@ -499,20 +500,22 @@ export abstract class TwingEnvironment extends EventEmitter {
      *
      * @throws {TwingErrorSyntax} When the code is syntactically wrong
      */
-    tokenize(source: TwingSource) {
+    tokenize(source: TwingSource): TwingTokenStream {
         if (!this.lexer) {
             this.lexer = new TwingLexer(this);
         }
 
-        return this.lexer.tokenize(source);
+        let stream = this.lexer.tokenizeSource(source);
+
+        return new TwingTokenStream(stream.toAst(), stream.getSourceContext());
     }
 
     setParser(parser: TwingParser) {
         this.parser = parser;
     }
 
     /**
-     * Converts a token stream to a template.
+     * Converts a token list to a template.
      *
      * @param {TwingTokenStream} stream
      * @returns {TwingNodeModule}

src/lib/extension-set.ts

@@ -7,7 +7,6 @@ import {TwingTest} from "./test";
 import {merge} from "./helpers/merge";
 import {TwingOperator, TwingOperatorType} from "./operator";
 import {TwingNodeType} from "./node";
-import {TwingEnvironment} from "./environment";
 import {TwingSourceMapNodeFactory} from "./source-map/node-factory";
 
 export class TwingExtensionSet {