Skip to content
Browse files

Change text tokens to proper tokens (instead of strings)

  • Loading branch information...
1 parent 1430b27 commit 2328615772f223ffabc6deaa62dcec8c4c90f0aa @bobthecow committed
View
6 src/Mustache/Compiler.php
@@ -55,7 +55,7 @@ private function walk(array $tree, $level = 0) {
$code = '';
$level++;
foreach ($tree as $node) {
- switch (is_string($node) ? 'text' : $node[Mustache_Tokenizer::TYPE]) {
+ switch ($node[Mustache_Tokenizer::TYPE]) {
case Mustache_Tokenizer::T_SECTION:
$code .= $this->section(
$node[Mustache_Tokenizer::NODES],
@@ -98,8 +98,8 @@ private function walk(array $tree, $level = 0) {
break;
- case 'text':
- $code .= $this->text($node, $level);
+ case Mustache_Tokenizer::T_TEXT:
+ $code .= $this->text($node[Mustache_Tokenizer::VALUE], $level);
break;
default:
View
4 src/Mustache/Parser.php
@@ -46,7 +46,7 @@ private function buildTree(ArrayIterator $tokens, array $parent = null) {
if ($token === null) {
continue;
- } elseif (is_array($token)) {
+ } else {
switch ($token[Mustache_Tokenizer::TYPE]) {
case Mustache_Tokenizer::T_SECTION:
case Mustache_Tokenizer::T_INVERTED:
@@ -72,8 +72,6 @@ private function buildTree(ArrayIterator $tokens, array $parent = null) {
$nodes[] = $token;
break;
}
- } else {
- $nodes[] = $token;
}
} while ($tokens->valid());
View
18 src/Mustache/Tokenizer.php
@@ -32,6 +32,7 @@ class Mustache_Tokenizer {
const T_ESCAPED = '_v';
const T_UNESCAPED = '{';
const T_UNESCAPED_2 = '&';
+ const T_TEXT = '_t';
// Valid token types
private static $tagTypes = array(
@@ -63,6 +64,7 @@ class Mustache_Tokenizer {
const END = 'end';
const INDENT = 'indent';
const NODES = 'nodes';
+ const VALUE = 'value';
private $state;
private $tagType;
@@ -187,7 +189,7 @@ private function reset() {
*/
private function flushBuffer() {
if (!empty($this->buffer)) {
- $this->tokens[] = $this->buffer;
+ $this->tokens[] = array(self::TYPE => self::T_TEXT, self::VALUE => $this->buffer);
$this->buffer = '';
}
}
@@ -201,12 +203,12 @@ private function lineIsWhitespace() {
$tokensCount = count($this->tokens);
for ($j = $this->lineStart; $j < $tokensCount; $j++) {
$token = $this->tokens[$j];
- if (is_array($token) && isset(self::$tagTypes[$token[self::TYPE]])) {
+ if (isset(self::$tagTypes[$token[self::TYPE]])) {
if (isset(self::$interpolatedTags[$token[self::TYPE]])) {
return false;
}
- } elseif (is_string($token)) {
- if (preg_match('/\S/', $token)) {
+ } elseif ($token[self::TYPE] == self::T_TEXT) {
+ if (preg_match('/\S/', $token[self::VALUE])) {
return false;
}
}
@@ -225,16 +227,16 @@ private function filterLine($noNewLine = false) {
if ($this->seenTag && $this->lineIsWhitespace()) {
$tokensCount = count($this->tokens);
for ($j = $this->lineStart; $j < $tokensCount; $j++) {
- if (!is_array($this->tokens[$j])) {
- if (isset($this->tokens[$j+1]) && is_array($this->tokens[$j+1]) && $this->tokens[$j+1][self::TYPE] == self::T_PARTIAL) {
- $this->tokens[$j+1][self::INDENT] = (string) $this->tokens[$j];
+ if ($this->tokens[$j][self::TYPE] == self::T_TEXT) {
+ if (isset($this->tokens[$j+1]) && $this->tokens[$j+1][self::TYPE] == self::T_PARTIAL) {
+ $this->tokens[$j+1][self::INDENT] = $this->tokens[$j][self::VALUE];
}
$this->tokens[$j] = null;
}
}
} elseif (!$noNewLine) {
- $this->tokens[] = "\n";
+ $this->tokens[] = array(self::TYPE => self::T_TEXT, self::VALUE => "\n");
}
$this->seenTag = false;
View
17 test/Mustache/Test/CompilerTest.php
@@ -34,14 +34,14 @@ public function getCompileValues() {
'return $buffer;',
)),
- array('', array('TEXT'), 'Monkey', false, 'UTF-8', array(
+ array('', array($this->createTextToken('TEXT')), 'Monkey', false, 'UTF-8', array(
"\nclass Monkey extends Mustache_Template",
'return htmlspecialchars($buffer, ENT_COMPAT, \'UTF-8\');',
'$buffer .= $indent . \'TEXT\';',
'return $buffer;',
)),
- array('', array('TEXT'), 'Monkey', true, 'ISO-8859-1', array(
+ array('', array($this->createTextToken('TEXT')), 'Monkey', true, 'ISO-8859-1', array(
"\nclass Monkey extends Mustache_Template",
'$buffer .= $indent . \'TEXT\';',
'return call_user_func($this->mustache->getEscape(), $buffer);',
@@ -51,8 +51,8 @@ public function getCompileValues() {
array(
'',
array(
- 'foo',
- "\n",
+ $this->createTextToken('foo'),
+ $this->createTextToken("\n"),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
Mustache_Tokenizer::NAME => 'name',
@@ -61,7 +61,7 @@ public function getCompileValues() {
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
Mustache_Tokenizer::NAME => '.',
),
- "'bar'",
+ $this->createTextToken("'bar'"),
),
'Monkey',
false,
@@ -88,4 +88,11 @@ public function testCompilerThrowsUnknownNodeTypeException() {
$compiler = new Mustache_Compiler;
$compiler->compile('', array(array(Mustache_Tokenizer::TYPE => 'invalid')), 'SomeClass');
}
+
+ private function createTextToken($value) {
+ return array(
+ Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
+ Mustache_Tokenizer::VALUE => $value,
+ );
+ }
}
View
30 test/Mustache/Test/ParserTest.php
@@ -32,8 +32,14 @@ public function getTokenSets()
),
array(
- array('text'),
- array('text')
+ array(array(
+ Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
+ Mustache_Tokenizer::VALUE => 'text'
+ )),
+ array(array(
+ Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
+ Mustache_Tokenizer::VALUE => 'text'
+ )),
),
array(
@@ -49,7 +55,10 @@ public function getTokenSets()
array(
array(
- 'foo',
+ array(
+ Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
+ Mustache_Tokenizer::VALUE => 'foo'
+ ),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED,
Mustache_Tokenizer::INDEX => 123,
@@ -64,10 +73,16 @@ public function getTokenSets()
Mustache_Tokenizer::INDEX => 456,
Mustache_Tokenizer::NAME => 'parent'
),
- 'bar',
+ array(
+ Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
+ Mustache_Tokenizer::VALUE => 'bar'
+ ),
),
array(
- 'foo',
+ array(
+ Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
+ Mustache_Tokenizer::VALUE => 'foo'
+ ),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED,
Mustache_Tokenizer::NAME => 'parent',
@@ -80,7 +95,10 @@ public function getTokenSets()
),
),
),
- 'bar',
+ array(
+ Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
+ Mustache_Tokenizer::VALUE => 'bar'
+ ),
),
),
View
31 test/Mustache/Test/TokenizerTest.php
@@ -27,13 +27,23 @@ public function getTokens() {
array(
'text',
null,
- array('text'),
+ array(
+ array(
+ Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
+ Mustache_Tokenizer::VALUE => 'text',
+ ),
+ ),
),
array(
'text',
'<<< >>>',
- array('text'),
+ array(
+ array(
+ Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
+ Mustache_Tokenizer::VALUE => 'text',
+ ),
+ ),
),
array(
@@ -53,7 +63,12 @@ public function getTokens() {
array(
'{{ name }}',
'<<< >>>',
- array('{{ name }}'),
+ array(
+ array(
+ Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
+ Mustache_Tokenizer::VALUE => '{{ name }}',
+ ),
+ ),
),
array(
@@ -81,7 +96,10 @@ public function getTokens() {
Mustache_Tokenizer::CTAG => '}}',
Mustache_Tokenizer::INDEX => 8,
),
- "\n",
+ array(
+ Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
+ Mustache_Tokenizer::VALUE => "\n",
+ ),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION,
Mustache_Tokenizer::NAME => 'b',
@@ -104,7 +122,10 @@ public function getTokens() {
Mustache_Tokenizer::CTAG => '|',
Mustache_Tokenizer::INDEX => 37,
),
- "\n",
+ array(
+ Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
+ Mustache_Tokenizer::VALUE => "\n",
+ ),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_UNESCAPED,
Mustache_Tokenizer::NAME => 'd',

0 comments on commit 2328615

Please sign in to comment.
Something went wrong with that request. Please try again.