Skip to content

Commit

Permalink
Nette\Utils\Tokenizer & TokenIterator renamed to Nette\Latte\Tokenize…
Browse files Browse the repository at this point in the history 
…r & TokenIterator (BC break, but they are internal classes)
  • Loading branch information
@dg
dg committed Mar 21, 2014
1 parent 5f4b5b3 commit 86fa9a7
Show file tree
Hide file tree
Showing 14 changed files with 35 additions and 46 deletions.
6 changes: 3 additions & 3 deletions Nette/Latte/MacroTokens.php
View file
Expand Up @@ -15,7 +15,7 @@
*
* @author David Grudl
*/
class MacroTokens extends Nette\Utils\TokenIterator
class MacroTokens extends TokenIterator
{
const T_WHITESPACE = 1,
T_COMMENT = 2,
Expand All @@ -27,7 +27,7 @@ class MacroTokens extends Nette\Utils\TokenIterator
T_KEYWORD = 8,
T_CHAR = 9;

/** @var Nette\Utils\Tokenizer */
/** @var Tokenizer */
private static $tokenizer;

/** @var int */
Expand All @@ -43,7 +43,7 @@ public function __construct($input = NULL)

public function parse($s)
{
self::$tokenizer = self::$tokenizer ?: new Nette\Utils\Tokenizer(array(
self::$tokenizer = self::$tokenizer ?: new Tokenizer(array(
self::T_WHITESPACE => '\s+',
self::T_COMMENT => '(?s)/\*.*?\*/',
self::T_STRING => Parser::RE_STRING,
Expand Down
2 changes: 1 addition & 1 deletion Nette/Utils/TokenIterator.php → Nette/Latte/TokenIterator.php
View file
Expand Up @@ -5,7 +5,7 @@
* Copyright (c) 2004 David Grudl (http://davidgrudl.com)
*/

namespace Nette\Utils;
namespace Nette\Latte;

use Nette;

Expand Down
13 changes: 3 additions & 10 deletions Nette/Utils/Tokenizer.php → Nette/Latte/Tokenizer.php
View file
Expand Up @@ -5,9 +5,10 @@
* Copyright (c) 2004 David Grudl (http://davidgrudl.com)
*/

namespace Nette\Utils;
namespace Nette\Latte;

use Nette;
use Nette,
Nette\Utils\Strings;


/**
Expand Down Expand Up @@ -97,11 +98,3 @@ public static function getCoordinates($text, $offset)
}

}


/**
* The exception that indicates tokenizer error.
*/
class TokenizerException extends \Exception
{
}
7 changes: 7 additions & 0 deletions Nette/Latte/exceptions.php
View file
Expand Up @@ -20,4 +20,11 @@ class CompileException extends Nette\Templating\FilterException
}


/**
* The exception that indicates tokenizer error.
*/
class TokenizerException extends \Exception
{
}

class_alias('Nette\Latte\CompileException', 'Nette\Latte\ParseException');
4 changes: 3 additions & 1 deletion Nette/Loaders/NetteLoader.php
View file
Expand Up @@ -33,6 +33,8 @@ class NetteLoader
'Nette\Utils\PhpGenerator\Parameter' => 'Nette\PhpGenerator\Parameter',
'Nette\Utils\PhpGenerator\PhpLiteral' => 'Nette\PhpGenerator\PhpLiteral',
'Nette\Utils\PhpGenerator\Property' => 'Nette\PhpGenerator\Property',
'Nette\Utils\Tokenizer' => 'Nette\Latte\Tokenizer',
'Nette\Utils\TokenIterator' => 'Nette\Latte\TokenIterator',
'Nette\Iterators\CachingIterator' => 'Nette\Latte\Runtime\CachingIterator',
);

Expand Down Expand Up @@ -65,6 +67,7 @@ class NetteLoader
'Nette\InvalidArgumentException' => 'common/exceptions',
'Nette\InvalidStateException' => 'common/exceptions',
'Nette\Latte\CompileException' => 'Latte/exceptions',
'Nette\Latte\TokenizerException' => 'Latte/exceptions',
'Nette\Mail\SmtpException' => 'Mail/SmtpMailer',
'Nette\MemberAccessException' => 'common/exceptions',
'Nette\NotImplementedException' => 'common/exceptions',
Expand All @@ -80,7 +83,6 @@ class NetteLoader
'Nette\Utils\NeonEntity' => 'Utils/Neon',
'Nette\Utils\NeonException' => 'Utils/Neon',
'Nette\Utils\RegexpException' => 'Utils/Strings',
'Nette\Utils\TokenizerException' => 'Utils/Tokenizer',
);


Expand Down
2 changes: 1 addition & 1 deletion tests/Nette/Latte/CoreMacros.var.default.phpt
View file
Expand Up @@ -27,7 +27,7 @@ test(function() use ($compiler) { // {var ... }

Assert::exception(function() use ($compiler) {
$compiler->expandMacro('var', '$var => "123', '');
}, 'Nette\Utils\TokenizerException', 'Unexpected %a% on line 1, column 9.');
}, 'Nette\Latte\TokenizerException', 'Unexpected %a% on line 1, column 9.');
});


Expand Down
2 changes: 1 addition & 1 deletion tests/Nette/Latte/PhpWriter.formatArgs().phpt
View file
Expand Up @@ -38,7 +38,7 @@ test(function() { // strings
Assert::same( "('hello')", formatArgs('(hello)') );
Assert::exception(function() {
formatArgs("'\\\\'1, 2, symbol1, symbol2'");
}, 'Nette\Utils\TokenizerException', 'Unexpected %a% on line 1, column 27.');
}, 'Nette\Latte\TokenizerException', 'Unexpected %a% on line 1, column 27.');
});


Expand Down
2 changes: 1 addition & 1 deletion tests/Nette/Latte/PhpWriter.formatModifiers().phpt
View file
Expand Up @@ -28,7 +28,7 @@ test(function() { // special
}, 'Nette\InvalidStateException', 'Modifier name must be alphanumeric string%a%');
Assert::exception(function() {
Assert::same( '$template->mod(@, \'\\\\\', "a", "b", "c", "arg2")', formatModifiers('@', "mod:'\\\\':a:b:c':arg2") );
}, 'Nette\Utils\TokenizerException', 'Unexpected %a% on line 1, column 15.');
}, 'Nette\Latte\TokenizerException', 'Unexpected %a% on line 1, column 15.');
});


Expand Down
2 changes: 1 addition & 1 deletion tests/Nette/Latte/PhpWriter.formatWord().phpt
View file
Expand Up @@ -34,4 +34,4 @@ Assert::same( '$x["[x]"]', $writer->formatWord('$x["[x]"]') );

Assert::exception(function() use ($writer) {
$writer->formatWord("'var\"");
}, 'Nette\Utils\TokenizerException', "Unexpected ''var\"' on line 1, column 1.");
}, 'Nette\Latte\TokenizerException', "Unexpected ''var\"' on line 1, column 1.");
6 changes: 3 additions & 3 deletions tests/Nette/Utils/TokenIterator.join.phpt → tests/Nette/Latte/TokenIterator.join.phpt
View file
@@ -1,13 +1,13 @@
<?php

/**
* Test: Nette\Utils\TokenIterator traversing
* Test: Nette\Latte\TokenIterator traversing
*
* @author David Grudl
*/

use Nette\Utils\Tokenizer,
Nette\Utils\TokenIterator,
use Nette\Latte\Tokenizer,
Nette\Latte\TokenIterator,
Tester\Assert;


Expand Down
19 changes: 3 additions & 16 deletions ...Nette/Utils/TokenIterator.traversing.phpt → ...Nette/Latte/TokenIterator.traversing.phpt
View file
@@ -1,13 +1,13 @@
<?php

/**
* Test: Nette\Utils\TokenIterator traversing
* Test: Nette\Latte\TokenIterator traversing
*
* @author David Grudl
*/

use Nette\Utils\Tokenizer,
Nette\Utils\TokenIterator,
use Nette\Latte\Tokenizer,
Nette\Latte\TokenIterator,
Tester\Assert;


Expand Down Expand Up @@ -121,16 +121,3 @@ test(function() {
Assert::same( array(), $traverser->nextUntil(T_STRING, T_DNUMBER, T_WHITESPACE) );
Assert::same( 2, $traverser->position );
});


test(function() {
$tokenizer = new Tokenizer(array(
'\d+',
'\s+',
'\w+',
));
$traverser = new TokenIterator($tokenizer->tokenize('say 123'));
Assert::null( $traverser->nextToken('s') );
Assert::same( array('say', 0), $traverser->nextToken('say') );
Assert::same( array(' ', 3), $traverser->nextToken() );
});
4 changes: 2 additions & 2 deletions ...Nette/Utils/Tokenizer.getCoordinates.phpt → ...Nette/Latte/Tokenizer.getCoordinates.phpt
View file
@@ -1,12 +1,12 @@
<?php

/**
* Test: Nette\Utils\Tokenizer::getCoordinates
* Test: Nette\Latte\Tokenizer::getCoordinates
*
* @author David Grudl
*/

use Nette\Utils\Tokenizer,
use Nette\Latte\Tokenizer,
Tester\Assert;


Expand Down
6 changes: 3 additions & 3 deletions tests/Nette/Utils/Tokenizer.namedTokens.phpt → tests/Nette/Latte/Tokenizer.namedTokens.phpt
View file
@@ -1,12 +1,12 @@
<?php

/**
* Test: Nette\Utils\Tokenizer::tokenize with names
* Test: Nette\Latte\Tokenizer::tokenize with names
*
* @author David Grudl
*/

use Nette\Utils\Tokenizer,
use Nette\Latte\Tokenizer,
Tester\Assert;


Expand All @@ -27,4 +27,4 @@ Assert::same( array(

Assert::exception(function() use ($tokenizer) {
$tokenizer->tokenize('say 123;');
}, 'Nette\Utils\TokenizerException', "Unexpected ';' on line 1, column 8.");
}, 'Nette\Latte\TokenizerException', "Unexpected ';' on line 1, column 8.");
6 changes: 3 additions & 3 deletions tests/Nette/Utils/Tokenizer.simple.phpt → tests/Nette/Latte/Tokenizer.simple.phpt
View file
@@ -1,12 +1,12 @@
<?php

/**
* Test: Nette\Utils\Tokenizer::tokenize simple
* Test: Nette\Latte\Tokenizer::tokenize simple
*
* @author David Grudl
*/

use Nette\Utils\Tokenizer,
use Nette\Latte\Tokenizer,
Tester\Assert;


Expand All @@ -27,4 +27,4 @@ Assert::same( array(

Assert::exception(function() use ($tokenizer) {
$tokenizer->tokenize('say 123;');
}, 'Nette\Utils\TokenizerException', "Unexpected ';' on line 1, column 8.");
}, 'Nette\Latte\TokenizerException', "Unexpected ';' on line 1, column 8.");

8 comments on commit 86fa9a7

@fprochazka
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Such a usefull class! 😢 https://travis-ci.org/Kdyby/Aop/jobs/21328814

@fprochazka
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you could add alias in nette/nette to preserve BC.

@Majkl578
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wait what? How is this Latte-only thing?

@dg
Copy link
Member Author

@dg dg commented on 86fa9a7 Mar 23, 2014

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is internal thing.

@xificurk
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I use it as well - it is non-trivial class that is very useful not only in Latte, thus it would be nice to have it in utils.

@mishak87
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is useful set of classes. Might I say a killer feature? With basic documentation and own package it might even become popular tool for parsing as neon is for configuration.

@dg
Copy link
Member Author

@dg dg commented on 86fa9a7 Mar 24, 2014

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mishak87 ok. Can you write basic documentation here https://github.com/nette/tokenizer?

@mishak87
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://github.com/nette/tokenizer/issues/1

Please sign in to comment.