Skip to content

Commit

Permalink
entity/reflection: upgraded to new Tokenizer in ModifierParser
Browse files Browse the repository at this point in the history
  • Loading branch information
@hrach
hrach committed Dec 31, 2018
1 parent 83b8f22 commit 05ccd97
Showing 1 changed file with 46 additions and 49 deletions.
95 changes: 46 additions & 49 deletions src/Entity/Reflection/ModifierParser.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,32 +8,26 @@

namespace Nextras\Orm\Entity\Reflection;

use Nette\Tokenizer\Exception;
use Nette\Tokenizer\Stream;
use Nette\Tokenizer\Token;
use Nette\Tokenizer\Tokenizer;
use Nette\Utils\Reflection;
use Nette\Utils\TokenIterator;
use Nette\Utils\Tokenizer;
use Nette\Utils\TokenizerException;
use Nextras\Orm\InvalidModifierDefinitionException;
use ReflectionClass;


class ModifierParser
{
/** @internal */
const TOKEN_KEYWORD = 1;
/** @internal */
const TOKEN_STRING = 2;
/** @internal */
const TOKEN_LBRACKET = 3;
/** @internal */
const TOKEN_RBRACKET = 4;
/** @internal */
const TOKEN_EQUAL = 5;
/** @internal */
const TOKEN_SEPARATOR = 6;
/** @internal */
const TOKEN_WHITESPACE = 7;
/** @internal regular expression for single & double quoted PHP string */
const RE_STRING = '\'(?:\\\\.|[^\'\\\\])*\'|"(?:\\\\.|[^"\\\\])*"';
private const TOKEN_KEYWORD = 1;
private const TOKEN_STRING = 2;
private const TOKEN_LBRACKET = 3;
private const TOKEN_RBRACKET = 4;
private const TOKEN_EQUAL = 5;
private const TOKEN_SEPARATOR = 6;
private const TOKEN_WHITESPACE = 7;
/** @const regular expression for single & double quoted PHP string */
private const RE_STRING = '\'(?:\\\\.|[^\'\\\\])*\'|"(?:\\\\.|[^"\\\\])*"';

/** @var Tokenizer */
private $tokenizer;
Expand Down Expand Up @@ -71,83 +65,84 @@ public function matchModifiers(string $input): array
*/
public function parse(string $string, ReflectionClass $reflectionClass): array
{
$tokens = $this->lex($string, $reflectionClass);
$iterator = new TokenIterator($tokens);
$iterator = $this->lex($string, $reflectionClass);
return [
$name = $this->processName($iterator),
$this->processArgs($iterator, $name, false),
];
}


private function lex(string $input, ReflectionClass $reflectionClass): array
private function lex(string $input, ReflectionClass $reflectionClass): Stream
{
try {
$tokens = $this->tokenizer->tokenize($input);
} catch (TokenizerException $e) {
$tokens = $this->tokenizer->tokenize($input)->tokens;
} catch (Exception $e) {
throw new InvalidModifierDefinitionException('Unable to tokenize the modifier.', 0, $e);
}

$tokens = array_filter($tokens, function ($pair) {
return $pair[2] !== self::TOKEN_WHITESPACE && $pair[2] !== null;
$tokens = array_filter($tokens, function ($token) {
return $token->type !== self::TOKEN_WHITESPACE;
});
$tokens = array_values($tokens);

$expanded = [];
foreach ($tokens as $token) {
if ($token[2] === self::TOKEN_STRING) {
$token[0] = stripslashes(substr($token[0], 1, -1));
if ($token->type === self::TOKEN_STRING) {
$token->value = stripslashes(substr($token->value, 1, -1));
$expanded[] = $token;

} elseif ($token[2] === self::TOKEN_KEYWORD) {
$values = $this->processKeyword($token[0], $reflectionClass);
} elseif ($token->type === self::TOKEN_KEYWORD) {
$values = $this->processKeyword($token->value, $reflectionClass);
if (is_array($values)) {
$count = count($values) - 1;
foreach ($values as $i => $value) {
$expanded[] = [$value, $token[1], $token[2]];
$expanded[] = new Token($value, $token->type, $token->offset);
if ($i !== $count) {
$expanded[] = [',', -1, self::TOKEN_SEPARATOR];
$expanded[] = new Token(',', self::TOKEN_SEPARATOR, -1);
}
}
} else {
$expanded[] = [$values, $token[1], $token[2]];
$expanded[] = new Token($values, $token->type, $token->offset);
}

} else {
$expanded[] = $token;
}
}
return $expanded;

return new Stream($expanded);
}


private function processName(TokenIterator $iterator): string
private function processName(Stream $iterator): string
{
$iterator->position++;
if (!isset($iterator->tokens[$iterator->position])) {
throw new InvalidModifierDefinitionException("Modifier does not have a name.");
}
[$value, , $type] = $iterator->currentToken();
if ($type !== self::TOKEN_KEYWORD) {
$currentToken = $iterator->currentToken();
if ($currentToken->type !== self::TOKEN_KEYWORD) {
throw new InvalidModifierDefinitionException("Modifier does not have a name.");
} elseif (isset($iterator->tokens[$iterator->position + 1])) {
[, , $type] = $iterator->tokens[$iterator->position + 1];
if ($type === self::TOKEN_SEPARATOR) {
throw new InvalidModifierDefinitionException("After the {{$value}}'s modifier name cannot be a comma separator.");
$nextToken = $iterator->tokens[$iterator->position + 1];
if ($nextToken->type === self::TOKEN_SEPARATOR) {
throw new InvalidModifierDefinitionException("After the {{$currentToken->value}}'s modifier name cannot be a comma separator.");
}
}

return $value;
return $currentToken->value;
}


private function processArgs(TokenIterator $iterator, string $modifierName, bool $inArray)
private function processArgs(Stream $iterator, string $modifierName, bool $inArray)
{
$result = [];
$iterator->position++;
while (isset($iterator->tokens[$iterator->position])) {
/** @var int|null $type */
[$value, , $type] = $iterator->currentToken();
$currentToken = $iterator->currentToken();
$type = $currentToken->type;
$value = $currentToken->value;

if ($type === self::TOKEN_RBRACKET) {
if ($inArray) {
Expand All @@ -157,17 +152,18 @@ private function processArgs(TokenIterator $iterator, string $modifierName, bool
}
} elseif ($type === self::TOKEN_STRING || $type === self::TOKEN_KEYWORD) {
$iterator->position++;
[, , $nextTokenType] = $iterator->currentToken();
$nextToken = $iterator->currentToken();
$nextTokenType = $nextToken ? $nextToken->type : null;

if ($nextTokenType === self::TOKEN_EQUAL) {
$iterator->position++;
[, , $nextTokenType] = $iterator->currentToken();
$nextValue = $iterator->currentValue();
$nextToken = $iterator->currentToken();
$nextTokenType = $nextToken ? $nextToken->type : null;

if ($nextTokenType === self::TOKEN_LBRACKET) {
$result[$value] = $this->processArgs($iterator, $modifierName, true);
} elseif ($nextTokenType === self::TOKEN_STRING || $nextTokenType === self::TOKEN_KEYWORD) {
$result[$value] = $nextValue;
$result[$value] = $iterator->currentValue();
} elseif ($nextTokenType !== null) {
throw new InvalidModifierDefinitionException("Modifier {{$modifierName}} has invalid token after =.");
}
Expand All @@ -182,7 +178,8 @@ private function processArgs(TokenIterator $iterator, string $modifierName, bool
}

$iterator->position++;
[, , $type] = $iterator->currentToken();
$currentToken = $iterator->currentToken();
$type = $currentToken ? $currentToken->type : null;
if ($type === self::TOKEN_RBRACKET && $inArray) {
return $result;
} elseif ($type !== null && $type !== self::TOKEN_SEPARATOR) {
Expand Down

0 comments on commit 05ccd97

Please sign in to comment.