Skip to content

Commit

Permalink
Add Token utility object (#1466)
Browse files Browse the repository at this point in the history 
 Add Token value object and use it in all places we do tokenisation to remove magic arrays

---------

Co-authored-by: Jacob Dreesen <jacob@hdreesen.de>
  • Loading branch information
@ciaranmcnulty @jdreesen
ciaranmcnulty and jdreesen committed Jan 31, 2024
1 parent b3ed048 commit 5f8ebb6
Show file tree
Hide file tree
Showing 12 changed files with 377 additions and 161 deletions.
58 changes: 58 additions & 0 deletions spec/PhpSpec/Util/Token/ArrayTokenSpec.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
<?php

namespace spec\PhpSpec\Util\Token;

use PhpSpec\ObjectBehavior;
use PhpSpec\Util\Token;
use PhpSpec\Util\Token\ArrayToken;

class ArrayTokenSpec extends ObjectBehavior
{
function let()
{
$this->beConstructedThrough(
[Token::class, 'fromPhpToken'],
[[T_CATCH, 'catch', 100]]
);
}

function it_equals_the_string_in_token_array_it_was_created_with()
{
$this->equals('catch')->shouldBe(true);
}

function it_stringifys_as_the_string_in_token_array_it_was_created_with()
{
$this->asString()->shouldBe('catch');
}

function it_does_not_equal_string_different_to_one_in_token_array_it_was_created_with()
{
$this->equals('bar')->shouldBe(false);
}

function it_has_same_type_as_token_array_it_was_created_with()
{
$this->hasType(T_CATCH)->shouldBe(true);
}

function it_does_not_have_same_type_as_token_different_to_array_it_was_created_with()
{
$this->hasType(T_FINALLY)->shouldBe(false);
}

function it_is_in_a_matching_type_list()
{
$this->isInTypes([T_CATCH])->shouldBe(true);
}

function it_is_not_in_a_non_matching_type_list()
{
$this->isInTypes([T_FINALLY])->shouldBe(false);
}

function it_has_a_line()
{
$this->getLine()->shouldBe(100);
}
}
48 changes: 48 additions & 0 deletions spec/PhpSpec/Util/Token/StringTokenSpec.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
<?php

namespace spec\PhpSpec\Util\Token;

use PhpSpec\ObjectBehavior;
use PhpSpec\Util\Token;
use PhpSpec\Util\Token\StringToken;

class StringTokenSpec extends ObjectBehavior
{
function let()
{
$this->beConstructedThrough(
[Token::class, 'fromPhpToken'],
['f']
);
}

function it_equals_the_string_it_was_created_with()
{
$this->equals('f')->shouldBe(true);
}

function it_stringifys_as_the_string_it_was_created_with()
{
$this->asString()->shouldBe('f');
}

function it_does_not_equal_different_string()
{
$this->equals('b')->shouldBe(false);
}

function it_does_not_have_type()
{
$this->hasType(T_CATCH)->shouldBe(false);
}

function it_is_never_in_a_type_list()
{
$this->isInTypes([T_CATCH])->shouldBe(false);
}

function it_does_not_have_a_line()
{
$this->getLine()->shouldBeNull();
}
}
37 changes: 21 additions & 16 deletions src/PhpSpec/CodeAnalysis/TokenizedNamespaceResolver.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@

namespace PhpSpec\CodeAnalysis;

use PhpSpec\Util\Token;

final class TokenizedNamespaceResolver implements NamespaceResolver
{
const STATE_DEFAULT = 0;
Expand All @@ -34,56 +36,59 @@ public function analyse(string $code): void
$this->currentUseGroup = '';
$this->uses = [];

$tokens = token_get_all($code);
$tokens = Token::getAll($code);

foreach ($tokens as $index => $token) {
foreach ($tokens as $token) {

switch ($this->state) {

case self::STATE_READING_NAMESPACE:
if (';' == $token) {
if ($token->equals(';')) {
$this->currentNamespace = trim($this->currentNamespace);
$this->state = self::STATE_DEFAULT;
}
elseif (\is_array($token)) {
$this->currentNamespace .= $token[1];
else {
$this->currentNamespace .= $token->asString();
}
break;

case self::STATE_READING_USE_GROUP:
if ('}' == $token) {
if ($token->equals('}')) {
$this->state = self::STATE_READING_USE;
$this->currentUseGroup = '';
}
elseif (',' == $token) {
elseif ($token->equals(',')) {
$this->storeCurrentUse();
}
elseif (\is_array($token)) {
$this->currentUse = $this->currentUseGroup . trim($token[1]);
else {
$this->currentUse = $this->currentUseGroup . trim($token->asString());
}
break;

case self::STATE_READING_USE:
if (';' == $token) {
if ($token->equals(';')) {
$this->storeCurrentUse();
$this->state = self::STATE_DEFAULT;
}
if ('{' == $token) {
if ($token->equals('{')) {
$this->currentUseGroup = trim($this->currentUse);
$this->state = self::STATE_READING_USE_GROUP;
}
elseif (',' == $token) {
elseif ($token->equals(',')) {
$this->storeCurrentUse();
}
elseif (\is_array($token)) {
$this->currentUse .= $token[1];
else {
$this->currentUse .= $token->asString();
}
break;

default:
if (\is_array($token) && T_NAMESPACE == $token[0]) {
if ($token->hasType(T_NAMESPACE)) {
$this->state = self::STATE_READING_NAMESPACE;
$this->currentNamespace = '';
$this->uses = array();
}
elseif (\is_array($token) && T_USE == $token[0]) {
if ($token->hasType(T_USE)) {
$this->state = self::STATE_READING_USE;
$this->currentUse = '';
}
Expand Down
83 changes: 36 additions & 47 deletions src/PhpSpec/CodeAnalysis/TokenizedTypeHintRewriter.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@
namespace PhpSpec\CodeAnalysis;

use PhpSpec\Loader\Transformer\TypeHintIndex;
use PhpSpec\Util\Token;
use function array_map;
use function join;

final class TokenizedTypeHintRewriter implements TypeHintRewriter
{
Expand Down Expand Up @@ -47,13 +50,14 @@ public function __construct(

public function rewrite(string $classDefinition): string
{
$this->reset();

$this->namespaceResolver->analyse($classDefinition);

$this->reset();
$tokens = $this->stripTypeHints(token_get_all($classDefinition));
$tokensToString = $this->tokensToString($tokens);
$tokens = Token::getAll($classDefinition);
$strippedTokens = $this->stripTypeHints($tokens);

return $tokensToString;
return join('', array_map(fn(Token $token) => $token->asString(), $strippedTokens));
}

private function reset(): void
Expand All @@ -63,58 +67,60 @@ private function reset(): void
$this->currentFunction = '';
}

/** @param list<Token> $tokens */
private function stripTypeHints(array $tokens): array
{
foreach ($tokens as $index => $token) {
if ($this->isToken($token, '{')) {

if ($token->equals('{')) {
$this->currentBodyLevel++;
}
elseif ($this->isToken($token, '}')) {
elseif ($token->equals('}')) {
$this->currentBodyLevel--;
}

switch ($this->state) {
case self::STATE_READING_ARGUMENTS:
if (')' == $token) {
if ($token->equals(')')) {
$this->state = self::STATE_READING_CLASS;
}
elseif ($this->tokenHasType($token, T_VARIABLE)) {
elseif ($token->hasType(T_VARIABLE)) {
$this->extractTypehints($tokens, $index, $token);
}
break;
case self::STATE_READING_FUNCTION:
if ('(' == $token) {
if ($token->equals('(')) {
$this->state = self::STATE_READING_ARGUMENTS;
}
elseif ($this->tokenHasType($token, T_STRING) && !$this->currentFunction) {
$this->currentFunction = $token[1];
elseif ($token->hasType(T_STRING) && !$this->currentFunction) {
$this->currentFunction = $token->asString();
}
break;
case self::STATE_READING_CLASS:
if ('{' == $token && $this->currentFunction) {
if ($token->equals('{') && $this->currentFunction) {
$this->state = self::STATE_READING_FUNCTION_BODY;
$this->currentBodyLevel = 1;
}
elseif ('}' == $token && $this->currentClass) {
elseif ($token->equals('}') && $this->currentClass) {
$this->state = self::STATE_DEFAULT;
$this->currentClass = '';
}
elseif ($this->tokenHasType($token, T_STRING) && !$this->currentClass && $this->shouldExtractTokensOfClass($token[1])) {
$this->currentClass = $token[1];
elseif ($token->hasType(T_STRING) && !$this->currentClass && $this->shouldExtractTokensOfClass($token->asString())) {
$this->currentClass = $token->asString();
}
elseif ($this->tokenHasType($token, T_FUNCTION) && $this->currentClass) {
elseif ($token->hasType( T_FUNCTION) && $this->currentClass) {
$this->state = self::STATE_READING_FUNCTION;
}
break;
case self::STATE_READING_FUNCTION_BODY:
if ('}' == $token && $this->currentBodyLevel === 0) {
if ($token->equals('}') && $this->currentBodyLevel === 0) {
$this->currentFunction = '';
$this->state = self::STATE_READING_CLASS;
}

break;
default:
if ($this->tokenHasType($token, T_CLASS)) {
if ($token->hasType( T_CLASS)) {
$this->state = self::STATE_READING_CLASS;
}
}
Expand All @@ -123,21 +129,14 @@ private function stripTypeHints(array $tokens): array
return $tokens;
}


private function tokensToString(array $tokens): string
{
return join('', array_map(function ($token) {
return \is_array($token) ? $token[1] : $token;
}, $tokens));
}

private function extractTypehints(array &$tokens, int $index, array $token): void
private function extractTypehints(array &$tokens, int $variableNameIndex, Token $variableName): void
{
$typehint = '';
for ($i = $index - 1; !$this->haveNotReachedEndOfTypeHint($tokens[$i]); $i--) {
$typehint = (is_array($tokens[$i]) ? $tokens[$i][1] : $tokens[$i]) . $typehint;
for ($i = $variableNameIndex - 1; !$this->haveNotReachedEndOfTypeHint($tokens[$i]); $i--) {
$scanningToken = $tokens[$i];
$typehint = $scanningToken->asString() . $typehint;

if (T_WHITESPACE !== $tokens[$i][0]) {
if (!$scanningToken->hasType(T_WHITESPACE)) {
unset($tokens[$i]);
}
}
Expand All @@ -150,7 +149,7 @@ private function extractTypehints(array &$tokens, int $index, array $token): voi
$this->typeHintIndex->addInvalid(
$class,
trim($this->currentFunction),
$token[1],
$variableName->asString(),
new DisallowedUnionTypehintException("Union type $typehint cannot be used to create a double")
);

Expand All @@ -161,7 +160,7 @@ private function extractTypehints(array &$tokens, int $index, array $token): voi
$this->typeHintIndex->addInvalid(
$class,
trim($this->currentFunction),
$token[1],
$variableName->asString(),
new DisallowedUnionTypehintException("Intersection type $typehint cannot be used to create a double")
);

Expand All @@ -173,41 +172,31 @@ private function extractTypehints(array &$tokens, int $index, array $token): voi
$this->typeHintIndex->add(
$class,
trim($this->currentFunction),
$token[1],
$variableName->asString(),
$typehintFcqn
);
} catch (DisallowedNonObjectTypehintException $e) {
$this->typeHintIndex->addInvalid(
$class,
trim($this->currentFunction),
$token[1],
$variableName->asString(),
$e
);
}
}
}

private function haveNotReachedEndOfTypeHint(string|array $token) : bool
private function haveNotReachedEndOfTypeHint(Token $token) : bool
{
if ($token == '|' || is_array($token) && $token[1] == '&') {
if ($token->equals('|') || $token->equals('&')) {
return false;
}

return !\in_array($token[0], $this->typehintTokens);
}

private function tokenHasType(array|string $token, int $type): bool
{
return \is_array($token) && $type == $token[0];
return !$token->isInTypes($this->typehintTokens);
}

private function shouldExtractTokensOfClass(string $className): bool
{
return substr($className, -4) == 'Spec';
}

private function isToken(array|string $token, string $string): bool
{
return $token == $string || (\is_array($token) && $token[1] == $string);
}
}
Loading

0 comments on commit 5f8ebb6

Please sign in to comment.