Skip to content

Commit

Permalink
Merge branch 'php-8/backfill-named-function-call-parameters-tokenizat…
Browse files Browse the repository at this point in the history 
…ion' of https://github.com/jrfnl/PHP_CodeSniffer
  • Loading branch information
@gsherwood
gsherwood committed Jan 13, 2021
2 parents 7f5df08 + 845335a commit 0859719
Show file tree
Hide file tree
Showing 5 changed files with 1,410 additions and 45 deletions.
6 changes: 6 additions & 0 deletions package.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<file baseinstalldir="" name="BitwiseOrTest.php" role="test" />
<file baseinstalldir="" name="GotoLabelTest.inc" role="test" />
<file baseinstalldir="" name="GotoLabelTest.php" role="test" />
<file baseinstalldir="" name="NamedFunctionCallArgumentsTest.inc" role="test" />
<file baseinstalldir="" name="NamedFunctionCallArgumentsTest.php" role="test" />
<file baseinstalldir="" name="NullsafeObjectOperatorTest.inc" role="test" />
<file baseinstalldir="" name="NullsafeObjectOperatorTest.php" role="test" />
<file baseinstalldir="" name="ScopeSettingWithNamespaceOperatorTest.inc" role="test" />
Expand Down Expand Up @@ -2074,6 +2076,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.inc" name="tests/Core/Tokenizer/BitwiseOrTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.php" name="tests/Core/Tokenizer/GotoLabelTest.php" />
<install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.inc" name="tests/Core/Tokenizer/GotoLabelTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" />
<install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.inc" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.inc" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/ScopeSettingWithNamespaceOperatorTest.php" name="tests/Core/Tokenizer/ScopeSettingWithNamespaceOperatorTest.php" />
Expand Down Expand Up @@ -2148,6 +2152,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Tokenizer/BitwiseOrTest.inc" name="tests/Core/Tokenizer/BitwiseOrTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.php" name="tests/Core/Tokenizer/GotoLabelTest.php" />
<install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.inc" name="tests/Core/Tokenizer/GotoLabelTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" />
<install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.inc" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.inc" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/ScopeSettingWithNamespaceOperatorTest.php" name="tests/Core/Tokenizer/ScopeSettingWithNamespaceOperatorTest.php" />
Expand Down
168 changes: 123 additions & 45 deletions src/Tokenizers/PHP.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -893,6 +893,62 @@ protected function tokenize($string)
continue;
}//end if

/*
Tokenize the parameter labels for PHP 8.0 named parameters as a special T_PARAM_NAME
token and ensure that the colon after it is always T_COLON.
*/

if ($tokenIsArray === true
&& preg_match('`^[a-zA-Z_\x80-\xff]`', $token[1]) === 1
) {
// Get the next non-empty token.
for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
if (is_array($tokens[$i]) === false
|| isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === false
) {
break;
}
}

if (isset($tokens[$i]) === true
&& is_array($tokens[$i]) === false
&& $tokens[$i] === ':'
) {
// Get the previous non-empty token.
for ($j = ($stackPtr - 1); $j > 0; $j--) {
if (is_array($tokens[$j]) === false
|| isset(Util\Tokens::$emptyTokens[$tokens[$j][0]]) === false
) {
break;
}
}

if (is_array($tokens[$j]) === false
&& ($tokens[$j] === '('
|| $tokens[$j] === ',')
) {
$newToken = [];
$newToken['code'] = T_PARAM_NAME;
$newToken['type'] = 'T_PARAM_NAME';
$newToken['content'] = $token[1];
$finalTokens[$newStackPtr] = $newToken;

$newStackPtr++;

// Modify the original token stack so that future checks, like
// determining T_COLON vs T_INLINE_ELSE can handle this correctly.
$tokens[$stackPtr][0] = T_PARAM_NAME;

if (PHP_CODESNIFFER_VERBOSITY > 1) {
$type = Util\Tokens::tokenName($token[0]);
echo "\t\t* token $stackPtr changed from $type to T_PARAM_NAME".PHP_EOL;
}

continue;
}
}//end if
}//end if

/*
Before PHP 7.0, the "yield from" was tokenized as
T_YIELD, T_WHITESPACE and T_STRING. So look for
Expand Down Expand Up @@ -1701,76 +1757,98 @@ function return types. We want to keep the parenthesis map clean,
// Convert colons that are actually the ELSE component of an
// inline IF statement.
if (empty($insideInlineIf) === false && $newToken['code'] === T_COLON) {
// Make sure this isn't a return type separator.
$isInlineIf = true;

// Make sure this isn't a named parameter label.
// Get the previous non-empty token.
for ($i = ($stackPtr - 1); $i > 0; $i--) {
if (is_array($tokens[$i]) === false
|| ($tokens[$i][0] !== T_DOC_COMMENT
&& $tokens[$i][0] !== T_COMMENT
&& $tokens[$i][0] !== T_WHITESPACE)
|| isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === false
) {
break;
}
}

if ($tokens[$i] === ')') {
$parenCount = 1;
for ($i--; $i > 0; $i--) {
if ($tokens[$i] === '(') {
$parenCount--;
if ($parenCount === 0) {
break;
}
} else if ($tokens[$i] === ')') {
$parenCount++;
}
if ($tokens[$i][0] === T_PARAM_NAME) {
$isInlineIf = false;
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* token is parameter label, not T_INLINE_ELSE".PHP_EOL;
}
}

// We've found the open parenthesis, so if the previous
// non-empty token is FUNCTION or USE, this is a return type.
// Note that we need to skip T_STRING tokens here as these
// can be function names.
for ($i--; $i > 0; $i--) {
if ($isInlineIf === true) {
// Make sure this isn't a return type separator.
for ($i = ($stackPtr - 1); $i > 0; $i--) {
if (is_array($tokens[$i]) === false
|| ($tokens[$i][0] !== T_DOC_COMMENT
&& $tokens[$i][0] !== T_COMMENT
&& $tokens[$i][0] !== T_WHITESPACE
&& $tokens[$i][0] !== T_STRING)
&& $tokens[$i][0] !== T_WHITESPACE)
) {
break;
}
}

if ($tokens[$i][0] === T_FUNCTION || $tokens[$i][0] === T_FN || $tokens[$i][0] === T_USE) {
$isInlineIf = false;
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* token is return type, not T_INLINE_ELSE".PHP_EOL;
if ($tokens[$i] === ')') {
$parenCount = 1;
for ($i--; $i > 0; $i--) {
if ($tokens[$i] === '(') {
$parenCount--;
if ($parenCount === 0) {
break;
}
} else if ($tokens[$i] === ')') {
$parenCount++;
}
}
}

// We've found the open parenthesis, so if the previous
// non-empty token is FUNCTION or USE, this is a return type.
// Note that we need to skip T_STRING tokens here as these
// can be function names.
for ($i--; $i > 0; $i--) {
if (is_array($tokens[$i]) === false
|| ($tokens[$i][0] !== T_DOC_COMMENT
&& $tokens[$i][0] !== T_COMMENT
&& $tokens[$i][0] !== T_WHITESPACE
&& $tokens[$i][0] !== T_STRING)
) {
break;
}
}

if ($tokens[$i][0] === T_FUNCTION || $tokens[$i][0] === T_FN || $tokens[$i][0] === T_USE) {
$isInlineIf = false;
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* token is return type, not T_INLINE_ELSE".PHP_EOL;
}
}
}//end if
}//end if

// Check to see if this is a CASE or DEFAULT opener.
$inlineIfToken = $insideInlineIf[(count($insideInlineIf) - 1)];
for ($i = $stackPtr; $i > $inlineIfToken; $i--) {
if (is_array($tokens[$i]) === true
&& ($tokens[$i][0] === T_CASE
|| $tokens[$i][0] === T_DEFAULT)
) {
$isInlineIf = false;
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* token is T_CASE or T_DEFAULT opener, not T_INLINE_ELSE".PHP_EOL;
}
if ($isInlineIf === true) {
$inlineIfToken = $insideInlineIf[(count($insideInlineIf) - 1)];
for ($i = $stackPtr; $i > $inlineIfToken; $i--) {
if (is_array($tokens[$i]) === true
&& ($tokens[$i][0] === T_CASE
|| $tokens[$i][0] === T_DEFAULT)
) {
$isInlineIf = false;
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* token is T_CASE or T_DEFAULT opener, not T_INLINE_ELSE".PHP_EOL;
}

break;
}
break;
}

if (is_array($tokens[$i]) === false
&& ($tokens[$i] === ';'
|| $tokens[$i] === '{')
) {
break;
if (is_array($tokens[$i]) === false
&& ($tokens[$i] === ';'
|| $tokens[$i] === '{')
) {
break;
}
}
}
}//end if

if ($isInlineIf === true) {
array_pop($insideInlineIf);
Expand Down
1 change: 1 addition & 0 deletions src/Util/Tokens.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@
define('T_ZSR_EQUAL', 'PHPCS_T_ZSR_EQUAL');
define('T_FN_ARROW', 'T_FN_ARROW');
define('T_TYPE_UNION', 'T_TYPE_UNION');
define('T_PARAM_NAME', 'T_PARAM_NAME');

// Some PHP 5.5 tokens, replicated for lower versions.
if (defined('T_FINALLY') === false) {
Expand Down
Loading

0 comments on commit 0859719

Please sign in to comment.