Skip to content

Commit

Permalink
Merge branch 'feature/php-8-nullsafe-operator' of https://github.com/…
Browse files Browse the repository at this point in the history
  • Loading branch information
gsherwood committed Sep 1, 2020
2 parents 48d5bae + 5aa9f1a commit f09e28a
Show file tree
Hide file tree
Showing 8 changed files with 230 additions and 19 deletions.
6 changes: 6 additions & 0 deletions package.xml
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<file baseinstalldir="" name="BackfillFnTokenTest.php" role="test" />
<file baseinstalldir="" name="BackfillNumericSeparatorTest.inc" role="test" />
<file baseinstalldir="" name="BackfillNumericSeparatorTest.php" role="test" />
<file baseinstalldir="" name="NullsafeObjectOperatorTest.inc" role="test" />
<file baseinstalldir="" name="NullsafeObjectOperatorTest.php" role="test" />
<file baseinstalldir="" name="ShortArrayTest.inc" role="test" />
<file baseinstalldir="" name="ShortArrayTest.php" role="test" />
<file baseinstalldir="" name="StableCommentWhitespaceTest.inc" role="test" />
Expand Down Expand Up @@ -1978,6 +1980,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.inc" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/ShortArrayTest.php" name="tests/Core/Tokenizer/ShortArrayTest.php" />
<install as="CodeSniffer/Core/Tokenizer/ShortArrayTest.inc" name="tests/Core/Tokenizer/ShortArrayTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.php" />
Expand Down Expand Up @@ -2037,6 +2041,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.inc" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/ShortArrayTest.php" name="tests/Core/Tokenizer/ShortArrayTest.php" />
<install as="CodeSniffer/Core/Tokenizer/ShortArrayTest.inc" name="tests/Core/Tokenizer/ShortArrayTest.inc" />
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.php" />
Expand Down
55 changes: 41 additions & 14 deletions src/Tokenizers/PHP.php
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,7 @@ class PHP extends Tokenizer
T_NS_C => 13,
T_NS_SEPARATOR => 1,
T_NEW => 3,
T_NULLSAFE_OBJECT_OPERATOR => 3,
T_OBJECT_OPERATOR => 2,
T_OPEN_TAG_WITH_ECHO => 3,
T_OR_EQUAL => 2,
Expand Down Expand Up @@ -1017,6 +1018,29 @@ protected function tokenize($string)
continue;
}

/*
Before PHP 8, the ?-> operator was tokenized as
T_INLINE_THEN followed by T_OBJECT_OPERATOR.
So look for and combine these tokens in earlier versions.
*/

if ($tokenIsArray === false
&& $token[0] === '?'
&& isset($tokens[($stackPtr + 1)]) === true
&& is_array($tokens[($stackPtr + 1)]) === true
&& $tokens[($stackPtr + 1)][0] === T_OBJECT_OPERATOR
) {
$newToken = [];
$newToken['code'] = T_NULLSAFE_OBJECT_OPERATOR;
$newToken['type'] = 'T_NULLSAFE_OBJECT_OPERATOR';
$newToken['content'] = '?->';
$finalTokens[$newStackPtr] = $newToken;

$newStackPtr++;
$stackPtr++;
continue;
}

/*
Before PHP 7.4, underscores inside T_LNUMBER and T_DNUMBER
tokens split the token with a T_STRING. So look for
Expand Down Expand Up @@ -1513,17 +1537,18 @@ function return types. We want to keep the parenthesis map clean,
// Some T_STRING tokens should remain that way
// due to their context.
$context = [
T_OBJECT_OPERATOR => true,
T_FUNCTION => true,
T_CLASS => true,
T_EXTENDS => true,
T_IMPLEMENTS => true,
T_NEW => true,
T_CONST => true,
T_NS_SEPARATOR => true,
T_USE => true,
T_NAMESPACE => true,
T_PAAMAYIM_NEKUDOTAYIM => true,
T_OBJECT_OPERATOR => true,
T_NULLSAFE_OBJECT_OPERATOR => true,
T_FUNCTION => true,
T_CLASS => true,
T_EXTENDS => true,
T_IMPLEMENTS => true,
T_NEW => true,
T_CONST => true,
T_NS_SEPARATOR => true,
T_USE => true,
T_NAMESPACE => true,
T_PAAMAYIM_NEKUDOTAYIM => true,
];

if (isset($context[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
Expand Down Expand Up @@ -2018,6 +2043,7 @@ protected function processAdditional()
T_CLOSE_PARENTHESIS => T_CLOSE_PARENTHESIS,
T_VARIABLE => T_VARIABLE,
T_OBJECT_OPERATOR => T_OBJECT_OPERATOR,
T_NULLSAFE_OBJECT_OPERATOR => T_NULLSAFE_OBJECT_OPERATOR,
T_STRING => T_STRING,
T_CONSTANT_ENCAPSED_STRING => T_CONSTANT_ENCAPSED_STRING,
];
Expand Down Expand Up @@ -2087,9 +2113,10 @@ protected function processAdditional()
}

$context = [
T_OBJECT_OPERATOR => true,
T_NS_SEPARATOR => true,
T_PAAMAYIM_NEKUDOTAYIM => true,
T_OBJECT_OPERATOR => true,
T_NULLSAFE_OBJECT_OPERATOR => true,
T_NS_SEPARATOR => true,
T_PAAMAYIM_NEKUDOTAYIM => true,
];
if (isset($context[$this->tokens[$x]['code']]) === true) {
if (PHP_CODESNIFFER_VERBOSITY > 1) {
Expand Down
11 changes: 6 additions & 5 deletions src/Tokenizers/Tokenizer.php
Original file line number Diff line number Diff line change
Expand Up @@ -1291,11 +1291,12 @@ private function recurseScopeMap($stackPtr, $depth=1, &$ignore=0)
// a new statement, it isn't a scope opener.
$disallowed = Util\Tokens::$assignmentTokens;
$disallowed += [
T_DOLLAR => true,
T_VARIABLE => true,
T_OBJECT_OPERATOR => true,
T_COMMA => true,
T_OPEN_PARENTHESIS => true,
T_DOLLAR => true,
T_VARIABLE => true,
T_OBJECT_OPERATOR => true,
T_NULLSAFE_OBJECT_OPERATOR => true,
T_COMMA => true,
T_OPEN_PARENTHESIS => true,
];

if (isset($disallowed[$this->tokens[$x]['code']]) === true) {
Expand Down
5 changes: 5 additions & 0 deletions src/Util/Tokens.php
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,11 @@
define('T_FN', 'PHPCS_T_FN');
}

// Some PHP 8.0 tokens, replicated for lower versions.
if (defined('T_NULLSAFE_OBJECT_OPERATOR') === false) {
define('T_NULLSAFE_OBJECT_OPERATOR', 'PHPCS_T_NULLSAFE_OBJECT_OPERATOR');
}

// Tokens used for parsing doc blocks.
define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');
Expand Down
29 changes: 29 additions & 0 deletions tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
<?php

/*
* Null safe operator.
*/

/* testObjectOperator */
echo $obj->foo;

/* testNullsafeObjectOperator */
echo $obj?->foo;

/* testNullsafeObjectOperatorWriteContext */
// Intentional parse error, but not the concern of the tokenizer.
$foo?->bar->baz = 'baz';

/* testTernaryThen */
echo $obj ? $obj->prop : $other->prop;

/* testParseErrorWhitespaceNotAllowed */
echo $obj ?
-> foo;

/* testParseErrorCommentNotAllowed */
echo $obj ?/*comment*/-> foo;

/* testLiveCoding */
// Intentional parse error. This has to be the last test in the file.
echo $obj?
140 changes: 140 additions & 0 deletions tests/Core/Tokenizer/NullsafeObjectOperatorTest.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
<?php
/**
* Tests the backfill for the PHP >= 8.0 nullsafe object operator.
*
* @author Juliette Reinders Folmer <phpcs_nospam@adviesenzo.nl>
* @copyright 2020 Squiz Pty Ltd (ABN 77 084 670 600)
* @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
*/

namespace PHP_CodeSniffer\Tests\Core\Tokenizer;

use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest;
use PHP_CodeSniffer\Util\Tokens;

class NullsafeObjectOperatorTest extends AbstractMethodUnitTest
{

/**
* Tokens to search for.
*
* @var array
*/
protected $find = [
T_NULLSAFE_OBJECT_OPERATOR,
T_OBJECT_OPERATOR,
T_INLINE_THEN,
];


/**
* Test that a normal object operator is still tokenized as such.
*
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
*
* @return void
*/
public function testObjectOperator()
{
$tokens = self::$phpcsFile->getTokens();

$operator = $this->getTargetToken('/* testObjectOperator */', $this->find);
$this->assertSame(T_OBJECT_OPERATOR, $tokens[$operator]['code'], 'Failed asserting code is object operator');
$this->assertSame('T_OBJECT_OPERATOR', $tokens[$operator]['type'], 'Failed asserting type is object operator');

}//end testObjectOperator()


/**
* Test that a nullsafe object operator is tokenized as such.
*
* @param string $testMarker The comment which prefaces the target token in the test file.
*
* @dataProvider dataNullsafeObjectOperator
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
*
* @return void
*/
public function testNullsafeObjectOperator($testMarker)
{
$tokens = self::$phpcsFile->getTokens();

$operator = $this->getTargetToken($testMarker, $this->find);
$this->assertSame(T_NULLSAFE_OBJECT_OPERATOR, $tokens[$operator]['code'], 'Failed asserting code is nullsafe object operator');
$this->assertSame('T_NULLSAFE_OBJECT_OPERATOR', $tokens[$operator]['type'], 'Failed asserting type is nullsafe object operator');

}//end testNullsafeObjectOperator()


/**
* Data provider.
*
* @see testNullsafeObjectOperator()
*
* @return array
*/
public function dataNullsafeObjectOperator()
{
return [
['/* testNullsafeObjectOperator */'],
['/* testNullsafeObjectOperatorWriteContext */'],
];

}//end dataNullsafeObjectOperator()


/**
* Test that a question mark not followed by an object operator is tokenized as T_TERNARY_THEN.
*
* @param string $testMarker The comment which prefaces the target token in the test file.
* @param bool $testObjectOperator Whether to test for the next non-empty token being tokenized
* as an object operator.
*
* @dataProvider dataTernaryThen
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
*
* @return void
*/
public function testTernaryThen($testMarker, $testObjectOperator=false)
{
$tokens = self::$phpcsFile->getTokens();

$operator = $this->getTargetToken($testMarker, $this->find);
$this->assertSame(T_INLINE_THEN, $tokens[$operator]['code'], 'Failed asserting code is inline then');
$this->assertSame('T_INLINE_THEN', $tokens[$operator]['type'], 'Failed asserting type is inline then');

if ($testObjectOperator === true) {
$next = self::$phpcsFile->findNext(Tokens::$emptyTokens, ($operator + 1), null, true);
$this->assertSame(T_OBJECT_OPERATOR, $tokens[$next]['code'], 'Failed asserting code is object operator');
$this->assertSame('T_OBJECT_OPERATOR', $tokens[$next]['type'], 'Failed asserting type is object operator');
}

}//end testTernaryThen()


/**
* Data provider.
*
* @see testTernaryThen()
*
* @return array
*/
public function dataTernaryThen()
{
return [
['/* testTernaryThen */'],
[
'/* testParseErrorWhitespaceNotAllowed */',
true,
],
[
'/* testParseErrorCommentNotAllowed */',
true,
],
['/* testLiveCoding */'],
];

}//end dataTernaryThen()


}//end class
2 changes: 2 additions & 0 deletions tests/Core/Tokenizer/ShortArrayTest.inc
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ $a = (new Foo( array(1, array(4, 5), 3) ))[1][0];
/* testClassMemberDereferencingOnClone */
echo (clone $iterable)[20];

/* testNullsafeMethodCallDereferencing */
$var = $obj?->function_call()[$x];

/*
* Short array brackets.
Expand Down
1 change: 1 addition & 0 deletions tests/Core/Tokenizer/ShortArrayTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ public function dataSquareBrackets()
['/* testClassMemberDereferencingOnInstantiation1 */'],
['/* testClassMemberDereferencingOnInstantiation2 */'],
['/* testClassMemberDereferencingOnClone */'],
['/* testNullsafeMethodCallDereferencing */'],
['/* testLiveCoding */'],
];

Expand Down

0 comments on commit f09e28a

Please sign in to comment.