From 5aa9f1a74be6c608a8a414fcc7f2f26fe7a891fd Mon Sep 17 00:00:00 2001 From: jrfnl Date: Sat, 1 Aug 2020 00:03:34 +0200 Subject: [PATCH] PHP 8.0 | Tokenizer/PHP: add support for nullsafe object operator PHP 8 introduces a new object chaining operator `?->` which short-circuits moving to the next expression if the left-hand side evaluates to `null`. This operator can not be used in write-context, but that is not the concern of the PHPCS `Tokenizers\PHP` class. This commit: * Defines the token constant for PHP < 8.0. * Adds a backfill for the nullsafe object operator for PHP < 8.0 to the PHP tokenizer. * Adds the token to applicable token lists in the PHP and base tokenizer class, like the one used in the short array re-tokenization. * Adds perfunctory unit tests for the nullsafe object operator backfill. * Adds a unit test using the operator to the tokenizer tests for the short array re-tokenization. Refs: * https://wiki.php.net/rfc/nullsafe_operator * https://github.com/php/php-src/commit/9bf119832dbf625174794834c71b1e793450d87f --- package.xml | 6 + src/Tokenizers/PHP.php | 55 +++++-- src/Tokenizers/Tokenizer.php | 11 +- src/Util/Tokens.php | 5 + .../Tokenizer/NullsafeObjectOperatorTest.inc | 29 ++++ .../Tokenizer/NullsafeObjectOperatorTest.php | 140 ++++++++++++++++++ tests/Core/Tokenizer/ShortArrayTest.inc | 2 + tests/Core/Tokenizer/ShortArrayTest.php | 1 + 8 files changed, 230 insertions(+), 19 deletions(-) create mode 100644 tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc create mode 100644 tests/Core/Tokenizer/NullsafeObjectOperatorTest.php diff --git a/package.xml b/package.xml index fc3dde3350..0192b57f5f 100644 --- a/package.xml +++ b/package.xml @@ -121,6 +121,8 @@ http://pear.php.net/dtd/package-2.0.xsd"> + + @@ -1991,6 +1993,8 @@ http://pear.php.net/dtd/package-2.0.xsd"> + + @@ -2050,6 +2054,8 @@ http://pear.php.net/dtd/package-2.0.xsd"> + + diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php index 61f9c2f798..de1fe98236 100644 --- a/src/Tokenizers/PHP.php +++ b/src/Tokenizers/PHP.php @@ -370,6 +370,7 @@ class PHP extends Tokenizer T_NS_C => 13, T_NS_SEPARATOR => 1, T_NEW => 3, + T_NULLSAFE_OBJECT_OPERATOR => 3, T_OBJECT_OPERATOR => 2, T_OPEN_TAG_WITH_ECHO => 3, T_OR_EQUAL => 2, @@ -1015,6 +1016,29 @@ protected function tokenize($string) continue; } + /* + Before PHP 8, the ?-> operator was tokenized as + T_INLINE_THEN followed by T_OBJECT_OPERATOR. + So look for and combine these tokens in earlier versions. + */ + + if ($tokenIsArray === false + && $token[0] === '?' + && isset($tokens[($stackPtr + 1)]) === true + && is_array($tokens[($stackPtr + 1)]) === true + && $tokens[($stackPtr + 1)][0] === T_OBJECT_OPERATOR + ) { + $newToken = []; + $newToken['code'] = T_NULLSAFE_OBJECT_OPERATOR; + $newToken['type'] = 'T_NULLSAFE_OBJECT_OPERATOR'; + $newToken['content'] = '?->'; + $finalTokens[$newStackPtr] = $newToken; + + $newStackPtr++; + $stackPtr++; + continue; + } + /* Before PHP 7.4, underscores inside T_LNUMBER and T_DNUMBER tokens split the token with a T_STRING. So look for @@ -1510,17 +1534,18 @@ function return types. We want to keep the parenthesis map clean, // Some T_STRING tokens should remain that way // due to their context. $context = [ - T_OBJECT_OPERATOR => true, - T_FUNCTION => true, - T_CLASS => true, - T_EXTENDS => true, - T_IMPLEMENTS => true, - T_NEW => true, - T_CONST => true, - T_NS_SEPARATOR => true, - T_USE => true, - T_NAMESPACE => true, - T_PAAMAYIM_NEKUDOTAYIM => true, + T_OBJECT_OPERATOR => true, + T_NULLSAFE_OBJECT_OPERATOR => true, + T_FUNCTION => true, + T_CLASS => true, + T_EXTENDS => true, + T_IMPLEMENTS => true, + T_NEW => true, + T_CONST => true, + T_NS_SEPARATOR => true, + T_USE => true, + T_NAMESPACE => true, + T_PAAMAYIM_NEKUDOTAYIM => true, ]; if (isset($context[$finalTokens[$lastNotEmptyToken]['code']]) === true) { @@ -2012,6 +2037,7 @@ protected function processAdditional() T_CLOSE_PARENTHESIS => T_CLOSE_PARENTHESIS, T_VARIABLE => T_VARIABLE, T_OBJECT_OPERATOR => T_OBJECT_OPERATOR, + T_NULLSAFE_OBJECT_OPERATOR => T_NULLSAFE_OBJECT_OPERATOR, T_STRING => T_STRING, T_CONSTANT_ENCAPSED_STRING => T_CONSTANT_ENCAPSED_STRING, ]; @@ -2081,9 +2107,10 @@ protected function processAdditional() } $context = [ - T_OBJECT_OPERATOR => true, - T_NS_SEPARATOR => true, - T_PAAMAYIM_NEKUDOTAYIM => true, + T_OBJECT_OPERATOR => true, + T_NULLSAFE_OBJECT_OPERATOR => true, + T_NS_SEPARATOR => true, + T_PAAMAYIM_NEKUDOTAYIM => true, ]; if (isset($context[$this->tokens[$x]['code']]) === true) { if (PHP_CODESNIFFER_VERBOSITY > 1) { diff --git a/src/Tokenizers/Tokenizer.php b/src/Tokenizers/Tokenizer.php index 82b2b9cc78..24f11d2505 100644 --- a/src/Tokenizers/Tokenizer.php +++ b/src/Tokenizers/Tokenizer.php @@ -1291,11 +1291,12 @@ private function recurseScopeMap($stackPtr, $depth=1, &$ignore=0) // a new statement, it isn't a scope opener. $disallowed = Util\Tokens::$assignmentTokens; $disallowed += [ - T_DOLLAR => true, - T_VARIABLE => true, - T_OBJECT_OPERATOR => true, - T_COMMA => true, - T_OPEN_PARENTHESIS => true, + T_DOLLAR => true, + T_VARIABLE => true, + T_OBJECT_OPERATOR => true, + T_NULLSAFE_OBJECT_OPERATOR => true, + T_COMMA => true, + T_OPEN_PARENTHESIS => true, ]; if (isset($disallowed[$this->tokens[$x]['code']]) === true) { diff --git a/src/Util/Tokens.php b/src/Util/Tokens.php index bc3a32414a..3994264067 100644 --- a/src/Util/Tokens.php +++ b/src/Util/Tokens.php @@ -124,6 +124,11 @@ define('T_FN', 'PHPCS_T_FN'); } +// Some PHP 8.0 tokens, replicated for lower versions. +if (defined('T_NULLSAFE_OBJECT_OPERATOR') === false) { + define('T_NULLSAFE_OBJECT_OPERATOR', 'PHPCS_T_NULLSAFE_OBJECT_OPERATOR'); +} + // Tokens used for parsing doc blocks. define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR'); define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE'); diff --git a/tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc b/tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc new file mode 100644 index 0000000000..982841eac4 --- /dev/null +++ b/tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc @@ -0,0 +1,29 @@ +foo; + +/* testNullsafeObjectOperator */ +echo $obj?->foo; + +/* testNullsafeObjectOperatorWriteContext */ +// Intentional parse error, but not the concern of the tokenizer. +$foo?->bar->baz = 'baz'; + +/* testTernaryThen */ +echo $obj ? $obj->prop : $other->prop; + +/* testParseErrorWhitespaceNotAllowed */ +echo $obj ? + -> foo; + +/* testParseErrorCommentNotAllowed */ +echo $obj ?/*comment*/-> foo; + +/* testLiveCoding */ +// Intentional parse error. This has to be the last test in the file. +echo $obj? diff --git a/tests/Core/Tokenizer/NullsafeObjectOperatorTest.php b/tests/Core/Tokenizer/NullsafeObjectOperatorTest.php new file mode 100644 index 0000000000..8e465a3be1 --- /dev/null +++ b/tests/Core/Tokenizer/NullsafeObjectOperatorTest.php @@ -0,0 +1,140 @@ += 8.0 nullsafe object operator. + * + * @author Juliette Reinders Folmer + * @copyright 2020 Squiz Pty Ltd (ABN 77 084 670 600) + * @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace PHP_CodeSniffer\Tests\Core\Tokenizer; + +use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest; +use PHP_CodeSniffer\Util\Tokens; + +class NullsafeObjectOperatorTest extends AbstractMethodUnitTest +{ + + /** + * Tokens to search for. + * + * @var array + */ + protected $find = [ + T_NULLSAFE_OBJECT_OPERATOR, + T_OBJECT_OPERATOR, + T_INLINE_THEN, + ]; + + + /** + * Test that a normal object operator is still tokenized as such. + * + * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize + * + * @return void + */ + public function testObjectOperator() + { + $tokens = self::$phpcsFile->getTokens(); + + $operator = $this->getTargetToken('/* testObjectOperator */', $this->find); + $this->assertSame(T_OBJECT_OPERATOR, $tokens[$operator]['code'], 'Failed asserting code is object operator'); + $this->assertSame('T_OBJECT_OPERATOR', $tokens[$operator]['type'], 'Failed asserting type is object operator'); + + }//end testObjectOperator() + + + /** + * Test that a nullsafe object operator is tokenized as such. + * + * @param string $testMarker The comment which prefaces the target token in the test file. + * + * @dataProvider dataNullsafeObjectOperator + * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize + * + * @return void + */ + public function testNullsafeObjectOperator($testMarker) + { + $tokens = self::$phpcsFile->getTokens(); + + $operator = $this->getTargetToken($testMarker, $this->find); + $this->assertSame(T_NULLSAFE_OBJECT_OPERATOR, $tokens[$operator]['code'], 'Failed asserting code is nullsafe object operator'); + $this->assertSame('T_NULLSAFE_OBJECT_OPERATOR', $tokens[$operator]['type'], 'Failed asserting type is nullsafe object operator'); + + }//end testNullsafeObjectOperator() + + + /** + * Data provider. + * + * @see testNullsafeObjectOperator() + * + * @return array + */ + public function dataNullsafeObjectOperator() + { + return [ + ['/* testNullsafeObjectOperator */'], + ['/* testNullsafeObjectOperatorWriteContext */'], + ]; + + }//end dataNullsafeObjectOperator() + + + /** + * Test that a question mark not followed by an object operator is tokenized as T_TERNARY_THEN. + * + * @param string $testMarker The comment which prefaces the target token in the test file. + * @param bool $testObjectOperator Whether to test for the next non-empty token being tokenized + * as an object operator. + * + * @dataProvider dataTernaryThen + * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize + * + * @return void + */ + public function testTernaryThen($testMarker, $testObjectOperator=false) + { + $tokens = self::$phpcsFile->getTokens(); + + $operator = $this->getTargetToken($testMarker, $this->find); + $this->assertSame(T_INLINE_THEN, $tokens[$operator]['code'], 'Failed asserting code is inline then'); + $this->assertSame('T_INLINE_THEN', $tokens[$operator]['type'], 'Failed asserting type is inline then'); + + if ($testObjectOperator === true) { + $next = self::$phpcsFile->findNext(Tokens::$emptyTokens, ($operator + 1), null, true); + $this->assertSame(T_OBJECT_OPERATOR, $tokens[$next]['code'], 'Failed asserting code is object operator'); + $this->assertSame('T_OBJECT_OPERATOR', $tokens[$next]['type'], 'Failed asserting type is object operator'); + } + + }//end testTernaryThen() + + + /** + * Data provider. + * + * @see testTernaryThen() + * + * @return array + */ + public function dataTernaryThen() + { + return [ + ['/* testTernaryThen */'], + [ + '/* testParseErrorWhitespaceNotAllowed */', + true, + ], + [ + '/* testParseErrorCommentNotAllowed */', + true, + ], + ['/* testLiveCoding */'], + ]; + + }//end dataTernaryThen() + + +}//end class diff --git a/tests/Core/Tokenizer/ShortArrayTest.inc b/tests/Core/Tokenizer/ShortArrayTest.inc index a864869af9..b052ee5681 100644 --- a/tests/Core/Tokenizer/ShortArrayTest.inc +++ b/tests/Core/Tokenizer/ShortArrayTest.inc @@ -62,6 +62,8 @@ $a = (new Foo( array(1, array(4, 5), 3) ))[1][0]; /* testClassMemberDereferencingOnClone */ echo (clone $iterable)[20]; +/* testNullsafeMethodCallDereferencing */ +$var = $obj?->function_call()[$x]; /* * Short array brackets. diff --git a/tests/Core/Tokenizer/ShortArrayTest.php b/tests/Core/Tokenizer/ShortArrayTest.php index 04aaf3db26..45f6c4cf88 100644 --- a/tests/Core/Tokenizer/ShortArrayTest.php +++ b/tests/Core/Tokenizer/ShortArrayTest.php @@ -72,6 +72,7 @@ public function dataSquareBrackets() ['/* testClassMemberDereferencingOnInstantiation1 */'], ['/* testClassMemberDereferencingOnInstantiation2 */'], ['/* testClassMemberDereferencingOnClone */'], + ['/* testNullsafeMethodCallDereferencing */'], ['/* testLiveCoding */'], ];