Skip to content

Commit

Permalink
ConstExprParser: support numeric literal separator
Browse files Browse the repository at this point in the history
  • Loading branch information
jiripudil authored and ondrejmirtes committed Apr 28, 2023
1 parent 308c57c commit 0b4de96
Show file tree
Hide file tree
Showing 6 changed files with 52 additions and 13 deletions.
16 changes: 8 additions & 8 deletions doc/grammars/type.abnf
Original file line number Diff line number Diff line change
Expand Up @@ -85,18 +85,18 @@ ConstantExpr
/ ConstantFetch *ByteHorizontalWs

ConstantFloat
= ["-"] 1*ByteDecDigit "." *ByteDecDigit [ConstantFloatExp]
/ ["-"] 1*ByteDecDigit ConstantFloatExp
/ ["-"] "." 1*ByteDecDigit [ConstantFloatExp]
= ["-"] 1*ByteDecDigit *("_" 1*ByteDecDigit) "." [1*ByteDecDigit *("_" 1*ByteDecDigit)] [ConstantFloatExp]
/ ["-"] 1*ByteDecDigit *("_" 1*ByteDecDigit) ConstantFloatExp
/ ["-"] "." 1*ByteDecDigit *("_" 1*ByteDecDigit) [ConstantFloatExp]

ConstantFloatExp
= "e" ["-"] 1*ByteDecDigit
= "e" ["-"] 1*ByteDecDigit *("_" 1*ByteDecDigit)

ConstantInt
= ["-"] "0b" 1*ByteBinDigit
/ ["-"] "0o" 1*ByteOctDigit
/ ["-"] "0x" 1*ByteHexDigit
/ ["-"] 1*ByteDecDigit
= ["-"] "0b" 1*ByteBinDigit *("_" 1*ByteBinDigit)
/ ["-"] "0o" 1*ByteOctDigit *("_" 1*ByteOctDigit)
/ ["-"] "0x" 1*ByteHexDigit *("_" 1*ByteHexDigit)
/ ["-"] 1*ByteDecDigit *("_" 1*ByteDecDigit)

ConstantTrue
= "true"
Expand Down
4 changes: 2 additions & 2 deletions src/Lexer/Lexer.php
Original file line number Diff line number Diff line change
Expand Up @@ -160,8 +160,8 @@ private function generateRegexp(): string
self::TOKEN_PHPDOC_TAG => '@(?:[a-z][a-z0-9-\\\\]+:)?[a-z][a-z0-9-\\\\]*+',
self::TOKEN_PHPDOC_EOL => '\\r?+\\n[\\x09\\x20]*+(?:\\*(?!/)\\x20?+)?',

self::TOKEN_FLOAT => '(?:-?[0-9]++\\.[0-9]*+(?:e-?[0-9]++)?)|(?:-?[0-9]*+\\.[0-9]++(?:e-?[0-9]++)?)|(?:-?[0-9]++e-?[0-9]++)',
self::TOKEN_INTEGER => '-?(?:(?:0b[0-1]++)|(?:0o[0-7]++)|(?:0x[0-9a-f]++)|(?:[0-9]++))',
self::TOKEN_FLOAT => '(?:-?[0-9]++(_[0-9]++)*\\.[0-9]*(_[0-9]++)*+(?:e-?[0-9]++(_[0-9]++)*)?)|(?:-?[0-9]*+(_[0-9]++)*\\.[0-9]++(_[0-9]++)*(?:e-?[0-9]++(_[0-9]++)*)?)|(?:-?[0-9]++(_[0-9]++)*e-?[0-9]++(_[0-9]++)*)',
self::TOKEN_INTEGER => '-?(?:(?:0b[0-1]++(_[0-1]++)*)|(?:0o[0-7]++(_[0-7]++)*)|(?:0x[0-9a-f]++(_[0-9a-f]++)*)|(?:[0-9]++(_[0-9]++)*))',
self::TOKEN_SINGLE_QUOTED_STRING => '\'(?:\\\\[^\\r\\n]|[^\'\\r\\n\\\\])*+\'',
self::TOKEN_DOUBLE_QUOTED_STRING => '"(?:\\\\[^\\r\\n]|[^"\\r\\n\\\\])*+"',

Expand Down
5 changes: 3 additions & 2 deletions src/Parser/ConstExprParser.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

use PHPStan\PhpDocParser\Ast;
use PHPStan\PhpDocParser\Lexer\Lexer;
use function str_replace;
use function strtolower;
use function substr;

Expand Down Expand Up @@ -47,7 +48,7 @@ public function parse(TokenIterator $tokens, bool $trimStrings = false): Ast\Con

return $this->enrichWithAttributes(
$tokens,
new Ast\ConstExpr\ConstExprFloatNode($value),
new Ast\ConstExpr\ConstExprFloatNode(str_replace('_', '', $value)),
$startLine,
$startIndex
);
Expand All @@ -59,7 +60,7 @@ public function parse(TokenIterator $tokens, bool $trimStrings = false): Ast\Con

return $this->enrichWithAttributes(
$tokens,
new Ast\ConstExpr\ConstExprIntegerNode($value),
new Ast\ConstExpr\ConstExprIntegerNode(str_replace('_', '', $value)),
$startLine,
$startIndex
);
Expand Down
3 changes: 2 additions & 1 deletion src/Parser/TypeParser.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
use PHPStan\PhpDocParser\Ast;
use PHPStan\PhpDocParser\Lexer\Lexer;
use function in_array;
use function str_replace;
use function strpos;
use function trim;

Expand Down Expand Up @@ -710,7 +711,7 @@ private function parseArrayShapeKey(TokenIterator $tokens)
$startLine = $tokens->currentTokenLine();

if ($tokens->isCurrentTokenType(Lexer::TOKEN_INTEGER)) {
$key = new Ast\ConstExpr\ConstExprIntegerNode($tokens->currentTokenValue());
$key = new Ast\ConstExpr\ConstExprIntegerNode(str_replace('_', '', $tokens->currentTokenValue()));
$tokens->next();

} elseif ($tokens->isCurrentTokenType(Lexer::TOKEN_SINGLE_QUOTED_STRING)) {
Expand Down
20 changes: 20 additions & 0 deletions tests/PHPStan/Parser/ConstExprParserTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,21 @@ public function provideIntegerNodeParseData(): Iterator
'-0X7Fb4',
new ConstExprIntegerNode('-0X7Fb4'),
];

yield [
'123_456',
new ConstExprIntegerNode('123456'),
];

yield [
'0b01_01_01',
new ConstExprIntegerNode('0b010101'),
];

yield [
'-0X7_Fb_4',
new ConstExprIntegerNode('-0X7Fb4'),
];
}


Expand Down Expand Up @@ -240,6 +255,11 @@ public function provideFloatNodeParseData(): Iterator
'-12.3e-4',
new ConstExprFloatNode('-12.3e-4'),
];

yield [
'-1_2.3_4e5_6',
new ConstExprFloatNode('-12.34e56'),
];
}


Expand Down
17 changes: 17 additions & 0 deletions tests/PHPStan/Parser/TypeParserTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -1035,10 +1035,27 @@ public function provideParseData(): array
'123',
new ConstTypeNode(new ConstExprIntegerNode('123')),
],
[
'123_456',
new ConstTypeNode(new ConstExprIntegerNode('123456')),
],
[
'_123',
new IdentifierTypeNode('_123'),
],
[
'123_',
new ConstTypeNode(new ConstExprIntegerNode('123')),
Lexer::TOKEN_IDENTIFIER,
],
[
'123.2',
new ConstTypeNode(new ConstExprFloatNode('123.2')),
],
[
'123_456.789_012',
new ConstTypeNode(new ConstExprFloatNode('123456.789012')),
],
[
'"bar"',
new ConstTypeNode(new QuoteAwareConstExprStringNode('bar', QuoteAwareConstExprStringNode::DOUBLE_QUOTED)),
Expand Down

0 comments on commit 0b4de96

Please sign in to comment.