Skip to content

Commit

Permalink
Implemented lexical and visual distinction for differently based numbers
Browse files Browse the repository at this point in the history
Added lexing, parsing and highlighting

Fixes #2225
  • Loading branch information
hurricup committed Jun 29, 2020
1 parent e94b094 commit e4a0e76
Show file tree
Hide file tree
Showing 25 changed files with 179 additions and 39 deletions.
6 changes: 4 additions & 2 deletions plugin/core/grammar/Perl.flex
Expand Up @@ -88,6 +88,7 @@ NUMBER_EXP = [eE][+-]?[0-9_]+
NUMBER_INT = [0-9][0-9_]*
NUMBER_HEX = "0"[xX][0-9a-fA-F_]+
NUMBER_BIN = "0"[bB][01_]+
NUMBER_OCT = "0"[0-7_]+

SPECIAL_VARIABLE_NAME = [\"\'\[\]\`\\\!\%\&\(\)\+\,\-\.\/\;\<\=\>\|\~\?\:\*\^\@\_\$]
NUMERIC_VARIABLE_NAME = "0" | [1-9] \d*
Expand Down Expand Up @@ -1165,8 +1166,9 @@ POSIX_CHARGROUP_ANY = {POSIX_CHARGROUP}|{POSIX_CHARGROUP_DOUBLE}
// fixme optimize via merging?
{BAREWORD_MINUS} / {MAY_BE_SPACES_OR_COMMENTS}* {FARROW} {yybegin(AFTER_VALUE);return STRING_CONTENT;}

{NUMBER_BIN} {yybegin(AFTER_VALUE);return NUMBER;}
{NUMBER_HEX} {yybegin(AFTER_VALUE);return NUMBER;}
{NUMBER_OCT} {yybegin(AFTER_VALUE);return NUMBER_OCT;}
{NUMBER_BIN} {yybegin(AFTER_VALUE);return NUMBER_BIN;}
{NUMBER_HEX} {yybegin(AFTER_VALUE);return NUMBER_HEX;}
{NUMBER_INT} / ".." {yybegin(AFTER_VALUE);return NUMBER;}
"." {NUMBER_INT} {NUMBER_EXP}? {yybegin(AFTER_VALUE);return NUMBER;}
{NUMBER_INT} ("." {NUMBER_INT}? )? {NUMBER_EXP}? {yybegin(AFTER_VALUE);return NUMBER;}
Expand Down
14 changes: 7 additions & 7 deletions plugin/core/grammar/Perl5.bnf
Expand Up @@ -81,7 +81,9 @@

NUMBER_VERSION='NUMBER_VERSION'
NUMBER='NUMBER'
NUMBER_SIMPLE='NUMBER_SIMPLE' // fixme get rid of
NUMBER_HEX = 'NUMBER_HEX'
NUMBER_OCT = 'NUMBER_OCT'
NUMBER_BIN = 'NUMBER_BIN'

STRING_SPECIAL_TAB='\t'
STRING_SPECIAL_NEWLINE='\n'
Expand All @@ -105,8 +107,6 @@
STRING_SPECIAL_UNICODE_CODE_PREFIX='U+'

STRING_CHAR_NAME='charname'
STRING_CHAR_CODE_HEX='hexcode'
STRING_CHAR_CODE_OCT='octcode'

STRING_SPECIAL_LEFT_BRACE='"{'
STRING_SPECIAL_RIGHT_BRACE='}"'
Expand Down Expand Up @@ -1228,7 +1228,7 @@ perl_handle_expr ::= [QUALIFYING_PACKAGE] HANDLE

tag_scalar ::= TAG

number_constant ::= NUMBER | NUMBER_VERSION | NUMBER_SIMPLE
number_constant ::= NUMBER | NUMBER_VERSION | NUMBER_HEX | NUMBER_OCT | NUMBER_BIN

private string ::= string_bare | string_sq | string_dq | string_xq | heredoc_opener

Expand All @@ -1247,17 +1247,17 @@ private special_constructs ::= '\t'|'\n'|'\r'|'\f'|'\b'|'\a'|'\e'|'\l'|'\\u'|'\c
private unicode_char ::= '\N' unicode_char_body {pin=1}
private unicode_char_body ::= '"{' unicode_char_body_content '}"' {pin=1}
private unicode_char_body_content ::= unicode_char_body_named | 'charname'
private unicode_char_body_named ::= 'U+' 'hexcode' {pin=1}
private unicode_char_body_named ::= 'U+' NUMBER_HEX {pin=1}

private hex_char ::= '\x' hex_char_body {pin=1}
private hex_char_body ::= hex_char_body_content | hex_char_body_braced
private hex_char_body_braced ::= '"{' hex_char_body_content '}"' {pin=1}
private hex_char_body_content ::= 'hexcode'
private hex_char_body_content ::= NUMBER_HEX

private oct_char ::= '\o' oct_char_body {pin=1}
private oct_char_body ::= oct_char_body_content | oct_char_body_braced
private oct_char_body_braced ::= '"{' oct_char_body_content '}"' {pin=1}
private oct_char_body_content ::= 'octcode'
private oct_char_body_content ::= NUMBER_OCT

string_xq ::= [ 'qx'] quoted_xq_string
private quoted_xq_string ::= QUOTE_TICK_OPEN [<<consumeTokens 'LP_STRING_QX'>> | qx_string_content] QUOTE_TICK_CLOSE {pin=1}
Expand Down
Expand Up @@ -226,7 +226,7 @@ public boolean isNewLineForbiddenAt(@NotNull ASTNode node) {

if (run != null) {
IElementType runType = run.getElementType();
if (runType == NUMBER_SIMPLE || runType == NUMBER && StringUtil.endsWith(run.getText(), ".")) {
if (runType == NUMBER && StringUtil.endsWith(run.getText(), ".")) {
return Spacing.createSpacing(1, 1, 0, true, 1);
}
}
Expand Down
Expand Up @@ -37,6 +37,9 @@ public class PerlColorSettingsPage implements ColorSettingsPage {
private static final AttributesDescriptor[] DESCRIPTORS = new AttributesDescriptor[]{
new AttributesDescriptor("Version", PerlSyntaxHighlighter.PERL_VERSION),
new AttributesDescriptor("Number", PerlSyntaxHighlighter.PERL_NUMBER),
new AttributesDescriptor("Number, binary", PerlSyntaxHighlighter.PERL_NUMBER_BIN),
new AttributesDescriptor("Number, octal", PerlSyntaxHighlighter.PERL_NUMBER_OCT),
new AttributesDescriptor("Number, hexadecimal", PerlSyntaxHighlighter.PERL_NUMBER_HEX),

new AttributesDescriptor("Line comment", PerlSyntaxHighlighter.PERL_COMMENT),

Expand Down Expand Up @@ -139,6 +142,7 @@ public class PerlColorSettingsPage implements ColorSettingsPage {
"<kw>say</kw> <qq>\"Double quoted string\"</qq>;\n" +
"<kw>say</kw> __LINE__;\n" +
"say \"Something \\Qhere\\E \\uthere \";\n" +
"my $sum = 42 + 0x42 + 0b1010101 + 042;" +
"<kw>say</kw> <<<q>'MOJO'</q>;\n" +
" <em>%=</em> <kw>print</kw> <qq>\"Mojo perl code\"</qq>\n" +
" <em><%</em> <kw>print</kw> <qq>\"Some more Mojo code\"</qq>; <em>%></em>\n" +
Expand Down
Expand Up @@ -47,6 +47,9 @@ public class PerlSyntaxHighlighter extends SyntaxHighlighterBase implements Perl
createTextAttributesKey("PERL_EMBED_MARKER", DefaultLanguageHighlighterColors.TEMPLATE_LANGUAGE_COLOR);
public static final TextAttributesKey[] EMBED_MARKER_KEYS = new TextAttributesKey[]{EMBED_MARKER_KEY};
public static final TextAttributesKey PERL_NUMBER = createTextAttributesKey("PERL_NUMBER", DefaultLanguageHighlighterColors.NUMBER);
public static final TextAttributesKey PERL_NUMBER_HEX = createTextAttributesKey("PERL_NUMBER_HEX", PERL_NUMBER);
public static final TextAttributesKey PERL_NUMBER_OCT = createTextAttributesKey("PERL_NUMBER_OCT", PERL_NUMBER);
public static final TextAttributesKey PERL_NUMBER_BIN = createTextAttributesKey("PERL_NUMBER_BIN", PERL_NUMBER);
public static final TextAttributesKey PERL_VERSION = createTextAttributesKey("PERL_VERSION", PERL_NUMBER);
public static final TextAttributesKey PERL_COMMENT =
createTextAttributesKey("PERL_COMMENT", DefaultLanguageHighlighterColors.LINE_COMMENT);
Expand Down Expand Up @@ -162,7 +165,10 @@ public class PerlSyntaxHighlighter extends SyntaxHighlighterBase implements Perl
RIGHT_BRACKET); //, REGEX_LEFT_BRACKET, REGEX_RIGHT_BRACKET, REGEX_POSIX_LEFT_BRACKET, REGEX_POSIX_RIGHT_BRACKET
safeMap(PERL_PAREN, LEFT_PAREN, RIGHT_PAREN); // , REGEX_LEFT_PAREN, REGEX_RIGHT_PAREN

safeMap(PERL_NUMBER, NUMBER, NUMBER_SIMPLE);
safeMap(PERL_NUMBER, NUMBER);
safeMap(PERL_NUMBER_HEX, NUMBER_HEX);
safeMap(PERL_NUMBER_OCT, NUMBER_OCT);
safeMap(PERL_NUMBER_BIN, NUMBER_BIN);
safeMap(NUMBER_VERSION, PERL_VERSION);

safeMap(PERL_TAG, TAG, TAG_END, TAG_DATA);
Expand Down
2 changes: 0 additions & 2 deletions plugin/core/src/com/perl5/lang/perl/lexer/PerlTokenSets.java
Expand Up @@ -446,8 +446,6 @@ public interface PerlTokenSets extends PerlElementTypes, MooseElementTypes {
STRING_SPECIAL_UNICODE_CODE_PREFIX,

STRING_CHAR_NAME,
STRING_CHAR_CODE_HEX,
STRING_CHAR_CODE_OCT,

STRING_SPECIAL_LEFT_BRACE,
STRING_SPECIAL_RIGHT_BRACE,
Expand Down
Expand Up @@ -40,7 +40,6 @@ public class PerlParserUtil extends GeneratedParserUtilBase implements PerlEleme

public static final TokenSet VERSION_TOKENS = TokenSet.create(
NUMBER,
NUMBER_SIMPLE,
NUMBER_VERSION
);
public static final TokenSet OPEN_QUOTES = TokenSet.create(
Expand Down
2 changes: 2 additions & 0 deletions plugin/parts/pluginChanges.html
Expand Up @@ -35,6 +35,8 @@
<li>Improved parsing for readline operator (diamond operator), <a href="https://github.com/Camelcade/Perl5-IDEA/issues/2213">#2213</a>
<li>Improved <code>Install Perl</code> action for perlbrew, now showing all available distributions, <a
href="https://github.com/Camelcade/Perl5-IDEA/issues/2221">#2221</a>
<li>It's now possible to configure different colors for differently based numbers: decimal, hexadecimal, octal and binary, <a
href="https://github.com/Camelcade/Perl5-IDEA/issues/2225">#2225</a>
<li>Fixed number of parsing bugs, <a href="https://github.com/Camelcade/Perl5-IDEA/issues/2215">#2215</a>
</li>
<li>Perl 5.22:
Expand Down
3 changes: 3 additions & 0 deletions plugin/test/highlighting/PerlSyntaxHighlightingTest.java
Expand Up @@ -636,6 +636,9 @@ public void testInterpolatedElements() {
@Test
public void testOperatorsAfterLoopControl() {doTest();}

@Test
public void testNumbers() {doTest();}

private void doTest() {
doTest(true);
}
Expand Down
3 changes: 3 additions & 0 deletions plugin/test/parser/PerlParserTest.java
Expand Up @@ -707,4 +707,7 @@ public void testInterpolatedElements() {

@Test
public void testAsyncStatement() {doTest();}

@Test
public void testNumbers() {doTest();}
}
Expand Up @@ -5,7 +5,7 @@ a
=
PERL_OPERATOR => DEFAULT_OPERATION_SIGN
0x123abc
PERL_NUMBER => DEFAULT_NUMBER
PERL_NUMBER_HEX => PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
$
Expand All @@ -15,7 +15,7 @@ a
=
PERL_OPERATOR => DEFAULT_OPERATION_SIGN
0X123abc
PERL_NUMBER => DEFAULT_NUMBER
PERL_NUMBER_HEX => PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
$
Expand All @@ -25,7 +25,7 @@ a
=
PERL_OPERATOR => DEFAULT_OPERATION_SIGN
0b010101
PERL_NUMBER => DEFAULT_NUMBER
PERL_NUMBER_BIN => PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
$
Expand All @@ -35,7 +35,7 @@ a
=
PERL_OPERATOR => DEFAULT_OPERATION_SIGN
0B010101
PERL_NUMBER => DEFAULT_NUMBER
PERL_NUMBER_BIN => PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
$
Expand All @@ -55,7 +55,7 @@ a
=
PERL_OPERATOR => DEFAULT_OPERATION_SIGN
01
PERL_NUMBER => DEFAULT_NUMBER
PERL_NUMBER_OCT => PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
$
Expand All @@ -65,7 +65,7 @@ a
=
PERL_OPERATOR => DEFAULT_OPERATION_SIGN
0Xabcdef
PERL_NUMBER => DEFAULT_NUMBER
PERL_NUMBER_HEX => PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
"
Expand Down Expand Up @@ -125,7 +125,7 @@ a
=
PERL_OPERATOR => DEFAULT_OPERATION_SIGN
0XFEDCBA
PERL_NUMBER => DEFAULT_NUMBER
PERL_NUMBER_HEX => PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
"
Expand Down Expand Up @@ -185,7 +185,7 @@ a
=
PERL_OPERATOR => DEFAULT_OPERATION_SIGN
0B1101
PERL_NUMBER => DEFAULT_NUMBER
PERL_NUMBER_BIN => PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
"
Expand Down
10 changes: 10 additions & 0 deletions plugin/testData/highlighting/perl/syntax/numbers.code
@@ -0,0 +1,10 @@
12345;
12345.67;
.23E-10; # a very small number
3.14_15_92; # a very important number
4_294_967_296; # underscore for legibility
0xff; # hex
0xdead_beef; # more hex
0377; # octal (only numbers, begins with 0)
0b011011; # binary
# 0x1.999ap-4 # hexadecimal floating point (the 'p' is required)
52 changes: 52 additions & 0 deletions plugin/testData/highlighting/perl/syntax/numbers.pl.txt
@@ -0,0 +1,52 @@
12345
PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
12345.67
PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
.23E-10
PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
# a very small number
PERL_COMMENT => DEFAULT_LINE_COMMENT
3.14_15_92
PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
# a very important number
PERL_COMMENT => DEFAULT_LINE_COMMENT
4_294_967_296
PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
# underscore for legibility
PERL_COMMENT => DEFAULT_LINE_COMMENT
0xff
PERL_NUMBER_HEX => PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
# hex
PERL_COMMENT => DEFAULT_LINE_COMMENT
0xdead_beef
PERL_NUMBER_HEX => PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
# more hex
PERL_COMMENT => DEFAULT_LINE_COMMENT
0377
PERL_NUMBER_OCT => PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
# octal (only numbers, begins with 0)
PERL_COMMENT => DEFAULT_LINE_COMMENT
0b011011
PERL_NUMBER_BIN => PERL_NUMBER => DEFAULT_NUMBER
;
PERL_SEMICOLON => DEFAULT_SEMICOLON
# binary
PERL_COMMENT => DEFAULT_LINE_COMMENT
# 0x1.999ap-4 # hexadecimal floating point (the 'p' is required)
PERL_COMMENT => DEFAULT_LINE_COMMENT
Expand Up @@ -27,7 +27,7 @@ Perl5
PerlVariableNameElementImpl(Perl5: SCALAR_NAME)('color')
PsiElement(Perl5: =)('=')
PsiPerlNumberConstantImpl(Perl5: NUMBER_CONSTANT)
PsiElement(Perl5: NUMBER)('0x00_00_00')
PsiElement(Perl5: NUMBER_HEX)('0x00_00_00')
PsiElement(Perl5: ))(')')
PsiPerlBlockImpl(Perl5: BLOCK)
PsiElement(Perl5: {)('{')
Expand Down
Expand Up @@ -27,7 +27,7 @@ Perl5
PerlVariableNameElementImpl(Perl5: SCALAR_NAME)('color')
PsiElement(Perl5: =)('=')
PsiPerlNumberConstantImpl(Perl5: NUMBER_CONSTANT)
PsiElement(Perl5: NUMBER)('0x00_00_00')
PsiElement(Perl5: NUMBER_HEX)('0x00_00_00')
PsiElement(Perl5: ))(')')
PsiPerlBlockImpl(Perl5: BLOCK)
PsiElement(Perl5: {)('{')
Expand Down
Expand Up @@ -27,7 +27,7 @@ Perl5
PerlVariableNameElementImpl(Perl5: SCALAR_NAME)('color')
PsiElement(Perl5: =)('=')
PsiPerlNumberConstantImpl(Perl5: NUMBER_CONSTANT)
PsiElement(Perl5: NUMBER)('0x00_00_00')
PsiElement(Perl5: NUMBER_HEX)('0x00_00_00')
PsiElement(Perl5: ))(')')
PsiPerlBlockImpl(Perl5: BLOCK)
PsiElement(Perl5: {)('{')
Expand Down
Expand Up @@ -27,7 +27,7 @@ Perl5
PerlVariableNameElementImpl(Perl5: SCALAR_NAME)('color')
PsiElement(Perl5: =)('=')
PsiPerlNumberConstantImpl(Perl5: NUMBER_CONSTANT)
PsiElement(Perl5: NUMBER)('0x00_00_00')
PsiElement(Perl5: NUMBER_HEX)('0x00_00_00')
PsiElement(Perl5: ))(')')
PsiPerlBlockImpl(Perl5: BLOCK)
PsiElement(Perl5: {)('{')
Expand Down
Expand Up @@ -27,7 +27,7 @@ Perl5
PerlVariableNameElementImpl(Perl5: SCALAR_NAME)('color')
PsiElement(Perl5: =)('=')
PsiPerlNumberConstantImpl(Perl5: NUMBER_CONSTANT)
PsiElement(Perl5: NUMBER)('0x00_00_00')
PsiElement(Perl5: NUMBER_HEX)('0x00_00_00')
PsiElement(Perl5: ))(')')
PsiPerlBlockImpl(Perl5: BLOCK)
PsiElement(Perl5: {)('{')
Expand Down
Expand Up @@ -27,7 +27,7 @@ Perl5
PerlVariableNameElementImpl(Perl5: SCALAR_NAME)('color')
PsiElement(Perl5: =)('=')
PsiPerlNumberConstantImpl(Perl5: NUMBER_CONSTANT)
PsiElement(Perl5: NUMBER)('0x00_00_00')
PsiElement(Perl5: NUMBER_HEX)('0x00_00_00')
PsiElement(Perl5: ))(')')
PsiPerlBlockImpl(Perl5: BLOCK)
PsiElement(Perl5: {)('{')
Expand Down
Expand Up @@ -27,7 +27,7 @@ Perl5
PerlVariableNameElementImpl(Perl5: SCALAR_NAME)('color')
PsiElement(Perl5: =)('=')
PsiPerlNumberConstantImpl(Perl5: NUMBER_CONSTANT)
PsiElement(Perl5: NUMBER)('0x00_00_00')
PsiElement(Perl5: NUMBER_HEX)('0x00_00_00')
PsiElement(Perl5: ))(')')
PsiPerlBlockImpl(Perl5: BLOCK)
PsiElement(Perl5: {)('{')
Expand Down
Expand Up @@ -27,7 +27,7 @@ Perl5
PerlVariableNameElementImpl(Perl5: SCALAR_NAME)('color')
PsiElement(Perl5: =)('=')
PsiPerlNumberConstantImpl(Perl5: NUMBER_CONSTANT)
PsiElement(Perl5: NUMBER)('0x00_00_00')
PsiElement(Perl5: NUMBER_HEX)('0x00_00_00')
PsiElement(Perl5: ))(')')
PsiPerlBlockImpl(Perl5: BLOCK)
PsiElement(Perl5: {)('{')
Expand Down
Expand Up @@ -27,7 +27,7 @@ Perl5
PerlVariableNameElementImpl(Perl5: SCALAR_NAME)('color')
PsiElement(Perl5: =)('=')
PsiPerlNumberConstantImpl(Perl5: NUMBER_CONSTANT)
PsiElement(Perl5: NUMBER)('0x00_00_00')
PsiElement(Perl5: NUMBER_HEX)('0x00_00_00')
PsiElement(Perl5: ))(')')
PsiPerlBlockImpl(Perl5: BLOCK)
PsiElement(Perl5: {)('{')
Expand Down

0 comments on commit e4a0e76

Please sign in to comment.