Skip to content

Fix parsing of constant and path expressions #27

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 26, 2021
Merged

Fix parsing of constant and path expressions #27

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b418be9
The equal part of a constant item is optional
elegoff Jun 25, 2021
ba1f2fe
fix when semi colon is preceded by space
elegoff Jun 25, 2021
8c7c6c0
Add forgotten path expressions to expressions
elegoff Jun 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 30 additions & 27 deletions rust-frontend/src/main/java/org/sonar/rust/RustGrammar.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,17 @@
* Sonar Rust Plugin (Community)
* Copyright (C) 2021 Eric Le Goff
* http://github.com/elegoff/sonar-rust
*
* <p>
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* <p>
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* <p>
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
Expand Down Expand Up @@ -593,13 +593,13 @@ private static void aliasItem(LexerlessGrammarBuilder b) {
RustKeyword.KW_TYPE, SPC, IDENTIFIER, SPC, b.optional(GENERIC_PARAMS), SPC,
b.optional(WHERE_CLAUSE, SPC),
b.optional(RustPunctuator.EQ, SPC, TYPE, SPC),
b.optional(RustPunctuator.COLON, b.optional(TYPE_PARAM_BOUNDS)),
b.optional(RustPunctuator.COLON, b.optional(SPC, TYPE_PARAM_BOUNDS, SPC)),
RustPunctuator.SEMI
);
}

private static void useItem(LexerlessGrammarBuilder b) {
b.rule(USE_DECLARATION).is("use", SPC, USE_TREE, ";");
b.rule(USE_DECLARATION).is(RustKeyword.KW_USE, SPC, USE_TREE, SPC, RustPunctuator.SEMI);

b.rule(USE_TREE).is(b.firstOf(
b.sequence(b.optional(b.optional(SIMPLE_PATH), RustPunctuator.PATHSEP), RustPunctuator.STAR),
Expand Down Expand Up @@ -688,9 +688,9 @@ private static void unionsItem(LexerlessGrammarBuilder b) {
/* https://doc.rust-lang.org/reference/items/constant-items.html */
private static void constantsItem(LexerlessGrammarBuilder b) {
b.rule(CONSTANT_ITEM).is(
RustKeyword.KW_CONST, SPC, b.firstOf(IDENTIFIER, RustPunctuator.UNDERSCORE),SPC,
RustKeyword.KW_CONST, SPC, b.firstOf(IDENTIFIER, RustPunctuator.UNDERSCORE), SPC,
RustPunctuator.COLON, SPC, TYPE,
SPC, RustPunctuator.EQ, SPC, EXPRESSION, RustPunctuator.SEMI
SPC, b.optional(RustPunctuator.EQ, SPC, EXPRESSION, SPC), RustPunctuator.SEMI
);

}
Expand All @@ -700,7 +700,7 @@ private static void staticItem(LexerlessGrammarBuilder b) {
b.rule(STATIC_ITEM).is(
RustKeyword.KW_STATIC, SPC, b.optional(RustKeyword.KW_MUT, SPC), IDENTIFIER,
SPC, RustPunctuator.COLON, SPC, TYPE, SPC, b.optional(RustPunctuator.EQ, SPC,
EXPRESSION), RustPunctuator.SEMI
EXPRESSION, SPC), RustPunctuator.SEMI
);
}

Expand Down Expand Up @@ -828,7 +828,7 @@ private static void visibilityItem(LexerlessGrammarBuilder b) {

private static void externcrates(LexerlessGrammarBuilder b) {
b.rule(EXTERN_CRATE).is(
RustKeyword.KW_EXTERN, SPC, RustKeyword.KW_CRATE, SPC, CRATE_REF, b.optional(SPC, AS_CLAUSE), RustPunctuator.SEMI
RustKeyword.KW_EXTERN, SPC, RustKeyword.KW_CRATE, SPC, CRATE_REF, SPC, b.optional(SPC, AS_CLAUSE, SPC), RustPunctuator.SEMI
);
b.rule(CRATE_REF).is(b.firstOf(RustKeyword.KW_SELFVALUE, IDENTIFIER));
b.rule(AS_CLAUSE).is(RustKeyword.KW_AS, SPC, b.firstOf(RustPunctuator.UNDERSCORE, IDENTIFIER));
Expand Down Expand Up @@ -886,8 +886,8 @@ private static void macrosByExample(LexerlessGrammarBuilder b) {
"macro_rules!", SPC, IDENTIFIER, SPC, MACRO_RULES_DEF
);
b.rule(MACRO_RULES_DEF).is(b.firstOf(
b.sequence("(", SPC, MACRO_RULES, SPC, ")", RustPunctuator.SEMI),
b.sequence("[", SPC, MACRO_RULES, SPC, "]", RustPunctuator.SEMI),
b.sequence("(", SPC, MACRO_RULES, SPC, ")", SPC, RustPunctuator.SEMI),
b.sequence("[", SPC, MACRO_RULES, SPC, "]", SPC, RustPunctuator.SEMI),
b.sequence("{", SPC, MACRO_RULES, SPC, "}")
));
b.rule(MACRO_RULES).is(
Expand Down Expand Up @@ -962,7 +962,7 @@ private static void macrosByExample(LexerlessGrammarBuilder b) {
RustPunctuator.FATARROW,
RustPunctuator.POUND,
RustPunctuator.DOLLAR
)); // Token except delimiters and repetition operators
)); // Token except delimiters and repetition operators
b.rule(MACRO_REP_OP).is(b.firstOf(RustPunctuator.STAR, RustPunctuator.PLUS, RustPunctuator.QUESTION));
b.rule(MACRO_TRANSCRIBER).is(DELIM_TOKEN_TREE);

Expand All @@ -978,7 +978,6 @@ private static void patterns(LexerlessGrammarBuilder b) {
MACRO_INVOCATION,



//unambigous PATH_PATTERN,
b.firstOf(
b.sequence(b.optional(RustPunctuator.PATHSEP),
Expand All @@ -1000,7 +999,7 @@ private static void patterns(LexerlessGrammarBuilder b) {

STRING_LITERAL,

b.sequence(b.optional("-"), INTEGER_LITERAL, b.nextNot(RustPunctuator.DOTDOT)),
b.sequence(b.optional("-"), INTEGER_LITERAL, b.nextNot(RustPunctuator.DOTDOT)),
b.sequence(b.optional("-"), FLOAT_LITERAL),

WILDCARD_PATTERN,
Expand Down Expand Up @@ -1116,7 +1115,7 @@ private static void types(LexerlessGrammarBuilder b) {
private static void functionpointer(LexerlessGrammarBuilder b) {
b.rule(BARE_FUNCTION_TYPE).is(
b.optional(FOR_LIFETIMES), FUNCTION_QUALIFIERS, SPC, RustKeyword.KW_FN,
"(", SPC, b.optional(FUNCTION_PARAMETERS_MAYBE_NAMED_VARIADIC),SPC, ")",SPC,
"(", SPC, b.optional(FUNCTION_PARAMETERS_MAYBE_NAMED_VARIADIC), SPC, ")", SPC,
b.optional(BARE_FUNCTION_RETURN_TYPE)
);
b.rule(BARE_FUNCTION_RETURN_TYPE).is(RustPunctuator.RARROW, SPC, TYPE_NO_BOUNDS);
Expand All @@ -1142,7 +1141,7 @@ public static void statement(LexerlessGrammarBuilder b) {
b.rule(STATEMENT).is(b.firstOf(
RustPunctuator.SEMI,
b.sequence(EXPRESSION_WITHOUT_BLOCK, SPC, RustPunctuator.SEMI),
b.sequence(EXPRESSION_WITH_BLOCK, b.optional(SPC, RustPunctuator.SEMI)),
b.sequence(EXPRESSION_WITH_BLOCK, SPC, b.optional(SPC, RustPunctuator.SEMI)),
ITEM,
LET_STATEMENT,
MACRO_INVOCATION_SEMI
Expand Down Expand Up @@ -1197,6 +1196,7 @@ public static void expressions(LexerlessGrammarBuilder b) {
b.firstOf(
b.sequence(LITERAL_EXPRESSION, b.zeroOrMore(SPC, EXPRESSION_TERM)),
b.sequence(BREAK_EXPRESSION, b.zeroOrMore(SPC, EXPRESSION_TERM)),

b.sequence(BOX_EXPRESSION, b.zeroOrMore(SPC, EXPRESSION_TERM)),
b.sequence(RustPunctuator.DOTDOT, b.next(b.firstOf(")", "]", "}"))),
b.sequence(RANGE_TO_INCLUSIVE_EXPR, b.zeroOrMore(SPC, EXPRESSION_TERM)),
Expand All @@ -1217,6 +1217,7 @@ public static void expressions(LexerlessGrammarBuilder b) {
b.sequence(MACRO_INVOCATION, b.zeroOrMore(SPC, EXPRESSION_TERM)),
b.sequence(RETURN_EXPRESSION, b.zeroOrMore(SPC, EXPRESSION_TERM)),
b.sequence(STRUCT_EXPRESSION, b.zeroOrMore(SPC, EXPRESSION_TERM)),
b.sequence(PATH_EXPRESSION, b.zeroOrMore(SPC, EXPRESSION_TERM)),
b.sequence(GROUPED_EXPRESSION, b.zeroOrMore(SPC, EXPRESSION_TERM)),
b.sequence(ARRAY_EXPRESSION, b.zeroOrMore(SPC, EXPRESSION_TERM)),
b.sequence(TUPLE_EXPRESSION, b.zeroOrMore(SPC, EXPRESSION_TERM)),
Expand Down Expand Up @@ -1252,7 +1253,7 @@ public static void expressions(LexerlessGrammarBuilder b) {
b.sequence(RETURN_EXPRESSION, b.zeroOrMore(SPC, EXPRESSION_TERM_EXCEPT_STRUCT)),
b.sequence(PATH_EXPRESSION, b.zeroOrMore(SPC, EXPRESSION_TERM_EXCEPT_STRUCT)),
b.sequence(STRUCT_EXPRESSION, b.oneOrMore(SPC, EXPRESSION_TERM_EXCEPT_STRUCT)), //NB : one or more

b.sequence(PATH_EXPRESSION, b.zeroOrMore(SPC, EXPRESSION_TERM_EXCEPT_STRUCT)),
b.sequence(GROUPED_EXPRESSION, b.zeroOrMore(SPC, EXPRESSION_TERM_EXCEPT_STRUCT)),
b.sequence(ARRAY_EXPRESSION, b.zeroOrMore(SPC, EXPRESSION_TERM_EXCEPT_STRUCT)),
b.sequence(TUPLE_EXPRESSION, b.zeroOrMore(SPC, EXPRESSION_TERM_EXCEPT_STRUCT)),
Expand Down Expand Up @@ -1632,7 +1633,7 @@ private static void array(LexerlessGrammarBuilder b) {
"]");

b.rule(ARRAY_ELEMENTS).is(b.firstOf(
b.sequence(SPC, EXPRESSION, RustPunctuator.SEMI, SPC, EXPRESSION),
b.sequence(SPC, EXPRESSION, SPC, RustPunctuator.SEMI, SPC, EXPRESSION),
b.sequence(SPC, EXPRESSION, SPC, b.zeroOrMore(RustPunctuator.COMMA, SPC, EXPRESSION), b.optional(RustPunctuator.COMMA, SPC))

));
Expand Down Expand Up @@ -1785,7 +1786,7 @@ private static void block(LexerlessGrammarBuilder b) {
b.firstOf(
b.sequence(RustPunctuator.SEMI, SPC),
b.sequence(EXPRESSION_WITHOUT_BLOCK, SPC, RustPunctuator.SEMI, SPC),
b.sequence(EXPRESSION_WITH_BLOCK, b.nextNot(SPC, RustPunctuator.DOT), b.optional(RustPunctuator.SEMI), SPC),
b.sequence(EXPRESSION_WITH_BLOCK, b.nextNot(SPC, RustPunctuator.DOT), b.optional(SPC, RustPunctuator.SEMI), SPC),
b.sequence(ITEM, SPC),
b.sequence(LET_STATEMENT, SPC),
b.sequence(MACRO_INVOCATION_SEMI, SPC)
Expand All @@ -1801,7 +1802,7 @@ private static void block(LexerlessGrammarBuilder b) {
b.firstOf(
b.sequence(RustPunctuator.SEMI, SPC),
b.sequence(EXPRESSION_WITHOUT_BLOCK, SPC, RustPunctuator.SEMI, SPC),
b.sequence(EXPRESSION_WITH_BLOCK, b.nextNot(SPC, RustPunctuator.DOT), b.optional(RustPunctuator.SEMI), SPC),
b.sequence(EXPRESSION_WITH_BLOCK, b.nextNot(SPC, RustPunctuator.DOT), b.optional(SPC, RustPunctuator.SEMI), SPC),
b.sequence(ITEM, SPC),
b.sequence(LET_STATEMENT, SPC),
b.sequence(MACRO_INVOCATION_SEMI, SPC)
Expand Down Expand Up @@ -1917,10 +1918,9 @@ public static void closure(LexerlessGrammarBuilder b) {
/* https://doc.rust-lang.org/reference/types.html#type-expressions */
public static void type(LexerlessGrammarBuilder b) {
b.rule(TYPE).is(b.firstOf(
IMPL_TRAIT_TYPE
,BARE_FUNCTION_TYPE
,TRAIT_OBJECT_TYPE,

IMPL_TRAIT_TYPE,
BARE_FUNCTION_TYPE,
TRAIT_OBJECT_TYPE,
PARENTHESIZED_TYPE,
IMPL_TRAIT_TYPE_ONE_BOUND,
TRAIT_OBJECT_TYPE_ONE_BOUND,
Expand All @@ -1933,11 +1933,9 @@ public static void type(LexerlessGrammarBuilder b) {
SLICE_TYPE,
INFERRED_TYPE,
QUALIFIED_PATH_IN_TYPE,

MACRO_INVOCATION



));
b.rule(TYPE_NO_BOUNDS).is(b.firstOf(
BARE_FUNCTION_TYPE,
Expand Down Expand Up @@ -2024,8 +2022,13 @@ public static void lexicalpath(LexerlessGrammarBuilder b) {
b.rule(PATH_EXPR_SEGMENT).is(
PATH_IDENT_SEGMENT, b.optional(b.sequence(RustPunctuator.PATHSEP, GENERIC_ARGS))
);

b.rule(PATH_IDENT_SEGMENT).is(b.firstOf(
RustKeyword.KW_SUPER, b.regexp("^[sS]elf$"), RustKeyword.KW_CRATE, b.regexp(DOLLAR_CRATE_REGEX), IDENTIFIER
RustKeyword.KW_SUPER,
b.regexp("^[sS]elf$"),
RustKeyword.KW_CRATE,
b.regexp(DOLLAR_CRATE_REGEX),
IDENTIFIER
));

b.rule(GENERIC_ARGS).is(b.firstOf(
Expand Down
2 changes: 1 addition & 1 deletion rust-frontend/src/main/java/org/sonar/rust/RustLexer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public enum RustLexer implements GrammarRuleKey {
public static LexerlessGrammarBuilder create() {
LexerlessGrammarBuilder b =RustGrammar.create();

b.rule(TOKENS).is(RustGrammar.SPC, b.zeroOrMore(RustGrammar.ANY_TOKEN,b.optional(RustPunctuator.SEMI),RustGrammar.SPC), RustGrammar.EOF);
b.rule(TOKENS).is(RustGrammar.SPC, b.zeroOrMore(RustGrammar.ANY_TOKEN,b.optional(RustGrammar.SPC, RustPunctuator.SEMI),RustGrammar.SPC), RustGrammar.EOF);

b.setRootRule(TOKENS);

Expand Down
6 changes: 5 additions & 1 deletion rust-frontend/src/main/java/org/sonar/rust/metrics/LinesOfCodeVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.sonar.sslr.parser.ParserAdapter;

import java.util.HashSet;
import java.util.List;
import java.util.Set;

public class LinesOfCodeVisitor extends RustVisitor {
Expand All @@ -41,7 +42,10 @@ public LinesOfCodeVisitor(ParserAdapter<LexerlessGrammar> parser) {
@Override
public void visitFile(AstNode node) {
linesOfCode.clear();
for (AstNode token : lexer.parse(getContext().file().content()).getChildren(RustGrammar.ANY_TOKEN)) {
List<AstNode> tokens = lexer.parse(getContext().file().content()).getChildren(RustGrammar.ANY_TOKEN);


for (AstNode token : tokens) {
String[] tokenLines = token.getTokenValue().split("(\r)?\n|\r", -1);

for (int lineOffset = 0; lineOffset < tokenLines.length; lineOffset++) {
Expand Down
2 changes: 1 addition & 1 deletion rust-frontend/src/test/java/org/sonar/rust/RustLexerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public void testTokens() {
public void testParsing() {


String sexpr = "let prior = false_prior;";
String sexpr = "let zero = 0;";



Expand Down
3 changes: 3 additions & 0 deletions rust-frontend/src/test/java/org/sonar/rust/parser/CompilationUnitTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ public void testAnyTokens() {
.notMatches("use")
.matches("foo")
.matches("fnac")
.matches("test")
;
}

Expand Down Expand Up @@ -65,6 +66,8 @@ public void testCompilationUnit() {
"}")
.matches("mod foobar{#![crate_type = \"lib\"]\n" +
"}")
.matches("test();")
.matches("test() ;")
;
}
}
1 change: 1 addition & 0 deletions rust-frontend/src/test/java/org/sonar/rust/parser/expressions/CallExpressionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ public void testCallExpression() {
.matches("Ok(2)")
.matches("Ok(())")
.matches("f(stdout[..40])")
.matches("<X as Default>::default()")
;
}

Expand Down
2 changes: 2 additions & 0 deletions rust-frontend/src/test/java/org/sonar/rust/parser/expressions/ExpressionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ public void testExpression() {
.matches("3+2")
.matches("1 << 1")
.matches("foo")
.matches("u8")
.matches("Some(42)")
.matches("panic!()")
.notMatches("== b")
Expand Down Expand Up @@ -113,6 +114,7 @@ public void testExpression() {
.matches("break 42")
.matches("break Ok(Poll::Pending)")
.matches("true_prior")
.matches("<X as Default>::default()")

;
}
Expand Down
4 changes: 4 additions & 0 deletions rust-frontend/src/test/java/org/sonar/rust/parser/expressions/LoopExpressionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,10 @@ public void testLoopExpression() {
" println!(\"hello\");\n" +
" i = i + 1;\n" +
"}")
.matches("for (i, n) in hist.normalized_bins().enumerate() {\n" +
" let bin = (n as f64) / (N_SAMPLES as f64) ;\n" +
" diff[i] = (bin - expected[i]).abs();\n" +
" }")
;
}
}
1 change: 1 addition & 0 deletions rust-frontend/src/test/java/org/sonar/rust/parser/expressions/PathExpressionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ public void testPathExpression() {
.matches("Identifier::Numeric")
.matches("Vec::new")
.matches("StepPosition::JumpEnd")
.matches("<X as Default>::default")

;
}
Expand Down
1 change: 1 addition & 0 deletions rust-frontend/src/test/java/org/sonar/rust/parser/items/ConstantTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ public void testConstant() {
.matches("const BITS: [u32; 2] = [BIT1, BIT2];")
.matches("const STRING: &'static str = \"bitstring\";")
.matches("const WHITE: Color = Color(255, 255, 255);")
.matches("const MAX: Self;")
;

}
Expand Down
3 changes: 3 additions & 0 deletions rust-frontend/src/test/java/org/sonar/rust/parser/items/TraitTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ public void testTrait() {
" fn elt_at(&self, n: u32) -> T;\n" +
" fn iter<F>(&self, f:F) where F: Fn(T);\n" +
"}")
.matches("trait Thing {\n" +
" const FIELD : Self;\n" +
"}")


;
Expand Down
Loading