Skip to content

Improve parsing #29

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jul 15, 2021
Merged

Improve parsing #29

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e74ab23
optimize regexp for raw strings and raw byte strings
elegoff Jun 28, 2021
cf55186
macro invocation are types
elegoff Jun 28, 2021
b11f944
spacing in qualified path in type
elegoff Jun 28, 2021
2bdf77d
Expression with identifier starting with 'await'
elegoff Jun 29, 2021
1bd3f87
Simple path starting with keyword 'super'
elegoff Jul 6, 2021
8178cc9
match expression in if expression
elegoff Jul 15, 2021
b8a7ea4
if condition starting with 'if'
elegoff Jul 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 23 additions & 16 deletions rust-frontend/src/main/java/org/sonar/rust/RustGrammar.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -983,7 +983,8 @@ private static void patterns(LexerlessGrammarBuilder b) {
b.sequence(b.optional(RustPunctuator.PATHSEP),
//PATH_EXPR_SEGMENT,
b.sequence(b.firstOf(
RustKeyword.KW_SUPER, b.regexp("^[sS]elf$"), RustKeyword.KW_CRATE, b.regexp(DOLLAR_CRATE_REGEX), IDENTIFIER
b.sequence(RustKeyword.KW_SUPER,b.nextNot(IDENTIFIER)),
b.regexp("^[sS]elf$"), RustKeyword.KW_CRATE, b.regexp(DOLLAR_CRATE_REGEX), IDENTIFIER
)
, b.optional(b.sequence(RustPunctuator.PATHSEP, GENERIC_ARGS))),
b.oneOrMore(b.sequence(RustPunctuator.PATHSEP, PATH_EXPR_SEGMENT)))
Expand Down Expand Up @@ -1267,7 +1268,7 @@ public static void expressions(LexerlessGrammarBuilder b) {
b.sequence(RustPunctuator.DOTDOT, b.nextNot(RustPunctuator.EQ), b.endOfInput()),
b.sequence(RustPunctuator.DOTDOTEQ, EXPRESSION),
b.sequence(RustPunctuator.DOTDOT, b.nextNot(RustPunctuator.EQ), b.optional(EXPRESSION)),
b.sequence(RustPunctuator.DOT, RustKeyword.KW_AWAIT, SPC, EXPRESSION_TERM),
b.sequence(RustPunctuator.DOT, RustKeyword.KW_AWAIT,b.nextNot(IDENTIFIER), SPC, EXPRESSION_TERM),
b.sequence(RustPunctuator.DOT, PATH_EXPR_SEGMENT, SPC, "(", SPC, b.optional(CALL_PARAMS, SPC), ")", SPC, EXPRESSION_TERM),
b.sequence(RustPunctuator.DOT, TUPLE_INDEX, SPC, EXPRESSION_TERM),
b.sequence(RustPunctuator.DOT, IDENTIFIER, SPC, EXPRESSION_TERM),
Expand Down Expand Up @@ -1303,7 +1304,7 @@ public static void expressions(LexerlessGrammarBuilder b) {
b.sequence(RustPunctuator.CARETEQ, SPC, EXPRESSION, EXPRESSION_TERM),
b.sequence(RustPunctuator.SHLEQ, SPC, EXPRESSION, EXPRESSION_TERM),
b.sequence(RustPunctuator.SHREQ, SPC, EXPRESSION, EXPRESSION_TERM),
b.sequence(RustPunctuator.DOT, RustKeyword.KW_AWAIT),
b.sequence(RustPunctuator.DOT, RustKeyword.KW_AWAIT, b.nextNot(IDENTIFIER)),
b.sequence(RustPunctuator.DOT, PATH_EXPR_SEGMENT, SPC, "(", SPC, b.optional(CALL_PARAMS, SPC), ")"),
b.sequence(RustPunctuator.DOT, TUPLE_INDEX),
b.sequence(RustPunctuator.DOT, IDENTIFIER),
Expand Down Expand Up @@ -1475,7 +1476,9 @@ private static void returnExpr(LexerlessGrammarBuilder b) {
//https://doc.rust-lang.org/reference/expressions/match-expr.html
private static void match(LexerlessGrammarBuilder b) {
b.rule(MATCH_EXPRESSION).is(
RustKeyword.KW_MATCH, SPC, b.optional(RustKeyword.KW_MATCH), EXPRESSION_EXCEPT_STRUCT,
RustKeyword.KW_MATCH, SPC,
b.optional( RustKeyword.KW_MATCH, b.next(IDENTIFIER)),
EXPRESSION_EXCEPT_STRUCT,
SPC, "{", SPC,
b.zeroOrMore(INNER_ATTRIBUTE, SPC),
b.optional(MATCH_ARMS, SPC),
Expand Down Expand Up @@ -1506,7 +1509,9 @@ private static void match(LexerlessGrammarBuilder b) {

private static void ifExpr(LexerlessGrammarBuilder b) {
b.rule(IF_EXPRESSION).is(
RustKeyword.KW_IF, SPC, b.optional(b.firstOf(RustKeyword.KW_IF, RustKeyword.KW_MATCH)), EXPRESSION_EXCEPT_STRUCT, b.next(SPC, "{")
RustKeyword.KW_IF, SPC,
b.optional( RustKeyword.KW_IF,b.next(IDENTIFIER)),
EXPRESSION_EXCEPT_STRUCT, b.next(SPC, "{")
, SPC, BLOCK_EXPRESSION, SPC,
b.optional(

Expand Down Expand Up @@ -1918,6 +1923,7 @@ public static void closure(LexerlessGrammarBuilder b) {
/* https://doc.rust-lang.org/reference/types.html#type-expressions */
public static void type(LexerlessGrammarBuilder b) {
b.rule(TYPE).is(b.firstOf(
MACRO_INVOCATION,
IMPL_TRAIT_TYPE,
BARE_FUNCTION_TYPE,
TRAIT_OBJECT_TYPE,
Expand All @@ -1932,12 +1938,13 @@ public static void type(LexerlessGrammarBuilder b) {
ARRAY_TYPE,
SLICE_TYPE,
INFERRED_TYPE,
QUALIFIED_PATH_IN_TYPE,
MACRO_INVOCATION
QUALIFIED_PATH_IN_TYPE



));
b.rule(TYPE_NO_BOUNDS).is(b.firstOf(
MACRO_INVOCATION,
BARE_FUNCTION_TYPE,
PARENTHESIZED_TYPE,
IMPL_TRAIT_TYPE_ONE_BOUND,
Expand All @@ -1950,9 +1957,8 @@ public static void type(LexerlessGrammarBuilder b) {
ARRAY_TYPE,
SLICE_TYPE,
INFERRED_TYPE,
QUALIFIED_PATH_IN_TYPE,
QUALIFIED_PATH_IN_TYPE

MACRO_INVOCATION
));
b.rule(PARENTHESIZED_TYPE).is("(", TYPE, ")");
b.rule(TRAIT_OBJECT_TYPE).is(b.optional(RustKeyword.KW_DYN, SPC), TYPE_PARAM_BOUNDS);
Expand Down Expand Up @@ -2010,7 +2016,8 @@ public static void lexicalpath(LexerlessGrammarBuilder b) {
b.zeroOrMore(b.sequence(RustPunctuator.PATHSEP, SIMPLE_PATH_SEGMENT))
);
b.rule(SIMPLE_PATH_SEGMENT).is(b.firstOf(
RustKeyword.KW_SUPER, RustKeyword.KW_SELFVALUE, b.regexp("^crate$"), b.regexp(DOLLAR_CRATE_REGEX), IDENTIFIER
b.sequence(RustKeyword.KW_SUPER,b.nextNot(IDENTIFIER)),
RustKeyword.KW_SELFVALUE, b.regexp("^crate$"), b.regexp(DOLLAR_CRATE_REGEX), IDENTIFIER
));

b.rule(PATH_IN_EXPRESSION).is(
Expand All @@ -2024,7 +2031,7 @@ public static void lexicalpath(LexerlessGrammarBuilder b) {
);

b.rule(PATH_IDENT_SEGMENT).is(b.firstOf(
RustKeyword.KW_SUPER,
b.sequence(RustKeyword.KW_SUPER,b.nextNot(IDENTIFIER)),
b.regexp("^[sS]elf$"),
RustKeyword.KW_CRATE,
b.regexp(DOLLAR_CRATE_REGEX),
Expand Down Expand Up @@ -2063,7 +2070,7 @@ public static void lexicalpath(LexerlessGrammarBuilder b) {
QUALIFIED_PATH_TYPE, b.oneOrMore(b.sequence(RustPunctuator.PATHSEP, PATH_EXPR_SEGMENT)));

b.rule(QUALIFIED_PATH_TYPE).is(
RustPunctuator.LT, TYPE, b.optional(RustKeyword.KW_AS, SPC, TYPE_PATH), RustPunctuator.GT
RustPunctuator.LT, SPC, TYPE, b.optional(SPC, RustKeyword.KW_AS, SPC, TYPE_PATH),SPC, RustPunctuator.GT
);

b.rule(QUALIFIED_PATH_IN_TYPE).is(QUALIFIED_PATH_TYPE, b.oneOrMore(
Expand Down Expand Up @@ -2181,8 +2188,8 @@ private static void bytes(LexerlessGrammarBuilder b) {

b.rule(RAW_BYTE_STRING_CONTENT).is(
b.firstOf(
b.regexp("(?=\"+)((.|\\n)+?\"+)"),
b.regexp("(#\"(.|\\n)+?\\\"#)"),
b.regexp("(?=\"+)([\\s\\S]+?\"+)"),
b.regexp("(#\"[\\s\\S]+?\\\"#)"),
b.sequence("#", RAW_BYTE_STRING_CONTENT, "#")
));

Expand Down Expand Up @@ -2237,8 +2244,8 @@ private static void characters(LexerlessGrammarBuilder b) {

b.rule(RAW_STRING_CONTENT).is(
b.firstOf(
b.regexp("(?=\"+)((.|\\n)+?\"+)"),
b.regexp("(#\"(.|\\n)+?\\\"#)"),
b.regexp("(?=\"+)([\\s\\S]+?\"+)"),
b.regexp("(#\"[\\s\\S]+?\\\"#)"),
b.sequence("#", RAW_STRING_CONTENT, "#")

));
Expand Down
2 changes: 1 addition & 1 deletion rust-frontend/src/test/java/org/sonar/rust/RustLexerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public void testTokens() {
public void testParsing() {


String sexpr = "let zero = 0;";
String sexpr = "if sif_ok {}";



Expand Down
33 changes: 24 additions & 9 deletions rust-frontend/src/test/java/org/sonar/rust/parser/expressions/ExpressionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,17 @@
* Sonar Rust Plugin (Community)
* Copyright (C) 2021 Eric Le Goff
* http://github.com/elegoff/sonar-rust
*
* <p>
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* <p>
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* <p>
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
Expand Down Expand Up @@ -45,6 +45,19 @@ public void testExpressionExceptStruct() {
.matches("!c")
.notMatches("!c { None }")
.matches("continue 'outer")
.matches("match foo {\n" +
" is_ok(foo)\n" +
" if true =>\n" +
" {\n" +
" match is_really_ok(foo) {\n" +
" val => true,\n" +
" _ => false,\n" +
" }\n" +
" }\n" +
" _ => false,\n" +
" }")
.matches("if_ok")
.matches("match_ok")

;
}
Expand Down Expand Up @@ -81,9 +94,9 @@ public void testExpression() {
.matches("Identifier::Numeric")
.matches("Vec::new")
.matches("MediaElementAudioSourceNode {\n" +
" node,\n" +
" media_element,\n" +
" }")
" node,\n" +
" media_element,\n" +
" }")
.matches("StepPosition::JumpEnd")
.matches("*position == StepPosition::JumpEnd || *position == StepPosition::End")
.matches("move |state : Rc<RefCell<OpState>>, bufs: BufVec| -> Op {\n" +
Expand All @@ -108,14 +121,16 @@ public void testExpression() {
" }\n" +
" .boxed_local()")
.notMatches("is_ok {\n" +
" // empty block" +
" } ")
" // empty block" +
" } ")
.matches("..")
.matches("break 42")
.matches("break Ok(Poll::Pending)")
.matches("true_prior")
.matches("<X as Default>::default()")
.matches("formatter.field(\"await_token\", &self.await_token)")
.matches("supertraits.push_value(input.parse()?)")

;
;
}
}
13 changes: 12 additions & 1 deletion rust-frontend/src/test/java/org/sonar/rust/parser/expressions/IfExpressionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public void tesIfExpression() {
" } ")
.matches("if is_ok {} ")
.matches("if if_ok {} ")
.matches("if match_ok {} ")
//TODO .matches("if match_ok {} ")
.matches("if async_ok {} ")
.matches("if is_red || is_black {let cpt = 1 ;} else {let cpt = 0 ;}")
.matches("if is_red || is_black {}")
Expand Down Expand Up @@ -114,6 +114,17 @@ public void tesIfExpression() {
.matches("if state.get_state() == MyState::KO {\n" +
" continue 'outer;\n" +
" }")
.matches("if match foo {\n" +
" is_ok(foo)\n" +
" if true =>\n" +
" {\n" +
" match is_really_ok(foo) {\n" +
" val => true,\n" +
" _ => false,\n" +
" }\n" +
" }\n" +
" _ => false,\n" +
" } {}")


;
Expand Down
14 changes: 14 additions & 0 deletions rust-frontend/src/test/java/org/sonar/rust/parser/expressions/MatchExpressionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,9 @@ public void tesMatchExpression() {
.matches("match new_state {\n" +
" 42 => break foo\n" +
" }")
.matches("match match_suffix {\n" +
" 42 => break foo\n" +
" }")
.matches("match new_state {\n" +
" PollState::Idle => break Ok(Poll::Pending),\n" +
" }")
Expand All @@ -257,6 +260,17 @@ public void tesMatchExpression() {
" PollState::Parked => thread::park(), // Park the thread.\n" +
" _ => unreachable!(),\n" +
" }")
.matches("match foo {\n" +
" is_ok(foo)\n" +
" if true =>\n" +
" {\n" +
" match is_really_ok(foo) {\n" +
" val => true,\n" +
" _ => false,\n" +
" }\n" +
" }\n" +
" _ => false,\n" +
" }")



Expand Down
3 changes: 3 additions & 0 deletions rust-frontend/src/test/java/org/sonar/rust/parser/lexer/IdentifierTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ public void testIdentifier() {
.matches("Abc")
.notMatches("as")
.notMatches("trait")
.notMatches("super")
.notMatches("foo ")
.notMatches("r#")
.matches("r#a")
Expand All @@ -74,6 +75,8 @@ public void testIdentifier() {
.notMatches("s#52")
.matches("phenotype")
.matches("crate_type")
.matches("await_token")
.matches("if_ok")

;

Expand Down
27 changes: 23 additions & 4 deletions rust-frontend/src/test/java/org/sonar/rust/parser/lexer/PathTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,26 @@ public void testGenericArgsBinding() {
}


@Test
public void testGenericArg() {
assertThat(RustGrammar.create().build().rule(RustGrammar.GENERIC_ARG))
.matches("T")
.matches("i32")
.matches("Circle")
.matches("U=i32")
.matches("V=f64")
.matches("Pin<T>")
.matches("Pin<Box<T>>")
.matches("Pin<Box<(dyn Future<T>)>>")
.matches("Pin<Box<(dyn Future<A = B>)>>")
.matches("Pin<Box<(dyn Future<A = Result<T>>)>>")
.matches("Pin<Box<(dyn Future<A = Result<T,(U,V)>>)>>")


;
}


@Test
public void testGenericArgs() {
assertThat(RustGrammar.create().build().rule(RustGrammar.GENERIC_ARGS))
Expand All @@ -98,15 +118,14 @@ public void testGenericArgs() {
.matches("<Circle, f64, u8>")
.matches("<U=i32>")
.matches("<V=f64>")
//FIXME.matches("<T,U,V=f64>")
.matches("<T,U,V=f64>")
.matches("<Pin<T>>")
.matches("<Pin<Box<T>>>")
.matches("<Pin<Box<(dyn Future<T>)>>>")
.matches("<Pin<Box<(dyn Future<A = B>)>>>")
.matches("<Pin<Box<(dyn Future<A = Result<T>>)>>>")
.matches("<Pin<Box<(dyn Future<A = Result<T,(U,V)>>)>>>")


;
}

Expand All @@ -117,6 +136,7 @@ public void testQualifiedPathType() {
.matches("<T1 as T>")
.matches("<X as Default>")


;
}

Expand All @@ -135,8 +155,7 @@ public void testQualifiedPathInType() {
assertThat(RustGrammar.create().build().rule(RustGrammar.QUALIFIED_PATH_IN_TYPE))
.matches("<S as T1>::f")
.matches("<X as Default>::default()")


.matches("<[T] as SpanlessEq>::eq(self, other)")
;
}

Expand Down
1 change: 1 addition & 0 deletions rust-frontend/src/test/java/org/sonar/rust/parser/macro/MacroTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ public void testMacroInvocation() {
.matches("panic!()")
.matches("println!(\"{}, {}\", word, j)")
.notMatches("")
.matches("Token![#]")
;
}

Expand Down
3 changes: 3 additions & 0 deletions rust-frontend/src/test/java/org/sonar/rust/parser/types/TypeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ public void testTypeNoBounds() {
.notMatches("Circle{")
.matches("[u8]")
.matches("extern \"C\" fn(this: *mut iasset) -> i32")
.matches("Token![#]")



Expand Down Expand Up @@ -182,6 +183,8 @@ public void testType() {
" + Send")
.matches("impl FnOnce()")
.matches("<X as Default>::default()")
.matches("Token![#]")




Expand Down