Skip to content

Reduce copying during parsing #60

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
May 19, 2023
Merged

Reduce copying during parsing #60

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
851f236
Use named parameters to reducue copying
MichaReiser May 17, 2023
726884b
Avoid allocating `vec`s for each statement
MichaReiser May 18, 2023
4ff779c
Update parser/src/python.lalrpop
MichaReiser May 18, 2023
fac0c25
Update `python.rs`
MichaReiser May 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 59 additions & 36 deletions parser/src/python.lalrpop
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,32 +26,56 @@ pub Top: ast::Mod = {
};

Program: ast::Suite = {
<lines:FileLine*> => {
lines.into_iter().flatten().collect()
=> vec![],
// Compound statements
<mut statements:Program> <next:CompoundStatement> => {
statements.push(next);
statements
},

// Small statements
<mut statements:Program> <small:(<SmallStatement> ";")*> <last:SmallStatement> ";"? "\n" => {
statements.extend(small);
statements.push(last);
statements
},
};

// A file line either has a declaration, or an empty newline:
FileLine: ast::Suite = {
Statement,
"\n" => vec![],
// Empty lines
<s:Program> "\n" => s,
};

Suite: ast::Suite = {
SimpleStatement,
"\n" Indent <s:Statement+> Dedent => s.into_iter().flatten().collect(),
<mut statements:(<SmallStatement> ";")*> <last:SmallStatement> ";"? "\n" => {
statements.push(last);
statements
},
"\n" Indent <s:Statements> Dedent => s,
};

Statement: ast::Suite = {
SimpleStatement,

// One or more statements
Statements: Vec<ast::Stmt> = {
// First simple statement
<mut head:(<SmallStatement> ";")*> <last:SmallStatement> ";"? "\n" => {
head.push(last);
head
},

// The first compound statement
<s:CompoundStatement> => vec![s],
Copy link
Contributor Author

@MichaReiser MichaReiser May 18, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The old grammar created a vec for every statement. The new grammar simply appends statements at the end of the vec.

};

SimpleStatement: ast::Suite = {
<mut statements:(<SmallStatement> ";")*> <last:SmallStatement> ";"? "\n" => {
// Any subsequent compound statements
<mut statements:Statements> <next:CompoundStatement> => {
statements.push(next);
statements
},

// Any subsequent small statements
<mut statements:Statements> <small:(<SmallStatement> ";")*> <last:SmallStatement> ";"? "\n" => {
statements.extend(small);
statements.push(last);
statements
}
},
};

SmallStatement: ast::Stmt = {
Expand Down Expand Up @@ -734,19 +758,19 @@ ClassPattern: ast::Pattern = {
}

IfStatement: ast::Stmt = {
<location:@L> "if" <test:NamedExpressionTest> ":" <body:Suite> <s2:(@L "elif" NamedExpressionTest ":" Suite)*> <s3:("else" ":" Suite)?> => {
<location:@L> "if" <test:NamedExpressionTest> ":" <body:Suite> <s2:(<@L> "elif" <NamedExpressionTest> ":" <Suite>)*> <s3:("else" ":" <Suite>)?> => {
// Determine last else:
let mut last = s3.map(|s| s.2).unwrap_or_default();
let mut last = s3.unwrap_or_default();
let end_location = last
.last()
.or_else(|| s2.last().and_then(|last| last.4.last()))
.or_else(|| s2.last().and_then(|last| last.2.last()))
.or_else(|| body.last())
.unwrap()
.end();
// handle elif:
for i in s2.into_iter().rev() {
let x = ast::Stmt::If(
ast::StmtIf { test: Box::new(i.2), body: i.4, orelse: last, range: (i.0..end_location).into() }
ast::StmtIf { test: Box::new(i.1), body: i.2, orelse: last, range: (i.0..end_location).into() }
);
last = vec![x];
}
Expand All @@ -758,8 +782,8 @@ IfStatement: ast::Stmt = {
};

WhileStatement: ast::Stmt = {
<location:@L> "while" <test:NamedExpressionTest> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
let orelse = s2.map(|s| s.2).unwrap_or_default();
<location:@L> "while" <test:NamedExpressionTest> ":" <body:Suite> <s2:("else" ":" <Suite>)?> => {
let orelse = s2.unwrap_or_default();
let end_location = orelse
.last()
.or_else(|| body.last())
Expand All @@ -777,8 +801,8 @@ WhileStatement: ast::Stmt = {
};

ForStatement: ast::Stmt = {
<location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
let orelse = s2.map(|s| s.2).unwrap_or_default();
<location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <orelse:("else" ":" <Suite>)?> => {
let orelse = orelse.unwrap_or_default();
let end_location = orelse
.last()
.or_else(|| body.last())
Expand All @@ -796,9 +820,9 @@ ForStatement: ast::Stmt = {
};

TryStatement: ast::Stmt = {
<location:@L> "try" ":" <body:Suite> <handlers:ExceptClause+> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> <end_location:@R> => {
let orelse = else_suite.map(|s| s.2).unwrap_or_default();
let finalbody = finally.map(|s| s.2).unwrap_or_default();
<location:@L> "try" ":" <body:Suite> <handlers:ExceptClause+> <orelse:("else" ":" <Suite>)?> <finalbody:("finally" ":" <Suite>)?> <end_location:@R> => {
let orelse = orelse.unwrap_or_default();
let finalbody = finalbody.unwrap_or_default();
let end_location = finalbody
.last()
.map(|last| last.end())
Expand All @@ -815,9 +839,9 @@ TryStatement: ast::Stmt = {
},
)
},
<location:@L> "try" ":" <body:Suite> <handlers:ExceptStarClause+> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> <end_location:@R> => {
let orelse = else_suite.map(|s| s.2).unwrap_or_default();
let finalbody = finally.map(|s| s.2).unwrap_or_default();
<location:@L> "try" ":" <body:Suite> <handlers:ExceptStarClause+> <orelse:("else" ":" <Suite>)?> <finalbody:("finally" ":" <Suite>)?> <end_location:@R> => {
let orelse = orelse.unwrap_or_default();
let finalbody = finalbody.unwrap_or_default();
let end_location = finalbody
.last()
.or_else(|| orelse.last())
Expand All @@ -834,10 +858,9 @@ TryStatement: ast::Stmt = {
},
)
},
<location:@L> "try" ":" <body:Suite> <finally:("finally" ":" Suite)> => {
<location:@L> "try" ":" <body:Suite> <finalbody:("finally" ":" <Suite>)> => {
let handlers = vec![];
let orelse = vec![];
let finalbody = finally.2;
let end_location = finalbody.last().unwrap().end();
ast::Stmt::Try(
ast::StmtTry {
Expand All @@ -863,12 +886,12 @@ ExceptStarClause: ast::Excepthandler = {
},
)
},
<location:@L> "except" "*" <x:(Test<"all"> "as" Identifier)> ":" <body:Suite> => {
<location:@L> "except" "*" <x:(<Test<"all">> "as" <Identifier>)> ":" <body:Suite> => {
let end_location = body.last().unwrap().end();
ast::Excepthandler::ExceptHandler(
ast::ExcepthandlerExceptHandler {
type_: Some(Box::new(x.0)),
name: Some(x.2),
name: Some(x.1),
body,
range: (location..end_location).into()
},
Expand All @@ -889,12 +912,12 @@ ExceptClause: ast::Excepthandler = {
},
)
},
<location:@L> "except" <x:(Test<"all"> "as" Identifier)> ":" <body:Suite> => {
<location:@L> "except" <x:(<Test<"all">> "as" <Identifier>)> ":" <body:Suite> => {
let end_location = body.last().unwrap().end();
ast::Excepthandler::ExceptHandler(
ast::ExcepthandlerExceptHandler {
type_: Some(Box::new(x.0)),
name: Some(x.2),
name: Some(x.1),
body,
range: (location..end_location).into()
},
Expand Down Expand Up @@ -941,7 +964,7 @@ WithItem<Goal>: ast::Withitem = {
};

FuncDef: ast::Stmt = {
<decorator_list:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" <Test<"all">>)?> ":" <body:Suite> => {
<decorator_list:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" <Test<"all">>)?> ":" <body:Suite> => {
let args = Box::new(args);
let returns = r.map(|x| Box::new(x));
let end_location = body.last().unwrap().end();
Expand Down
Loading