Flatten rustpython_parser interface

Author: youknowone

benches/execution.rs

@@ -3,7 +3,7 @@ use criterion::{
     criterion_group, criterion_main, Bencher, BenchmarkGroup, BenchmarkId, Criterion, Throughput,
 };
 use rustpython_compiler::Mode;
-use rustpython_parser::parser::parse_program;
+use rustpython_parser::parse_program;
 use rustpython_vm::{Interpreter, PyResult};
 use std::collections::HashMap;
 use std::path::Path;

compiler/codegen/src/compile.rs

@@ -86,7 +86,7 @@ impl CompileContext {
     }
 }
 
-/// Compile an ast::Mod produced from rustpython_parser::parser::parse()
+/// Compile an ast::Mod produced from rustpython_parser::parse()
 pub fn compile_top(
     ast: &ast::Mod,
     source_path: String,
@@ -2846,7 +2846,7 @@ mod tests {
     use super::{CompileOpts, Compiler};
     use crate::symboltable::SymbolTable;
     use rustpython_compiler_core::CodeObject;
-    use rustpython_parser::parser;
+    use rustpython_parser as parser;
 
     fn compile_exec(source: &str) -> CodeObject {
         let mut compiler: Compiler = Compiler::new(

compiler/parser/python.lalrpop

@@ -7,10 +7,9 @@ use crate::{
     ast,
     lexer::{LexicalError, LexicalErrorType},
     function::{ArgumentList, parse_args, parse_params, validate_arguments},
-    lexer,
     context::set_context,
     string::parse_strings,
-    token::StringKind,
+    token::{self, StringKind},
 };
 use num_bigint::BigInt;
 
@@ -1937,106 +1936,106 @@ extern {
     type Location = ast::Location;
     type Error = LexicalError;
 
-    enum lexer::Tok {
-        Indent => lexer::Tok::Indent,
-        Dedent => lexer::Tok::Dedent,
-        StartModule => lexer::Tok::StartModule,
-        StartInteractive => lexer::Tok::StartInteractive,
-        StartExpression => lexer::Tok::StartExpression,
-        "+" => lexer::Tok::Plus,
-        "-" => lexer::Tok::Minus,
-        "~" => lexer::Tok::Tilde,
-        ":" => lexer::Tok::Colon,
-        "." => lexer::Tok::Dot,
-        "..." => lexer::Tok::Ellipsis,
-        "," => lexer::Tok::Comma,
-        "*" => lexer::Tok::Star,
-        "**" => lexer::Tok::DoubleStar,
-        "&" => lexer::Tok::Amper,
-        "@" => lexer::Tok::At,
-        "%" => lexer::Tok::Percent,
-        "//" => lexer::Tok::DoubleSlash,
-        "^" => lexer::Tok::CircumFlex,
-        "|" => lexer::Tok::Vbar,
-        "<<" => lexer::Tok::LeftShift,
-        ">>" => lexer::Tok::RightShift,
-        "/" => lexer::Tok::Slash,
-        "(" => lexer::Tok::Lpar,
-        ")" => lexer::Tok::Rpar,
-        "[" => lexer::Tok::Lsqb,
-        "]" => lexer::Tok::Rsqb,
-        "{" => lexer::Tok::Lbrace,
-        "}" => lexer::Tok::Rbrace,
-        "=" => lexer::Tok::Equal,
-        "+=" => lexer::Tok::PlusEqual,
-        "-=" => lexer::Tok::MinusEqual,
-        "*=" => lexer::Tok::StarEqual,
-        "@=" => lexer::Tok::AtEqual,
-        "/=" => lexer::Tok::SlashEqual,
-        "%=" => lexer::Tok::PercentEqual,
-        "&=" => lexer::Tok::AmperEqual,
-        "|=" => lexer::Tok::VbarEqual,
-        "^=" => lexer::Tok::CircumflexEqual,
-        "<<=" => lexer::Tok::LeftShiftEqual,
-        ">>=" => lexer::Tok::RightShiftEqual,
-        "**=" => lexer::Tok::DoubleStarEqual,
-        "//=" => lexer::Tok::DoubleSlashEqual,
-        ":=" => lexer::Tok::ColonEqual,
-        "==" => lexer::Tok::EqEqual,
-        "!=" => lexer::Tok::NotEqual,
-        "<" => lexer::Tok::Less,
-        "<=" => lexer::Tok::LessEqual,
-        ">" => lexer::Tok::Greater,
-        ">=" => lexer::Tok::GreaterEqual,
-        "->" => lexer::Tok::Rarrow,
-        "and" => lexer::Tok::And,
-        "as" => lexer::Tok::As,
-        "assert" => lexer::Tok::Assert,
-        "async" => lexer::Tok::Async,
-        "await" => lexer::Tok::Await,
-        "break" => lexer::Tok::Break,
-        "class" => lexer::Tok::Class,
-        "continue" => lexer::Tok::Continue,
-        "def" => lexer::Tok::Def,
-        "del" => lexer::Tok::Del,
-        "elif" => lexer::Tok::Elif,
-        "else" => lexer::Tok::Else,
-        "except" => lexer::Tok::Except,
-        "finally" => lexer::Tok::Finally,
-        "for" => lexer::Tok::For,
-        "from" => lexer::Tok::From,
-        "global" => lexer::Tok::Global,
-        "if" => lexer::Tok::If,
-        "import" => lexer::Tok::Import,
-        "in" => lexer::Tok::In,
-        "is" => lexer::Tok::Is,
-        "lambda" => lexer::Tok::Lambda,
-        "nonlocal" => lexer::Tok::Nonlocal,
-        "not" => lexer::Tok::Not,
-        "or" => lexer::Tok::Or,
-        "pass" => lexer::Tok::Pass,
-        "raise" => lexer::Tok::Raise,
-        "return" => lexer::Tok::Return,
-        "try" => lexer::Tok::Try,
-        "while" => lexer::Tok::While,
-        "match" => lexer::Tok::Match,
-        "case" => lexer::Tok::Case,
-        "with" => lexer::Tok::With,
-        "yield" => lexer::Tok::Yield,
-        "True" => lexer::Tok::True,
-        "False" => lexer::Tok::False,
-        "None" => lexer::Tok::None,
-        int => lexer::Tok::Int { value: <BigInt> },
-        float => lexer::Tok::Float { value: <f64> },
-        complex => lexer::Tok::Complex { real: <f64>, imag: <f64> },
-        string => lexer::Tok::String {
+    enum token::Tok {
+        Indent => token::Tok::Indent,
+        Dedent => token::Tok::Dedent,
+        StartModule => token::Tok::StartModule,
+        StartInteractive => token::Tok::StartInteractive,
+        StartExpression => token::Tok::StartExpression,
+        "+" => token::Tok::Plus,
+        "-" => token::Tok::Minus,
+        "~" => token::Tok::Tilde,
+        ":" => token::Tok::Colon,
+        "." => token::Tok::Dot,
+        "..." => token::Tok::Ellipsis,
+        "," => token::Tok::Comma,
+        "*" => token::Tok::Star,
+        "**" => token::Tok::DoubleStar,
+        "&" => token::Tok::Amper,
+        "@" => token::Tok::At,
+        "%" => token::Tok::Percent,
+        "//" => token::Tok::DoubleSlash,
+        "^" => token::Tok::CircumFlex,
+        "|" => token::Tok::Vbar,
+        "<<" => token::Tok::LeftShift,
+        ">>" => token::Tok::RightShift,
+        "/" => token::Tok::Slash,
+        "(" => token::Tok::Lpar,
+        ")" => token::Tok::Rpar,
+        "[" => token::Tok::Lsqb,
+        "]" => token::Tok::Rsqb,
+        "{" => token::Tok::Lbrace,
+        "}" => token::Tok::Rbrace,
+        "=" => token::Tok::Equal,
+        "+=" => token::Tok::PlusEqual,
+        "-=" => token::Tok::MinusEqual,
+        "*=" => token::Tok::StarEqual,
+        "@=" => token::Tok::AtEqual,
+        "/=" => token::Tok::SlashEqual,
+        "%=" => token::Tok::PercentEqual,
+        "&=" => token::Tok::AmperEqual,
+        "|=" => token::Tok::VbarEqual,
+        "^=" => token::Tok::CircumflexEqual,
+        "<<=" => token::Tok::LeftShiftEqual,
+        ">>=" => token::Tok::RightShiftEqual,
+        "**=" => token::Tok::DoubleStarEqual,
+        "//=" => token::Tok::DoubleSlashEqual,
+        ":=" => token::Tok::ColonEqual,
+        "==" => token::Tok::EqEqual,
+        "!=" => token::Tok::NotEqual,
+        "<" => token::Tok::Less,
+        "<=" => token::Tok::LessEqual,
+        ">" => token::Tok::Greater,
+        ">=" => token::Tok::GreaterEqual,
+        "->" => token::Tok::Rarrow,
+        "and" => token::Tok::And,
+        "as" => token::Tok::As,
+        "assert" => token::Tok::Assert,
+        "async" => token::Tok::Async,
+        "await" => token::Tok::Await,
+        "break" => token::Tok::Break,
+        "class" => token::Tok::Class,
+        "continue" => token::Tok::Continue,
+        "def" => token::Tok::Def,
+        "del" => token::Tok::Del,
+        "elif" => token::Tok::Elif,
+        "else" => token::Tok::Else,
+        "except" => token::Tok::Except,
+        "finally" => token::Tok::Finally,
+        "for" => token::Tok::For,
+        "from" => token::Tok::From,
+        "global" => token::Tok::Global,
+        "if" => token::Tok::If,
+        "import" => token::Tok::Import,
+        "in" => token::Tok::In,
+        "is" => token::Tok::Is,
+        "lambda" => token::Tok::Lambda,
+        "nonlocal" => token::Tok::Nonlocal,
+        "not" => token::Tok::Not,
+        "or" => token::Tok::Or,
+        "pass" => token::Tok::Pass,
+        "raise" => token::Tok::Raise,
+        "return" => token::Tok::Return,
+        "try" => token::Tok::Try,
+        "while" => token::Tok::While,
+        "match" => token::Tok::Match,
+        "case" => token::Tok::Case,
+        "with" => token::Tok::With,
+        "yield" => token::Tok::Yield,
+        "True" => token::Tok::True,
+        "False" => token::Tok::False,
+        "None" => token::Tok::None,
+        int => token::Tok::Int { value: <BigInt> },
+        float => token::Tok::Float { value: <f64> },
+        complex => token::Tok::Complex { real: <f64>, imag: <f64> },
+        string => token::Tok::String {
             value: <String>,
             kind: <StringKind>,
             triple_quoted: <bool>
         },
-        name => lexer::Tok::Name { name: <String> },
-        "\n" => lexer::Tok::Newline,
-        ";" => lexer::Tok::Semi,
-        "#" => lexer::Tok::Comment(_),
+        name => token::Tok::Name { name: <String> },
+        "\n" => token::Tok::Newline,
+        ";" => token::Tok::Semi,
+        "#" => token::Tok::Comment(_),
     }
 }

compiler/parser/src/function.rs

@@ -1,7 +1,9 @@
 // Contains functions that perform validation and parsing of arguments and parameters.
 // Checks apply both to functions and to lambdas.
-use crate::ast;
-use crate::lexer::{LexicalError, LexicalErrorType};
+use crate::{
+    ast,
+    lexer::{LexicalError, LexicalErrorType},
+};
 use rustc_hash::FxHashSet;
 
 pub(crate) struct ArgumentList {

compiler/parser/src/lexer.rs

@@ -12,9 +12,7 @@
 //! # Example
 //!
 //! ```
-//! use rustpython_parser::lexer::{lex, Tok};
-//! use rustpython_parser::mode::Mode;
-//! use rustpython_parser::token::StringKind;
+//! use rustpython_parser::{lexer::lex, Tok, Mode, StringKind};
 //!
 //! let source = "x = 'RustPython'";
 //! let tokens = lex(source, Mode::Module)
@@ -33,19 +31,16 @@
 //! ```
 //!
 //! [Lexical analysis]: https://docs.python.org/3/reference/lexical_analysis.html
-pub use super::token::{StringKind, Tok};
-use crate::ast::Location;
-use crate::mode::Mode;
-use crate::soft_keywords::SoftKeywordTransformer;
-use crate::string::FStringErrorType;
+use crate::{
+    ast::Location,
+    mode::Mode,
+    soft_keywords::SoftKeywordTransformer,
+    string::FStringErrorType,
+    token::{StringKind, Tok},
+};
 use num_bigint::BigInt;
-use num_traits::identities::Zero;
-use num_traits::Num;
-use std::char;
-use std::cmp::Ordering;
-use std::ops::Index;
-use std::slice::SliceIndex;
-use std::str::FromStr;
+use num_traits::{Num, Zero};
+use std::{char, cmp::Ordering, ops::Index, slice::SliceIndex, str::FromStr};
 use unic_emoji_char::is_emoji_presentation;
 use unic_ucd_ident::{is_xid_continue, is_xid_start};
 
@@ -200,8 +195,7 @@ pub type LexResult = Result<Spanned, LexicalError>;
 /// # Examples
 ///
 /// ```
-/// use rustpython_parser::mode::Mode;
-/// use rustpython_parser::lexer::{lex};
+/// use rustpython_parser::{Mode, lexer::lex};
 ///
 /// let source = "def hello(): return 'world'";
 /// let lexer = lex(source, Mode::Module);
@@ -1320,8 +1314,7 @@ impl std::fmt::Display for LexicalErrorType {
 
 #[cfg(test)]
 mod tests {
-    use super::{lex, StringKind, Tok};
-    use crate::mode::Mode;
+    use super::*;
     use num_bigint::BigInt;
 
     const WINDOWS_EOL: &str = "\r\n";

compiler/parser/src/lib.rs

@@ -54,20 +54,18 @@
 //!
 //! The functionality of this crate is split into several modules:
 //!
-//! - [token]: This module contains the definition of the tokens that are generated by the lexer.
+//! - token: This module contains the definition of the tokens that are generated by the lexer.
 //! - [lexer]: This module contains the lexer and is responsible for generating the tokens.
-//! - [parser]: This module contains an interface to the parser and is responsible for generating the AST.
+//! - parser: This module contains an interface to the parser and is responsible for generating the AST.
 //!     - Functions and strings have special parsing requirements that are handled in additional files.
-//! - [mode]: This module contains the definition of the different modes that the parser can be in.
-//! - [error]: This module contains the definition of the errors that can be returned by the parser.
+//! - mode: This module contains the definition of the different modes that the parser can be in.
 //!
 //! # Examples
 //!
 //! For example, to get a stream of tokens from a given string, one could do this:
 //!
 //! ```
-//! use rustpython_parser::mode::Mode;
-//! use rustpython_parser::lexer::lex;
+//! use rustpython_parser::{lexer::lex, Mode};
 //!
 //! let python_source = r#"
 //! def is_odd(i):
@@ -80,9 +78,7 @@
 //! These tokens can be directly fed into the parser to generate an AST:
 //!
 //! ```
-//! use rustpython_parser::lexer::lex;
-//! use rustpython_parser::mode::Mode;
-//! use rustpython_parser::parser::parse_tokens;
+//! use rustpython_parser::{lexer::lex, Mode, parse_tokens};
 //!
 //! let python_source = r#"
 //! def is_odd(i):
@@ -98,7 +94,7 @@
 //! mode or tokenizing the source beforehand:
 //!
 //! ```
-//! use rustpython_parser::parser::parse_program;
+//! use rustpython_parser::parse_program;
 //!
 //! let python_source = r#"
 //! def is_odd(i):
@@ -111,11 +107,7 @@
 //!
 //! [lexical analysis]: https://en.wikipedia.org/wiki/Lexical_analysis
 //! [parsing]: https://en.wikipedia.org/wiki/Parsing
-//! [token]: crate::token
 //! [lexer]: crate::lexer
-//! [parser]: crate::parser
-//! [mode]: crate::mode
-//! [error]: crate::error
 
 #![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
 #![doc(html_root_url = "https://docs.rs/rustpython-parser/")]
@@ -125,12 +117,21 @@ extern crate log;
 pub use rustpython_ast as ast;
 
 mod function;
+// Skip flattening lexer to distinguish from full parser
 pub mod lexer;
-pub mod mode;
-pub mod parser;
+mod mode;
+mod parser;
 mod string;
 #[rustfmt::skip]
 mod python;
 mod context;
 mod soft_keywords;
-pub mod token;
+mod token;
+
+pub use mode::Mode;
+pub use parser::{
+    parse, parse_expression, parse_expression_located, parse_located, parse_program, parse_tokens,
+    ParseError, ParseErrorType,
+};
+pub use string::FStringErrorType;
+pub use token::{StringKind, Tok};