new_syntax_error working without full compiler

Fix RustPython#4100

Author: youknowone

Cargo.lock

@@ -760,8 +760,6 @@ def test_parse(self):
         b = compile('foo(1 + 1)', '<unknown>', 'exec', ast.PyCF_ONLY_AST)
         self.assertEqual(ast.dump(a), ast.dump(b))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_parse_in_error(self):
         try:
             1/0
@@ -1101,8 +1099,6 @@ def test_literal_eval_malformed_dict_nodes(self):
         malformed = ast.Dict(keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)])
         self.assertRaises(ValueError, ast.literal_eval, malformed)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_literal_eval_trailing_ws(self):
         self.assertEqual(ast.literal_eval("    -1"), -1)
         self.assertEqual(ast.literal_eval("\t\t-1"), -1)
@@ -1121,8 +1117,6 @@ def test_literal_eval_malformed_lineno(self):
         with self.assertRaisesRegex(ValueError, msg):
             ast.literal_eval(node)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_literal_eval_syntax_errors(self):
         with self.assertRaisesRegex(SyntaxError, "unexpected indent"):
             ast.literal_eval(r'''
@@ -1767,6 +1761,8 @@ def test_stdlib_validates(self):
         ast.MatchMapping([], [], rest="_"),
     ]
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_match_validation_pattern(self):
         name_x = ast.Name('x', ast.Load())
         for pattern in self._MATCH_PATTERNS:

Lib/test/test_ast.py

@@ -17,6 +17,7 @@ itertools = "0.10.3"
 log = "0.4.16"
 num-complex = { version = "0.4.0", features = ["serde"] }
 num-traits = "0.2.14"
+thiserror = "1.0"
 
 [dev-dependencies]
 rustpython-parser = { path = "../parser" }

compiler/codegen/Cargo.toml

@@ -1,8 +1,8 @@
-use std::{error::Error, fmt};
+use std::fmt;
 
 pub type CodegenError = rustpython_compiler_core::BaseError<CodegenErrorType>;
 
-#[derive(Debug)]
+#[derive(Debug, thiserror::Error)]
 #[non_exhaustive]
 pub enum CodegenErrorType {
     /// Invalid assignment, cannot store value in target.
@@ -79,5 +79,3 @@ impl fmt::Display for CodegenErrorType {
         }
     }
 }
-
-impl Error for CodegenErrorType {}

compiler/codegen/src/error.rs

@@ -1,6 +1,5 @@
-use std::fmt::Display;
-
 use crate::Location;
+use std::fmt::Display;
 
 #[derive(Debug, PartialEq, Eq)]
 pub struct BaseError<T> {
@@ -87,3 +86,24 @@ where
         }
     }
 }
+
+impl<T> CompileError<T> {
+    pub fn from<U>(error: BaseError<U>, source: &str) -> Self
+    where
+        T: From<U>,
+    {
+        let statement = get_statement(source, error.location);
+        CompileError {
+            body: error.into(),
+            statement,
+        }
+    }
+}
+
+fn get_statement(source: &str, loc: Location) -> Option<String> {
+    if loc.column() == 0 || loc.row() == 0 {
+        return None;
+    }
+    let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
+    Some(line + "\n")
+}

compiler/core/src/error.rs

@@ -117,8 +117,7 @@ impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
 }
 
 /// Represents an error during parsing
-#[derive(Debug, PartialEq)]
-pub struct ParseError(rustpython_compiler_core::BaseError<ParseErrorType>);
+pub type ParseError = rustpython_compiler_core::BaseError<ParseErrorType>;
 
 #[derive(Debug, PartialEq)]
 pub enum ParseErrorType {
@@ -134,66 +133,44 @@ pub enum ParseErrorType {
     Lexical(LexicalErrorType),
 }
 
-impl From<ParseError> for rustpython_compiler_core::BaseError<ParseErrorType> {
-    fn from(err: ParseError) -> Self {
-        err.0
-    }
-}
-
-impl From<ParseError> for ParseErrorType {
-    fn from(err: ParseError) -> Self {
-        err.0.error
-    }
-}
-
 /// Convert `lalrpop_util::ParseError` to our internal type
-impl ParseError {
-    fn new(error: ParseErrorType, location: Location, source_path: String) -> Self {
-        Self(rustpython_compiler_core::BaseError {
-            error,
+pub(crate) fn parse_error_from_lalrpop(
+    err: LalrpopError<Location, Tok, LexicalError>,
+    source_path: &str,
+) -> ParseError {
+    let source_path = source_path.to_owned();
+    match err {
+        // TODO: Are there cases where this isn't an EOF?
+        LalrpopError::InvalidToken { location } => ParseError {
+            error: ParseErrorType::Eof,
             location,
             source_path,
-        })
-    }
-
-    pub(crate) fn from_lalrpop(
-        err: LalrpopError<Location, Tok, LexicalError>,
-        source_path: &str,
-    ) -> Self {
-        let source_path = source_path.to_owned();
-        match err {
-            // TODO: Are there cases where this isn't an EOF?
-            LalrpopError::InvalidToken { location } => {
-                ParseError::new(ParseErrorType::Eof, location, source_path)
-            }
-            LalrpopError::ExtraToken { token } => {
-                ParseError::new(ParseErrorType::ExtraToken(token.1), token.0, source_path)
-            }
-            LalrpopError::User { error } => ParseError::new(
-                ParseErrorType::Lexical(error.error),
-                error.location,
+        },
+        LalrpopError::ExtraToken { token } => ParseError {
+            error: ParseErrorType::ExtraToken(token.1),
+            location: token.0,
+            source_path,
+        },
+        LalrpopError::User { error } => ParseError {
+            error: ParseErrorType::Lexical(error.error),
+            location: error.location,
+            source_path,
+        },
+        LalrpopError::UnrecognizedToken { token, expected } => {
+            // Hacky, but it's how CPython does it. See PyParser_AddToken,
+            // in particular "Only one possible expected token" comment.
+            let expected = (expected.len() == 1).then(|| expected[0].clone());
+            ParseError {
+                error: ParseErrorType::UnrecognizedToken(token.1, expected),
+                location: token.0,
                 source_path,
-            ),
-            LalrpopError::UnrecognizedToken { token, expected } => {
-                // Hacky, but it's how CPython does it. See PyParser_AddToken,
-                // in particular "Only one possible expected token" comment.
-                let expected = (expected.len() == 1).then(|| expected[0].clone());
-                ParseError::new(
-                    ParseErrorType::UnrecognizedToken(token.1, expected),
-                    token.0,
-                    source_path,
-                )
-            }
-            LalrpopError::UnrecognizedEOF { location, .. } => {
-                ParseError::new(ParseErrorType::Eof, location, source_path)
             }
         }
-    }
-}
-
-impl fmt::Display for ParseError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        self.0.fmt(f)
+        LalrpopError::UnrecognizedEOF { location, .. } => ParseError {
+            error: ParseErrorType::Eof,
+            location,
+            source_path,
+        },
     }
 }
 
@@ -237,16 +214,3 @@ impl ParseErrorType {
         )
     }
 }
-
-impl std::ops::Deref for ParseError {
-    type Target = rustpython_compiler_core::BaseError<ParseErrorType>;
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl Error for ParseError {
-    fn source(&self) -> Option<&(dyn Error + 'static)> {
-        None
-    }
-}

compiler/parser/src/error.rs

@@ -186,7 +186,7 @@ impl<'a> FStringParser<'a> {
                         vec![self.expr(ExprKind::FormattedValue {
                             value: Box::new(
                                 parse_fstring_expr(&expression)
-                                    .map_err(|e| InvalidExpression(Box::new(e.into())))?,
+                                    .map_err(|e| InvalidExpression(Box::new(e.error)))?,
                             ),
                             conversion: conversion as _,
                             format_spec: spec,
@@ -204,7 +204,7 @@ impl<'a> FStringParser<'a> {
                             self.expr(ExprKind::FormattedValue {
                                 value: Box::new(
                                     parse_fstring_expr(&expression)
-                                        .map_err(|e| InvalidExpression(Box::new(e.into())))?,
+                                        .map_err(|e| InvalidExpression(Box::new(e.error)))?,
                                 ),
                                 conversion: (if conversion == ConversionFlag::None && spec.is_none()
                                 {

compiler/parser/src/fstring.rs

@@ -74,7 +74,7 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, Pa
 
     python::TopParser::new()
         .parse(tokenizer)
-        .map_err(|e| ParseError::from_lalrpop(e, source_path))
+        .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
 }
 
 #[cfg(test)]

compiler/parser/src/parser.rs

@@ -1,7 +1,7 @@
 use rustpython_codegen::{compile, symboltable};
 use rustpython_compiler_core::CodeObject;
 use rustpython_parser::{
-    ast::{fold::Fold, ConstantOptimizer, Location},
+    ast::{fold::Fold, ConstantOptimizer},
     error::ParseErrorType,
     parser,
 };
@@ -12,31 +12,16 @@ pub use rustpython_compiler_core::{BaseError as CompileErrorBody, Mode};
 #[derive(Debug, thiserror::Error)]
 pub enum CompileErrorType {
     #[error(transparent)]
-    Compile(#[from] rustpython_codegen::error::CodegenErrorType),
+    Codegen(#[from] rustpython_codegen::error::CodegenErrorType),
     #[error(transparent)]
     Parse(#[from] rustpython_parser::error::ParseErrorType),
 }
 
 pub type CompileError = rustpython_compiler_core::CompileError<CompileErrorType>;
 
-fn error_from_codegen(
-    error: rustpython_codegen::error::CodegenError,
-    source: &str,
-) -> CompileError {
-    let statement = get_statement(source, error.location);
-    CompileError {
-        body: error.into(),
-        statement,
-    }
-}
-
 fn error_from_parse(error: rustpython_parser::error::ParseError, source: &str) -> CompileError {
     let error: CompileErrorBody<ParseErrorType> = error.into();
-    let statement = get_statement(source, error.location);
-    CompileError {
-        body: error.into(),
-        statement,
-    }
+    CompileError::from(error, source)
 }
 
 /// Compile a given sourcecode into a bytecode object.
@@ -60,7 +45,7 @@ pub fn compile(
             .fold_mod(ast)
             .unwrap_or_else(|e| match e {});
     }
-    compile::compile_top(&ast, source_path, mode, opts).map_err(|e| error_from_codegen(e, source))
+    compile::compile_top(&ast, source_path, mode, opts).map_err(|e| CompileError::from(e, source))
 }
 
 pub fn compile_symtable(
@@ -79,13 +64,5 @@ pub fn compile_symtable(
             symboltable::SymbolTable::scan_expr(&expr)
         }
     };
-    res.map_err(|e| error_from_codegen(e.into_codegen_error(source_path.to_owned()), source))
-}
-
-fn get_statement(source: &str, loc: Location) -> Option<String> {
-    if loc.column() == 0 || loc.row() == 0 {
-        return None;
-    }
-    let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
-    Some(line + "\n")
+    res.map_err(|e| CompileError::from(e.into_codegen_error(source_path.to_owned()), source))
 }

compiler/src/lib.rs

@@ -13,8 +13,7 @@ extern crate env_logger;
 extern crate log;
 
 use clap::{App, Arg};
-
-use rustpython_compiler as compile;
+use rustpython_compiler as compiler;
 use std::error::Error;
 use std::fs;
 use std::path::Path;
@@ -62,7 +61,7 @@ fn main() {
     let optimize = matches.occurrences_of("optimize") as u8;
     let scripts = matches.values_of_os("scripts").unwrap();
 
-    let opts = compile::CompileOpts {
+    let opts = compiler::CompileOpts {
         optimize,
         ..Default::default()
     };
@@ -81,12 +80,12 @@ fn main() {
 
 fn display_script(
     path: &Path,
-    mode: compile::Mode,
-    opts: compile::CompileOpts,
+    mode: compiler::Mode,
+    opts: compiler::CompileOpts,
     expand_codeobjects: bool,
 ) -> Result<(), Box<dyn Error>> {
     let source = fs::read_to_string(path)?;
-    let code = compile::compile(&source, mode, path.to_string_lossy().into_owned(), opts)?;
+    let code = compiler::compile(&source, mode, path.to_string_lossy().into_owned(), opts)?;
     println!("{}:", path.display());
     if expand_codeobjects {
         println!("{}", code.display_expand_codeobjects());

examples/dis.rs

@@ -7,7 +7,7 @@ fn main() -> vm::PyResult<()> {
         let code_obj = vm
             .compile(
                 r#"print("Hello World!")"#,
-                vm::compile::Mode::Exec,
+                vm::compiler::Mode::Exec,
                 "<embedded>".to_owned(),
             )
             .map_err(|err| vm.new_syntax_error(&err))?;

examples/hello_embed.rs

@@ -62,9 +62,9 @@ def fib(n):
             .expect("Failed to read line of input");
 
         // this line also automatically prints the output
-        // (note that this is only the case when compile::Mode::Single is passed to vm.compile)
+        // (note that this is only the case when compiler::Mode::Single is passed to vm.compile)
         match vm
-            .compile(&input, vm::compile::Mode::Single, "<embedded>".to_owned())
+            .compile(&input, vm::compiler::Mode::Single, "<embedded>".to_owned())
             .map_err(|err| vm.new_syntax_error(&err))
             .and_then(|code_obj| vm.run_code_obj(code_obj, scope.clone()))
         {

examples/mini_repl.rs

@@ -17,9 +17,9 @@ flame-it = ["flame", "flamer"]
 freeze-stdlib = ["rustpython-pylib/freeze-stdlib"]
 jit = ["rustpython-jit"]
 threading = ["rustpython-common/threading"]
-compiler = ["parser", "codegen"]
+compiler = ["parser", "codegen", "rustpython-compiler"]
 ast = ["rustpython-ast"]
-codegen = ["rustpython-compiler", "rustpython-codegen", "ast"]
+codegen = ["rustpython-codegen", "ast"]
 parser = ["rustpython-parser", "ast"]
 
 [dependencies]