Skip to content

Commit 55061c7

Browse files
youknowoneyouknowone
committed
new_syntax_error working without full compiler
Fix RustPython#4100
1 parent 07e339a commit 55061c7

24 files changed

+147
-173
lines changed

‎Cargo.lock

+1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Lib/test/test_ast.py

+2-6
Original file line numberDiff line numberDiff line change
@@ -760,8 +760,6 @@ def test_parse(self):
760760
b = compile('foo(1 + 1)', '<unknown>', 'exec', ast.PyCF_ONLY_AST)
761761
self.assertEqual(ast.dump(a), ast.dump(b))
762762

763-
# TODO: RUSTPYTHON
764-
@unittest.expectedFailure
765763
def test_parse_in_error(self):
766764
try:
767765
1/0
@@ -1101,8 +1099,6 @@ def test_literal_eval_malformed_dict_nodes(self):
11011099
malformed = ast.Dict(keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)])
11021100
self.assertRaises(ValueError, ast.literal_eval, malformed)
11031101

1104-
# TODO: RUSTPYTHON
1105-
@unittest.expectedFailure
11061102
def test_literal_eval_trailing_ws(self):
11071103
self.assertEqual(ast.literal_eval(" -1"), -1)
11081104
self.assertEqual(ast.literal_eval("\t\t-1"), -1)
@@ -1121,8 +1117,6 @@ def test_literal_eval_malformed_lineno(self):
11211117
with self.assertRaisesRegex(ValueError, msg):
11221118
ast.literal_eval(node)
11231119

1124-
# TODO: RUSTPYTHON
1125-
@unittest.expectedFailure
11261120
def test_literal_eval_syntax_errors(self):
11271121
with self.assertRaisesRegex(SyntaxError, "unexpected indent"):
11281122
ast.literal_eval(r'''
@@ -1767,6 +1761,8 @@ def test_stdlib_validates(self):
17671761
ast.MatchMapping([], [], rest="_"),
17681762
]
17691763

1764+
# TODO: RUSTPYTHON
1765+
@unittest.expectedFailure
17701766
def test_match_validation_pattern(self):
17711767
name_x = ast.Name('x', ast.Load())
17721768
for pattern in self._MATCH_PATTERNS:

‎compiler/codegen/Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ itertools = "0.10.3"
1717
log = "0.4.16"
1818
num-complex = { version = "0.4.0", features = ["serde"] }
1919
num-traits = "0.2.14"
20+
thiserror = "1.0"
2021

2122
[dev-dependencies]
2223
rustpython-parser = { path = "../parser" }

‎compiler/codegen/src/error.rs

+2-4
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
use std::{error::Error, fmt};
1+
use std::fmt;
22

33
pub type CodegenError = rustpython_compiler_core::BaseError<CodegenErrorType>;
44

5-
#[derive(Debug)]
5+
#[derive(Debug, thiserror::Error)]
66
#[non_exhaustive]
77
pub enum CodegenErrorType {
88
/// Invalid assignment, cannot store value in target.
@@ -79,5 +79,3 @@ impl fmt::Display for CodegenErrorType {
7979
}
8080
}
8181
}
82-
83-
impl Error for CodegenErrorType {}

‎compiler/core/src/error.rs

+22-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
use std::fmt::Display;
2-
31
use crate::Location;
2+
use std::fmt::Display;
43

54
#[derive(Debug, PartialEq, Eq)]
65
pub struct BaseError<T> {
@@ -87,3 +86,24 @@ where
8786
}
8887
}
8988
}
89+
90+
impl<T> CompileError<T> {
91+
pub fn from<U>(error: BaseError<U>, source: &str) -> Self
92+
where
93+
T: From<U>,
94+
{
95+
let statement = get_statement(source, error.location);
96+
CompileError {
97+
body: error.into(),
98+
statement,
99+
}
100+
}
101+
}
102+
103+
fn get_statement(source: &str, loc: Location) -> Option<String> {
104+
if loc.column() == 0 || loc.row() == 0 {
105+
return None;
106+
}
107+
let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
108+
Some(line + "\n")
109+
}

‎compiler/parser/src/error.rs

+33-69
Original file line numberDiff line numberDiff line change
@@ -117,8 +117,7 @@ impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
117117
}
118118

119119
/// Represents an error during parsing
120-
#[derive(Debug, PartialEq)]
121-
pub struct ParseError(rustpython_compiler_core::BaseError<ParseErrorType>);
120+
pub type ParseError = rustpython_compiler_core::BaseError<ParseErrorType>;
122121

123122
#[derive(Debug, PartialEq)]
124123
pub enum ParseErrorType {
@@ -134,66 +133,44 @@ pub enum ParseErrorType {
134133
Lexical(LexicalErrorType),
135134
}
136135

137-
impl From<ParseError> for rustpython_compiler_core::BaseError<ParseErrorType> {
138-
fn from(err: ParseError) -> Self {
139-
err.0
140-
}
141-
}
142-
143-
impl From<ParseError> for ParseErrorType {
144-
fn from(err: ParseError) -> Self {
145-
err.0.error
146-
}
147-
}
148-
149136
/// Convert `lalrpop_util::ParseError` to our internal type
150-
impl ParseError {
151-
fn new(error: ParseErrorType, location: Location, source_path: String) -> Self {
152-
Self(rustpython_compiler_core::BaseError {
153-
error,
137+
pub(crate) fn parse_error_from_lalrpop(
138+
err: LalrpopError<Location, Tok, LexicalError>,
139+
source_path: &str,
140+
) -> ParseError {
141+
let source_path = source_path.to_owned();
142+
match err {
143+
// TODO: Are there cases where this isn't an EOF?
144+
LalrpopError::InvalidToken { location } => ParseError {
145+
error: ParseErrorType::Eof,
154146
location,
155147
source_path,
156-
})
157-
}
158-
159-
pub(crate) fn from_lalrpop(
160-
err: LalrpopError<Location, Tok, LexicalError>,
161-
source_path: &str,
162-
) -> Self {
163-
let source_path = source_path.to_owned();
164-
match err {
165-
// TODO: Are there cases where this isn't an EOF?
166-
LalrpopError::InvalidToken { location } => {
167-
ParseError::new(ParseErrorType::Eof, location, source_path)
168-
}
169-
LalrpopError::ExtraToken { token } => {
170-
ParseError::new(ParseErrorType::ExtraToken(token.1), token.0, source_path)
171-
}
172-
LalrpopError::User { error } => ParseError::new(
173-
ParseErrorType::Lexical(error.error),
174-
error.location,
148+
},
149+
LalrpopError::ExtraToken { token } => ParseError {
150+
error: ParseErrorType::ExtraToken(token.1),
151+
location: token.0,
152+
source_path,
153+
},
154+
LalrpopError::User { error } => ParseError {
155+
error: ParseErrorType::Lexical(error.error),
156+
location: error.location,
157+
source_path,
158+
},
159+
LalrpopError::UnrecognizedToken { token, expected } => {
160+
// Hacky, but it's how CPython does it. See PyParser_AddToken,
161+
// in particular "Only one possible expected token" comment.
162+
let expected = (expected.len() == 1).then(|| expected[0].clone());
163+
ParseError {
164+
error: ParseErrorType::UnrecognizedToken(token.1, expected),
165+
location: token.0,
175166
source_path,
176-
),
177-
LalrpopError::UnrecognizedToken { token, expected } => {
178-
// Hacky, but it's how CPython does it. See PyParser_AddToken,
179-
// in particular "Only one possible expected token" comment.
180-
let expected = (expected.len() == 1).then(|| expected[0].clone());
181-
ParseError::new(
182-
ParseErrorType::UnrecognizedToken(token.1, expected),
183-
token.0,
184-
source_path,
185-
)
186-
}
187-
LalrpopError::UnrecognizedEOF { location, .. } => {
188-
ParseError::new(ParseErrorType::Eof, location, source_path)
189167
}
190168
}
191-
}
192-
}
193-
194-
impl fmt::Display for ParseError {
195-
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
196-
self.0.fmt(f)
169+
LalrpopError::UnrecognizedEOF { location, .. } => ParseError {
170+
error: ParseErrorType::Eof,
171+
location,
172+
source_path,
173+
},
197174
}
198175
}
199176

@@ -237,16 +214,3 @@ impl ParseErrorType {
237214
)
238215
}
239216
}
240-
241-
impl std::ops::Deref for ParseError {
242-
type Target = rustpython_compiler_core::BaseError<ParseErrorType>;
243-
fn deref(&self) -> &Self::Target {
244-
&self.0
245-
}
246-
}
247-
248-
impl Error for ParseError {
249-
fn source(&self) -> Option<&(dyn Error + 'static)> {
250-
None
251-
}
252-
}

‎compiler/parser/src/fstring.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ impl<'a> FStringParser<'a> {
186186
vec![self.expr(ExprKind::FormattedValue {
187187
value: Box::new(
188188
parse_fstring_expr(&expression)
189-
.map_err(|e| InvalidExpression(Box::new(e.into())))?,
189+
.map_err(|e| InvalidExpression(Box::new(e.error)))?,
190190
),
191191
conversion: conversion as _,
192192
format_spec: spec,
@@ -204,7 +204,7 @@ impl<'a> FStringParser<'a> {
204204
self.expr(ExprKind::FormattedValue {
205205
value: Box::new(
206206
parse_fstring_expr(&expression)
207-
.map_err(|e| InvalidExpression(Box::new(e.into())))?,
207+
.map_err(|e| InvalidExpression(Box::new(e.error)))?,
208208
),
209209
conversion: (if conversion == ConversionFlag::None && spec.is_none()
210210
{

‎compiler/parser/src/parser.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, Pa
7474

7575
python::TopParser::new()
7676
.parse(tokenizer)
77-
.map_err(|e| ParseError::from_lalrpop(e, source_path))
77+
.map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
7878
}
7979

8080
#[cfg(test)]

‎compiler/src/lib.rs

+5-28
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use rustpython_codegen::{compile, symboltable};
22
use rustpython_compiler_core::CodeObject;
33
use rustpython_parser::{
4-
ast::{fold::Fold, ConstantOptimizer, Location},
4+
ast::{fold::Fold, ConstantOptimizer},
55
error::ParseErrorType,
66
parser,
77
};
@@ -12,31 +12,16 @@ pub use rustpython_compiler_core::{BaseError as CompileErrorBody, Mode};
1212
#[derive(Debug, thiserror::Error)]
1313
pub enum CompileErrorType {
1414
#[error(transparent)]
15-
Compile(#[from] rustpython_codegen::error::CodegenErrorType),
15+
Codegen(#[from] rustpython_codegen::error::CodegenErrorType),
1616
#[error(transparent)]
1717
Parse(#[from] rustpython_parser::error::ParseErrorType),
1818
}
1919

2020
pub type CompileError = rustpython_compiler_core::CompileError<CompileErrorType>;
2121

22-
fn error_from_codegen(
23-
error: rustpython_codegen::error::CodegenError,
24-
source: &str,
25-
) -> CompileError {
26-
let statement = get_statement(source, error.location);
27-
CompileError {
28-
body: error.into(),
29-
statement,
30-
}
31-
}
32-
3322
fn error_from_parse(error: rustpython_parser::error::ParseError, source: &str) -> CompileError {
3423
let error: CompileErrorBody<ParseErrorType> = error.into();
35-
let statement = get_statement(source, error.location);
36-
CompileError {
37-
body: error.into(),
38-
statement,
39-
}
24+
CompileError::from(error, source)
4025
}
4126

4227
/// Compile a given sourcecode into a bytecode object.
@@ -60,7 +45,7 @@ pub fn compile(
6045
.fold_mod(ast)
6146
.unwrap_or_else(|e| match e {});
6247
}
63-
compile::compile_top(&ast, source_path, mode, opts).map_err(|e| error_from_codegen(e, source))
48+
compile::compile_top(&ast, source_path, mode, opts).map_err(|e| CompileError::from(e, source))
6449
}
6550

6651
pub fn compile_symtable(
@@ -79,13 +64,5 @@ pub fn compile_symtable(
7964
symboltable::SymbolTable::scan_expr(&expr)
8065
}
8166
};
82-
res.map_err(|e| error_from_codegen(e.into_codegen_error(source_path.to_owned()), source))
83-
}
84-
85-
fn get_statement(source: &str, loc: Location) -> Option<String> {
86-
if loc.column() == 0 || loc.row() == 0 {
87-
return None;
88-
}
89-
let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
90-
Some(line + "\n")
67+
res.map_err(|e| CompileError::from(e.into_codegen_error(source_path.to_owned()), source))
9168
}

‎examples/dis.rs

+5-6
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,7 @@ extern crate env_logger;
1313
extern crate log;
1414

1515
use clap::{App, Arg};
16-
17-
use rustpython_compiler as compile;
16+
use rustpython_compiler as compiler;
1817
use std::error::Error;
1918
use std::fs;
2019
use std::path::Path;
@@ -62,7 +61,7 @@ fn main() {
6261
let optimize = matches.occurrences_of("optimize") as u8;
6362
let scripts = matches.values_of_os("scripts").unwrap();
6463

65-
let opts = compile::CompileOpts {
64+
let opts = compiler::CompileOpts {
6665
optimize,
6766
..Default::default()
6867
};
@@ -81,12 +80,12 @@ fn main() {
8180

8281
fn display_script(
8382
path: &Path,
84-
mode: compile::Mode,
85-
opts: compile::CompileOpts,
83+
mode: compiler::Mode,
84+
opts: compiler::CompileOpts,
8685
expand_codeobjects: bool,
8786
) -> Result<(), Box<dyn Error>> {
8887
let source = fs::read_to_string(path)?;
89-
let code = compile::compile(&source, mode, path.to_string_lossy().into_owned(), opts)?;
88+
let code = compiler::compile(&source, mode, path.to_string_lossy().into_owned(), opts)?;
9089
println!("{}:", path.display());
9190
if expand_codeobjects {
9291
println!("{}", code.display_expand_codeobjects());

‎examples/hello_embed.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ fn main() -> vm::PyResult<()> {
77
let code_obj = vm
88
.compile(
99
r#"print("Hello World!")"#,
10-
vm::compile::Mode::Exec,
10+
vm::compiler::Mode::Exec,
1111
"<embedded>".to_owned(),
1212
)
1313
.map_err(|err| vm.new_syntax_error(&err))?;

‎examples/mini_repl.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -62,9 +62,9 @@ def fib(n):
6262
.expect("Failed to read line of input");
6363

6464
// this line also automatically prints the output
65-
// (note that this is only the case when compile::Mode::Single is passed to vm.compile)
65+
// (note that this is only the case when compiler::Mode::Single is passed to vm.compile)
6666
match vm
67-
.compile(&input, vm::compile::Mode::Single, "<embedded>".to_owned())
67+
.compile(&input, vm::compiler::Mode::Single, "<embedded>".to_owned())
6868
.map_err(|err| vm.new_syntax_error(&err))
6969
.and_then(|code_obj| vm.run_code_obj(code_obj, scope.clone()))
7070
{

‎vm/Cargo.toml

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@ flame-it = ["flame", "flamer"]
1717
freeze-stdlib = ["rustpython-pylib/freeze-stdlib"]
1818
jit = ["rustpython-jit"]
1919
threading = ["rustpython-common/threading"]
20-
compiler = ["parser", "codegen"]
20+
compiler = ["parser", "codegen", "rustpython-compiler"]
2121
ast = ["rustpython-ast"]
22-
codegen = ["rustpython-compiler", "rustpython-codegen", "ast"]
22+
codegen = ["rustpython-codegen", "ast"]
2323
parser = ["rustpython-parser", "ast"]
2424

2525
[dependencies]

0 commit comments

Comments
 (0)