From 11de6dd5def0935e9a473eec0fe525ffe27b4e34 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Thu, 8 Feb 2024 21:36:22 +0100 Subject: [PATCH] Reduce `Result` size by using `Box` instead of `String` (#9885) --- .../flake8_pytest_style/rules/parametrize.rs | 23 +- .../rules/flake8_simplify/rules/ast_expr.rs | 2 +- .../flynt/rules/static_join_to_fstring.rs | 3 +- .../rules/invalid_escape_sequence.rs | 2 +- .../pylint/rules/unspecified_encoding.rs | 2 +- .../ruff_linter/src/rules/pyupgrade/fixes.rs | 2 +- .../src/rules/ruff/rules/sequence_sorting.rs | 16 +- crates/ruff_python_ast/src/comparable.rs | 12 +- crates/ruff_python_ast/src/nodes.rs | 16 +- crates/ruff_python_formatter/src/lib.rs | 4 +- crates/ruff_python_formatter/src/range.rs | 4 +- .../ruff_python_formatter/tests/normalizer.rs | 12 +- crates/ruff_python_parser/src/function.rs | 64 ++-- crates/ruff_python_parser/src/invalid.rs | 2 +- crates/ruff_python_parser/src/lexer.rs | 244 +++++++-------- crates/ruff_python_parser/src/parser.rs | 8 +- crates/ruff_python_parser/src/python.lalrpop | 108 +++---- crates/ruff_python_parser/src/python.rs | 278 +++++++++--------- .../ruff_python_parser/src/soft_keywords.rs | 2 +- crates/ruff_python_parser/src/string.rs | 50 ++-- crates/ruff_python_parser/src/token.rs | 21 +- fuzz/fuzz_targets/ruff_parse_simple.rs | 2 +- 22 files changed, 453 insertions(+), 424 deletions(-) diff --git a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs index d71eb361fce40..daf31d2e2b1f0 100644 --- a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs +++ b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs @@ -257,15 +257,18 @@ fn elts_to_csv(elts: &[Expr], generator: Generator) -> Option { } let node = Expr::from(ast::StringLiteral { - value: elts.iter().fold(String::new(), |mut acc, elt| { - if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = elt { - if !acc.is_empty() { - acc.push(','); + value: elts + .iter() + .fold(String::new(), |mut acc, elt| { + if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = elt { + if !acc.is_empty() { + acc.push(','); + } + acc.push_str(value.to_str()); } - acc.push_str(value.to_str()); - } - acc - }), + acc + }) + .into_boxed_str(), ..ast::StringLiteral::default() }); Some(generator.expr(&node)) @@ -327,7 +330,7 @@ fn check_names(checker: &mut Checker, decorator: &Decorator, expr: &Expr) { .iter() .map(|name| { Expr::from(ast::StringLiteral { - value: (*name).to_string(), + value: (*name).to_string().into_boxed_str(), ..ast::StringLiteral::default() }) }) @@ -360,7 +363,7 @@ fn check_names(checker: &mut Checker, decorator: &Decorator, expr: &Expr) { .iter() .map(|name| { Expr::from(ast::StringLiteral { - value: (*name).to_string(), + value: (*name).to_string().into_boxed_str(), ..ast::StringLiteral::default() }) }) diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_expr.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_expr.rs index 46d41465bb8c6..669be14149ccc 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_expr.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_expr.rs @@ -217,7 +217,7 @@ fn check_os_environ_subscript(checker: &mut Checker, expr: &Expr) { slice.range(), ); let node = ast::StringLiteral { - value: capital_env_var, + value: capital_env_var.into_boxed_str(), unicode: env_var.is_unicode(), ..ast::StringLiteral::default() }; diff --git a/crates/ruff_linter/src/rules/flynt/rules/static_join_to_fstring.rs b/crates/ruff_linter/src/rules/flynt/rules/static_join_to_fstring.rs index 86c77bbb0ed73..bf0ca3d0565a1 100644 --- a/crates/ruff_linter/src/rules/flynt/rules/static_join_to_fstring.rs +++ b/crates/ruff_linter/src/rules/flynt/rules/static_join_to_fstring.rs @@ -72,7 +72,8 @@ fn build_fstring(joiner: &str, joinees: &[Expr]) -> Option { None } }) - .join(joiner), + .join(joiner) + .into_boxed_str(), ..ast::StringLiteral::default() }; return Some(node.into()); diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/invalid_escape_sequence.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/invalid_escape_sequence.rs index c227c536c7b23..5571d059deec6 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/invalid_escape_sequence.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/invalid_escape_sequence.rs @@ -74,7 +74,7 @@ pub(crate) fn invalid_escape_sequence( let Some(range) = indexer.fstring_ranges().innermost(token_range.start()) else { return; }; - (value.as_str(), range.start()) + (&**value, range.start()) } Tok::String { kind, .. } => { if kind.is_raw() { diff --git a/crates/ruff_linter/src/rules/pylint/rules/unspecified_encoding.rs b/crates/ruff_linter/src/rules/pylint/rules/unspecified_encoding.rs index b6728df692415..e1dd8284055d8 100644 --- a/crates/ruff_linter/src/rules/pylint/rules/unspecified_encoding.rs +++ b/crates/ruff_linter/src/rules/pylint/rules/unspecified_encoding.rs @@ -110,7 +110,7 @@ fn generate_keyword_fix(checker: &Checker, call: &ast::ExprCall) -> Fix { .generator() .expr(&Expr::StringLiteral(ast::ExprStringLiteral { value: ast::StringLiteralValue::single(ast::StringLiteral { - value: "locale".to_string(), + value: "locale".to_string().into_boxed_str(), unicode: false, range: TextRange::default(), }), diff --git a/crates/ruff_linter/src/rules/pyupgrade/fixes.rs b/crates/ruff_linter/src/rules/pyupgrade/fixes.rs index 59acb3f2ebfdb..7f259e2f9a30f 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/fixes.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/fixes.rs @@ -21,7 +21,7 @@ pub(crate) fn remove_import_members(contents: &str, members: &[&str]) -> String let last_range = names.last_mut().unwrap(); *last_range = TextRange::new(last_range.start(), range.end()); } else { - if members.contains(&name.as_str()) { + if members.contains(&&**name) { removal_indices.push(names.len()); } names.push(range); diff --git a/crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs b/crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs index da082966d1d21..f75fecd730a88 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs @@ -559,14 +559,14 @@ fn collect_string_sequence_lines( /// `self` and produces the classification for the line. #[derive(Debug, Default)] struct LineState { - first_item_in_line: Option<(String, TextRange)>, - following_items_in_line: Vec<(String, TextRange)>, + first_item_in_line: Option<(Box, TextRange)>, + following_items_in_line: Vec<(Box, TextRange)>, comment_range_start: Option, comment_in_line: Option, } impl LineState { - fn visit_string_token(&mut self, token_value: String, token_range: TextRange) { + fn visit_string_token(&mut self, token_value: Box, token_range: TextRange) { if self.first_item_in_line.is_none() { self.first_item_in_line = Some((token_value, token_range)); } else { @@ -631,8 +631,8 @@ struct LineWithItems { // For elements in the list, we keep track of the value of the // value of the element as well as the source-code range of the element. // (We need to know the actual value so that we can sort the items.) - first_item: (String, TextRange), - following_items: Vec<(String, TextRange)>, + first_item: (Box, TextRange), + following_items: Vec<(Box, TextRange)>, // For comments, we only need to keep track of the source-code range. trailing_comment_range: Option, } @@ -753,7 +753,7 @@ fn collect_string_sequence_items( /// source-code range of `"a"`. #[derive(Debug)] struct StringSequenceItem { - value: String, + value: Box, preceding_comment_ranges: Vec, element_range: TextRange, // total_range incorporates the ranges of preceding comments @@ -766,7 +766,7 @@ struct StringSequenceItem { impl StringSequenceItem { fn new( - value: String, + value: Box, preceding_comment_ranges: Vec, element_range: TextRange, end_of_line_comments: Option, @@ -787,7 +787,7 @@ impl StringSequenceItem { } } - fn with_no_comments(value: String, element_range: TextRange) -> Self { + fn with_no_comments(value: Box, element_range: TextRange) -> Self { Self::new(value, vec![], element_range, None) } } diff --git a/crates/ruff_python_ast/src/comparable.rs b/crates/ruff_python_ast/src/comparable.rs index b3c7faf116a5c..bc6327f01dca0 100644 --- a/crates/ruff_python_ast/src/comparable.rs +++ b/crates/ruff_python_ast/src/comparable.rs @@ -631,7 +631,7 @@ pub struct ComparableStringLiteral<'a> { impl<'a> From<&'a ast::StringLiteral> for ComparableStringLiteral<'a> { fn from(string_literal: &'a ast::StringLiteral) -> Self { Self { - value: string_literal.value.as_str(), + value: &string_literal.value, } } } @@ -1089,10 +1089,7 @@ impl<'a> From<&'a ast::Expr> for ComparableExpr<'a> { kind, value, range: _, - }) => Self::IpyEscapeCommand(ExprIpyEscapeCommand { - kind: *kind, - value: value.as_str(), - }), + }) => Self::IpyEscapeCommand(ExprIpyEscapeCommand { kind: *kind, value }), } } } @@ -1537,10 +1534,7 @@ impl<'a> From<&'a ast::Stmt> for ComparableStmt<'a> { kind, value, range: _, - }) => Self::IpyEscapeCommand(StmtIpyEscapeCommand { - kind: *kind, - value: value.as_str(), - }), + }) => Self::IpyEscapeCommand(StmtIpyEscapeCommand { kind: *kind, value }), ast::Stmt::Expr(ast::StmtExpr { value, range: _ }) => Self::Expr(StmtExpr { value: value.into(), }), diff --git a/crates/ruff_python_ast/src/nodes.rs b/crates/ruff_python_ast/src/nodes.rs index 6057d3d64acaa..09f4bf8ddd410 100644 --- a/crates/ruff_python_ast/src/nodes.rs +++ b/crates/ruff_python_ast/src/nodes.rs @@ -160,7 +160,7 @@ pub enum Stmt { pub struct StmtIpyEscapeCommand { pub range: TextRange, pub kind: IpyEscapeKind, - pub value: String, + pub value: Box, } impl From for Stmt { @@ -671,7 +671,7 @@ impl Expr { pub struct ExprIpyEscapeCommand { pub range: TextRange, pub kind: IpyEscapeKind, - pub value: String, + pub value: Box, } impl From for Expr { @@ -1384,7 +1384,7 @@ impl Default for StringLiteralValueInner { #[derive(Clone, Debug, Default, PartialEq)] pub struct StringLiteral { pub range: TextRange, - pub value: String, + pub value: Box, pub unicode: bool, } @@ -1398,7 +1398,7 @@ impl Deref for StringLiteral { type Target = str; fn deref(&self) -> &Self::Target { - self.value.as_str() + &self.value } } @@ -1426,14 +1426,16 @@ struct ConcatenatedStringLiteral { /// Each string literal that makes up the concatenated string. strings: Vec, /// The concatenated string value. - value: OnceCell, + value: OnceCell>, } impl ConcatenatedStringLiteral { /// Extracts a string slice containing the entire concatenated string. fn to_str(&self) -> &str { - self.value - .get_or_init(|| self.strings.iter().map(StringLiteral::as_str).collect()) + self.value.get_or_init(|| { + let concatenated: String = self.strings.iter().map(StringLiteral::as_str).collect(); + concatenated.into_boxed_str() + }) } } diff --git a/crates/ruff_python_formatter/src/lib.rs b/crates/ruff_python_formatter/src/lib.rs index 69daf521090f9..011ba245a91eb 100644 --- a/crates/ruff_python_formatter/src/lib.rs +++ b/crates/ruff_python_formatter/src/lib.rs @@ -134,8 +134,8 @@ pub fn format_module_source( let source_type = options.source_type(); let (tokens, comment_ranges) = tokens_and_ranges(source, source_type).map_err(|err| ParseError { - offset: err.location, - error: ParseErrorType::Lexical(err.error), + offset: err.location(), + error: ParseErrorType::Lexical(err.into_error()), })?; let module = parse_tokens(tokens, source, source_type.as_mode())?; let formatted = format_module_ast(&module, &comment_ranges, source, options)?; diff --git a/crates/ruff_python_formatter/src/range.rs b/crates/ruff_python_formatter/src/range.rs index 2bdb34f71a266..77a17c55873dc 100644 --- a/crates/ruff_python_formatter/src/range.rs +++ b/crates/ruff_python_formatter/src/range.rs @@ -73,8 +73,8 @@ pub fn format_range( let (tokens, comment_ranges) = tokens_and_ranges(source, options.source_type()).map_err(|err| ParseError { - offset: err.location, - error: ParseErrorType::Lexical(err.error), + offset: err.location(), + error: ParseErrorType::Lexical(err.into_error()), })?; assert_valid_char_boundaries(range, source); diff --git a/crates/ruff_python_formatter/tests/normalizer.rs b/crates/ruff_python_formatter/tests/normalizer.rs index 2bab8915cc054..5a7b769f3e054 100644 --- a/crates/ruff_python_formatter/tests/normalizer.rs +++ b/crates/ruff_python_formatter/tests/normalizer.rs @@ -95,19 +95,22 @@ impl Transformer for Normalizer { &string_literal.value, "\n", ) - .into_owned(); + .into_owned() + .into_boxed_str(); string_literal.value = STRIP_RST_BLOCKS .replace_all( &string_literal.value, "\n", ) - .into_owned(); + .into_owned() + .into_boxed_str(); string_literal.value = STRIP_MARKDOWN_BLOCKS .replace_all( &string_literal.value, "\n", ) - .into_owned(); + .into_owned() + .into_boxed_str(); // Normalize a string by (2) stripping any leading and trailing space from each // line, and (3) removing any blank lines from the start and end of the string. string_literal.value = string_literal @@ -117,6 +120,7 @@ impl Transformer for Normalizer { .collect::>() .join("\n") .trim() - .to_owned(); + .to_owned() + .into_boxed_str(); } } diff --git a/crates/ruff_python_parser/src/function.rs b/crates/ruff_python_parser/src/function.rs index 633b62132d626..1700066165e4b 100644 --- a/crates/ruff_python_parser/src/function.rs +++ b/crates/ruff_python_parser/src/function.rs @@ -39,10 +39,10 @@ pub(crate) fn validate_arguments(arguments: &ast::Parameters) -> Result<(), Lexi let range = arg.range; let arg_name = arg.name.as_str(); if !all_arg_names.insert(arg_name) { - return Err(LexicalError { - error: LexicalErrorType::DuplicateArgumentError(arg_name.to_string()), - location: range.start(), - }); + return Err(LexicalError::new( + LexicalErrorType::DuplicateArgumentError(arg_name.to_string().into_boxed_str()), + range.start(), + )); } } @@ -64,10 +64,10 @@ pub(crate) fn validate_pos_params( .skip_while(|arg| arg.default.is_some()) // and then args with default .next(); // there must not be any more args without default if let Some(invalid) = first_invalid { - return Err(LexicalError { - error: LexicalErrorType::DefaultArgumentError, - location: invalid.parameter.start(), - }); + return Err(LexicalError::new( + LexicalErrorType::DefaultArgumentError, + invalid.parameter.start(), + )); } Ok(()) } @@ -94,12 +94,12 @@ pub(crate) fn parse_arguments( // Check for duplicate keyword arguments in the call. if let Some(keyword_name) = &name { if !keyword_names.insert(keyword_name.to_string()) { - return Err(LexicalError { - error: LexicalErrorType::DuplicateKeywordArgumentError( - keyword_name.to_string(), + return Err(LexicalError::new( + LexicalErrorType::DuplicateKeywordArgumentError( + keyword_name.to_string().into_boxed_str(), ), - location: start, - }); + start, + )); } } else { double_starred = true; @@ -113,17 +113,17 @@ pub(crate) fn parse_arguments( } else { // Positional arguments mustn't follow keyword arguments. if !keywords.is_empty() && !is_starred(&value) { - return Err(LexicalError { - error: LexicalErrorType::PositionalArgumentError, - location: value.start(), - }); + return Err(LexicalError::new( + LexicalErrorType::PositionalArgumentError, + value.start(), + )); // Allow starred arguments after keyword arguments but // not after double-starred arguments. } else if double_starred { - return Err(LexicalError { - error: LexicalErrorType::UnpackedArgumentError, - location: value.start(), - }); + return Err(LexicalError::new( + LexicalErrorType::UnpackedArgumentError, + value.start(), + )); } args.push(value); @@ -202,22 +202,22 @@ mod tests { function_and_lambda_error! { // Check definitions - test_duplicates_f1: "def f(a, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_f2: "def f(a, *, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_f3: "def f(a, a=20): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_f4: "def f(a, *a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_f5: "def f(a, *, **a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_l1: "lambda a, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_l2: "lambda a, *, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_l3: "lambda a, a=20: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_l4: "lambda a, *a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_l5: "lambda a, *, **a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), + test_duplicates_f1: "def f(a, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_f2: "def f(a, *, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_f3: "def f(a, a=20): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_f4: "def f(a, *a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_f5: "def f(a, *, **a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_l1: "lambda a, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_l2: "lambda a, *, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_l3: "lambda a, a=20: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_l4: "lambda a, *a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_l5: "lambda a, *, **a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), test_default_arg_error_f: "def f(a, b=20, c): pass", LexicalErrorType::DefaultArgumentError, test_default_arg_error_l: "lambda a, b=20, c: 1", LexicalErrorType::DefaultArgumentError, // Check some calls. test_positional_arg_error_f: "f(b=20, c)", LexicalErrorType::PositionalArgumentError, test_unpacked_arg_error_f: "f(**b, *c)", LexicalErrorType::UnpackedArgumentError, - test_duplicate_kw_f1: "f(a=20, a=30)", LexicalErrorType::DuplicateKeywordArgumentError("a".to_string()), + test_duplicate_kw_f1: "f(a=20, a=30)", LexicalErrorType::DuplicateKeywordArgumentError("a".to_string().into_boxed_str()), } } diff --git a/crates/ruff_python_parser/src/invalid.rs b/crates/ruff_python_parser/src/invalid.rs index 2075a6e08917a..909e6faf17e35 100644 --- a/crates/ruff_python_parser/src/invalid.rs +++ b/crates/ruff_python_parser/src/invalid.rs @@ -39,7 +39,7 @@ pub(crate) fn assignment_target(target: &Expr) -> Result<(), LexicalError> { let err = |location: TextSize| -> LexicalError { let error = LexicalErrorType::AssignmentError; - LexicalError { error, location } + LexicalError::new(error, location) }; match *target { BoolOp(ref e) => Err(err(e.range.start())), diff --git a/crates/ruff_python_parser/src/lexer.rs b/crates/ruff_python_parser/src/lexer.rs index 8d5a20b03a628..9e3ab6d34f2c7 100644 --- a/crates/ruff_python_parser/src/lexer.rs +++ b/crates/ruff_python_parser/src/lexer.rs @@ -107,10 +107,10 @@ where fn next(&mut self) -> Option { let result = match self.inner.next()? { Ok((tok, range)) => Ok((tok, range + self.start_offset)), - Err(error) => Err(LexicalError { - location: error.location + self.start_offset, - ..error - }), + Err(error) => { + let location = error.location() + self.start_offset; + Err(LexicalError::new(error.into_error(), location)) + } }; Some(result) @@ -241,7 +241,7 @@ impl<'source> Lexer<'source> { "yield" => Tok::Yield, _ => { return Ok(Tok::Name { - name: text.to_string(), + name: text.to_string().into_boxed_str(), }) } }; @@ -284,10 +284,10 @@ impl<'source> Lexer<'source> { let value = match Int::from_str_radix(number.as_str(), radix.as_u32(), token) { Ok(int) => int, Err(err) => { - return Err(LexicalError { - error: LexicalErrorType::OtherError(format!("{err:?}")), - location: self.token_range().start(), - }); + return Err(LexicalError::new( + LexicalErrorType::OtherError(format!("{err:?}").into_boxed_str()), + self.token_range().start(), + )); } }; Ok(Tok::Int { value }) @@ -309,10 +309,10 @@ impl<'source> Lexer<'source> { number.push('.'); if self.cursor.eat_char('_') { - return Err(LexicalError { - error: LexicalErrorType::OtherError("Invalid Syntax".to_owned()), - location: self.offset() - TextSize::new(1), - }); + return Err(LexicalError::new( + LexicalErrorType::OtherError("Invalid Syntax".to_string().into_boxed_str()), + self.offset() - TextSize::new(1), + )); } self.radix_run(&mut number, Radix::Decimal); @@ -340,9 +340,13 @@ impl<'source> Lexer<'source> { if is_float { // Improvement: Use `Cow` instead of pushing to value text - let value = f64::from_str(number.as_str()).map_err(|_| LexicalError { - error: LexicalErrorType::OtherError("Invalid decimal literal".to_owned()), - location: self.token_start(), + let value = f64::from_str(number.as_str()).map_err(|_| { + LexicalError::new( + LexicalErrorType::OtherError( + "Invalid decimal literal".to_string().into_boxed_str(), + ), + self.token_start(), + ) })?; // Parse trailing 'j': @@ -364,18 +368,20 @@ impl<'source> Lexer<'source> { Ok(value) => { if start_is_zero && value.as_u8() != Some(0) { // Leading zeros in decimal integer literals are not permitted. - return Err(LexicalError { - error: LexicalErrorType::OtherError("Invalid Token".to_owned()), - location: self.token_range().start(), - }); + return Err(LexicalError::new( + LexicalErrorType::OtherError( + "Invalid Token".to_string().into_boxed_str(), + ), + self.token_range().start(), + )); } value } Err(err) => { - return Err(LexicalError { - error: LexicalErrorType::OtherError(format!("{err:?}")), - location: self.token_range().start(), - }) + return Err(LexicalError::new( + LexicalErrorType::OtherError(format!("{err:?}").into_boxed_str()), + self.token_range().start(), + )) } }; Ok(Tok::Int { value }) @@ -411,7 +417,7 @@ impl<'source> Lexer<'source> { let offset = memchr::memchr2(b'\n', b'\r', bytes).unwrap_or(bytes.len()); self.cursor.skip_bytes(offset); - Tok::Comment(self.token_text().to_string()) + Tok::Comment(self.token_text().to_string().into_boxed_str()) } /// Lex a single IPython escape command. @@ -508,12 +514,15 @@ impl<'source> Lexer<'source> { 2 => IpyEscapeKind::Help2, _ => unreachable!("`question_count` is always 1 or 2"), }; - return Tok::IpyEscapeCommand { kind, value }; + return Tok::IpyEscapeCommand { + kind, + value: value.into_boxed_str(), + }; } '\n' | '\r' | EOF_CHAR => { return Tok::IpyEscapeCommand { kind: escape_kind, - value, + value: value.into_boxed_str(), }; } c => { @@ -584,10 +593,10 @@ impl<'source> Lexer<'source> { } else { FStringErrorType::UnterminatedString }; - return Err(LexicalError { - error: LexicalErrorType::FStringError(error), - location: self.offset(), - }); + return Err(LexicalError::new( + LexicalErrorType::FStringError(error), + self.offset(), + )); } '\n' | '\r' if !fstring.is_triple_quoted() => { // If we encounter a newline while we're in a format spec, then @@ -597,10 +606,10 @@ impl<'source> Lexer<'source> { if in_format_spec { break; } - return Err(LexicalError { - error: LexicalErrorType::FStringError(FStringErrorType::UnterminatedString), - location: self.offset(), - }); + return Err(LexicalError::new( + LexicalErrorType::FStringError(FStringErrorType::UnterminatedString), + self.offset(), + )); } '\\' => { self.cursor.bump(); // '\' @@ -673,7 +682,7 @@ impl<'source> Lexer<'source> { normalized }; Ok(Some(Tok::FStringMiddle { - value, + value: value.into_boxed_str(), is_raw: fstring.is_raw_string(), triple_quoted: fstring.is_triple_quoted(), })) @@ -705,18 +714,16 @@ impl<'source> Lexer<'source> { if fstring.quote_char() == quote && fstring.is_triple_quoted() == triple_quoted { - return Err(LexicalError { - error: LexicalErrorType::FStringError( - FStringErrorType::UnclosedLbrace, - ), - location: self.cursor.text_len(), - }); + return Err(LexicalError::new( + LexicalErrorType::FStringError(FStringErrorType::UnclosedLbrace), + self.cursor.text_len(), + )); } } - return Err(LexicalError { - error: LexicalErrorType::Eof, - location: self.cursor.text_len(), - }); + return Err(LexicalError::new( + LexicalErrorType::Eof, + self.cursor.text_len(), + )); }; // Rare case: if there are an odd number of backslashes before the quote, then @@ -756,18 +763,16 @@ impl<'source> Lexer<'source> { if fstring.quote_char() == quote && fstring.is_triple_quoted() == triple_quoted { - return Err(LexicalError { - error: LexicalErrorType::FStringError( - FStringErrorType::UnclosedLbrace, - ), - location: self.offset(), - }); + return Err(LexicalError::new( + LexicalErrorType::FStringError(FStringErrorType::UnclosedLbrace), + self.offset(), + )); } } - return Err(LexicalError { - error: LexicalErrorType::StringError, - location: self.offset(), - }); + return Err(LexicalError::new( + LexicalErrorType::StringError, + self.offset(), + )); }; // Rare case: if there are an odd number of backslashes before the quote, then @@ -797,20 +802,22 @@ impl<'source> Lexer<'source> { // matches with f-strings quotes and if it is, then this must be a // missing '}' token so raise the proper error. if fstring.quote_char() == quote && !fstring.is_triple_quoted() { - return Err(LexicalError { - error: LexicalErrorType::FStringError( + return Err(LexicalError::new( + LexicalErrorType::FStringError( FStringErrorType::UnclosedLbrace, ), - location: self.offset() - TextSize::new(1), - }); + self.offset() - TextSize::new(1), + )); } } - return Err(LexicalError { - error: LexicalErrorType::OtherError( - "EOL while scanning string literal".to_owned(), + return Err(LexicalError::new( + LexicalErrorType::OtherError( + "EOL while scanning string literal" + .to_string() + .into_boxed_str(), ), - location: self.offset() - TextSize::new(1), - }); + self.offset() - TextSize::new(1), + )); } Some(ch) if ch == quote => { break self.offset() - TextSize::new(1); @@ -821,7 +828,9 @@ impl<'source> Lexer<'source> { }; Ok(Tok::String { - value: self.source[TextRange::new(value_start, value_end)].to_string(), + value: self.source[TextRange::new(value_start, value_end)] + .to_string() + .into_boxed_str(), kind, triple_quoted, }) @@ -889,10 +898,10 @@ impl<'source> Lexer<'source> { Ok((identifier, self.token_range())) } else { - Err(LexicalError { - error: LexicalErrorType::UnrecognizedToken { tok: c }, - location: self.token_start(), - }) + Err(LexicalError::new( + LexicalErrorType::UnrecognizedToken { tok: c }, + self.token_start(), + )) } } else { // Reached the end of the file. Emit a trailing newline token if not at the beginning of a logical line, @@ -915,15 +924,12 @@ impl<'source> Lexer<'source> { if self.cursor.eat_char('\r') { self.cursor.eat_char('\n'); } else if self.cursor.is_eof() { - return Err(LexicalError { - error: LexicalErrorType::Eof, - location: self.token_start(), - }); + return Err(LexicalError::new(LexicalErrorType::Eof, self.token_start())); } else if !self.cursor.eat_char('\n') { - return Err(LexicalError { - error: LexicalErrorType::LineContinuationError, - location: self.token_start(), - }); + return Err(LexicalError::new( + LexicalErrorType::LineContinuationError, + self.token_start(), + )); } } // Form feed @@ -956,15 +962,12 @@ impl<'source> Lexer<'source> { if self.cursor.eat_char('\r') { self.cursor.eat_char('\n'); } else if self.cursor.is_eof() { - return Err(LexicalError { - error: LexicalErrorType::Eof, - location: self.token_start(), - }); + return Err(LexicalError::new(LexicalErrorType::Eof, self.token_start())); } else if !self.cursor.eat_char('\n') { - return Err(LexicalError { - error: LexicalErrorType::LineContinuationError, - location: self.token_start(), - }); + return Err(LexicalError::new( + LexicalErrorType::LineContinuationError, + self.token_start(), + )); } indentation = Indentation::root(); } @@ -1015,10 +1018,10 @@ impl<'source> Lexer<'source> { Some((Tok::Indent, self.token_range())) } Err(_) => { - return Err(LexicalError { - error: LexicalErrorType::IndentationError, - location: self.offset(), - }); + return Err(LexicalError::new( + LexicalErrorType::IndentationError, + self.offset(), + )); } }; @@ -1031,10 +1034,7 @@ impl<'source> Lexer<'source> { if self.nesting > 0 { // Reset the nesting to avoid going into infinite loop. self.nesting = 0; - return Err(LexicalError { - error: LexicalErrorType::Eof, - location: self.offset(), - }); + return Err(LexicalError::new(LexicalErrorType::Eof, self.offset())); } // Next, insert a trailing newline, if required. @@ -1199,10 +1199,10 @@ impl<'source> Lexer<'source> { '}' => { if let Some(fstring) = self.fstrings.current_mut() { if fstring.nesting() == self.nesting { - return Err(LexicalError { - error: LexicalErrorType::FStringError(FStringErrorType::SingleRbrace), - location: self.token_start(), - }); + return Err(LexicalError::new( + LexicalErrorType::FStringError(FStringErrorType::SingleRbrace), + self.token_start(), + )); } fstring.try_end_format_spec(self.nesting); } @@ -1293,10 +1293,10 @@ impl<'source> Lexer<'source> { _ => { self.state = State::Other; - return Err(LexicalError { - error: LexicalErrorType::UnrecognizedToken { tok: c }, - location: self.token_start(), - }); + return Err(LexicalError::new( + LexicalErrorType::UnrecognizedToken { tok: c }, + self.token_start(), + )); } }; @@ -1357,9 +1357,9 @@ impl FusedIterator for Lexer<'_> {} #[derive(Debug, Clone, PartialEq)] pub struct LexicalError { /// The type of error that occurred. - pub error: LexicalErrorType, + error: LexicalErrorType, /// The location of the error. - pub location: TextSize, + location: TextSize, } impl LexicalError { @@ -1367,19 +1367,31 @@ impl LexicalError { pub fn new(error: LexicalErrorType, location: TextSize) -> Self { Self { error, location } } + + pub fn error(&self) -> &LexicalErrorType { + &self.error + } + + pub fn into_error(self) -> LexicalErrorType { + self.error + } + + pub fn location(&self) -> TextSize { + self.location + } } impl std::ops::Deref for LexicalError { type Target = LexicalErrorType; fn deref(&self) -> &Self::Target { - &self.error + self.error() } } impl std::error::Error for LexicalError { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - Some(&self.error) + Some(self.error()) } } @@ -1388,8 +1400,8 @@ impl std::fmt::Display for LexicalError { write!( f, "{} at byte offset {}", - &self.error, - u32::from(self.location) + self.error(), + u32::from(self.location()) ) } } @@ -1397,6 +1409,9 @@ impl std::fmt::Display for LexicalError { /// Represents the different types of errors that can occur during lexing. #[derive(Debug, Clone, PartialEq)] pub enum LexicalErrorType { + /// A duplicate argument was found in a function definition. + DuplicateArgumentError(Box), + // TODO: Can probably be removed, the places it is used seem to be able // to use the `UnicodeError` variant instead. #[doc(hidden)] @@ -1414,14 +1429,13 @@ pub enum LexicalErrorType { TabsAfterSpaces, /// A non-default argument follows a default argument. DefaultArgumentError, - /// A duplicate argument was found in a function definition. - DuplicateArgumentError(String), + /// A positional argument follows a keyword argument. PositionalArgumentError, /// An iterable argument unpacking `*args` follows keyword argument unpacking `**kwargs`. UnpackedArgumentError, /// A keyword argument was repeated. - DuplicateKeywordArgumentError(String), + DuplicateKeywordArgumentError(Box), /// An unrecognized token was encountered. UnrecognizedToken { tok: char }, /// An f-string error containing the [`FStringErrorType`]. @@ -1433,7 +1447,7 @@ pub enum LexicalErrorType { /// Occurs when a syntactically invalid assignment was encountered. AssignmentError, /// An unexpected error occurred. - OtherError(String), + OtherError(Box), } impl std::error::Error for LexicalErrorType {} @@ -2053,8 +2067,8 @@ def f(arg=%timeit a = b): match lexed.as_slice() { [Err(error)] => { assert_eq!( - error.error, - LexicalErrorType::UnrecognizedToken { tok: '🐦' } + error.error(), + &LexicalErrorType::UnrecognizedToken { tok: '🐦' } ); } result => panic!("Expected an error token but found {result:?}"), @@ -2267,7 +2281,7 @@ f"{(lambda x:{x})}" } fn lex_fstring_error(source: &str) -> FStringErrorType { - match lex_error(source).error { + match lex_error(source).into_error() { LexicalErrorType::FStringError(error) => error, err => panic!("Expected FStringError: {err:?}"), } diff --git a/crates/ruff_python_parser/src/parser.rs b/crates/ruff_python_parser/src/parser.rs index 2eb0b4bd61bcd..a73b6d12e16b8 100644 --- a/crates/ruff_python_parser/src/parser.rs +++ b/crates/ruff_python_parser/src/parser.rs @@ -285,8 +285,8 @@ fn parse_error_from_lalrpop(err: LalrpopError) -> P offset: token.0, }, LalrpopError::User { error } => ParseError { - error: ParseErrorType::Lexical(error.error), - offset: error.location, + offset: error.location(), + error: ParseErrorType::Lexical(error.into_error()), }, LalrpopError::UnrecognizedToken { token, expected } => { // Hacky, but it's how CPython does it. See PyParser_AddToken, @@ -359,8 +359,8 @@ impl ParseErrorType { impl From for ParseError { fn from(error: LexicalError) -> Self { ParseError { - error: ParseErrorType::Lexical(error.error), - offset: error.location, + offset: error.location(), + error: ParseErrorType::Lexical(error.into_error()), } } } diff --git a/crates/ruff_python_parser/src/python.lalrpop b/crates/ruff_python_parser/src/python.lalrpop index cc9bf71e8a110..386574b0001b7 100644 --- a/crates/ruff_python_parser/src/python.lalrpop +++ b/crates/ruff_python_parser/src/python.lalrpop @@ -289,7 +289,7 @@ ImportAsAlias: ast::Alias = { DottedName: ast::Identifier = { => ast::Identifier::new(n, (location..end_location).into()), => { - let mut r = n; + let mut r = String::from(n); for x in n2 { r.push('.'); r.push_str(x.1.as_str()); @@ -337,10 +337,10 @@ IpyEscapeCommandStatement: ast::Stmt = { } )) } else { - Err(LexicalError { - error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), + Err(LexicalError::new( + LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()), location, - })? + ))? } } } @@ -350,10 +350,10 @@ IpyEscapeCommandExpr: crate::parser::ParenthesizedExpr = { if mode == Mode::Ipython { // This should never occur as the lexer won't allow it. if !matches!(c.0, IpyEscapeKind::Magic | IpyEscapeKind::Shell) { - return Err(LexicalError { - error: LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string()), + return Err(LexicalError::new( + LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string().into_boxed_str()), location, - })?; + ))?; } Ok(ast::ExprIpyEscapeCommand { kind: c.0, @@ -361,10 +361,10 @@ IpyEscapeCommandExpr: crate::parser::ParenthesizedExpr = { range: (location..end_location).into() }.into()) } else { - Err(LexicalError { - error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), + Err(LexicalError::new( + LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()), location, - })? + ))? } } } @@ -381,10 +381,10 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = { }, ast::Expr::Subscript(ast::ExprSubscript { value, slice, range, .. }) => { let ast::Expr::NumberLiteral(ast::ExprNumberLiteral { value: ast::Number::Int(integer), .. }) = slice.as_ref() else { - return Err(LexicalError { - error: LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string()), - location: range.start(), - }); + return Err(LexicalError::new( + LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string().into_boxed_str()), + range.start(), + )); }; unparse_expr(value, buffer)?; buffer.push('['); @@ -397,10 +397,10 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = { buffer.push_str(attr.as_str()); }, _ => { - return Err(LexicalError { - error: LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string()), - location: expr.start(), - }); + return Err(LexicalError::new( + LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string().into_boxed_str()), + expr.start(), + )); } } Ok(()) @@ -408,10 +408,10 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = { if mode != Mode::Ipython { return Err(ParseError::User { - error: LexicalError { - error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), + error: LexicalError::new( + LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()), location, - }, + ), }); } @@ -420,10 +420,10 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = { 2 => IpyEscapeKind::Help2, _ => { return Err(ParseError::User { - error: LexicalError { - error: LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string()), + error: LexicalError::new( + LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string().into_boxed_str()), location, - }, + ), }); } }; @@ -434,7 +434,7 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = { Ok(ast::Stmt::IpyEscapeCommand( ast::StmtIpyEscapeCommand { kind, - value, + value: value.into_boxed_str(), range: (location..end_location).into() } )) @@ -561,10 +561,10 @@ Pattern: ast::Pattern = { AsPattern: ast::Pattern = { "as" =>? { if name.as_str() == "_" { - Err(LexicalError { - error: LexicalErrorType::OtherError("cannot use '_' as a target".to_string()), + Err(LexicalError::new( + LexicalErrorType::OtherError("cannot use '_' as a target".to_string().into_boxed_str()), location, - })? + ))? } else { Ok(ast::Pattern::MatchAs( ast::PatternMatchAs { @@ -1247,10 +1247,10 @@ DoubleStarTypedParameter: ast::Parameter = { ParameterListStarArgs: (Option>, Vec, Option>) = { "*" >)*> >)?> =>? { if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() { - return Err(LexicalError { - error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()), + return Err(LexicalError::new( + LexicalErrorType::OtherError("named arguments must follow bare *".to_string().into_boxed_str()), location, - })?; + ))?; } let kwarg = kwarg.flatten(); @@ -1364,10 +1364,10 @@ NamedExpression: crate::parser::ParenthesizedExpr = { LambdaDef: crate::parser::ParenthesizedExpr = { "lambda" ?> ":" > =>? { if fstring_middle.is_some() { - return Err(LexicalError { - error: LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses), + return Err(LexicalError::new( + LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses), location, - })?; + ))?; } parameters.as_ref().map(validate_arguments).transpose()?; @@ -1630,10 +1630,10 @@ FStringMiddlePattern: ast::FStringElement = { FStringReplacementField: ast::FStringElement = { "{" "}" =>? { if value.expr.is_lambda_expr() && !value.is_parenthesized() { - return Err(LexicalError { - error: LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses), - location: value.start(), - })?; + return Err(LexicalError::new( + LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses), + value.start(), + ))?; } let debug_text = debug.map(|_| { let start_offset = location + "{".text_len(); @@ -1677,14 +1677,14 @@ FStringFormatSpec: ast::FStringFormatSpec = { FStringConversion: (TextSize, ast::ConversionFlag) = { "!" =>? { - let conversion = match s.as_str() { + let conversion = match s.as_ref() { "s" => ast::ConversionFlag::Str, "r" => ast::ConversionFlag::Repr, "a" => ast::ConversionFlag::Ascii, - _ => Err(LexicalError { - error: LexicalErrorType::FStringError(FStringErrorType::InvalidConversionFlag), - location: name_location, - })? + _ => Err(LexicalError::new( + LexicalErrorType::FStringError(FStringErrorType::InvalidConversionFlag), + name_location, + ))? }; Ok((location, conversion)) } @@ -1722,10 +1722,10 @@ Atom: crate::parser::ParenthesizedExpr = { "(" >> ",")?> )*> ")" =>? { if left.is_none() && right.is_empty() && trailing_comma.is_none() { if mid.expr.is_starred_expr() { - return Err(LexicalError{ - error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()), - location: mid.start(), - })?; + return Err(LexicalError::new( + LexicalErrorType::OtherError("cannot use starred expression here".to_string().into_boxed_str()), + mid.start(), + ))?; } Ok(crate::parser::ParenthesizedExpr { expr: mid.into(), @@ -1751,10 +1751,10 @@ Atom: crate::parser::ParenthesizedExpr = { range: (location..end_location).into(), }.into(), "(" "**" > ")" =>? { - Err(LexicalError{ - error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()), + Err(LexicalError::new( + LexicalErrorType::OtherError("cannot use double starred expression here".to_string().into_boxed_str()), location, - }.into()) + ).into()) }, "{" "}" => { let (keys, values) = e @@ -2061,19 +2061,19 @@ extern { float => token::Tok::Float { value: }, complex => token::Tok::Complex { real: , imag: }, string => token::Tok::String { - value: , + value: >, kind: , triple_quoted: }, fstring_middle => token::Tok::FStringMiddle { - value: , + value: >, is_raw: , triple_quoted: }, - name => token::Tok::Name { name: }, + name => token::Tok::Name { name: > }, ipy_escape_command => token::Tok::IpyEscapeCommand { kind: , - value: + value: > }, "\n" => token::Tok::Newline, ";" => token::Tok::Semi, diff --git a/crates/ruff_python_parser/src/python.rs b/crates/ruff_python_parser/src/python.rs index c409f91eeebf4..abe55991b201a 100644 --- a/crates/ruff_python_parser/src/python.rs +++ b/crates/ruff_python_parser/src/python.rs @@ -1,5 +1,5 @@ // auto-generated: "lalrpop 0.20.0" -// sha3: aa0540221d25f4eadfc9e043fb4fc631d537b672b8a96785dfec2407e0524b79 +// sha3: fd05d84d3b654796ff740a7f905ec0ae8915f43f952428717735481947ab55e1 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use ruff_python_ast::{self as ast, Int, IpyEscapeKind}; use crate::{ @@ -50,11 +50,11 @@ mod __parse__Top { Variant0(token::Tok), Variant1((f64, f64)), Variant2(f64), - Variant3((String, bool, bool)), + Variant3((Box, bool, bool)), Variant4(Int), - Variant5((IpyEscapeKind, String)), - Variant6(String), - Variant7((String, StringKind, bool)), + Variant5((IpyEscapeKind, Box)), + Variant6(Box), + Variant7((Box, StringKind, bool)), Variant8(core::option::Option), Variant9(Option>), Variant10(core::option::Option>>), @@ -151,7 +151,7 @@ mod __parse__Top { Variant101(ast::TypeParams), Variant102(core::option::Option), Variant103(ast::UnaryOp), - Variant104(core::option::Option<(String, bool, bool)>), + Variant104(core::option::Option<(Box, bool, bool)>), } const __ACTION: &[i16] = &[ // State 0 @@ -18323,73 +18323,73 @@ mod __parse__Top { fn __symbol_type_mismatch() -> ! { panic!("symbol type mismatch") } - fn __pop_Variant5< + fn __pop_Variant7< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, (IpyEscapeKind, String), TextSize) + ) -> (TextSize, (Box, StringKind, bool), TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant5(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant7(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } - fn __pop_Variant31< + fn __pop_Variant3< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, (Option<(TextSize, TextSize, Option)>, ast::Expr), TextSize) + ) -> (TextSize, (Box, bool, bool), TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant31(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant3(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } - fn __pop_Variant13< + fn __pop_Variant5< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, (Option>, Vec, Option>), TextSize) + ) -> (TextSize, (IpyEscapeKind, Box), TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant13(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant5(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } - fn __pop_Variant59< + fn __pop_Variant31< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, (Option>, crate::parser::ParenthesizedExpr), TextSize) + ) -> (TextSize, (Option<(TextSize, TextSize, Option)>, ast::Expr), TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant59(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant31(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } - fn __pop_Variant79< + fn __pop_Variant13< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, (Option, Option), TextSize) + ) -> (TextSize, (Option>, Vec, Option>), TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant79(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant13(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } - fn __pop_Variant7< + fn __pop_Variant59< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, (String, StringKind, bool), TextSize) + ) -> (TextSize, (Option>, crate::parser::ParenthesizedExpr), TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant7(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant59(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } - fn __pop_Variant3< + fn __pop_Variant79< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, (String, bool, bool), TextSize) + ) -> (TextSize, (Option, Option), TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant3(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant79(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } @@ -18493,6 +18493,16 @@ mod __parse__Top { _ => __symbol_type_mismatch() } } + fn __pop_Variant6< + >( + __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> + ) -> (TextSize, Box, TextSize) + { + match __symbols.pop() { + Some((__l, __Symbol::Variant6(__v), __r)) => (__l, __v, __r), + _ => __symbol_type_mismatch() + } + } fn __pop_Variant4< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> @@ -18523,16 +18533,6 @@ mod __parse__Top { _ => __symbol_type_mismatch() } } - fn __pop_Variant6< - >( - __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, String, TextSize) - { - match __symbols.pop() { - Some((__l, __Symbol::Variant6(__v), __r)) => (__l, __v, __r), - _ => __symbol_type_mismatch() - } - } fn __pop_Variant69< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> @@ -19113,33 +19113,33 @@ mod __parse__Top { _ => __symbol_type_mismatch() } } - fn __pop_Variant74< + fn __pop_Variant104< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, core::option::Option<(Option<(TextSize, TextSize, Option)>, ast::Expr)>, TextSize) + ) -> (TextSize, core::option::Option<(Box, bool, bool)>, TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant74(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant104(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } - fn __pop_Variant14< + fn __pop_Variant74< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, core::option::Option<(Option>, Vec, Option>)>, TextSize) + ) -> (TextSize, core::option::Option<(Option<(TextSize, TextSize, Option)>, ast::Expr)>, TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant14(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant74(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } - fn __pop_Variant104< + fn __pop_Variant14< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, core::option::Option<(String, bool, bool)>, TextSize) + ) -> (TextSize, core::option::Option<(Option>, Vec, Option>)>, TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant104(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant14(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } @@ -33541,7 +33541,7 @@ fn __action69< source_code: &str, mode: Mode, (_, location, _): (TextSize, TextSize, TextSize), - (_, n, _): (TextSize, String, TextSize), + (_, n, _): (TextSize, Box, TextSize), (_, end_location, _): (TextSize, TextSize, TextSize), ) -> ast::Identifier { @@ -33555,13 +33555,13 @@ fn __action70< source_code: &str, mode: Mode, (_, location, _): (TextSize, TextSize, TextSize), - (_, n, _): (TextSize, String, TextSize), + (_, n, _): (TextSize, Box, TextSize), (_, n2, _): (TextSize, alloc::vec::Vec<(token::Tok, ast::Identifier)>, TextSize), (_, end_location, _): (TextSize, TextSize, TextSize), ) -> ast::Identifier { { - let mut r = n; + let mut r = String::from(n); for x in n2 { r.push('.'); r.push_str(x.1.as_str()); @@ -33639,7 +33639,7 @@ fn __action74< source_code: &str, mode: Mode, (_, location, _): (TextSize, TextSize, TextSize), - (_, c, _): (TextSize, (IpyEscapeKind, String), TextSize), + (_, c, _): (TextSize, (IpyEscapeKind, Box), TextSize), (_, end_location, _): (TextSize, TextSize, TextSize), ) -> Result> { @@ -33653,10 +33653,10 @@ fn __action74< } )) } else { - Err(LexicalError { - error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), + Err(LexicalError::new( + LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()), location, - })? + ))? } } } @@ -33668,7 +33668,7 @@ fn __action75< source_code: &str, mode: Mode, (_, location, _): (TextSize, TextSize, TextSize), - (_, c, _): (TextSize, (IpyEscapeKind, String), TextSize), + (_, c, _): (TextSize, (IpyEscapeKind, Box), TextSize), (_, end_location, _): (TextSize, TextSize, TextSize), ) -> Result> { @@ -33676,10 +33676,10 @@ fn __action75< if mode == Mode::Ipython { // This should never occur as the lexer won't allow it. if !matches!(c.0, IpyEscapeKind::Magic | IpyEscapeKind::Shell) { - return Err(LexicalError { - error: LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string()), + return Err(LexicalError::new( + LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string().into_boxed_str()), location, - })?; + ))?; } Ok(ast::ExprIpyEscapeCommand { kind: c.0, @@ -33687,10 +33687,10 @@ fn __action75< range: (location..end_location).into() }.into()) } else { - Err(LexicalError { - error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), + Err(LexicalError::new( + LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()), location, - })? + ))? } } } @@ -33715,10 +33715,10 @@ fn __action76< }, ast::Expr::Subscript(ast::ExprSubscript { value, slice, range, .. }) => { let ast::Expr::NumberLiteral(ast::ExprNumberLiteral { value: ast::Number::Int(integer), .. }) = slice.as_ref() else { - return Err(LexicalError { - error: LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string()), - location: range.start(), - }); + return Err(LexicalError::new( + LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string().into_boxed_str()), + range.start(), + )); }; unparse_expr(value, buffer)?; buffer.push('['); @@ -33731,10 +33731,10 @@ fn __action76< buffer.push_str(attr.as_str()); }, _ => { - return Err(LexicalError { - error: LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string()), - location: expr.start(), - }); + return Err(LexicalError::new( + LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string().into_boxed_str()), + expr.start(), + )); } } Ok(()) @@ -33742,10 +33742,10 @@ fn __action76< if mode != Mode::Ipython { return Err(ParseError::User { - error: LexicalError { - error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), + error: LexicalError::new( + LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()), location, - }, + ), }); } @@ -33754,10 +33754,10 @@ fn __action76< 2 => IpyEscapeKind::Help2, _ => { return Err(ParseError::User { - error: LexicalError { - error: LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string()), + error: LexicalError::new( + LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string().into_boxed_str()), location, - }, + ), }); } }; @@ -33768,7 +33768,7 @@ fn __action76< Ok(ast::Stmt::IpyEscapeCommand( ast::StmtIpyEscapeCommand { kind, - value, + value: value.into_boxed_str(), range: (location..end_location).into() } )) @@ -34126,10 +34126,10 @@ fn __action95< { { if name.as_str() == "_" { - Err(LexicalError { - error: LexicalErrorType::OtherError("cannot use '_' as a target".to_string()), + Err(LexicalError::new( + LexicalErrorType::OtherError("cannot use '_' as a target".to_string().into_boxed_str()), location, - })? + ))? } else { Ok(ast::Pattern::MatchAs( ast::PatternMatchAs { @@ -35910,17 +35910,17 @@ fn __action184< (_, parameters, _): (TextSize, core::option::Option, TextSize), (_, end_location_args, _): (TextSize, TextSize, TextSize), (_, _, _): (TextSize, token::Tok, TextSize), - (_, fstring_middle, _): (TextSize, core::option::Option<(String, bool, bool)>, TextSize), + (_, fstring_middle, _): (TextSize, core::option::Option<(Box, bool, bool)>, TextSize), (_, body, _): (TextSize, crate::parser::ParenthesizedExpr, TextSize), (_, end_location, _): (TextSize, TextSize, TextSize), ) -> Result> { { if fstring_middle.is_some() { - return Err(LexicalError { - error: LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses), + return Err(LexicalError::new( + LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses), location, - })?; + ))?; } parameters.as_ref().map(validate_arguments).transpose()?; @@ -36363,7 +36363,7 @@ fn __action217< source_code: &str, mode: Mode, (_, location, _): (TextSize, TextSize, TextSize), - (_, string, _): (TextSize, (String, StringKind, bool), TextSize), + (_, string, _): (TextSize, (Box, StringKind, bool), TextSize), (_, end_location, _): (TextSize, TextSize, TextSize), ) -> Result> { @@ -36413,7 +36413,7 @@ fn __action220< source_code: &str, mode: Mode, (_, location, _): (TextSize, TextSize, TextSize), - (_, fstring_middle, _): (TextSize, (String, bool, bool), TextSize), + (_, fstring_middle, _): (TextSize, (Box, bool, bool), TextSize), (_, end_location, _): (TextSize, TextSize, TextSize), ) -> Result> { @@ -36441,10 +36441,10 @@ fn __action221< { { if value.expr.is_lambda_expr() && !value.is_parenthesized() { - return Err(LexicalError { - error: LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses), - location: value.start(), - })?; + return Err(LexicalError::new( + LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses), + value.start(), + ))?; } let debug_text = debug.map(|_| { let start_offset = location + "{".text_len(); @@ -36514,18 +36514,18 @@ fn __action224< (_, location, _): (TextSize, TextSize, TextSize), (_, _, _): (TextSize, token::Tok, TextSize), (_, name_location, _): (TextSize, TextSize, TextSize), - (_, s, _): (TextSize, String, TextSize), + (_, s, _): (TextSize, Box, TextSize), ) -> Result<(TextSize, ast::ConversionFlag),__lalrpop_util::ParseError> { { - let conversion = match s.as_str() { + let conversion = match s.as_ref() { "s" => ast::ConversionFlag::Str, "r" => ast::ConversionFlag::Repr, "a" => ast::ConversionFlag::Ascii, - _ => Err(LexicalError { - error: LexicalErrorType::FStringError(FStringErrorType::InvalidConversionFlag), - location: name_location, - })? + _ => Err(LexicalError::new( + LexicalErrorType::FStringError(FStringErrorType::InvalidConversionFlag), + name_location, + ))? }; Ok((location, conversion)) } @@ -36899,7 +36899,7 @@ fn __action249< source_code: &str, mode: Mode, (_, location, _): (TextSize, TextSize, TextSize), - (_, s, _): (TextSize, String, TextSize), + (_, s, _): (TextSize, Box, TextSize), (_, end_location, _): (TextSize, TextSize, TextSize), ) -> ast::Identifier { @@ -37357,8 +37357,8 @@ fn __action281< >( source_code: &str, mode: Mode, - (_, __0, _): (TextSize, (String, bool, bool), TextSize), -) -> core::option::Option<(String, bool, bool)> + (_, __0, _): (TextSize, (Box, bool, bool), TextSize), +) -> core::option::Option<(Box, bool, bool)> { Some(__0) } @@ -37371,7 +37371,7 @@ fn __action282< mode: Mode, __lookbehind: &TextSize, __lookahead: &TextSize, -) -> core::option::Option<(String, bool, bool)> +) -> core::option::Option<(Box, bool, bool)> { None } @@ -39668,10 +39668,10 @@ fn __action445< { { if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() { - return Err(LexicalError { - error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()), + return Err(LexicalError::new( + LexicalErrorType::OtherError("named arguments must follow bare *".to_string().into_boxed_str()), location, - })?; + ))?; } let kwarg = kwarg.flatten(); @@ -39793,10 +39793,10 @@ fn __action453< { { if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() { - return Err(LexicalError { - error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()), + return Err(LexicalError::new( + LexicalErrorType::OtherError("named arguments must follow bare *".to_string().into_boxed_str()), location, - })?; + ))?; } let kwarg = kwarg.flatten(); @@ -41296,10 +41296,10 @@ fn __action554< { if left.is_none() && right.is_empty() && trailing_comma.is_none() { if mid.expr.is_starred_expr() { - return Err(LexicalError{ - error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()), - location: mid.start(), - })?; + return Err(LexicalError::new( + LexicalErrorType::OtherError("cannot use starred expression here".to_string().into_boxed_str()), + mid.start(), + ))?; } Ok(crate::parser::ParenthesizedExpr { expr: mid.into(), @@ -41386,10 +41386,10 @@ fn __action558< ) -> Result> { { - Err(LexicalError{ - error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()), + Err(LexicalError::new( + LexicalErrorType::OtherError("cannot use double starred expression here".to_string().into_boxed_str()), location, - }.into()) + ).into()) } } @@ -41994,10 +41994,10 @@ fn __action596< { if left.is_none() && right.is_empty() && trailing_comma.is_none() { if mid.expr.is_starred_expr() { - return Err(LexicalError{ - error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()), - location: mid.start(), - })?; + return Err(LexicalError::new( + LexicalErrorType::OtherError("cannot use starred expression here".to_string().into_boxed_str()), + mid.start(), + ))?; } Ok(crate::parser::ParenthesizedExpr { expr: mid.into(), @@ -42084,10 +42084,10 @@ fn __action600< ) -> Result> { { - Err(LexicalError{ - error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()), + Err(LexicalError::new( + LexicalErrorType::OtherError("cannot use double starred expression here".to_string().into_boxed_str()), location, - }.into()) + ).into()) } } @@ -48027,7 +48027,7 @@ fn __action789< >( source_code: &str, mode: Mode, - __0: (TextSize, String, TextSize), + __0: (TextSize, Box, TextSize), __1: (TextSize, TextSize, TextSize), ) -> ast::Identifier { @@ -48055,7 +48055,7 @@ fn __action790< >( source_code: &str, mode: Mode, - __0: (TextSize, String, TextSize), + __0: (TextSize, Box, TextSize), __1: (TextSize, alloc::vec::Vec<(token::Tok, ast::Identifier)>, TextSize), __2: (TextSize, TextSize, TextSize), ) -> ast::Identifier @@ -48408,7 +48408,7 @@ fn __action801< source_code: &str, mode: Mode, __0: (TextSize, token::Tok, TextSize), - __1: (TextSize, String, TextSize), + __1: (TextSize, Box, TextSize), ) -> Result<(TextSize, ast::ConversionFlag),__lalrpop_util::ParseError> { let __start0 = __0.0; @@ -48505,7 +48505,7 @@ fn __action804< >( source_code: &str, mode: Mode, - __0: (TextSize, (String, bool, bool), TextSize), + __0: (TextSize, (Box, bool, bool), TextSize), __1: (TextSize, TextSize, TextSize), ) -> Result> { @@ -49209,7 +49209,7 @@ fn __action826< >( source_code: &str, mode: Mode, - __0: (TextSize, String, TextSize), + __0: (TextSize, Box, TextSize), __1: (TextSize, TextSize, TextSize), ) -> ast::Identifier { @@ -49519,7 +49519,7 @@ fn __action836< >( source_code: &str, mode: Mode, - __0: (TextSize, (IpyEscapeKind, String), TextSize), + __0: (TextSize, (IpyEscapeKind, Box), TextSize), __1: (TextSize, TextSize, TextSize), ) -> Result> { @@ -49547,7 +49547,7 @@ fn __action837< >( source_code: &str, mode: Mode, - __0: (TextSize, (IpyEscapeKind, String), TextSize), + __0: (TextSize, (IpyEscapeKind, Box), TextSize), __1: (TextSize, TextSize, TextSize), ) -> Result> { @@ -49609,7 +49609,7 @@ fn __action839< __1: (TextSize, core::option::Option, TextSize), __2: (TextSize, TextSize, TextSize), __3: (TextSize, token::Tok, TextSize), - __4: (TextSize, core::option::Option<(String, bool, bool)>, TextSize), + __4: (TextSize, core::option::Option<(Box, bool, bool)>, TextSize), __5: (TextSize, crate::parser::ParenthesizedExpr, TextSize), __6: (TextSize, TextSize, TextSize), ) -> Result> @@ -52719,7 +52719,7 @@ fn __action937< >( source_code: &str, mode: Mode, - __0: (TextSize, (String, StringKind, bool), TextSize), + __0: (TextSize, (Box, StringKind, bool), TextSize), __1: (TextSize, TextSize, TextSize), ) -> Result> { @@ -64211,7 +64211,7 @@ fn __action1304< >( source_code: &str, mode: Mode, - __0: (TextSize, String, TextSize), + __0: (TextSize, Box, TextSize), ) -> ast::Identifier { let __start0 = __0.2; @@ -64237,7 +64237,7 @@ fn __action1305< >( source_code: &str, mode: Mode, - __0: (TextSize, String, TextSize), + __0: (TextSize, Box, TextSize), __1: (TextSize, alloc::vec::Vec<(token::Tok, ast::Identifier)>, TextSize), ) -> ast::Identifier { @@ -64527,7 +64527,7 @@ fn __action1315< >( source_code: &str, mode: Mode, - __0: (TextSize, (String, bool, bool), TextSize), + __0: (TextSize, (Box, bool, bool), TextSize), ) -> Result> { let __start0 = __0.2; @@ -65035,7 +65035,7 @@ fn __action1333< >( source_code: &str, mode: Mode, - __0: (TextSize, String, TextSize), + __0: (TextSize, Box, TextSize), ) -> ast::Identifier { let __start0 = __0.2; @@ -65347,7 +65347,7 @@ fn __action1344< >( source_code: &str, mode: Mode, - __0: (TextSize, (IpyEscapeKind, String), TextSize), + __0: (TextSize, (IpyEscapeKind, Box), TextSize), ) -> Result> { let __start0 = __0.2; @@ -65373,7 +65373,7 @@ fn __action1345< >( source_code: &str, mode: Mode, - __0: (TextSize, (IpyEscapeKind, String), TextSize), + __0: (TextSize, (IpyEscapeKind, Box), TextSize), ) -> Result> { let __start0 = __0.2; @@ -65430,7 +65430,7 @@ fn __action1347< __0: (TextSize, token::Tok, TextSize), __1: (TextSize, core::option::Option, TextSize), __2: (TextSize, token::Tok, TextSize), - __3: (TextSize, core::option::Option<(String, bool, bool)>, TextSize), + __3: (TextSize, core::option::Option<(Box, bool, bool)>, TextSize), __4: (TextSize, crate::parser::ParenthesizedExpr, TextSize), ) -> Result> { @@ -69997,7 +69997,7 @@ fn __action1494< >( source_code: &str, mode: Mode, - __0: (TextSize, (String, StringKind, bool), TextSize), + __0: (TextSize, (Box, StringKind, bool), TextSize), ) -> Result> { let __start0 = __0.2; @@ -77662,7 +77662,7 @@ fn __action1727< __0: (TextSize, token::Tok, TextSize), __1: (TextSize, ast::Parameters, TextSize), __2: (TextSize, token::Tok, TextSize), - __3: (TextSize, core::option::Option<(String, bool, bool)>, TextSize), + __3: (TextSize, core::option::Option<(Box, bool, bool)>, TextSize), __4: (TextSize, crate::parser::ParenthesizedExpr, TextSize), ) -> Result> { @@ -77693,7 +77693,7 @@ fn __action1728< mode: Mode, __0: (TextSize, token::Tok, TextSize), __1: (TextSize, token::Tok, TextSize), - __2: (TextSize, core::option::Option<(String, bool, bool)>, TextSize), + __2: (TextSize, core::option::Option<(Box, bool, bool)>, TextSize), __3: (TextSize, crate::parser::ParenthesizedExpr, TextSize), ) -> Result> { @@ -79598,7 +79598,7 @@ fn __action1785< __0: (TextSize, token::Tok, TextSize), __1: (TextSize, ast::Parameters, TextSize), __2: (TextSize, token::Tok, TextSize), - __3: (TextSize, (String, bool, bool), TextSize), + __3: (TextSize, (Box, bool, bool), TextSize), __4: (TextSize, crate::parser::ParenthesizedExpr, TextSize), ) -> Result> { @@ -79661,7 +79661,7 @@ fn __action1787< mode: Mode, __0: (TextSize, token::Tok, TextSize), __1: (TextSize, token::Tok, TextSize), - __2: (TextSize, (String, bool, bool), TextSize), + __2: (TextSize, (Box, bool, bool), TextSize), __3: (TextSize, crate::parser::ParenthesizedExpr, TextSize), ) -> Result> { diff --git a/crates/ruff_python_parser/src/soft_keywords.rs b/crates/ruff_python_parser/src/soft_keywords.rs index 379ae1c08db38..e4bff73edc28c 100644 --- a/crates/ruff_python_parser/src/soft_keywords.rs +++ b/crates/ruff_python_parser/src/soft_keywords.rs @@ -203,7 +203,7 @@ fn soft_to_name(tok: &Tok) -> Tok { _ => unreachable!("other tokens never reach here"), }; Tok::Name { - name: name.to_owned(), + name: name.to_string().into_boxed_str(), } } diff --git a/crates/ruff_python_parser/src/string.rs b/crates/ruff_python_parser/src/string.rs index 80f42e453b089..5b15474cf2dd6 100644 --- a/crates/ruff_python_parser/src/string.rs +++ b/crates/ruff_python_parser/src/string.rs @@ -151,10 +151,10 @@ impl<'a> StringParser<'a> { fn parse_escaped_char(&mut self, string: &mut String) -> Result<(), LexicalError> { let Some(first_char) = self.next_char() else { - return Err(LexicalError { - error: LexicalErrorType::StringError, - location: self.get_pos(), - }); + return Err(LexicalError::new( + LexicalErrorType::StringError, + self.get_pos(), + )); }; let new_char = match first_char { @@ -184,12 +184,14 @@ impl<'a> StringParser<'a> { } _ => { if self.kind.is_any_bytes() && !first_char.is_ascii() { - return Err(LexicalError { - error: LexicalErrorType::OtherError( - "bytes can only contain ASCII literal characters".to_owned(), + return Err(LexicalError::new( + LexicalErrorType::OtherError( + "bytes can only contain ASCII literal characters" + .to_string() + .into_boxed_str(), ), - location: self.get_pos(), - }); + self.get_pos(), + )); } string.push('\\'); @@ -257,7 +259,9 @@ impl<'a> StringParser<'a> { if !ch.is_ascii() { return Err(LexicalError::new( LexicalErrorType::OtherError( - "bytes can only contain ASCII literal characters".to_string(), + "bytes can only contain ASCII literal characters" + .to_string() + .into_boxed_str(), ), self.get_pos(), )); @@ -291,7 +295,7 @@ impl<'a> StringParser<'a> { } } Ok(StringType::Str(ast::StringLiteral { - value, + value: value.into_boxed_str(), unicode: self.kind.is_unicode(), range: self.range, })) @@ -354,12 +358,14 @@ pub(crate) fn concatenated_strings( let has_bytes = byte_literal_count > 0; if has_bytes && byte_literal_count < strings.len() { - return Err(LexicalError { - error: LexicalErrorType::OtherError( - "cannot mix bytes and nonbytes literals".to_owned(), + return Err(LexicalError::new( + LexicalErrorType::OtherError( + "cannot mix bytes and nonbytes literals" + .to_string() + .into_boxed_str(), ), - location: range.start(), - }); + range.start(), + )); } if has_bytes { @@ -418,15 +424,12 @@ struct FStringError { impl From for LexicalError { fn from(err: FStringError) -> Self { - LexicalError { - error: LexicalErrorType::FStringError(err.error), - location: err.location, - } + LexicalError::new(LexicalErrorType::FStringError(err.error), err.location) } } /// Represents the different types of errors that can occur during parsing of an f-string. -#[derive(Debug, Clone, PartialEq)] +#[derive(Copy, Debug, Clone, PartialEq)] pub enum FStringErrorType { /// Expected a right brace after an opened left brace. UnclosedLbrace, @@ -466,10 +469,7 @@ impl std::fmt::Display for FStringErrorType { impl From for crate::parser::LalrpopError { fn from(err: FStringError) -> Self { lalrpop_util::ParseError::User { - error: LexicalError { - error: LexicalErrorType::FStringError(err.error), - location: err.location, - }, + error: LexicalError::new(LexicalErrorType::FStringError(err.error), err.location), } } } diff --git a/crates/ruff_python_parser/src/token.rs b/crates/ruff_python_parser/src/token.rs index 059901b7efbcc..d3d51452cf9c0 100644 --- a/crates/ruff_python_parser/src/token.rs +++ b/crates/ruff_python_parser/src/token.rs @@ -16,7 +16,7 @@ pub enum Tok { /// Token value for a name, commonly known as an identifier. Name { /// The name value. - name: String, + name: Box, }, /// Token value for an integer. Int { @@ -38,7 +38,7 @@ pub enum Tok { /// Token value for a string. String { /// The string value. - value: String, + value: Box, /// The kind of string. kind: StringKind, /// Whether the string is triple quoted. @@ -51,7 +51,7 @@ pub enum Tok { /// part of the expression part and isn't an opening or closing brace. FStringMiddle { /// The string value. - value: String, + value: Box, /// Whether the string is raw or not. is_raw: bool, /// Whether the string is triple quoted. @@ -63,12 +63,12 @@ pub enum Tok { /// only when the mode is [`Mode::Ipython`]. IpyEscapeCommand { /// The magic command value. - value: String, + value: Box, /// The kind of magic command. kind: IpyEscapeKind, }, /// Token value for a comment. These are filtered out of the token stream prior to parsing. - Comment(String), + Comment(Box), /// Token value for a newline. Newline, /// Token value for a newline that is not a logical line break. These are filtered out of @@ -912,3 +912,14 @@ impl From<&Tok> for TokenKind { Self::from_token(value) } } + +#[cfg(target_pointer_width = "64")] +mod sizes { + use crate::lexer::{LexicalError, LexicalErrorType}; + use crate::Tok; + use static_assertions::assert_eq_size; + + assert_eq_size!(Tok, [u8; 24]); + assert_eq_size!(LexicalErrorType, [u8; 24]); + assert_eq_size!(Result, [u8; 32]); +} diff --git a/fuzz/fuzz_targets/ruff_parse_simple.rs b/fuzz/fuzz_targets/ruff_parse_simple.rs index 5c628b10cd2b4..24a998336b110 100644 --- a/fuzz/fuzz_targets/ruff_parse_simple.rs +++ b/fuzz/fuzz_targets/ruff_parse_simple.rs @@ -47,7 +47,7 @@ fn do_fuzz(case: &[u8]) -> Corpus { ); } Err(err) => { - let offset = err.location.to_usize(); + let offset = err.location().to_usize(); assert!( code.is_char_boundary(offset), "Invalid error location {} (not at char boundary)",