Reduce Result<Tok, LexicalError> size by using Box<str> instead o…

…f `String` (astral-sh#9885)
nkxxll · Mar 4, 2024 · 11de6dd · 11de6dd
1 parent 85bd0b4
commit 11de6dd
Show file tree

Hide file tree

Showing 22 changed files with 453 additions and 424 deletions.
diff --git a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs
@@ -257,15 +257,18 @@ fn elts_to_csv(elts: &[Expr], generator: Generator) -> Option<String> {
     }
 
     let node = Expr::from(ast::StringLiteral {
-        value: elts.iter().fold(String::new(), |mut acc, elt| {
-            if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = elt {
-                if !acc.is_empty() {
-                    acc.push(',');
+        value: elts
+            .iter()
+            .fold(String::new(), |mut acc, elt| {
+                if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = elt {
+                    if !acc.is_empty() {
+                        acc.push(',');
+                    }
+                    acc.push_str(value.to_str());
                 }
-                acc.push_str(value.to_str());
-            }
-            acc
-        }),
+                acc
+            })
+            .into_boxed_str(),
         ..ast::StringLiteral::default()
     });
     Some(generator.expr(&node))
@@ -327,7 +330,7 @@ fn check_names(checker: &mut Checker, decorator: &Decorator, expr: &Expr) {
                                 .iter()
                                 .map(|name| {
                                     Expr::from(ast::StringLiteral {
-                                        value: (*name).to_string(),
+                                        value: (*name).to_string().into_boxed_str(),
                                         ..ast::StringLiteral::default()
                                     })
                                 })
@@ -360,7 +363,7 @@ fn check_names(checker: &mut Checker, decorator: &Decorator, expr: &Expr) {
                                 .iter()
                                 .map(|name| {
                                     Expr::from(ast::StringLiteral {
-                                        value: (*name).to_string(),
+                                        value: (*name).to_string().into_boxed_str(),
                                         ..ast::StringLiteral::default()
                                     })
                                 })

diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_expr.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_expr.rs
@@ -217,7 +217,7 @@ fn check_os_environ_subscript(checker: &mut Checker, expr: &Expr) {
         slice.range(),
     );
     let node = ast::StringLiteral {
-        value: capital_env_var,
+        value: capital_env_var.into_boxed_str(),
         unicode: env_var.is_unicode(),
         ..ast::StringLiteral::default()
     };

diff --git a/crates/ruff_linter/src/rules/flynt/rules/static_join_to_fstring.rs b/crates/ruff_linter/src/rules/flynt/rules/static_join_to_fstring.rs
@@ -72,7 +72,8 @@ fn build_fstring(joiner: &str, joinees: &[Expr]) -> Option<Expr> {
                         None
                     }
                 })
-                .join(joiner),
+                .join(joiner)
+                .into_boxed_str(),
             ..ast::StringLiteral::default()
         };
         return Some(node.into());

diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/invalid_escape_sequence.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/invalid_escape_sequence.rs
@@ -74,7 +74,7 @@ pub(crate) fn invalid_escape_sequence(
             let Some(range) = indexer.fstring_ranges().innermost(token_range.start()) else {
                 return;
             };
-            (value.as_str(), range.start())
+            (&**value, range.start())
         }
         Tok::String { kind, .. } => {
             if kind.is_raw() {

diff --git a/crates/ruff_linter/src/rules/pylint/rules/unspecified_encoding.rs b/crates/ruff_linter/src/rules/pylint/rules/unspecified_encoding.rs
@@ -110,7 +110,7 @@ fn generate_keyword_fix(checker: &Checker, call: &ast::ExprCall) -> Fix {
                 .generator()
                 .expr(&Expr::StringLiteral(ast::ExprStringLiteral {
                     value: ast::StringLiteralValue::single(ast::StringLiteral {
-                        value: "locale".to_string(),
+                        value: "locale".to_string().into_boxed_str(),
                         unicode: false,
                         range: TextRange::default(),
                     }),

diff --git a/crates/ruff_linter/src/rules/pyupgrade/fixes.rs b/crates/ruff_linter/src/rules/pyupgrade/fixes.rs
@@ -21,7 +21,7 @@ pub(crate) fn remove_import_members(contents: &str, members: &[&str]) -> String
                 let last_range = names.last_mut().unwrap();
                 *last_range = TextRange::new(last_range.start(), range.end());
             } else {
-                if members.contains(&name.as_str()) {
+                if members.contains(&&**name) {
                     removal_indices.push(names.len());
                 }
                 names.push(range);

diff --git a/crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs b/crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs
@@ -559,14 +559,14 @@ fn collect_string_sequence_lines(
 /// `self` and produces the classification for the line.
 #[derive(Debug, Default)]
 struct LineState {
-    first_item_in_line: Option<(String, TextRange)>,
-    following_items_in_line: Vec<(String, TextRange)>,
+    first_item_in_line: Option<(Box<str>, TextRange)>,
+    following_items_in_line: Vec<(Box<str>, TextRange)>,
     comment_range_start: Option<TextSize>,
     comment_in_line: Option<TextRange>,
 }
 
 impl LineState {
-    fn visit_string_token(&mut self, token_value: String, token_range: TextRange) {
+    fn visit_string_token(&mut self, token_value: Box<str>, token_range: TextRange) {
         if self.first_item_in_line.is_none() {
             self.first_item_in_line = Some((token_value, token_range));
         } else {
@@ -631,8 +631,8 @@ struct LineWithItems {
     // For elements in the list, we keep track of the value of the
     // value of the element as well as the source-code range of the element.
     // (We need to know the actual value so that we can sort the items.)
-    first_item: (String, TextRange),
-    following_items: Vec<(String, TextRange)>,
+    first_item: (Box<str>, TextRange),
+    following_items: Vec<(Box<str>, TextRange)>,
     // For comments, we only need to keep track of the source-code range.
     trailing_comment_range: Option<TextRange>,
 }
@@ -753,7 +753,7 @@ fn collect_string_sequence_items(
 /// source-code range of `"a"`.
 #[derive(Debug)]
 struct StringSequenceItem {
-    value: String,
+    value: Box<str>,
     preceding_comment_ranges: Vec<TextRange>,
     element_range: TextRange,
     // total_range incorporates the ranges of preceding comments
@@ -766,7 +766,7 @@ struct StringSequenceItem {
 
 impl StringSequenceItem {
     fn new(
-        value: String,
+        value: Box<str>,
         preceding_comment_ranges: Vec<TextRange>,
         element_range: TextRange,
         end_of_line_comments: Option<TextRange>,
@@ -787,7 +787,7 @@ impl StringSequenceItem {
         }
     }
 
-    fn with_no_comments(value: String, element_range: TextRange) -> Self {
+    fn with_no_comments(value: Box<str>, element_range: TextRange) -> Self {
         Self::new(value, vec![], element_range, None)
     }
 }

diff --git a/crates/ruff_python_ast/src/comparable.rs b/crates/ruff_python_ast/src/comparable.rs
@@ -631,7 +631,7 @@ pub struct ComparableStringLiteral<'a> {
 impl<'a> From<&'a ast::StringLiteral> for ComparableStringLiteral<'a> {
     fn from(string_literal: &'a ast::StringLiteral) -> Self {
         Self {
-            value: string_literal.value.as_str(),
+            value: &string_literal.value,
         }
     }
 }
@@ -1089,10 +1089,7 @@ impl<'a> From<&'a ast::Expr> for ComparableExpr<'a> {
                 kind,
                 value,
                 range: _,
-            }) => Self::IpyEscapeCommand(ExprIpyEscapeCommand {
-                kind: *kind,
-                value: value.as_str(),
-            }),
+            }) => Self::IpyEscapeCommand(ExprIpyEscapeCommand { kind: *kind, value }),
         }
     }
 }
@@ -1537,10 +1534,7 @@ impl<'a> From<&'a ast::Stmt> for ComparableStmt<'a> {
                 kind,
                 value,
                 range: _,
-            }) => Self::IpyEscapeCommand(StmtIpyEscapeCommand {
-                kind: *kind,
-                value: value.as_str(),
-            }),
+            }) => Self::IpyEscapeCommand(StmtIpyEscapeCommand { kind: *kind, value }),
             ast::Stmt::Expr(ast::StmtExpr { value, range: _ }) => Self::Expr(StmtExpr {
                 value: value.into(),
             }),

diff --git a/crates/ruff_python_ast/src/nodes.rs b/crates/ruff_python_ast/src/nodes.rs
@@ -160,7 +160,7 @@ pub enum Stmt {
 pub struct StmtIpyEscapeCommand {
     pub range: TextRange,
     pub kind: IpyEscapeKind,
-    pub value: String,
+    pub value: Box<str>,
 }
 
 impl From<StmtIpyEscapeCommand> for Stmt {
@@ -671,7 +671,7 @@ impl Expr {
 pub struct ExprIpyEscapeCommand {
     pub range: TextRange,
     pub kind: IpyEscapeKind,
-    pub value: String,
+    pub value: Box<str>,
 }
 
 impl From<ExprIpyEscapeCommand> for Expr {
@@ -1384,7 +1384,7 @@ impl Default for StringLiteralValueInner {
 #[derive(Clone, Debug, Default, PartialEq)]
 pub struct StringLiteral {
     pub range: TextRange,
-    pub value: String,
+    pub value: Box<str>,
     pub unicode: bool,
 }
 
@@ -1398,7 +1398,7 @@ impl Deref for StringLiteral {
     type Target = str;
 
     fn deref(&self) -> &Self::Target {
-        self.value.as_str()
+        &self.value
     }
 }
 
@@ -1426,14 +1426,16 @@ struct ConcatenatedStringLiteral {
     /// Each string literal that makes up the concatenated string.
     strings: Vec<StringLiteral>,
     /// The concatenated string value.
-    value: OnceCell<String>,
+    value: OnceCell<Box<str>>,
 }
 
 impl ConcatenatedStringLiteral {
     /// Extracts a string slice containing the entire concatenated string.
     fn to_str(&self) -> &str {
-        self.value
-            .get_or_init(|| self.strings.iter().map(StringLiteral::as_str).collect())
+        self.value.get_or_init(|| {
+            let concatenated: String = self.strings.iter().map(StringLiteral::as_str).collect();
+            concatenated.into_boxed_str()
+        })
     }
 }
 

diff --git a/crates/ruff_python_formatter/src/lib.rs b/crates/ruff_python_formatter/src/lib.rs
@@ -134,8 +134,8 @@ pub fn format_module_source(
     let source_type = options.source_type();
     let (tokens, comment_ranges) =
         tokens_and_ranges(source, source_type).map_err(|err| ParseError {
-            offset: err.location,
-            error: ParseErrorType::Lexical(err.error),
+            offset: err.location(),
+            error: ParseErrorType::Lexical(err.into_error()),
         })?;
     let module = parse_tokens(tokens, source, source_type.as_mode())?;
     let formatted = format_module_ast(&module, &comment_ranges, source, options)?;

diff --git a/crates/ruff_python_formatter/src/range.rs b/crates/ruff_python_formatter/src/range.rs
@@ -73,8 +73,8 @@ pub fn format_range(
 
     let (tokens, comment_ranges) =
         tokens_and_ranges(source, options.source_type()).map_err(|err| ParseError {
-            offset: err.location,
-            error: ParseErrorType::Lexical(err.error),
+            offset: err.location(),
+            error: ParseErrorType::Lexical(err.into_error()),
         })?;
 
     assert_valid_char_boundaries(range, source);

diff --git a/crates/ruff_python_formatter/tests/normalizer.rs b/crates/ruff_python_formatter/tests/normalizer.rs
@@ -95,19 +95,22 @@ impl Transformer for Normalizer {
                 &string_literal.value,
                 "<DOCTEST-CODE-SNIPPET: Removed by normalizer>\n",
             )
-            .into_owned();
+            .into_owned()
+            .into_boxed_str();
         string_literal.value = STRIP_RST_BLOCKS
             .replace_all(
                 &string_literal.value,
                 "<RSTBLOCK-CODE-SNIPPET: Removed by normalizer>\n",
             )
-            .into_owned();
+            .into_owned()
+            .into_boxed_str();
         string_literal.value = STRIP_MARKDOWN_BLOCKS
             .replace_all(
                 &string_literal.value,
                 "<MARKDOWN-CODE-SNIPPET: Removed by normalizer>\n",
             )
-            .into_owned();
+            .into_owned()
+            .into_boxed_str();
         // Normalize a string by (2) stripping any leading and trailing space from each
         // line, and (3) removing any blank lines from the start and end of the string.
         string_literal.value = string_literal
@@ -117,6 +120,7 @@ impl Transformer for Normalizer {
             .collect::<Vec<_>>()
             .join("\n")
             .trim()
-            .to_owned();
+            .to_owned()
+            .into_boxed_str();
     }
 }