Reduce Tok size by using Box<str> instead of String

crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs

@@ -257,15 +257,18 @@ fn elts_to_csv(elts: &[Expr], generator: Generator) -> Option<String> {
     }
 
     let node = Expr::from(ast::StringLiteral {
-        value: elts.iter().fold(String::new(), |mut acc, elt| {
-            if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = elt {
-                if !acc.is_empty() {
-                    acc.push(',');
+        value: elts
+            .iter()
+            .fold(String::new(), |mut acc, elt| {
+                if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = elt {
+                    if !acc.is_empty() {
+                        acc.push(',');
+                    }
+                    acc.push_str(value.to_str());
                 }
-                acc.push_str(value.to_str());
-            }
-            acc
-        }),
+                acc
+            })
+            .into_boxed_str(),
         ..ast::StringLiteral::default()
     });
     Some(generator.expr(&node))
@@ -327,7 +330,7 @@ fn check_names(checker: &mut Checker, decorator: &Decorator, expr: &Expr) {
                                 .iter()
                                 .map(|name| {
                                     Expr::from(ast::StringLiteral {
-                                        value: (*name).to_string(),
+                                        value: (*name).to_string().into_boxed_str(),
                                         ..ast::StringLiteral::default()
                                     })
                                 })
@@ -360,7 +363,7 @@ fn check_names(checker: &mut Checker, decorator: &Decorator, expr: &Expr) {
                                 .iter()
                                 .map(|name| {
                                     Expr::from(ast::StringLiteral {
-                                        value: (*name).to_string(),
+                                        value: (*name).to_string().into_boxed_str(),
                                         ..ast::StringLiteral::default()
                                     })
                                 })

crates/ruff_linter/src/rules/flake8_simplify/rules/ast_expr.rs

@@ -217,7 +217,7 @@ fn check_os_environ_subscript(checker: &mut Checker, expr: &Expr) {
         slice.range(),
     );
     let node = ast::StringLiteral {
-        value: capital_env_var,
+        value: capital_env_var.into_boxed_str(),
         unicode: env_var.is_unicode(),
         ..ast::StringLiteral::default()
     };

crates/ruff_linter/src/rules/flynt/rules/static_join_to_fstring.rs

@@ -72,7 +72,8 @@ fn build_fstring(joiner: &str, joinees: &[Expr]) -> Option<Expr> {
                         None
                     }
                 })
-                .join(joiner),
+                .join(joiner)
+                .into_boxed_str(),
             ..ast::StringLiteral::default()
         };
         return Some(node.into());

crates/ruff_linter/src/rules/pycodestyle/rules/invalid_escape_sequence.rs

@@ -74,7 +74,7 @@ pub(crate) fn invalid_escape_sequence(
             let Some(range) = indexer.fstring_ranges().innermost(token_range.start()) else {
                 return;
             };
-            (value.as_str(), range.start())
+            (value.as_ref(), range.start())
         }
         Tok::String { kind, .. } => {
             if kind.is_raw() {

crates/ruff_linter/src/rules/pylint/rules/unspecified_encoding.rs

@@ -110,7 +110,7 @@ fn generate_keyword_fix(checker: &Checker, call: &ast::ExprCall) -> Fix {
                 .generator()
                 .expr(&Expr::StringLiteral(ast::ExprStringLiteral {
                     value: ast::StringLiteralValue::single(ast::StringLiteral {
-                        value: "locale".to_string(),
+                        value: "locale".to_string().into_boxed_str(),
                         unicode: false,
                         range: TextRange::default(),
                     }),

crates/ruff_linter/src/rules/pyupgrade/fixes.rs

@@ -21,7 +21,7 @@ pub(crate) fn remove_import_members(contents: &str, members: &[&str]) -> String
                 let last_range = names.last_mut().unwrap();
                 *last_range = TextRange::new(last_range.start(), range.end());
             } else {
-                if members.contains(&name.as_str()) {
+                if members.contains(&name.as_ref()) {
                     removal_indices.push(names.len());
                 }
                 names.push(range);

crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs

@@ -559,14 +559,14 @@ fn collect_string_sequence_lines(
 /// `self` and produces the classification for the line.
 #[derive(Debug, Default)]
 struct LineState {
-    first_item_in_line: Option<(String, TextRange)>,
-    following_items_in_line: Vec<(String, TextRange)>,
+    first_item_in_line: Option<(Box<str>, TextRange)>,
+    following_items_in_line: Vec<(Box<str>, TextRange)>,
     comment_range_start: Option<TextSize>,
     comment_in_line: Option<TextRange>,
 }
 
 impl LineState {
-    fn visit_string_token(&mut self, token_value: String, token_range: TextRange) {
+    fn visit_string_token(&mut self, token_value: Box<str>, token_range: TextRange) {
         if self.first_item_in_line.is_none() {
             self.first_item_in_line = Some((token_value, token_range));
         } else {
@@ -631,8 +631,8 @@ struct LineWithItems {
     // For elements in the list, we keep track of the value of the
     // value of the element as well as the source-code range of the element.
     // (We need to know the actual value so that we can sort the items.)
-    first_item: (String, TextRange),
-    following_items: Vec<(String, TextRange)>,
+    first_item: (Box<str>, TextRange),
+    following_items: Vec<(Box<str>, TextRange)>,
     // For comments, we only need to keep track of the source-code range.
     trailing_comment_range: Option<TextRange>,
 }
@@ -753,7 +753,7 @@ fn collect_string_sequence_items(
 /// source-code range of `"a"`.
 #[derive(Debug)]
 struct StringSequenceItem {
-    value: String,
+    value: Box<str>,
     preceding_comment_ranges: Vec<TextRange>,
     element_range: TextRange,
     // total_range incorporates the ranges of preceding comments
@@ -766,7 +766,7 @@ struct StringSequenceItem {
 
 impl StringSequenceItem {
     fn new(
-        value: String,
+        value: Box<str>,
         preceding_comment_ranges: Vec<TextRange>,
         element_range: TextRange,
         end_of_line_comments: Option<TextRange>,
@@ -787,7 +787,7 @@ impl StringSequenceItem {
         }
     }
 
-    fn with_no_comments(value: String, element_range: TextRange) -> Self {
+    fn with_no_comments(value: Box<str>, element_range: TextRange) -> Self {
         Self::new(value, vec![], element_range, None)
     }
 }

crates/ruff_python_ast/src/comparable.rs

@@ -631,7 +631,7 @@ pub struct ComparableStringLiteral<'a> {
 impl<'a> From<&'a ast::StringLiteral> for ComparableStringLiteral<'a> {
     fn from(string_literal: &'a ast::StringLiteral) -> Self {
         Self {
-            value: string_literal.value.as_str(),
+            value: &string_literal.value,
         }
     }
 }
@@ -1089,10 +1089,7 @@ impl<'a> From<&'a ast::Expr> for ComparableExpr<'a> {
                 kind,
                 value,
                 range: _,
-            }) => Self::IpyEscapeCommand(ExprIpyEscapeCommand {
-                kind: *kind,
-                value: value.as_str(),
-            }),
+            }) => Self::IpyEscapeCommand(ExprIpyEscapeCommand { kind: *kind, value }),
         }
     }
 }
@@ -1537,10 +1534,7 @@ impl<'a> From<&'a ast::Stmt> for ComparableStmt<'a> {
                 kind,
                 value,
                 range: _,
-            }) => Self::IpyEscapeCommand(StmtIpyEscapeCommand {
-                kind: *kind,
-                value: value.as_str(),
-            }),
+            }) => Self::IpyEscapeCommand(StmtIpyEscapeCommand { kind: *kind, value }),
             ast::Stmt::Expr(ast::StmtExpr { value, range: _ }) => Self::Expr(StmtExpr {
                 value: value.into(),
             }),

crates/ruff_python_ast/src/nodes.rs

@@ -160,7 +160,7 @@ pub enum Stmt {
 pub struct StmtIpyEscapeCommand {
     pub range: TextRange,
     pub kind: IpyEscapeKind,
-    pub value: String,
+    pub value: Box<str>,
 }
 
 impl From<StmtIpyEscapeCommand> for Stmt {
@@ -671,7 +671,7 @@ impl Expr {
 pub struct ExprIpyEscapeCommand {
     pub range: TextRange,
     pub kind: IpyEscapeKind,
-    pub value: String,
+    pub value: Box<str>,
 }
 
 impl From<ExprIpyEscapeCommand> for Expr {
@@ -1384,7 +1384,7 @@ impl Default for StringLiteralValueInner {
 #[derive(Clone, Debug, Default, PartialEq)]
 pub struct StringLiteral {
     pub range: TextRange,
-    pub value: String,
+    pub value: Box<str>,
     pub unicode: bool,
 }
 
@@ -1398,7 +1398,7 @@ impl Deref for StringLiteral {
     type Target = str;
 
     fn deref(&self) -> &Self::Target {
-        self.value.as_str()
+        &self.value
     }
 }
 
@@ -1426,14 +1426,16 @@ struct ConcatenatedStringLiteral {
     /// Each string literal that makes up the concatenated string.
     strings: Vec<StringLiteral>,
     /// The concatenated string value.
-    value: OnceCell<String>,
+    value: OnceCell<Box<str>>,
 }
 
 impl ConcatenatedStringLiteral {
     /// Extracts a string slice containing the entire concatenated string.
     fn to_str(&self) -> &str {
-        self.value
-            .get_or_init(|| self.strings.iter().map(StringLiteral::as_str).collect())
+        self.value.get_or_init(|| {
+            let concatenated: String = self.strings.iter().map(StringLiteral::as_str).collect();
+            concatenated.into_boxed_str()
+        })
     }
 }
 

crates/ruff_python_formatter/tests/normalizer.rs

@@ -95,19 +95,22 @@ impl Transformer for Normalizer {
                 &string_literal.value,
                 "<DOCTEST-CODE-SNIPPET: Removed by normalizer>\n",
             )
-            .into_owned();
+            .into_owned()
+            .into_boxed_str();
         string_literal.value = STRIP_RST_BLOCKS
             .replace_all(
                 &string_literal.value,
                 "<RSTBLOCK-CODE-SNIPPET: Removed by normalizer>\n",
             )
-            .into_owned();
+            .into_owned()
+            .into_boxed_str();
         string_literal.value = STRIP_MARKDOWN_BLOCKS
             .replace_all(
                 &string_literal.value,
                 "<MARKDOWN-CODE-SNIPPET: Removed by normalizer>\n",
             )
-            .into_owned();
+            .into_owned()
+            .into_boxed_str();
         // Normalize a string by (2) stripping any leading and trailing space from each
         // line, and (3) removing any blank lines from the start and end of the string.
         string_literal.value = string_literal
@@ -117,6 +120,7 @@ impl Transformer for Normalizer {
             .collect::<Vec<_>>()
             .join("\n")
             .trim()
-            .to_owned();
+            .to_owned()
+            .into_boxed_str();
     }
 }

crates/ruff_python_parser/src/lexer.rs

@@ -241,7 +241,7 @@ impl<'source> Lexer<'source> {
             "yield" => Tok::Yield,
             _ => {
                 return Ok(Tok::Name {
-                    name: text.to_string(),
+                    name: text.to_string().into_boxed_str(),
                 })
             }
         };
@@ -413,7 +413,7 @@ impl<'source> Lexer<'source> {
         let offset = memchr::memchr2(b'\n', b'\r', bytes).unwrap_or(bytes.len());
         self.cursor.skip_bytes(offset);
 
-        Tok::Comment(self.token_text().to_string())
+        Tok::Comment(self.token_text().to_string().into_boxed_str())
     }
 
     /// Lex a single IPython escape command.
@@ -510,12 +510,15 @@ impl<'source> Lexer<'source> {
                         2 => IpyEscapeKind::Help2,
                         _ => unreachable!("`question_count` is always 1 or 2"),
                     };
-                    return Tok::IpyEscapeCommand { kind, value };
+                    return Tok::IpyEscapeCommand {
+                        kind,
+                        value: value.into_boxed_str(),
+                    };
                 }
                 '\n' | '\r' | EOF_CHAR => {
                     return Tok::IpyEscapeCommand {
                         kind: escape_kind,
-                        value,
+                        value: value.into_boxed_str(),
                     };
                 }
                 c => {
@@ -675,7 +678,7 @@ impl<'source> Lexer<'source> {
             normalized
         };
         Ok(Some(Tok::FStringMiddle {
-            value,
+            value: value.into_boxed_str(),
             is_raw: fstring.is_raw_string(),
             triple_quoted: fstring.is_triple_quoted(),
         }))
@@ -758,7 +761,9 @@ impl<'source> Lexer<'source> {
         };
 
         let tok = Tok::String {
-            value: self.source[TextRange::new(value_start, value_end)].to_string(),
+            value: self.source[TextRange::new(value_start, value_end)]
+                .to_string()
+                .into_boxed_str(),
             kind,
             triple_quoted,
         };

crates/ruff_python_parser/src/python.lalrpop

@@ -289,7 +289,7 @@ ImportAsAlias<I>: ast::Alias = {
 DottedName: ast::Identifier = {
     <location:@L> <n:name> <end_location:@R> => ast::Identifier::new(n, (location..end_location).into()),
     <location:@L> <n:name> <n2: ("." Identifier)+> <end_location:@R> => {
-        let mut r = n;
+        let mut r = String::from(n);
         for x in n2 {
             r.push('.');
             r.push_str(x.1.as_str());
@@ -434,7 +434,7 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = {
         Ok(ast::Stmt::IpyEscapeCommand(
             ast::StmtIpyEscapeCommand {
                 kind,
-                value,
+                value: value.into_boxed_str(),
                 range: (location..end_location).into()
             }
         ))
@@ -1677,7 +1677,7 @@ FStringFormatSpec: ast::FStringFormatSpec = {
 
 FStringConversion: (TextSize, ast::ConversionFlag) = {
     <location:@L> "!" <name_location:@L> <s:name> =>? {
-        let conversion = match s.as_str() {
+        let conversion = match s.as_ref() {
             "s" => ast::ConversionFlag::Str,
             "r" => ast::ConversionFlag::Repr,
             "a" => ast::ConversionFlag::Ascii,
@@ -2061,19 +2061,19 @@ extern {
         float => token::Tok::Float { value: <f64> },
         complex => token::Tok::Complex { real: <f64>, imag: <f64> },
         string => token::Tok::String {
-            value: <String>,
+            value: <Box<str>>,
             kind: <StringKind>,
             triple_quoted: <bool>
         },
         fstring_middle => token::Tok::FStringMiddle {
-            value: <String>,
+            value: <Box<str>>,
             is_raw: <bool>,
             triple_quoted: <bool>
         },
-        name => token::Tok::Name { name: <String> },
+        name => token::Tok::Name { name: <Box<str>> },
         ipy_escape_command => token::Tok::IpyEscapeCommand {
             kind: <IpyEscapeKind>,
-            value: <String>
+            value: <Box<str>>
         },
         "\n" => token::Tok::Newline,
         ";" => token::Tok::Semi,