From fcfab5fe4b33b9e9c76cd07c88b30502bd082a74 Mon Sep 17 00:00:00 2001 From: Jason Newcomb Date: Fri, 10 Oct 2025 17:04:01 -0400 Subject: [PATCH 1/3] `clippy_dev`: Move all parsing within a parse context. --- clippy_dev/src/deprecate_lint.rs | 8 +- clippy_dev/src/lib.rs | 5 +- clippy_dev/src/main.rs | 27 +++-- clippy_dev/src/parse.rs | 190 ++++++++++++++++--------------- clippy_dev/src/rename_lint.rs | 8 +- clippy_dev/src/update_lints.rs | 8 +- clippy_dev/src/utils.rs | 20 ++++ 7 files changed, 151 insertions(+), 115 deletions(-) diff --git a/clippy_dev/src/deprecate_lint.rs b/clippy_dev/src/deprecate_lint.rs index 4d99eb91e6f9..e2602f57f953 100644 --- a/clippy_dev/src/deprecate_lint.rs +++ b/clippy_dev/src/deprecate_lint.rs @@ -1,4 +1,4 @@ -use crate::parse::{DeprecatedLint, Lint, find_lint_decls, read_deprecated_lints}; +use crate::parse::{DeprecatedLint, Lint, ParseCx}; use crate::update_lints::generate_lint_files; use crate::utils::{UpdateMode, Version}; use std::ffi::OsStr; @@ -14,9 +14,9 @@ use std::{fs, io}; /// # Panics /// /// If a file path could not read from or written to -pub fn deprecate(clippy_version: Version, name: &str, reason: &str) { - let mut lints = find_lint_decls(); - let (mut deprecated_lints, renamed_lints) = read_deprecated_lints(); +pub fn deprecate<'cx>(cx: ParseCx<'cx>, clippy_version: Version, name: &'cx str, reason: &'cx str) { + let mut lints = cx.find_lint_decls(); + let (mut deprecated_lints, renamed_lints) = cx.read_deprecated_lints(); let Some(lint) = lints.iter().find(|l| l.name == name) else { eprintln!("error: failed to find lint `{name}`"); diff --git a/clippy_dev/src/lib.rs b/clippy_dev/src/lib.rs index fb8b2e1c91c1..16db6fa43dbe 100644 --- a/clippy_dev/src/lib.rs +++ b/clippy_dev/src/lib.rs @@ -34,7 +34,8 @@ pub mod setup; pub mod sync; pub mod update_lints; +mod parse; mod utils; -pub use utils::{ClippyInfo, UpdateMode}; -mod parse; +pub use self::parse::{ParseCx, new_parse_cx}; +pub use self::utils::{ClippyInfo, UpdateMode}; diff --git a/clippy_dev/src/main.rs b/clippy_dev/src/main.rs index 78fb44d7ad1b..392c3aabf193 100644 --- a/clippy_dev/src/main.rs +++ b/clippy_dev/src/main.rs @@ -4,8 +4,8 @@ use clap::{Args, Parser, Subcommand}; use clippy_dev::{ - ClippyInfo, UpdateMode, deprecate_lint, dogfood, fmt, lint, new_lint, release, rename_lint, serve, setup, sync, - update_lints, + ClippyInfo, UpdateMode, deprecate_lint, dogfood, fmt, lint, new_lint, new_parse_cx, release, rename_lint, serve, + setup, sync, update_lints, }; use std::env; @@ -27,7 +27,7 @@ fn main() { allow_no_vcs, } => dogfood::dogfood(fix, allow_dirty, allow_staged, allow_no_vcs), DevCommand::Fmt { check } => fmt::run(UpdateMode::from_check(check)), - DevCommand::UpdateLints { check } => update_lints::update(UpdateMode::from_check(check)), + DevCommand::UpdateLints { check } => new_parse_cx(|cx| update_lints::update(cx, UpdateMode::from_check(check))), DevCommand::NewLint { pass, name, @@ -35,7 +35,7 @@ fn main() { r#type, msrv, } => match new_lint::create(clippy.version, pass, &name, &category, r#type.as_deref(), msrv) { - Ok(()) => update_lints::update(UpdateMode::Change), + Ok(()) => new_parse_cx(|cx| update_lints::update(cx, UpdateMode::Change)), Err(e) => eprintln!("Unable to create lint: {e}"), }, DevCommand::Setup(SetupCommand { subcommand }) => match subcommand { @@ -78,13 +78,18 @@ fn main() { old_name, new_name, uplift, - } => rename_lint::rename( - clippy.version, - &old_name, - new_name.as_ref().unwrap_or(&old_name), - uplift, - ), - DevCommand::Deprecate { name, reason } => deprecate_lint::deprecate(clippy.version, &name, &reason), + } => new_parse_cx(|cx| { + rename_lint::rename( + cx, + clippy.version, + &old_name, + new_name.as_ref().unwrap_or(&old_name), + uplift, + ); + }), + DevCommand::Deprecate { name, reason } => { + new_parse_cx(|cx| deprecate_lint::deprecate(cx, clippy.version, &name, &reason)); + }, DevCommand::Sync(SyncCommand { subcommand }) => match subcommand { SyncSubcommand::UpdateNightly => sync::update_nightly(), }, diff --git a/clippy_dev/src/parse.rs b/clippy_dev/src/parse.rs index 5cea73d34af5..5c89f83dbd6a 100644 --- a/clippy_dev/src/parse.rs +++ b/clippy_dev/src/parse.rs @@ -1,12 +1,20 @@ pub mod cursor; use self::cursor::{Capture, Cursor}; -use crate::utils::{ErrAction, File, expect_action}; +use crate::utils::{ErrAction, File, Scoped, expect_action}; use core::range::Range; use std::fs; use std::path::{Path, PathBuf}; use walkdir::{DirEntry, WalkDir}; +pub struct ParseCxImpl; +pub type ParseCx<'cx> = &'cx mut ParseCxImpl; + +/// Calls the given function inside a newly created parsing context. +pub fn new_parse_cx<'env, T>(f: impl for<'cx> FnOnce(&'cx mut Scoped<'cx, 'env, ParseCxImpl>) -> T) -> T { + f(&mut Scoped::new(ParseCxImpl)) +} + pub struct Lint { pub name: String, pub group: String, @@ -27,33 +35,101 @@ pub struct RenamedLint { pub version: String, } -/// Finds all lint declarations (`declare_clippy_lint!`) -#[must_use] -pub fn find_lint_decls() -> Vec { - let mut lints = Vec::with_capacity(1000); - let mut contents = String::new(); - for e in expect_action(fs::read_dir("."), ErrAction::Read, ".") { - let e = expect_action(e, ErrAction::Read, "."); - if !expect_action(e.file_type(), ErrAction::Read, ".").is_dir() { - continue; +impl ParseCxImpl { + /// Finds all lint declarations (`declare_clippy_lint!`) + #[must_use] + pub fn find_lint_decls(&mut self) -> Vec { + let mut lints = Vec::with_capacity(1000); + let mut contents = String::new(); + for e in expect_action(fs::read_dir("."), ErrAction::Read, ".") { + let e = expect_action(e, ErrAction::Read, "."); + if !expect_action(e.file_type(), ErrAction::Read, ".").is_dir() { + continue; + } + let Ok(mut name) = e.file_name().into_string() else { + continue; + }; + if name.starts_with("clippy_lints") && name != "clippy_lints_internal" { + name.push_str("/src"); + for (file, module) in read_src_with_module(name.as_ref()) { + parse_clippy_lint_decls( + file.path(), + File::open_read_to_cleared_string(file.path(), &mut contents), + &module, + &mut lints, + ); + } + } } - let Ok(mut name) = e.file_name().into_string() else { - continue; - }; - if name.starts_with("clippy_lints") && name != "clippy_lints_internal" { - name.push_str("/src"); - for (file, module) in read_src_with_module(name.as_ref()) { - parse_clippy_lint_decls( - file.path(), - File::open_read_to_cleared_string(file.path(), &mut contents), - &module, - &mut lints, - ); + lints.sort_by(|lhs, rhs| lhs.name.cmp(&rhs.name)); + lints + } + + #[must_use] + pub fn read_deprecated_lints(&mut self) -> (Vec, Vec) { + #[allow(clippy::enum_glob_use)] + use cursor::Pat::*; + #[rustfmt::skip] + static DECL_TOKENS: &[cursor::Pat<'_>] = &[ + // #[clippy::version = "version"] + Pound, OpenBracket, Ident("clippy"), DoubleColon, Ident("version"), Eq, CaptureLitStr, CloseBracket, + // ("first", "second"), + OpenParen, CaptureLitStr, Comma, CaptureLitStr, CloseParen, Comma, + ]; + #[rustfmt::skip] + static DEPRECATED_TOKENS: &[cursor::Pat<'_>] = &[ + // !{ DEPRECATED(DEPRECATED_VERSION) = [ + Bang, OpenBrace, Ident("DEPRECATED"), OpenParen, Ident("DEPRECATED_VERSION"), CloseParen, Eq, OpenBracket, + ]; + #[rustfmt::skip] + static RENAMED_TOKENS: &[cursor::Pat<'_>] = &[ + // !{ RENAMED(RENAMED_VERSION) = [ + Bang, OpenBrace, Ident("RENAMED"), OpenParen, Ident("RENAMED_VERSION"), CloseParen, Eq, OpenBracket, + ]; + + let path = "clippy_lints/src/deprecated_lints.rs"; + let mut deprecated = Vec::with_capacity(30); + let mut renamed = Vec::with_capacity(80); + let mut contents = String::new(); + File::open_read_to_cleared_string(path, &mut contents); + + let mut cursor = Cursor::new(&contents); + let mut captures = [Capture::EMPTY; 3]; + + // First instance is the macro definition. + assert!( + cursor.find_ident("declare_with_version").is_some(), + "error reading deprecated lints" + ); + + if cursor.find_ident("declare_with_version").is_some() && cursor.match_all(DEPRECATED_TOKENS, &mut []) { + while cursor.match_all(DECL_TOKENS, &mut captures) { + deprecated.push(DeprecatedLint { + name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[1])), + reason: parse_str_single_line(path.as_ref(), cursor.get_text(captures[2])), + version: parse_str_single_line(path.as_ref(), cursor.get_text(captures[0])), + }); } + } else { + panic!("error reading deprecated lints"); } + + if cursor.find_ident("declare_with_version").is_some() && cursor.match_all(RENAMED_TOKENS, &mut []) { + while cursor.match_all(DECL_TOKENS, &mut captures) { + renamed.push(RenamedLint { + old_name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[1])), + new_name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[2])), + version: parse_str_single_line(path.as_ref(), cursor.get_text(captures[0])), + }); + } + } else { + panic!("error reading renamed lints"); + } + + deprecated.sort_by(|lhs, rhs| lhs.name.cmp(&rhs.name)); + renamed.sort_by(|lhs, rhs| lhs.old_name.cmp(&rhs.old_name)); + (deprecated, renamed) } - lints.sort_by(|lhs, rhs| lhs.name.cmp(&rhs.name)); - lints } /// Reads the source files from the given root directory @@ -116,72 +192,6 @@ fn parse_clippy_lint_decls(path: &Path, contents: &str, module: &str, lints: &mu } } -#[must_use] -pub fn read_deprecated_lints() -> (Vec, Vec) { - #[allow(clippy::enum_glob_use)] - use cursor::Pat::*; - #[rustfmt::skip] - static DECL_TOKENS: &[cursor::Pat<'_>] = &[ - // #[clippy::version = "version"] - Pound, OpenBracket, Ident("clippy"), DoubleColon, Ident("version"), Eq, CaptureLitStr, CloseBracket, - // ("first", "second"), - OpenParen, CaptureLitStr, Comma, CaptureLitStr, CloseParen, Comma, - ]; - #[rustfmt::skip] - static DEPRECATED_TOKENS: &[cursor::Pat<'_>] = &[ - // !{ DEPRECATED(DEPRECATED_VERSION) = [ - Bang, OpenBrace, Ident("DEPRECATED"), OpenParen, Ident("DEPRECATED_VERSION"), CloseParen, Eq, OpenBracket, - ]; - #[rustfmt::skip] - static RENAMED_TOKENS: &[cursor::Pat<'_>] = &[ - // !{ RENAMED(RENAMED_VERSION) = [ - Bang, OpenBrace, Ident("RENAMED"), OpenParen, Ident("RENAMED_VERSION"), CloseParen, Eq, OpenBracket, - ]; - - let path = "clippy_lints/src/deprecated_lints.rs"; - let mut deprecated = Vec::with_capacity(30); - let mut renamed = Vec::with_capacity(80); - let mut contents = String::new(); - File::open_read_to_cleared_string(path, &mut contents); - - let mut cursor = Cursor::new(&contents); - let mut captures = [Capture::EMPTY; 3]; - - // First instance is the macro definition. - assert!( - cursor.find_ident("declare_with_version").is_some(), - "error reading deprecated lints" - ); - - if cursor.find_ident("declare_with_version").is_some() && cursor.match_all(DEPRECATED_TOKENS, &mut []) { - while cursor.match_all(DECL_TOKENS, &mut captures) { - deprecated.push(DeprecatedLint { - name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[1])), - reason: parse_str_single_line(path.as_ref(), cursor.get_text(captures[2])), - version: parse_str_single_line(path.as_ref(), cursor.get_text(captures[0])), - }); - } - } else { - panic!("error reading deprecated lints"); - } - - if cursor.find_ident("declare_with_version").is_some() && cursor.match_all(RENAMED_TOKENS, &mut []) { - while cursor.match_all(DECL_TOKENS, &mut captures) { - renamed.push(RenamedLint { - old_name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[1])), - new_name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[2])), - version: parse_str_single_line(path.as_ref(), cursor.get_text(captures[0])), - }); - } - } else { - panic!("error reading renamed lints"); - } - - deprecated.sort_by(|lhs, rhs| lhs.name.cmp(&rhs.name)); - renamed.sort_by(|lhs, rhs| lhs.old_name.cmp(&rhs.old_name)); - (deprecated, renamed) -} - /// Removes the line splices and surrounding quotes from a string literal fn parse_str_lit(s: &str) -> String { let (s, is_raw) = if let Some(s) = s.strip_prefix("r") { diff --git a/clippy_dev/src/rename_lint.rs b/clippy_dev/src/rename_lint.rs index 207c9b2ff596..04942cd639d6 100644 --- a/clippy_dev/src/rename_lint.rs +++ b/clippy_dev/src/rename_lint.rs @@ -1,5 +1,5 @@ use crate::parse::cursor::{self, Capture, Cursor}; -use crate::parse::{RenamedLint, find_lint_decls, read_deprecated_lints}; +use crate::parse::{ParseCx, RenamedLint}; use crate::update_lints::generate_lint_files; use crate::utils::{ ErrAction, FileUpdater, UpdateMode, UpdateStatus, Version, delete_dir_if_exists, delete_file_if_exists, @@ -26,10 +26,10 @@ use std::path::Path; /// * If `old_name` doesn't name an existing lint. /// * If `old_name` names a deprecated or renamed lint. #[expect(clippy::too_many_lines)] -pub fn rename(clippy_version: Version, old_name: &str, new_name: &str, uplift: bool) { +pub fn rename<'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_name: &'cx str, new_name: &'cx str, uplift: bool) { let mut updater = FileUpdater::default(); - let mut lints = find_lint_decls(); - let (deprecated_lints, mut renamed_lints) = read_deprecated_lints(); + let mut lints = cx.find_lint_decls(); + let (deprecated_lints, mut renamed_lints) = cx.read_deprecated_lints(); let Ok(lint_idx) = lints.binary_search_by(|x| x.name.as_str().cmp(old_name)) else { panic!("could not find lint `{old_name}`"); diff --git a/clippy_dev/src/update_lints.rs b/clippy_dev/src/update_lints.rs index ef841891721d..5d0de45a2e26 100644 --- a/clippy_dev/src/update_lints.rs +++ b/clippy_dev/src/update_lints.rs @@ -1,5 +1,5 @@ use crate::parse::cursor::Cursor; -use crate::parse::{DeprecatedLint, Lint, RenamedLint, find_lint_decls, read_deprecated_lints}; +use crate::parse::{DeprecatedLint, Lint, ParseCx, RenamedLint}; use crate::utils::{FileUpdater, UpdateMode, UpdateStatus, update_text_region_fn}; use itertools::Itertools; use std::collections::HashSet; @@ -21,9 +21,9 @@ const DOCS_LINK: &str = "https://rust-lang.github.io/rust-clippy/master/index.ht /// # Panics /// /// Panics if a file path could not read from or then written to -pub fn update(update_mode: UpdateMode) { - let lints = find_lint_decls(); - let (deprecated, renamed) = read_deprecated_lints(); +pub fn update(cx: ParseCx<'_>, update_mode: UpdateMode) { + let lints = cx.find_lint_decls(); + let (deprecated, renamed) = cx.read_deprecated_lints(); generate_lint_files(update_mode, &lints, &deprecated, &renamed); } diff --git a/clippy_dev/src/utils.rs b/clippy_dev/src/utils.rs index 526613a53c77..c01b05ec1720 100644 --- a/clippy_dev/src/utils.rs +++ b/clippy_dev/src/utils.rs @@ -1,5 +1,7 @@ use core::fmt::{self, Display}; +use core::marker::PhantomData; use core::num::NonZero; +use core::ops::{Deref, DerefMut}; use core::range::Range; use core::str::FromStr; use std::ffi::OsStr; @@ -10,6 +12,24 @@ use std::process::{self, Command, Stdio}; use std::{env, thread}; use walkdir::WalkDir; +pub struct Scoped<'inner, 'outer: 'inner, T>(T, PhantomData<&'inner mut T>, PhantomData<&'outer mut ()>); +impl Scoped<'_, '_, T> { + pub fn new(value: T) -> Self { + Self(value, PhantomData, PhantomData) + } +} +impl Deref for Scoped<'_, '_, T> { + type Target = T; + fn deref(&self) -> &Self::Target { + &self.0 + } +} +impl DerefMut for Scoped<'_, '_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + #[derive(Clone, Copy)] pub enum ErrAction { Open, From 7579e71c1fdecd326cb82d8372b8708a7cceca9a Mon Sep 17 00:00:00 2001 From: Jason Newcomb Date: Sat, 11 Oct 2025 11:35:08 -0400 Subject: [PATCH 2/3] `clippy_dev`: Inline and simplify `read_src_with_module`. --- clippy_dev/src/fmt.rs | 2 +- clippy_dev/src/parse.rs | 74 +++++++++++++++-------------------- clippy_dev/src/rename_lint.rs | 2 +- clippy_dev/src/utils.rs | 4 +- 4 files changed, 35 insertions(+), 47 deletions(-) diff --git a/clippy_dev/src/fmt.rs b/clippy_dev/src/fmt.rs index 2b2138d3108d..781e37e6144e 100644 --- a/clippy_dev/src/fmt.rs +++ b/clippy_dev/src/fmt.rs @@ -268,7 +268,7 @@ fn run_rustfmt(update_mode: UpdateMode) { .expect("invalid rustfmt path"); rustfmt_path.truncate(rustfmt_path.trim_end().len()); - let args: Vec<_> = walk_dir_no_dot_or_target() + let args: Vec<_> = walk_dir_no_dot_or_target(".") .filter_map(|e| { let e = expect_action(e, ErrAction::Read, "."); e.path() diff --git a/clippy_dev/src/parse.rs b/clippy_dev/src/parse.rs index 5c89f83dbd6a..f151a3c8fee9 100644 --- a/clippy_dev/src/parse.rs +++ b/clippy_dev/src/parse.rs @@ -1,11 +1,10 @@ pub mod cursor; use self::cursor::{Capture, Cursor}; -use crate::utils::{ErrAction, File, Scoped, expect_action}; +use crate::utils::{ErrAction, File, Scoped, expect_action, walk_dir_no_dot_or_target}; use core::range::Range; use std::fs; -use std::path::{Path, PathBuf}; -use walkdir::{DirEntry, WalkDir}; +use std::path::{self, Path, PathBuf}; pub struct ParseCxImpl; pub type ParseCx<'cx> = &'cx mut ParseCxImpl; @@ -43,18 +42,38 @@ impl ParseCxImpl { let mut contents = String::new(); for e in expect_action(fs::read_dir("."), ErrAction::Read, ".") { let e = expect_action(e, ErrAction::Read, "."); - if !expect_action(e.file_type(), ErrAction::Read, ".").is_dir() { - continue; - } - let Ok(mut name) = e.file_name().into_string() else { + + // Skip if this isn't a lint crate's directory. + let mut crate_path = if expect_action(e.file_type(), ErrAction::Read, ".").is_dir() + && let Ok(crate_path) = e.file_name().into_string() + && crate_path.starts_with("clippy_lints") + && crate_path != "clippy_lints_internal" + { + crate_path + } else { continue; }; - if name.starts_with("clippy_lints") && name != "clippy_lints_internal" { - name.push_str("/src"); - for (file, module) in read_src_with_module(name.as_ref()) { + + crate_path.push(path::MAIN_SEPARATOR); + crate_path.push_str("src"); + for e in walk_dir_no_dot_or_target(&crate_path) { + let e = expect_action(e, ErrAction::Read, &crate_path); + if let Some(path) = e.path().to_str() + && let Some(path) = path.strip_suffix(".rs") + && let Some(path) = path.get(crate_path.len() + 1..) + { + let module = if path == "lib" { + String::new() + } else { + let path = path + .strip_suffix("mod") + .and_then(|x| x.strip_suffix(path::MAIN_SEPARATOR)) + .unwrap_or(path); + path.replace(['/', '\\'], "::") + }; parse_clippy_lint_decls( - file.path(), - File::open_read_to_cleared_string(file.path(), &mut contents), + e.path(), + File::open_read_to_cleared_string(e.path(), &mut contents), &module, &mut lints, ); @@ -132,37 +151,6 @@ impl ParseCxImpl { } } -/// Reads the source files from the given root directory -fn read_src_with_module(src_root: &Path) -> impl use<'_> + Iterator { - WalkDir::new(src_root).into_iter().filter_map(move |e| { - let e = expect_action(e, ErrAction::Read, src_root); - let path = e.path().as_os_str().as_encoded_bytes(); - if let Some(path) = path.strip_suffix(b".rs") - && let Some(path) = path.get(src_root.as_os_str().len() + 1..) - { - if path == b"lib" { - Some((e, String::new())) - } else { - let path = if let Some(path) = path.strip_suffix(b"mod") - && let Some(path) = path.strip_suffix(b"/").or_else(|| path.strip_suffix(b"\\")) - { - path - } else { - path - }; - if let Ok(path) = str::from_utf8(path) { - let path = path.replace(['/', '\\'], "::"); - Some((e, path)) - } else { - None - } - } - } else { - None - } - }) -} - /// Parse a source file looking for `declare_clippy_lint` macro invocations. fn parse_clippy_lint_decls(path: &Path, contents: &str, module: &str, lints: &mut Vec) { #[allow(clippy::enum_glob_use)] diff --git a/clippy_dev/src/rename_lint.rs b/clippy_dev/src/rename_lint.rs index 04942cd639d6..bbe5baa41cce 100644 --- a/clippy_dev/src/rename_lint.rs +++ b/clippy_dev/src/rename_lint.rs @@ -127,7 +127,7 @@ pub fn rename<'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_name: &'cx str } let mut update_fn = file_update_fn(old_name, new_name, mod_edit); - for e in walk_dir_no_dot_or_target() { + for e in walk_dir_no_dot_or_target(".") { let e = expect_action(e, ErrAction::Read, "."); if e.path().as_os_str().as_encoded_bytes().ends_with(b".rs") { updater.update_file(e.path(), &mut update_fn); diff --git a/clippy_dev/src/utils.rs b/clippy_dev/src/utils.rs index c01b05ec1720..52452dd86b49 100644 --- a/clippy_dev/src/utils.rs +++ b/clippy_dev/src/utils.rs @@ -593,8 +593,8 @@ pub fn delete_dir_if_exists(path: &Path) { } /// Walks all items excluding top-level dot files/directories and any target directories. -pub fn walk_dir_no_dot_or_target() -> impl Iterator> { - WalkDir::new(".").into_iter().filter_entry(|e| { +pub fn walk_dir_no_dot_or_target(p: impl AsRef) -> impl Iterator> { + WalkDir::new(p).into_iter().filter_entry(|e| { e.path() .file_name() .is_none_or(|x| x != "target" && x.as_encoded_bytes().first().copied() != Some(b'.')) From bae625fed002328b1aff4dc8ae60b26d5c643d83 Mon Sep 17 00:00:00 2001 From: Jason Newcomb Date: Sat, 11 Oct 2025 14:59:59 -0400 Subject: [PATCH 3/3] `clippy_dev`: Allocate onto an arena when parsing. --- clippy_dev/src/deprecate_lint.rs | 15 +- clippy_dev/src/lib.rs | 2 + clippy_dev/src/parse.rs | 251 ++++++++++++++++++++----------- clippy_dev/src/rename_lint.rs | 40 ++--- clippy_dev/src/update_lints.rs | 14 +- 5 files changed, 200 insertions(+), 122 deletions(-) diff --git a/clippy_dev/src/deprecate_lint.rs b/clippy_dev/src/deprecate_lint.rs index e2602f57f953..0401cfda7080 100644 --- a/clippy_dev/src/deprecate_lint.rs +++ b/clippy_dev/src/deprecate_lint.rs @@ -23,8 +23,11 @@ pub fn deprecate<'cx>(cx: ParseCx<'cx>, clippy_version: Version, name: &'cx str, return; }; - let prefixed_name = String::from_iter(["clippy::", name]); - match deprecated_lints.binary_search_by(|x| x.name.cmp(&prefixed_name)) { + let prefixed_name = cx.str_buf.with(|buf| { + buf.extend(["clippy::", name]); + cx.arena.alloc_str(buf) + }); + match deprecated_lints.binary_search_by(|x| x.name.cmp(prefixed_name)) { Ok(_) => { println!("`{name}` is already deprecated"); return; @@ -33,8 +36,8 @@ pub fn deprecate<'cx>(cx: ParseCx<'cx>, clippy_version: Version, name: &'cx str, idx, DeprecatedLint { name: prefixed_name, - reason: reason.into(), - version: clippy_version.rust_display().to_string(), + reason, + version: cx.str_buf.alloc_display(cx.arena, clippy_version.rust_display()), }, ), } @@ -58,8 +61,8 @@ pub fn deprecate<'cx>(cx: ParseCx<'cx>, clippy_version: Version, name: &'cx str, } } -fn remove_lint_declaration(name: &str, path: &Path, lints: &mut Vec) -> io::Result { - fn remove_lint(name: &str, lints: &mut Vec) { +fn remove_lint_declaration(name: &str, path: &Path, lints: &mut Vec>) -> io::Result { + fn remove_lint(name: &str, lints: &mut Vec>) { lints.iter().position(|l| l.name == name).map(|pos| lints.remove(pos)); } diff --git a/clippy_dev/src/lib.rs b/clippy_dev/src/lib.rs index 16db6fa43dbe..dcca08aee7e6 100644 --- a/clippy_dev/src/lib.rs +++ b/clippy_dev/src/lib.rs @@ -5,6 +5,7 @@ new_range_api, os_str_slice, os_string_truncate, + pattern, rustc_private, slice_split_once )] @@ -17,6 +18,7 @@ )] #![allow(clippy::missing_panics_doc)] +extern crate rustc_arena; #[expect(unused_extern_crates, reason = "required to link to rustc crates")] extern crate rustc_driver; extern crate rustc_lexer; diff --git a/clippy_dev/src/parse.rs b/clippy_dev/src/parse.rs index f151a3c8fee9..de5caf4e1ef6 100644 --- a/clippy_dev/src/parse.rs +++ b/clippy_dev/src/parse.rs @@ -2,42 +2,109 @@ pub mod cursor; use self::cursor::{Capture, Cursor}; use crate::utils::{ErrAction, File, Scoped, expect_action, walk_dir_no_dot_or_target}; +use core::fmt::{Display, Write as _}; use core::range::Range; +use rustc_arena::DroplessArena; use std::fs; use std::path::{self, Path, PathBuf}; +use std::str::pattern::Pattern; -pub struct ParseCxImpl; -pub type ParseCx<'cx> = &'cx mut ParseCxImpl; +pub struct ParseCxImpl<'cx> { + pub arena: &'cx DroplessArena, + pub str_buf: StrBuf, +} +pub type ParseCx<'cx> = &'cx mut ParseCxImpl<'cx>; /// Calls the given function inside a newly created parsing context. -pub fn new_parse_cx<'env, T>(f: impl for<'cx> FnOnce(&'cx mut Scoped<'cx, 'env, ParseCxImpl>) -> T) -> T { - f(&mut Scoped::new(ParseCxImpl)) +pub fn new_parse_cx<'env, T>(f: impl for<'cx> FnOnce(&'cx mut Scoped<'cx, 'env, ParseCxImpl<'cx>>) -> T) -> T { + let arena = DroplessArena::default(); + f(&mut Scoped::new(ParseCxImpl { + arena: &arena, + str_buf: StrBuf::with_capacity(128), + })) } -pub struct Lint { - pub name: String, - pub group: String, - pub module: String, +/// A string used as a temporary buffer used to avoid allocating for short lived strings. +pub struct StrBuf(String); +impl StrBuf { + /// Creates a new buffer with the specified initial capacity. + pub fn with_capacity(cap: usize) -> Self { + Self(String::with_capacity(cap)) + } + + /// Allocates the result of formatting the given value onto the arena. + pub fn alloc_display<'cx>(&mut self, arena: &'cx DroplessArena, value: impl Display) -> &'cx str { + self.0.clear(); + write!(self.0, "{value}").expect("`Display` impl returned an error"); + arena.alloc_str(&self.0) + } + + /// Allocates the string onto the arena with all ascii characters converted to + /// lowercase. + pub fn alloc_ascii_lower<'cx>(&mut self, arena: &'cx DroplessArena, s: &str) -> &'cx str { + self.0.clear(); + self.0.push_str(s); + self.0.make_ascii_lowercase(); + arena.alloc_str(&self.0) + } + + /// Allocates the result of replacing all instances the pattern with the given string + /// onto the arena. + pub fn alloc_replaced<'cx>( + &mut self, + arena: &'cx DroplessArena, + s: &str, + pat: impl Pattern, + replacement: &str, + ) -> &'cx str { + let mut parts = s.split(pat); + let Some(first) = parts.next() else { + return ""; + }; + self.0.clear(); + self.0.push_str(first); + for part in parts { + self.0.push_str(replacement); + self.0.push_str(part); + } + if self.0.is_empty() { + "" + } else { + arena.alloc_str(&self.0) + } + } + + /// Performs an operation with the freshly cleared buffer. + pub fn with(&mut self, f: impl FnOnce(&mut String) -> T) -> T { + self.0.clear(); + f(&mut self.0) + } +} + +pub struct Lint<'cx> { + pub name: &'cx str, + pub group: &'cx str, + pub module: &'cx str, pub path: PathBuf, pub declaration_range: Range, } -pub struct DeprecatedLint { - pub name: String, - pub reason: String, - pub version: String, +pub struct DeprecatedLint<'cx> { + pub name: &'cx str, + pub reason: &'cx str, + pub version: &'cx str, } -pub struct RenamedLint { - pub old_name: String, - pub new_name: String, - pub version: String, +pub struct RenamedLint<'cx> { + pub old_name: &'cx str, + pub new_name: &'cx str, + pub version: &'cx str, } -impl ParseCxImpl { +impl<'cx> ParseCxImpl<'cx> { /// Finds all lint declarations (`declare_clippy_lint!`) #[must_use] - pub fn find_lint_decls(&mut self) -> Vec { + pub fn find_lint_decls(&mut self) -> Vec> { let mut lints = Vec::with_capacity(1000); let mut contents = String::new(); for e in expect_action(fs::read_dir("."), ErrAction::Read, ".") { @@ -63,29 +130,59 @@ impl ParseCxImpl { && let Some(path) = path.get(crate_path.len() + 1..) { let module = if path == "lib" { - String::new() + "" } else { let path = path .strip_suffix("mod") .and_then(|x| x.strip_suffix(path::MAIN_SEPARATOR)) .unwrap_or(path); - path.replace(['/', '\\'], "::") + self.str_buf + .alloc_replaced(self.arena, path, path::MAIN_SEPARATOR, "::") }; - parse_clippy_lint_decls( + self.parse_clippy_lint_decls( e.path(), File::open_read_to_cleared_string(e.path(), &mut contents), - &module, + module, &mut lints, ); } } } - lints.sort_by(|lhs, rhs| lhs.name.cmp(&rhs.name)); + lints.sort_by(|lhs, rhs| lhs.name.cmp(rhs.name)); lints } + /// Parse a source file looking for `declare_clippy_lint` macro invocations. + fn parse_clippy_lint_decls(&mut self, path: &Path, contents: &str, module: &'cx str, lints: &mut Vec>) { + #[allow(clippy::enum_glob_use)] + use cursor::Pat::*; + #[rustfmt::skip] + static DECL_TOKENS: &[cursor::Pat<'_>] = &[ + // !{ /// docs + Bang, OpenBrace, AnyComment, + // #[clippy::version = "version"] + Pound, OpenBracket, Ident("clippy"), DoubleColon, Ident("version"), Eq, LitStr, CloseBracket, + // pub NAME, GROUP, + Ident("pub"), CaptureIdent, Comma, AnyComment, CaptureIdent, Comma, + ]; + + let mut cursor = Cursor::new(contents); + let mut captures = [Capture::EMPTY; 2]; + while let Some(start) = cursor.find_ident("declare_clippy_lint") { + if cursor.match_all(DECL_TOKENS, &mut captures) && cursor.find_pat(CloseBrace) { + lints.push(Lint { + name: self.str_buf.alloc_ascii_lower(self.arena, cursor.get_text(captures[0])), + group: self.arena.alloc_str(cursor.get_text(captures[1])), + module, + path: path.into(), + declaration_range: start as usize..cursor.pos() as usize, + }); + } + } + } + #[must_use] - pub fn read_deprecated_lints(&mut self) -> (Vec, Vec) { + pub fn read_deprecated_lints(&mut self) -> (Vec>, Vec>) { #[allow(clippy::enum_glob_use)] use cursor::Pat::*; #[rustfmt::skip] @@ -124,9 +221,9 @@ impl ParseCxImpl { if cursor.find_ident("declare_with_version").is_some() && cursor.match_all(DEPRECATED_TOKENS, &mut []) { while cursor.match_all(DECL_TOKENS, &mut captures) { deprecated.push(DeprecatedLint { - name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[1])), - reason: parse_str_single_line(path.as_ref(), cursor.get_text(captures[2])), - version: parse_str_single_line(path.as_ref(), cursor.get_text(captures[0])), + name: self.parse_str_single_line(path.as_ref(), cursor.get_text(captures[1])), + reason: self.parse_str_single_line(path.as_ref(), cursor.get_text(captures[2])), + version: self.parse_str_single_line(path.as_ref(), cursor.get_text(captures[0])), }); } } else { @@ -136,81 +233,53 @@ impl ParseCxImpl { if cursor.find_ident("declare_with_version").is_some() && cursor.match_all(RENAMED_TOKENS, &mut []) { while cursor.match_all(DECL_TOKENS, &mut captures) { renamed.push(RenamedLint { - old_name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[1])), - new_name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[2])), - version: parse_str_single_line(path.as_ref(), cursor.get_text(captures[0])), + old_name: self.parse_str_single_line(path.as_ref(), cursor.get_text(captures[1])), + new_name: self.parse_str_single_line(path.as_ref(), cursor.get_text(captures[2])), + version: self.parse_str_single_line(path.as_ref(), cursor.get_text(captures[0])), }); } } else { panic!("error reading renamed lints"); } - deprecated.sort_by(|lhs, rhs| lhs.name.cmp(&rhs.name)); - renamed.sort_by(|lhs, rhs| lhs.old_name.cmp(&rhs.old_name)); + deprecated.sort_by(|lhs, rhs| lhs.name.cmp(rhs.name)); + renamed.sort_by(|lhs, rhs| lhs.old_name.cmp(rhs.old_name)); (deprecated, renamed) } -} -/// Parse a source file looking for `declare_clippy_lint` macro invocations. -fn parse_clippy_lint_decls(path: &Path, contents: &str, module: &str, lints: &mut Vec) { - #[allow(clippy::enum_glob_use)] - use cursor::Pat::*; - #[rustfmt::skip] - static DECL_TOKENS: &[cursor::Pat<'_>] = &[ - // !{ /// docs - Bang, OpenBrace, AnyComment, - // #[clippy::version = "version"] - Pound, OpenBracket, Ident("clippy"), DoubleColon, Ident("version"), Eq, LitStr, CloseBracket, - // pub NAME, GROUP, - Ident("pub"), CaptureIdent, Comma, AnyComment, CaptureIdent, Comma, - ]; - - let mut cursor = Cursor::new(contents); - let mut captures = [Capture::EMPTY; 2]; - while let Some(start) = cursor.find_ident("declare_clippy_lint") { - if cursor.match_all(DECL_TOKENS, &mut captures) && cursor.find_pat(CloseBrace) { - lints.push(Lint { - name: cursor.get_text(captures[0]).to_lowercase(), - group: cursor.get_text(captures[1]).into(), - module: module.into(), - path: path.into(), - declaration_range: start as usize..cursor.pos() as usize, - }); + /// Removes the line splices and surrounding quotes from a string literal + fn parse_str_lit(&mut self, s: &str) -> &'cx str { + let (s, is_raw) = if let Some(s) = s.strip_prefix("r") { + (s.trim_matches('#'), true) + } else { + (s, false) + }; + let s = s + .strip_prefix('"') + .and_then(|s| s.strip_suffix('"')) + .unwrap_or_else(|| panic!("expected quoted string, found `{s}`")); + + if is_raw { + if s.is_empty() { "" } else { self.arena.alloc_str(s) } + } else { + self.str_buf.with(|buf| { + rustc_literal_escaper::unescape_str(s, &mut |_, ch| { + if let Ok(ch) = ch { + buf.push(ch); + } + }); + if buf.is_empty() { "" } else { self.arena.alloc_str(buf) } + }) } } -} -/// Removes the line splices and surrounding quotes from a string literal -fn parse_str_lit(s: &str) -> String { - let (s, is_raw) = if let Some(s) = s.strip_prefix("r") { - (s.trim_matches('#'), true) - } else { - (s, false) - }; - let s = s - .strip_prefix('"') - .and_then(|s| s.strip_suffix('"')) - .unwrap_or_else(|| panic!("expected quoted string, found `{s}`")); - - if is_raw { - s.into() - } else { - let mut res = String::with_capacity(s.len()); - rustc_literal_escaper::unescape_str(s, &mut |_, ch| { - if let Ok(ch) = ch { - res.push(ch); - } - }); - res + fn parse_str_single_line(&mut self, path: &Path, s: &str) -> &'cx str { + let value = self.parse_str_lit(s); + assert!( + !value.contains('\n'), + "error parsing `{}`: `{s}` should be a single line string", + path.display(), + ); + value } } - -fn parse_str_single_line(path: &Path, s: &str) -> String { - let value = parse_str_lit(s); - assert!( - !value.contains('\n'), - "error parsing `{}`: `{s}` should be a single line string", - path.display(), - ); - value -} diff --git a/clippy_dev/src/rename_lint.rs b/clippy_dev/src/rename_lint.rs index bbe5baa41cce..8e30eb7ce95b 100644 --- a/clippy_dev/src/rename_lint.rs +++ b/clippy_dev/src/rename_lint.rs @@ -31,24 +31,30 @@ pub fn rename<'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_name: &'cx str let mut lints = cx.find_lint_decls(); let (deprecated_lints, mut renamed_lints) = cx.read_deprecated_lints(); - let Ok(lint_idx) = lints.binary_search_by(|x| x.name.as_str().cmp(old_name)) else { + let Ok(lint_idx) = lints.binary_search_by(|x| x.name.cmp(old_name)) else { panic!("could not find lint `{old_name}`"); }; let lint = &lints[lint_idx]; - let old_name_prefixed = String::from_iter(["clippy::", old_name]); + let old_name_prefixed = cx.str_buf.with(|buf| { + buf.extend(["clippy::", old_name]); + cx.arena.alloc_str(buf) + }); let new_name_prefixed = if uplift { - new_name.to_owned() + new_name } else { - String::from_iter(["clippy::", new_name]) + cx.str_buf.with(|buf| { + buf.extend(["clippy::", new_name]); + cx.arena.alloc_str(buf) + }) }; for lint in &mut renamed_lints { if lint.new_name == old_name_prefixed { - lint.new_name.clone_from(&new_name_prefixed); + lint.new_name = new_name_prefixed; } } - match renamed_lints.binary_search_by(|x| x.old_name.cmp(&old_name_prefixed)) { + match renamed_lints.binary_search_by(|x| x.old_name.cmp(old_name_prefixed)) { Ok(_) => { println!("`{old_name}` already has a rename registered"); return; @@ -58,12 +64,8 @@ pub fn rename<'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_name: &'cx str idx, RenamedLint { old_name: old_name_prefixed, - new_name: if uplift { - new_name.to_owned() - } else { - String::from_iter(["clippy::", new_name]) - }, - version: clippy_version.rust_display().to_string(), + new_name: new_name_prefixed, + version: cx.str_buf.alloc_display(cx.arena, clippy_version.rust_display()), }, ); }, @@ -99,7 +101,7 @@ pub fn rename<'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_name: &'cx str } delete_test_files(old_name, change_prefixed_tests); lints.remove(lint_idx); - } else if lints.binary_search_by(|x| x.name.as_str().cmp(new_name)).is_err() { + } else if lints.binary_search_by(|x| x.name.cmp(new_name)).is_err() { let lint = &mut lints[lint_idx]; if lint.module.ends_with(old_name) && lint @@ -113,13 +115,15 @@ pub fn rename<'cx>(cx: ParseCx<'cx>, clippy_version: Version, old_name: &'cx str mod_edit = ModEdit::Rename; } - let mod_len = lint.module.len(); - lint.module.truncate(mod_len - old_name.len()); - lint.module.push_str(new_name); + lint.module = cx.str_buf.with(|buf| { + buf.push_str(&lint.module[..lint.module.len() - old_name.len()]); + buf.push_str(new_name); + cx.arena.alloc_str(buf) + }); } rename_test_files(old_name, new_name, change_prefixed_tests); - new_name.clone_into(&mut lints[lint_idx].name); - lints.sort_by(|lhs, rhs| lhs.name.cmp(&rhs.name)); + lints[lint_idx].name = new_name; + lints.sort_by(|lhs, rhs| lhs.name.cmp(rhs.name)); } else { println!("Renamed `clippy::{old_name}` to `clippy::{new_name}`"); println!("Since `{new_name}` already exists the existing code has not been changed"); diff --git a/clippy_dev/src/update_lints.rs b/clippy_dev/src/update_lints.rs index 5d0de45a2e26..3d0da6846114 100644 --- a/clippy_dev/src/update_lints.rs +++ b/clippy_dev/src/update_lints.rs @@ -30,9 +30,9 @@ pub fn update(cx: ParseCx<'_>, update_mode: UpdateMode) { #[expect(clippy::too_many_lines)] pub fn generate_lint_files( update_mode: UpdateMode, - lints: &[Lint], - deprecated: &[DeprecatedLint], - renamed: &[RenamedLint], + lints: &[Lint<'_>], + deprecated: &[DeprecatedLint<'_>], + renamed: &[RenamedLint<'_>], ) { let mut updater = FileUpdater::default(); updater.update_file_checked( @@ -61,7 +61,7 @@ pub fn generate_lint_files( |dst| { for lint in lints .iter() - .map(|l| &*l.name) + .map(|l| l.name) .chain(deprecated.iter().filter_map(|l| l.name.strip_prefix("clippy::"))) .chain(renamed.iter().filter_map(|l| l.old_name.strip_prefix("clippy::"))) .sorted() @@ -132,13 +132,13 @@ pub fn generate_lint_files( dst.push_str(GENERATED_FILE_COMMENT); dst.push_str("#![allow(clippy::duplicated_attributes)]\n"); for lint in renamed { - if seen_lints.insert(&lint.new_name) { + if seen_lints.insert(lint.new_name) { writeln!(dst, "#![allow({})]", lint.new_name).unwrap(); } } seen_lints.clear(); for lint in renamed { - if seen_lints.insert(&lint.old_name) { + if seen_lints.insert(lint.old_name) { writeln!(dst, "#![warn({})] //~ ERROR: lint `{}`", lint.old_name, lint.old_name).unwrap(); } } @@ -164,7 +164,7 @@ pub fn generate_lint_files( for lint_mod in lints .iter() .filter(|l| !l.module.is_empty()) - .map(|l| l.module.split_once("::").map_or(&*l.module, |x| x.0)) + .map(|l| l.module.split_once("::").map_or(l.module, |x| x.0)) .sorted() .dedup() {