-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: rayon bring us back to that 36% performance improvement we had
originally observed, and dramatically simplifies our code
- Loading branch information
Showing
6 changed files
with
82 additions
and
104 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,109 +1,39 @@ | ||
| use std::{path::PathBuf, thread}; | ||
| use std::path::PathBuf; | ||
|
|
||
| use log::*; | ||
| use mrp::MatchAndReplaceStrategy; | ||
| use rayon::prelude::*; | ||
|
|
||
| #[derive(Default)] | ||
| pub struct BulkRenameOptions { | ||
| pub no_rename: bool, | ||
| pub multi: bool, | ||
| } | ||
|
|
||
| pub fn in_bulk<'p: 'r, 'r, R: MatchAndReplaceStrategy<'r> + std::marker::Sync>( | ||
| paths: &'p [PathBuf], | ||
| rename: &R, | ||
| options: &BulkRenameOptions, | ||
| ) { | ||
| if paths.is_empty() { | ||
| return; | ||
| } | ||
|
|
||
| if options.multi { | ||
| let thread_count = num_cpus::get(); | ||
|
|
||
| if thread_count > paths.len() { | ||
| warn!("there are more threads that files to rename, so single threaded it is"); | ||
| } else if thread_count * 500 > paths.len() { | ||
| warn!("probably too few files to warrant multithreading, but here we go..."); | ||
| return in_bulk_multithreaded(paths, rename, thread_count, options.no_rename); | ||
| } else { | ||
| return in_bulk_multithreaded(paths, rename, thread_count, options.no_rename); | ||
| } | ||
| } | ||
| return in_bulk_single_thread(paths, rename, options.no_rename); | ||
| } | ||
|
|
||
| fn in_bulk_single_thread<'p: 'r, 'r, R: MatchAndReplaceStrategy<'r>>( | ||
| paths: &'p [PathBuf], | ||
| rename: &R, | ||
| no_rename: bool, | ||
| ) { | ||
| let values = paths.iter().filter_map(|p| { | ||
| let str = p.to_str(); | ||
|
|
||
| if str.is_none() { | ||
| error!("Path is invalid unicode: {:?}", p); | ||
| } | ||
|
|
||
| return str; | ||
| }); | ||
|
|
||
| for from in values { | ||
| if let Some(to) = rename.apply(from) { | ||
| if no_rename { | ||
| paths | ||
| .par_iter() | ||
| .filter_map(|p| { | ||
| let path_string = p.to_str(); | ||
|
|
||
| if path_string.is_none() { | ||
| error!("Path is invalid unicode: {:?}", p); | ||
| } | ||
|
|
||
| return match path_string { | ||
| Some(s) => rename.apply(s).map(|renamed| (s, renamed)), | ||
| None => None, | ||
| }; | ||
| }) | ||
| .for_each(|(from, to)| { | ||
| if options.no_rename { | ||
| println!("{:?} -> {:?}", from, to); | ||
| } else { | ||
| if let Err(err) = std::fs::rename(from, to.to_string()) { | ||
| error!("{:?}: {}", from, err); | ||
| } | ||
| }; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| fn in_bulk_multithreaded<'p: 'r, 'r, R: MatchAndReplaceStrategy<'r> + std::marker::Sync>( | ||
| paths: &'p [PathBuf], | ||
| rename: &R, | ||
| thread_count: usize, | ||
| no_rename: bool, | ||
| ) { | ||
| debug!("found {} threads available on this machine", thread_count); | ||
| let max_chunk_size = paths.len() / (thread_count - 1); | ||
|
|
||
| debug!( | ||
| "chunking work, to handle {} files in each of {} threads", | ||
| max_chunk_size, thread_count | ||
| ); | ||
|
|
||
| let chunks = paths.chunks(max_chunk_size); | ||
|
|
||
| thread::scope(|s| { | ||
| let mut join_handles = vec![]; | ||
|
|
||
| for (id, path_chunk) in chunks.enumerate() { | ||
| if let Ok(handle) = thread::Builder::new().spawn_scoped(s, || { | ||
| in_bulk_single_thread(path_chunk, rename, no_rename); | ||
| }) { | ||
| debug!( | ||
| "spawned thread {} with {} file to rename", | ||
| id, | ||
| path_chunk.len() | ||
| ); | ||
| join_handles.push(handle); | ||
| } else { | ||
| error!( | ||
| "failed to spawn thread {}, renaming the next {} files in the main thread", | ||
| id, | ||
| path_chunk.len() | ||
| ); | ||
| in_bulk_single_thread(path_chunk, rename, no_rename); | ||
| }; | ||
| } | ||
|
|
||
| for handle in join_handles { | ||
| handle | ||
| .join() | ||
| .expect("Couldn't join on the associated thread") | ||
| } | ||
| }) | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters