diff --git a/Cargo.lock b/Cargo.lock index 1923b57..5f0fc75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -201,7 +201,6 @@ dependencies = [ "crossbeam-channel", "ignore", "indicatif", - "memmap2", "rayon", "regex", "serde", @@ -543,15 +542,6 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" -[[package]] -name = "memmap2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" -dependencies = [ - "libc", -] - [[package]] name = "num-traits" version = "0.2.19" diff --git a/Cargo.toml b/Cargo.toml index a4af229..eb738d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ anyhow = "1.0" ahash = { version = "0.8", features = ["serde"] } clap = { version = "4.5", features = ["derive"] } ignore = "0.4" -memmap2 = "0.9" +tempfile = "3.10" rayon = "1.10" regex = { version = "1.11", default-features = false, features = ["std", "unicode-perl"] } serde = { version = "1.0", features = ["derive"] } @@ -64,7 +64,6 @@ lang-typescript = ["tree-sitter-typescript"] [dev-dependencies] assert_cmd = "2.0" -tempfile = "3.10" criterion = "0.8" [[bench]] diff --git a/README.md b/README.md index 3768c28..d619933 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,16 @@ c. **Anchoring**: The scanner looks for "library anchors" (e.g., `import` or `#include` statements) that match known cryptographic libraries defined in `patterns.toml`. d. **Algorithm Detection**: If an anchor is found, the scanner performs a deeper search within that file for specific algorithm usage patterns, such as function calls and constants. -All results are streamed as JSONL to the output, allowing for real-time monitoring and processing. +Findings are streamed as JSONL to stdout for monitoring and processing. For a deeper architecture overview, see `DESIGN.md`. +The CLI exits unsuccessfully if discovery, reading, parsing, or writing fails; +diagnostics go to stderr. Stdout may contain partial findings on failure. With +`--output`, results are staged beside the destination and replace it only after a +successful scan, preserving any previous inventory on failure. Output must be a +regular file, and existing source files or the custom patterns file cannot be +used as the destination. The destination directory must be writable. + ## Installation Ensure you have the Rust toolchain installed. You can install it from [rustup.rs](https://rustup.rs/). diff --git a/src/main.rs b/src/main.rs index 2e15ead..e7faf37 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,10 @@ use std::fs::File; -use std::io::{BufWriter, Write}; +use std::io::{BufWriter, Read, Write}; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use anyhow::{Context, Result, anyhow}; +use anyhow::{Context, Result, anyhow, bail}; use cipherscope::{DEFAULT_PATTERNS, Finding, patterns, scan, scan_with_patterns}; use clap::Parser; use crossbeam_channel as channel; @@ -12,8 +12,8 @@ use ignore::WalkBuilder; use ignore::overrides::OverrideBuilder; use ignore::types::TypesBuilder; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; -use memmap2::Mmap; use rayon::prelude::*; +use tempfile::NamedTempFile; #[derive(Parser, Debug)] #[command( @@ -141,32 +141,6 @@ fn main() -> Result<()> { // Bounded queues apply backpressure when discovery or scanning outruns its consumer. let queue_capacity = cli.threads.saturating_mul(4).max(1); let (tx, rx) = channel::bounded::(queue_capacity); - let (writer, output_scan_path): (Box, Option) = if cli.output == "-" - { - (Box::new(std::io::stdout()), None) - } else { - let file = File::create(&cli.output).with_context(|| format!("create {}", cli.output))?; - let output_scan_path = std::fs::canonicalize(&cli.output) - .with_context(|| format!("resolve output path: {}", cli.output))?; - (Box::new(BufWriter::new(file)), Some(output_scan_path)) - }; - let found_count_writer = found_count.clone(); - let scan_bar_writer = scan_bar.clone(); - let writer_handle = std::thread::spawn(move || -> Result<()> { - let mut writer = writer; - for finding in rx.iter() { - serde_json::to_writer(&mut writer, &finding)?; - writer.write_all(b"\n")?; - let count = found_count_writer.fetch_add(1, Ordering::Relaxed) + 1; - if let Some(pb) = &scan_bar_writer { - pb.set_message(format!("Found {} cryptographic items", count)); - } - } - // Flush any remaining buffered output - writer.flush()?; - Ok(()) - }); - // Streaming architecture: WalkBuilder sends files to channel, rayon workers process immediately // This eliminates the mutex contention and synchronous barrier of collect-then-process let (file_tx, file_rx) = channel::bounded::(queue_capacity); @@ -231,18 +205,47 @@ fn main() -> Result<()> { let max_bytes = cli.max_file_mb.map(|mb| mb.saturating_mul(1024 * 1024)); + // Validate all options before opening output or starting background threads. + let (writer, pending_output, output_scan_path) = prepare_output(&cli)?; + let error_count = Arc::new(AtomicUsize::new(0)); + let cancelled = Arc::new(AtomicBool::new(false)); + let found_count_writer = found_count.clone(); + let scan_bar_writer = scan_bar.clone(); + let cancelled_writer = cancelled.clone(); + let writer_handle = std::thread::spawn(move || { + let result = write_findings(writer, rx, &found_count_writer, scan_bar_writer.as_ref()); + if result.is_err() { + cancelled_writer.store(true, Ordering::Relaxed); + } + result + }); + // Spawn scanner workers that process files as they're discovered let scan_bar_for_workers = scan_bar.clone(); let scanned_count_for_workers = scanned_count.clone(); let patterns_for_workers = patterns.clone(); let tx_for_workers = tx.clone(); + let errors_for_workers = error_count.clone(); + let cancelled_workers = cancelled.clone(); + let skipped_for_workers = skipped_oversize_count.clone(); // Use a thread to run the parallel scanner on the receiving end let scanner_handle = std::thread::spawn(move || { // Process files as they arrive from the channel file_rx.into_iter().par_bridge().for_each(|path| { - if let Err(err) = process_file(&path, &patterns_for_workers, &tx_for_workers) { - eprintln!("Error processing {}: {err:#}", path.display()); + if cancelled_workers.load(Ordering::Relaxed) { + return; + } + match process_file(&path, &patterns_for_workers, &tx_for_workers, max_bytes) { + Ok(false) => { + skipped_for_workers.fetch_add(1, Ordering::Relaxed); + return; + } + Ok(true) => {} + Err(err) => { + errors_for_workers.fetch_add(1, Ordering::Relaxed); + eprintln!("Error processing {}: {err:#}", path.display()); + } } scanned_count_for_workers.fetch_add(1, Ordering::Relaxed); @@ -260,6 +263,7 @@ fn main() -> Result<()> { .git_exclude(cli.gitignore) .git_global(cli.gitignore) .follow_links(false) + .skip_stdout(true) .threads(cli.threads) .build_parallel() .run(|| { @@ -269,7 +273,18 @@ fn main() -> Result<()> { let discovery_bar = discovery_bar.clone(); let skipped_oversize = skipped_oversize_discovery.clone(); let output_scan_path = output_scan_path.clone(); + let errors = error_count.clone(); + let cancelled = cancelled.clone(); Box::new(move |entry| { + if cancelled.load(Ordering::Relaxed) { + return ignore::WalkState::Quit; + } + if let Ok(entry) = &entry + && let Some(err) = entry.error() + { + errors.fetch_add(1, Ordering::Relaxed); + eprintln!("walk error: {err}"); + } match entry { Ok(e) if e.file_type().map(|t| t.is_file()).unwrap_or(false) => { // Never scan the output while it is being written, even if it has a @@ -294,7 +309,9 @@ fn main() -> Result<()> { && patterns.supports_language(lang) { // Send to channel instead of pushing to mutex-protected Vec - let _ = file_tx.send(path); + if file_tx.send(path).is_err() { + return ignore::WalkState::Quit; + } let count = file_count.fetch_add(1, Ordering::Relaxed) + 1; // Batch progress updates: only update every 100 files if let Some(pb) = &discovery_bar @@ -309,7 +326,10 @@ fn main() -> Result<()> { } } Ok(_) => {} - Err(err) => eprintln!("walk error: {err}"), + Err(err) => { + errors.fetch_add(1, Ordering::Relaxed); + eprintln!("walk error: {err}"); + } } ignore::WalkState::Continue }) @@ -333,16 +353,28 @@ fn main() -> Result<()> { } // Wait for all scanning to complete - scanner_handle + let scanner_result = scanner_handle .join() - .map_err(|_| anyhow!("scanner thread panicked"))?; + .map_err(|_| anyhow!("scanner thread panicked")); // All files have been processed drop(tx); - writer_handle + let writer_result = writer_handle .join() - .map_err(|_| anyhow!("writer thread panicked"))??; + .map_err(|_| anyhow!("writer thread panicked")); + scanner_result?; + writer_result??; + + let errors = error_count.load(Ordering::Relaxed); + if errors != 0 { + bail!("scan incomplete: {errors} error(s); see diagnostics above"); + } + if let Some(output) = pending_output { + output + .persist(&cli.output) + .with_context(|| format!("save output: {}", cli.output))?; + } if let Some(pb) = &scan_bar { pb.finish_with_message(format!( @@ -366,7 +398,7 @@ fn main() -> Result<()> { /// Processes a single file to find cryptographic assets. /// /// This function performs the core analysis for each file: -/// 1. **Memory-maps** the file for efficient reading. +/// 1. Reads the file into owned memory, enforcing the size limit while reading. /// 2. Decodes the file content to UTF-8, with a fallback to a lossy conversion. /// 3. **Parses** the content into an Abstract Syntax Tree (AST) using `tree-sitter`. /// 4. **Finds library anchors**: Scans the AST for `import` or `include` statements that @@ -379,29 +411,133 @@ fn process_file( path: &Path, patterns: &patterns::PatternSet, tx: &channel::Sender, -) -> Result<()> { + max_bytes: Option, +) -> Result { let file = File::open(path).with_context(|| format!("open {}", path.display()))?; - if file.metadata()?.len() == 0 { - return Ok(()); - } - let mmap = unsafe { Mmap::map(&file)? }; - // Decode file contents safely; fall back to lossy if not valid UTF-8 to avoid UB - let content_owned; - let content: &str = match std::str::from_utf8(&mmap) { - Ok(s) => s, - Err(_) => { - content_owned = String::from_utf8_lossy(&mmap).into_owned(); - &content_owned - } + let Some(bytes) = read_source(file, max_bytes)? else { + return Ok(false); }; + let content = String::from_utf8_lossy(&bytes); let Some(lang) = scan::language_from_path(path) else { - return Ok(()); + return Ok(true); }; let source_label = path.to_string_lossy(); - for finding in scan_with_patterns(content, lang, &source_label, patterns)? { + for finding in scan_with_patterns(&content, lang, &source_label, patterns)? { tx.send(finding).context("writer thread stopped")?; } - Ok(()) + Ok(true) +} + +// Reading owned bytes avoids the undefined behavior of file-backed mmap when +// editors or build tools modify/truncate source files during a scan. +fn read_source(reader: impl Read, max_bytes: Option) -> Result>> { + let mut bytes = Vec::new(); + reader + .take(max_bytes.unwrap_or(u64::MAX).saturating_add(1)) + .read_to_end(&mut bytes) + .context("read source")?; + if max_bytes.is_some_and(|limit| bytes.len() as u64 > limit) { + return Ok(None); + } + Ok(Some(bytes)) +} + +type PreparedOutput = ( + Box, + Option, + Option, +); + +fn prepare_output(cli: &Cli) -> Result { + if cli.output == "-" { + return Ok((Box::new(BufWriter::new(std::io::stdout())), None, None)); + } + let destination = Path::new(&cli.output); + let existing = match std::fs::symlink_metadata(destination) { + Ok(metadata) => { + if !metadata.is_file() { + bail!("output must be a regular file: {}", destination.display()); + } + let resolved = destination.canonicalize()?; + let is_input = scan::language_from_path(destination).is_some() + || cli + .roots + .iter() + .chain(cli.patterns.iter()) + .any(|path| path.canonicalize().is_ok_and(|path| path == resolved)); + if is_input { + bail!( + "refusing to overwrite scan input: {}", + destination.display() + ); + } + Some(resolved) + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => None, + Err(err) => return Err(err).context("inspect output path"), + }; + let parent = destination + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let pending = tempfile::Builder::new() + .prefix(".cipherscope-") + .suffix(".tmp") + .tempfile_in(parent) + .with_context(|| format!("create output in {}", parent.display()))?; + let writer = BufWriter::new(pending.reopen()?); + Ok((Box::new(writer), Some(pending), existing)) +} + +fn write_findings( + mut writer: impl Write, + rx: channel::Receiver, + found_count: &AtomicUsize, + progress: Option<&ProgressBar>, +) -> Result<()> { + for finding in rx { + serde_json::to_writer(&mut writer, &finding)?; + writer.write_all(b"\n")?; + let count = found_count.fetch_add(1, Ordering::Relaxed) + 1; + if let Some(pb) = progress { + pb.set_message(format!("Found {count} cryptographic items")); + } + } + writer.flush().context("flush findings") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_at_most_limit_plus_one_bytes() { + let mut reader = std::io::Cursor::new(vec![b'x'; 100]); + assert!(read_source(&mut reader, Some(8)).unwrap().is_none()); + assert_eq!(reader.position(), 9); + assert_eq!( + read_source(&b"abcd"[..], Some(4)).unwrap().unwrap(), + b"abcd" + ); + assert!(read_source(&b""[..], Some(0)).unwrap().unwrap().is_empty()); + } + + #[test] + fn writer_propagates_flush_failure() { + struct FailingFlush; + impl Write for FailingFlush { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Err(std::io::Error::other("disk full")) + } + } + let (tx, rx) = channel::bounded(1); + drop(tx); + let error = write_findings(FailingFlush, rx, &AtomicUsize::new(0), None).unwrap_err(); + assert!(format!("{error:#}").contains("disk full")); + } } diff --git a/tests/cli_io.rs b/tests/cli_io.rs new file mode 100644 index 0000000..51ed931 --- /dev/null +++ b/tests/cli_io.rs @@ -0,0 +1,117 @@ +use std::{fs, process::Command}; +use tempfile::TempDir; + +fn scanner() -> Command { + Command::new(env!("CARGO_BIN_EXE_cipherscope")) +} + +#[test] +fn missing_root_fails_and_preserves_existing_output() { + let dir = TempDir::new().unwrap(); + let output_path = dir.path().join("inventory.jsonl"); + fs::write(&output_path, "previous inventory\n").unwrap(); + let output = scanner() + .arg("--roots") + .arg(dir.path().join("missing")) + .arg("--output") + .arg(&output_path) + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("scan incomplete")); + assert_eq!( + fs::read_to_string(output_path).unwrap(), + "previous inventory\n" + ); + assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 1); +} + +#[test] +fn invalid_exclusion_preserves_output() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("inventory.jsonl"); + fs::write(&path, "previous inventory").unwrap(); + let output = scanner() + .arg("--roots") + .arg(dir.path()) + .args(["--exclude", "["]) + .arg("--output") + .arg(&path) + .output() + .unwrap(); + assert!(!output.status.success()); + assert_eq!(fs::read_to_string(path).unwrap(), "previous inventory"); +} + +#[test] +fn output_cannot_replace_an_explicit_input() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("input.rs"); + let source = "use ring::digest;\n"; + fs::write(&path, source).unwrap(); + let output = scanner() + .arg("--roots") + .arg(&path) + .arg("--output") + .arg(&path) + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("refusing to overwrite scan input")); + assert_eq!(fs::read_to_string(path).unwrap(), source); +} + +#[test] +fn output_cannot_replace_custom_patterns() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("custom.toml"); + let patterns = "library = []\n"; + fs::write(&path, patterns).unwrap(); + let output = scanner() + .arg("--roots") + .arg(dir.path()) + .arg("--patterns") + .arg(&path) + .arg("--output") + .arg(&path) + .output() + .unwrap(); + assert!(!output.status.success()); + assert_eq!(fs::read_to_string(path).unwrap(), patterns); +} + +#[cfg(feature = "lang-c")] +#[test] +fn failed_scan_still_streams_successful_findings_to_stdout() { + let dir = TempDir::new().unwrap(); + fs::write(dir.path().join("source.c"), "#include \n").unwrap(); + let output = scanner() + .arg("--roots") + .arg(dir.path()) + .arg("--roots") + .arg(dir.path().join("missing")) + .args(["--threads", "1"]) + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stdout).contains("OpenSSL")); +} + +#[cfg(unix)] +#[test] +fn output_symlink_does_not_modify_its_target() { + let dir = TempDir::new().unwrap(); + let source = dir.path().join("source.rs"); + let output_path = dir.path().join("output.jsonl"); + fs::write(&source, "source text").unwrap(); + std::os::unix::fs::symlink(&source, &output_path).unwrap(); + let output = scanner() + .arg("--roots") + .arg(dir.path()) + .arg("--output") + .arg(output_path) + .output() + .unwrap(); + assert!(!output.status.success()); + assert_eq!(fs::read_to_string(source).unwrap(), "source text"); +}