From fed0b405594b3973906e6a91ac3a033390fb2aa9 Mon Sep 17 00:00:00 2001 From: J2ghz Date: Sun, 6 Sep 2020 00:28:36 +0200 Subject: [PATCH 01/20] try anyhow --- Cargo.lock | 7 +++++ Cargo.toml | 3 +- benches/bench.rs | 2 +- src/lib.rs | 74 ++++++++++++++++++++++++++++-------------------- src/main.rs | 10 ++++--- 5 files changed, 59 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ccde004..e348c6d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,6 +21,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "anyhow" +version = "1.0.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b602bfe940d21c130f3895acd65221e8a61270debe89d628b9cb4e3ccb8569b" + [[package]] name = "arrayvec" version = "0.5.1" @@ -883,6 +889,7 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" name = "sdr-heatmap" version = "0.1.5" dependencies = [ + "anyhow", "arrayvec", "clap", "criterion", diff --git a/Cargo.toml b/Cargo.toml index dba8996..03870c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,4 +35,5 @@ flate2 = '1.0.17' rayon = '1.4.0' webp = "0.1.0" arrayvec = "0.5" -itertools = "0.9" \ No newline at end of file +itertools = "0.9" +anyhow = "1.0" \ No newline at end of file diff --git a/benches/bench.rs b/benches/bench.rs index 4d84226..7a7c889 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -94,7 +94,7 @@ fn process_bench(c: &mut Criterion) { b.iter_with_large_setup( || read_csv_to_memory(file), |data| { - let summary = process(data, -1000.0, 1000.0, Palette::Default); + let summary = process(data, -1000.0, 1000.0, Palette::Default).unwrap(); black_box(summary); }, ) diff --git a/src/lib.rs b/src/lib.rs index 52dbc93..1aa78e3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,9 +6,10 @@ use std::io::prelude::*; use std::path::Path; use std::{cmp::Ordering, ffi::OsStr, fs::File}; mod palettes; +use anyhow::{Context, Result}; use arrayvec::ArrayVec; -use itertools::Itertools; use image::png::PngEncoder; +use itertools::Itertools; pub use palettes::{scale_tocolor, Palette}; #[derive(Debug)] @@ -35,22 +36,40 @@ impl Measurement { }) .collect() } - fn new(record: StringRecord) -> Measurement { + fn new(record: StringRecord) -> Result { let mut values: Vec<_> = record .iter() .skip(6) - .map(|s| parse_f32(s).unwrap()) - .collect(); + .map(|s| parse_f32(s)) + .collect::>>()?; values.truncate(record.len() - 7); - Measurement { - date: record.get(0).unwrap().to_string(), - time: record.get(1).unwrap().to_string(), - freq_low: parse(record.get(2).unwrap()).unwrap(), - freq_high: parse(record.get(3).unwrap()).unwrap(), - freq_step: parse(record.get(4).unwrap()).unwrap(), - samples: parse(record.get(5).unwrap()).unwrap(), + Ok(Measurement { + date: record + .get(0) + .context("Couldn't get date column")? + .to_string(), + time: record + .get(1) + .context("Couldn't get time column")? + .to_string(), + freq_low: record + .get(2) + .context("Couldn't get freq_low column")? + .parse()?, + freq_high: record + .get(3) + .context("Couldn't get freq_high column")? + .parse()?, + freq_step: record + .get(4) + .context("Couldn't get freq_step column")? + .parse()?, + samples: record + .get(5) + .context("Couldn't get samples column")? + .parse()?, values, - } + }) } } @@ -95,19 +114,11 @@ impl Summary { } } -fn parse( - string: &str, -) -> std::result::Result::Err> { - let parsed = string.parse::(); - debug_assert!(parsed.is_ok(), "Could not parse '{}'", string); - parsed -} - -fn parse_f32(s: &str) -> std::result::Result::Err> { +fn parse_f32(s: &str) -> Result { if s == "-nan" || s == "nan" { Ok(f32::NAN) } else { - parse(s) + Ok(s.parse::()?) } } @@ -126,7 +137,7 @@ fn read_file(file: T) -> csv::Reader { .from_reader(file) } -pub fn main>(path: P, palette: Palette) { +pub fn main>(path: P, palette: Palette) -> Result<()> { let path = path.as_ref(); info!("Loading: {}", path.display()); //Preprocess @@ -136,11 +147,12 @@ pub fn main>(path: P, palette: Palette) { //Process let file = open_file(path); let reader = read_file(file); - let (datawidth, dataheight, img) = process(reader, summary.min, summary.max, palette); + let (datawidth, dataheight, img) = process(reader, summary.min, summary.max, palette)?; //Draw let (height, imgdata) = create_image(datawidth, dataheight, img); let dest = path.with_extension("png"); save_image(datawidth, height, imgdata, dest.to_str().unwrap()).unwrap(); + Ok(()) } pub fn preprocess(file: Box) -> Summary { @@ -242,7 +254,7 @@ pub fn process( min: f32, max: f32, palette: Palette, -) -> (usize, usize, std::vec::Vec) { +) -> Result<(usize, usize, std::vec::Vec)> { let mut date: String = "".to_string(); let mut time: String = "".to_string(); let mut batch = 0; @@ -252,7 +264,7 @@ pub fn process( let mut record = result.unwrap(); record.trim(); assert!(record.len() > 7); - let m = Measurement::new(record); + let m = Measurement::new(record)?; let vals = m.get_values_with_freq(); if date == m.date && time == m.time { } else { @@ -274,7 +286,7 @@ pub fn process( datawidth = batch; } info!("Img data {}x{}", datawidth, batch); - (datawidth, img.len() / 3 / datawidth, img) + Ok((datawidth, img.len() / 3 / datawidth, img)) } pub fn process_iter( @@ -292,7 +304,7 @@ pub fn process_iter( record }) .map(Measurement::new) - .flat_map(|m| m.values.into_iter()) + .flat_map(|m| m.unwrap().values.into_iter()) .flat_map(|val| { let slice = scale_tocolor(Palette::Default, val, min, max); ArrayVec::from(slice).into_iter() @@ -384,14 +396,14 @@ mod tests { } #[test_resources("samples/*.csv.gz")] - fn process_implementations_equal(path: &str) { + fn process_implementations_equal(path: &str) { let sum = preprocess_iter(open_file(path)); let basic = process( read_file(open_file(path)), sum.min, sum.max, Palette::Default, - ); + ).unwrap(); let iter = process_iter(read_file(open_file(path)), sum.min, sum.max, sum.width); assert!(basic.2 == iter.2, "Results differ"); @@ -401,6 +413,6 @@ mod tests { #[test_resources("samples/*.csv.gz")] fn complete(path: &str) { - main(path, Palette::Default) + main(path, Palette::Default).unwrap() } } diff --git a/src/main.rs b/src/main.rs index 7d1fba7..8a1a07c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +use anyhow::Result; use clap::{App, Arg}; use sdr_heatmap::Palette; use std::path::Path; @@ -7,7 +8,7 @@ const VERSION: &str = env!("CARGO_PKG_VERSION"); const NAME: &str = env!("CARGO_PKG_NAME"); const AUTHOR: &str = env!("CARGO_PKG_AUTHORS"); -fn main() { +fn main() -> Result<()> { let matches = App::new(NAME) .version(VERSION) .author(AUTHOR) @@ -67,10 +68,11 @@ fn main() { let entry = entry.unwrap(); let name = entry.file_name().to_str().unwrap(); if exts.iter().any(|ext| name.ends_with(ext)) { - sdr_heatmap::main(entry.path(), palette); + sdr_heatmap::main(entry.path(), palette)?; } } } else { - sdr_heatmap::main(input, palette); - } + sdr_heatmap::main(input, palette)?; + }; + Ok(()) } From 7bd11d1a56e0593518fbb0b6e98ab4cf8060e0f1 Mon Sep 17 00:00:00 2001 From: J2ghz Date: Mon, 7 Sep 2020 17:47:29 +0200 Subject: [PATCH 02/20] warn on unwrap --- src/lib.rs | 50 +++++++++++++++++++++++++++++--------------------- src/main.rs | 8 ++++---- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1aa78e3..1a533d4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +#![warn(clippy::unwrap_used)] use csv::StringRecord; use flate2::read::GzDecoder; use log::*; @@ -122,12 +123,12 @@ fn parse_f32(s: &str) -> Result { } } -pub fn open_file>(path: P) -> Box { +pub fn open_file>(path: P) -> std::io::Result> { let path = path.as_ref(); - let file = File::open(path).unwrap(); + let file = File::open(path)?; match path.extension() { - Some(ext) if ext == OsStr::new("gz") => Box::new(GzDecoder::new(file)), - _ => Box::new(file), + Some(ext) if ext == OsStr::new("gz") => Ok(Box::new(GzDecoder::new(file))), + _ => Ok(Box::new(file)), } } @@ -141,17 +142,17 @@ pub fn main>(path: P, palette: Palette) -> Result<()> { let path = path.as_ref(); info!("Loading: {}", path.display()); //Preprocess - let file = open_file(path); + let file = open_file(path)?; let summary = preprocess_iter(file); info!("Color values {} to {}", summary.min, summary.max); //Process - let file = open_file(path); + let file = open_file(path)?; let reader = read_file(file); let (datawidth, dataheight, img) = process(reader, summary.min, summary.max, palette)?; //Draw let (height, imgdata) = create_image(datawidth, dataheight, img); let dest = path.with_extension("png"); - save_image(datawidth, height, imgdata, dest.to_str().unwrap()).unwrap(); + save_image(datawidth, height, imgdata, dest)?; Ok(()) } @@ -261,7 +262,7 @@ pub fn process( let mut datawidth = 0; let mut img = Vec::new(); for result in reader.into_records() { - let mut record = result.unwrap(); + let mut record = result?; record.trim(); assert!(record.len() > 7); let m = Measurement::new(record)?; @@ -340,20 +341,21 @@ fn create_image(width: usize, height: usize, mut img: Vec) -> (usize, std::v (height, imgdata) } -fn save_image( +fn save_image>( width: usize, height: usize, imgdata: Vec, - dest: &str, -) -> std::result::Result<(), image::error::ImageError> { - info!("Saving {} {}x{}", dest, width, height); - let f = std::fs::File::create(dest).unwrap(); + dest: P, +) -> Result<()> { + info!("Saving {} {}x{}", dest.as_ref().display(), width, height); + let f = std::fs::File::create(dest)?; PngEncoder::new(f).encode( &imgdata, width as u32, height as u32, image::ColorType::Rgb8, - ) + )?; + Ok(()) } #[cfg(test)] @@ -364,7 +366,7 @@ mod tests { #[test] fn preprocess_basic_result() { - let res = preprocess(open_file(Path::new("samples/46M.csv.gz"))); + let res = preprocess(open_file(Path::new("samples/46M.csv.gz")).unwrap()); assert_eq!( res, Summary { @@ -384,7 +386,7 @@ mod tests { #[test] fn preprocess_iter_result() { - let res = preprocess_iter(open_file(Path::new("samples/46M.csv.gz"))); + let res = preprocess_iter(open_file(Path::new("samples/46M.csv.gz")).unwrap()); assert_eq!( res, Summary { @@ -396,15 +398,21 @@ mod tests { } #[test_resources("samples/*.csv.gz")] - fn process_implementations_equal(path: &str) { - let sum = preprocess_iter(open_file(path)); + fn process_implementations_equal(path: &str) { + let sum = preprocess_iter(open_file(path).unwrap()); let basic = process( - read_file(open_file(path)), + read_file(open_file(path).unwrap()), sum.min, sum.max, Palette::Default, - ).unwrap(); - let iter = process_iter(read_file(open_file(path)), sum.min, sum.max, sum.width); + ) + .unwrap(); + let iter = process_iter( + read_file(open_file(path).unwrap()), + sum.min, + sum.max, + sum.width, + ); assert!(basic.2 == iter.2, "Results differ"); assert_eq!(basic.0, iter.0, "Widths differ"); diff --git a/src/main.rs b/src/main.rs index 8a1a07c..98ff3f9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +#![warn(clippy::unwrap_used)] use anyhow::Result; use clap::{App, Arg}; use sdr_heatmap::Palette; @@ -52,8 +53,7 @@ fn main() -> Result<()> { .quiet(quiet) .verbosity(verbose) //.timestamp(ts) - .init() - .unwrap(); + .init()?; let input = Path::new(matches.value_of("INPUT").unwrap()); let exts = vec![".csv", ".csv.gz"]; @@ -65,8 +65,8 @@ fn main() -> Result<()> { if matches.is_present("recursive") { for entry in WalkDir::new(input) { - let entry = entry.unwrap(); - let name = entry.file_name().to_str().unwrap(); + let entry = entry?; + let name = entry.file_name().to_string_lossy(); if exts.iter().any(|ext| name.ends_with(ext)) { sdr_heatmap::main(entry.path(), palette)?; } From 5379f0080a5a37957adc8ecaab1f10866f86a584 Mon Sep 17 00:00:00 2001 From: J2ghz Date: Tue, 8 Sep 2020 01:05:34 +0200 Subject: [PATCH 03/20] format, fix bench --- Cargo.lock | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 37 ++++++++++++------------- benches/bench.rs | 2 +- 3 files changed, 90 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e348c6d..89516c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -348,6 +348,15 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177" +[[package]] +name = "heck" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "hermit-abi" version = "0.1.15" @@ -652,6 +661,30 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2 1.0.19", + "quote 1.0.7", + "syn 1.0.39", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2 1.0.19", + "quote 1.0.7", + "version_check", +] + [[package]] name = "proc-macro2" version = "0.4.30" @@ -901,6 +934,7 @@ dependencies = [ "proptest", "rayon", "stderrlog", + "structopt", "test-generator", "walkdir", "webp", @@ -978,6 +1012,30 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +[[package]] +name = "structopt" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cc388d94ffabf39b5ed5fadddc40147cb21e605f53db6f8f36a625d27489ac5" +dependencies = [ + "clap", + "lazy_static 1.4.0", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e2513111825077552a6751dfad9e11ce0fba07d7276a3943a037d7e93e64c5f" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2 1.0.19", + "quote 1.0.7", + "syn 1.0.39", +] + [[package]] name = "syn" version = "0.15.44" @@ -1110,6 +1168,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "unicode-segmentation" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" + [[package]] name = "unicode-width" version = "0.1.8" @@ -1143,6 +1207,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" + [[package]] name = "void" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index 03870c1..7747446 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,23 +1,23 @@ [package] -name = 'sdr-heatmap' -version = '0.1.5' authors = ['J2ghz '] -edition = '2018' +categories = ['command-line-utilities'] description = 'Generates images from rtl_power csv files' -license = 'AGPL-3.0-or-later' -repository = 'https://github.com/j2ghz/sdr-heatmap/' -readme = 'README.md' +edition = '2018' +exclude = ['samples/*'] keywords = [ - 'sdr', - 'rtl-sdr', - 'rtl_power', + 'sdr', + 'rtl-sdr', + 'rtl_power', ] -categories = ['command-line-utilities'] -exclude = ['samples/*'] +license = 'AGPL-3.0-or-later' +name = 'sdr-heatmap' +readme = 'README.md' +repository = 'https://github.com/j2ghz/sdr-heatmap/' +version = '0.1.5' [[bench]] -name = 'bench' harness = false +name = 'bench' [dev-dependencies] criterion = '0.3' @@ -25,15 +25,16 @@ proptest = "0.10" test-generator = "0.3" [dependencies] +anyhow = "1.0" +arrayvec = "0.5" +clap = '2.33.3' csv = '1.1.3' +flate2 = '1.0.17' image = '0.23.9' -clap = '2.33.3' +itertools = "0.9" log = '0.4.11' +rayon = '1.4.0' stderrlog = '0.4.3' +structopt = "0.3" walkdir = '2' -flate2 = '1.0.17' -rayon = '1.4.0' webp = "0.1.0" -arrayvec = "0.5" -itertools = "0.9" -anyhow = "1.0" \ No newline at end of file diff --git a/benches/bench.rs b/benches/bench.rs index 7a7c889..bb0f34e 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -8,7 +8,7 @@ use std::{ fn read_file_to_memory(filename: &Path) -> std::boxed::Box>> { let mut buf = Vec::new(); - let mut file = open_file(filename); + let mut file = open_file(filename).expect("Can't open file"); let _ = file.read_to_end(&mut buf).unwrap(); Box::new(Cursor::new(buf)) From 25a9cca1f584b52d7c7477cdacc8fe9028a2b70a Mon Sep 17 00:00:00 2001 From: J2ghz Date: Tue, 8 Sep 2020 02:21:44 +0200 Subject: [PATCH 04/20] use structopt --- benches/bench.rs | 2 +- src/lib.rs | 15 +++--- src/main.rs | 118 +++++++++++++++++++++++++++-------------------- 3 files changed, 77 insertions(+), 58 deletions(-) diff --git a/benches/bench.rs b/benches/bench.rs index bb0f34e..5861213 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -24,7 +24,7 @@ fn read_csv_to_memory( } fn get_file_size(filename: &Path) -> u64 { - let mut file = open_file(filename); + let mut file = open_file(filename).expect("Couldn't open file"); let mut length: u64 = 0; let mut buf = vec![0; 1024]; loop { diff --git a/src/lib.rs b/src/lib.rs index 1a533d4..7ab1bab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -123,9 +123,9 @@ fn parse_f32(s: &str) -> Result { } } -pub fn open_file>(path: P) -> std::io::Result> { +pub fn open_file>(path: P) -> Result> { let path = path.as_ref(); - let file = File::open(path)?; + let file = File::open(path).context(format!("Couldn't open file '{}'", path.display()))?; match path.extension() { Some(ext) if ext == OsStr::new("gz") => Ok(Box::new(GzDecoder::new(file))), _ => Ok(Box::new(file)), @@ -146,9 +146,10 @@ pub fn main>(path: P, palette: Palette) -> Result<()> { let summary = preprocess_iter(file); info!("Color values {} to {}", summary.min, summary.max); //Process - let file = open_file(path)?; + let file = open_file(path).context("Couldn't preprocess file")?; let reader = read_file(file); - let (datawidth, dataheight, img) = process(reader, summary.min, summary.max, palette)?; + let (datawidth, dataheight, img) = + process(reader, summary.min, summary.max, palette).context("Couldn't process file")?; //Draw let (height, imgdata) = create_image(datawidth, dataheight, img); let dest = path.with_extension("png"); @@ -286,8 +287,10 @@ pub fn process( if datawidth == 0 { datawidth = batch; } - info!("Img data {}x{}", datawidth, batch); - Ok((datawidth, img.len() / 3 / datawidth, img)) + let w = datawidth; + let h = img.len() / 3 / datawidth; + info!("Img data {}x{}", w, h); + Ok((w, h, img)) } pub fn process_iter( diff --git a/src/main.rs b/src/main.rs index 98ff3f9..d2d0769 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,78 +1,94 @@ #![warn(clippy::unwrap_used)] use anyhow::Result; -use clap::{App, Arg}; +use anyhow::{anyhow, Context}; +use log::{debug, warn}; use sdr_heatmap::Palette; -use std::path::Path; +use std::{path::PathBuf, str::FromStr}; use walkdir::WalkDir; const VERSION: &str = env!("CARGO_PKG_VERSION"); const NAME: &str = env!("CARGO_PKG_NAME"); const AUTHOR: &str = env!("CARGO_PKG_AUTHORS"); +use structopt::StructOpt; + +#[derive(Debug, StructOpt)] +enum OptPalette { + Default, + Extended, +} + +impl FromStr for OptPalette { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "default" => Ok(OptPalette::Default), + "extended" => Ok(OptPalette::Extended), + _ => Err(anyhow!("{} is not a valid palette name", s)), + } + } +} +impl Into for OptPalette { + fn into(self) -> Palette { + match self { + OptPalette::Default => Palette::Default, + OptPalette::Extended => Palette::Extended, + } + } +} + +#[derive(Debug, StructOpt)] +#[structopt(name = NAME, about = "Render .csv from rtl_power into images. Based on heatmap.py", version = VERSION, author = AUTHOR)] +struct Opt { + /// Verbose mode (-v, -vv, -vvv, etc) + #[structopt(short = "v", long = "verbose", parse(from_occurrences))] + verbose: usize, + + /// Silence all output + #[structopt(short = "q", long = "quiet")] + quiet: bool, + + /// Finds .csv files in the specified folder and runs on all of them + #[structopt(short = "r", long = "recursive")] + recursive: bool, + + /// Input file + #[structopt(parse(from_os_str))] + input: PathBuf, + + /// Choose a function that converts signal value to a color. (Default: RGB: [0-255,0-255,50], Extended: like default, with more steps) + #[structopt(short, long, default_value = "default")] + palette: OptPalette, +} + fn main() -> Result<()> { - let matches = App::new(NAME) - .version(VERSION) - .author(AUTHOR) - .about("Render .csv from rtl_power into images. Based on heatmap.py") - .arg( - Arg::with_name("INPUT") - .help("Specify the .csv file to use") - .required(true) - .index(1), - ) - .arg( - Arg::with_name("verbose") - .short("v") - .multiple(true) - .help("Sets the level of verbosity"), - ) - .arg( - Arg::with_name("quiet") - .short("q") - .help("Silence all output"), - ) - .arg( - Arg::with_name("recursive") - .short("r") - .multiple(true) - .help("Finds .csv files in the specified folder and runs on all fo them"), - ) - .arg( - Arg::with_name("palette") - .short("p") - .default_value("default") - .possible_values(&["default","extended"]) - .help("Choose a function that converts signal value to a color.\nDefault: RGB: [0-255,0-255,50]\nExtended: like default, with more steps\n") - ) - .get_matches(); + let options: Opt = Opt::from_args(); - let verbose = matches.occurrences_of("verbose") as usize; - let quiet = matches.is_present("quiet"); stderrlog::new() .module(module_path!()) - .quiet(quiet) - .verbosity(verbose) - //.timestamp(ts) + .quiet(options.quiet) + .verbosity(options.verbose) .init()?; - let input = Path::new(matches.value_of("INPUT").unwrap()); + debug!("Options: {:?}", options); + + let input = options.input; let exts = vec![".csv", ".csv.gz"]; - let palette = match matches.value_of("palette").unwrap() { - "default" => Palette::Default, - "extended" => Palette::Extended, - _ => panic!("Clap should have caught other values earlier"), - }; + let palette = options.palette.into(); - if matches.is_present("recursive") { + if options.recursive { for entry in WalkDir::new(input) { let entry = entry?; let name = entry.file_name().to_string_lossy(); if exts.iter().any(|ext| name.ends_with(ext)) { - sdr_heatmap::main(entry.path(), palette)?; + sdr_heatmap::main(entry.path(), palette) + .context(format!("Error on file '{}'", entry.path().display()))?; } } } else { - sdr_heatmap::main(input, palette)?; + sdr_heatmap::main(&input, palette) + .context(format!("Error on file '{}'", input.display()))?; }; Ok(()) } From e688b5b2a532b96b9e6a99052ca029dd1b9c5134 Mon Sep 17 00:00:00 2001 From: J2ghz Date: Tue, 8 Sep 2020 13:35:52 +0200 Subject: [PATCH 05/20] add plain test --- samples/1k.csv | 3 +++ src/lib.rs | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 samples/1k.csv diff --git a/samples/1k.csv b/samples/1k.csv new file mode 100644 index 0000000..a4d32c8 --- /dev/null +++ b/samples/1k.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c765f521ee4a6cfa3446da1732b3e747173c73b37a8c92236b3dbd2eaf8d40e +size 62 diff --git a/src/lib.rs b/src/lib.rs index 7ab1bab..84a8712 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -423,7 +423,12 @@ mod tests { } #[test_resources("samples/*.csv.gz")] - fn complete(path: &str) { + fn complete_gzip(path: &str) { + main(path, Palette::Default).unwrap() + } + + #[test_resources("samples/*.csv")] + fn complete_plain(path: &str) { main(path, Palette::Default).unwrap() } } From d2b2f621811a9389754e4938783d83234b8df653 Mon Sep 17 00:00:00 2001 From: J2ghz Date: Tue, 8 Sep 2020 14:17:36 +0200 Subject: [PATCH 06/20] correct csv width check --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 84a8712..a4a9169 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -265,7 +265,7 @@ pub fn process( for result in reader.into_records() { let mut record = result?; record.trim(); - assert!(record.len() > 7); + assert!(record.len() >= 7); let m = Measurement::new(record)?; let vals = m.get_values_with_freq(); if date == m.date && time == m.time { @@ -303,7 +303,7 @@ pub fn process_iter( .into_records() .map(|res| { let mut record = res.expect("Invalid CSV record"); - debug_assert!(record.len() > 7); + debug_assert!(record.len() >= 7); record.trim(); record }) From 693520e7376307da9b8088fd37e44145ec5eb5c6 Mon Sep 17 00:00:00 2001 From: J2ghz Date: Wed, 9 Sep 2020 00:11:01 +0200 Subject: [PATCH 07/20] remove values limit --- src/lib.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a4a9169..bdfd308 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,12 +38,11 @@ impl Measurement { .collect() } fn new(record: StringRecord) -> Result { - let mut values: Vec<_> = record + let values: Vec<_> = record .iter() .skip(6) .map(|s| parse_f32(s)) .collect::>>()?; - values.truncate(record.len() - 7); Ok(Measurement { date: record .get(0) @@ -431,4 +430,9 @@ mod tests { fn complete_plain(path: &str) { main(path, Palette::Default).unwrap() } + + #[test] + fn test() { + main("samples/1k.csv", Palette::Default).unwrap() + } } From 88245e748917c70da7c83d1c940235146788a113 Mon Sep 17 00:00:00 2001 From: J2ghz Date: Wed, 9 Sep 2020 00:11:10 +0200 Subject: [PATCH 08/20] add vscode --- .vscode/launch.json | 83 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..978809a --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,83 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in library 'sdr-heatmap'", + "cargo": { + "args": [ + "test", + "--no-run", + "--lib", + "--package=sdr-heatmap" + ], + "filter": { + "name": "sdr-heatmap", + "kind": "lib" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug executable 'sdr-heatmap'", + "cargo": { + "args": [ + "build", + "--bin=sdr-heatmap", + "--package=sdr-heatmap" + ], + "filter": { + "name": "sdr-heatmap", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in executable 'sdr-heatmap'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bin=sdr-heatmap", + "--package=sdr-heatmap" + ], + "filter": { + "name": "sdr-heatmap", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug benchmark 'bench'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bench=bench", + "--package=sdr-heatmap" + ], + "filter": { + "name": "bench", + "kind": "bench" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + } + ] +} \ No newline at end of file From 974e835505e28bf02901f50207a13684094e45a6 Mon Sep 17 00:00:00 2001 From: J2ghz Date: Wed, 9 Sep 2020 00:24:27 +0200 Subject: [PATCH 09/20] conditionally remova last --- src/lib.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index bdfd308..d9fd076 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,11 +38,14 @@ impl Measurement { .collect() } fn new(record: StringRecord) -> Result { - let values: Vec<_> = record + let mut values: Vec<_> = record .iter() .skip(6) .map(|s| parse_f32(s)) .collect::>>()?; + if values.len() > 1 { + values.remove(values.len() - 1); + } Ok(Measurement { date: record .get(0) @@ -430,9 +433,4 @@ mod tests { fn complete_plain(path: &str) { main(path, Palette::Default).unwrap() } - - #[test] - fn test() { - main("samples/1k.csv", Palette::Default).unwrap() - } } From a0e853caaa2425667144e301a73f03f2570f5a5d Mon Sep 17 00:00:00 2001 From: J2ghz Date: Thu, 26 Nov 2020 22:25:40 +0100 Subject: [PATCH 10/20] use LTO for release --- Cargo.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 7747446..ae33673 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,10 @@ readme = 'README.md' repository = 'https://github.com/j2ghz/sdr-heatmap/' version = '0.1.5' +[profile.release] +lto = "fat" +codegen-units = 1 + [[bench]] harness = false name = 'bench' From 2289473d83921861fe2d15fc2804f3f48423261b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jozef=20Holl=C3=BD?= Date: Wed, 30 Dec 2020 00:47:17 +0100 Subject: [PATCH 11/20] process_iter result --- Cargo.lock | 15 ++++++++++++--- Cargo.toml | 4 ++-- src/lib.rs | 29 ++++++++++++++++++----------- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1beb6be..561fca8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -191,7 +191,7 @@ dependencies = [ "clap", "criterion-plot", "csv", - "itertools", + "itertools 0.9.0", "lazy_static", "num-traits", "oorandom", @@ -213,7 +213,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e022feadec601fba1649cfa83586381a4ad31c6bf3a9ab7d408118b05dd9889d" dependencies = [ "cast", - "itertools", + "itertools 0.9.0", ] [[package]] @@ -443,6 +443,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.6" @@ -927,7 +936,7 @@ dependencies = [ "csv", "flate2", "image 0.23.12", - "itertools", + "itertools 0.10.0", "log", "proptest", "rayon", diff --git a/Cargo.toml b/Cargo.toml index 1cac9a4..e74ae7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,8 +16,8 @@ repository = 'https://github.com/j2ghz/sdr-heatmap/' version = '0.1.5' [profile.release] -lto = "fat" codegen-units = 1 +lto = "fat" [[bench]] harness = false @@ -35,7 +35,7 @@ clap = '2.33.3' csv = '1.1.4' flate2 = '1.0.19' image = '0.23.12' -itertools = "0.9" +itertools = "0.10.0" log = '0.4.11' rayon = '1.5.0' stderrlog = '0.5.0' diff --git a/src/lib.rs b/src/lib.rs index d9fd076..b01a315 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -300,24 +300,30 @@ pub fn process_iter( min: f32, max: f32, width: usize, -) -> (usize, usize, std::vec::Vec) { +) -> Result<(usize, usize, std::vec::Vec)> { let img: Vec = reader .into_records() - .map(|res| { - let mut record = res.expect("Invalid CSV record"); + .map(|res| -> anyhow::Result { + let mut record = res.context("Invalid CSV record")?; debug_assert!(record.len() >= 7); record.trim(); - record + Ok(Measurement::new(record)?) + }) + .map(|m| match m { + Ok(m) => Ok(m.values.into_iter().flat_map(|val| { + ArrayVec::from(scale_tocolor(Palette::Default, val, min, max)).into_iter() + })), + Err(e) => Err(e), }) - .map(Measurement::new) - .flat_map(|m| m.unwrap().values.into_iter()) - .flat_map(|val| { - let slice = scale_tocolor(Palette::Default, val, min, max); - ArrayVec::from(slice).into_iter() + .flat_map(|r| match r { + Ok(inner) => inner, + Err(e) => { + panic!("{:?}", e) + } }) .collect(); - (width, img.len() / 3 / width, img) + Ok((width, img.len() / 3 / width, img)) } fn tape_measure(width: usize, imgdata: &mut Vec) { @@ -417,7 +423,8 @@ mod tests { sum.min, sum.max, sum.width, - ); + ) + .unwrap(); assert!(basic.2 == iter.2, "Results differ"); assert_eq!(basic.0, iter.0, "Widths differ"); From 61f065eebdf48cfdd911c9d32484ba77b7458203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jozef=20Holl=C3=BD?= Date: Sun, 7 Mar 2021 15:07:37 +0100 Subject: [PATCH 12/20] fix a few lints --- src/lib.rs | 2 +- src/main.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 5e31a66..64cbb67 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -307,7 +307,7 @@ pub fn process_iter( let mut record = res.context("Invalid CSV record")?; debug_assert!(record.len() >= 7); record.trim(); - Ok(Measurement::new(record)?) + Measurement::new(record) }) .map(|m| match m { Ok(m) => Ok(m.values.into_iter().flat_map(|val| { diff --git a/src/main.rs b/src/main.rs index 048b303..d4612e5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -29,9 +29,9 @@ impl FromStr for OptPalette { } } } -impl Into for OptPalette { - fn into(self) -> Palette { - match self { +impl From for Palette { + fn from(opt: OptPalette) -> Self { + match opt { OptPalette::Default => Palette::Default, OptPalette::Extended => Palette::Extended, } From d7308572fecb980b96a5e997bde0d121c1b72017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jozef=20Holl=C3=BD?= Date: Sat, 5 Jun 2021 00:11:31 +0200 Subject: [PATCH 13/20] lib update --- Cargo.lock | 230 ++++++++++++++++++++++++++++------------------------- Cargo.toml | 2 +- 2 files changed, 123 insertions(+), 109 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76db6fb..824f0c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,15 +25,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" +checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" [[package]] name = "arrayvec" -version = "0.5.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +checksum = "5a2f58b0bb10c380af2b26e57212856b8c9a59e0925b4c20f4a174a49734eaf7" [[package]] name = "atty" @@ -75,9 +75,9 @@ checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" [[package]] name = "bstr" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d" +checksum = "90682c8d613ad3373e66de8c6411e0ae2ab2571e879d2efbf73558cc66f21279" dependencies = [ "lazy_static", "memchr", @@ -87,9 +87,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.6.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" +checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" [[package]] name = "bytemuck" @@ -99,24 +99,24 @@ checksum = "bed57e2090563b83ba8f83366628ce535a7584c9afa4c9fc0612a03925c6df58" [[package]] name = "byteorder" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "cast" -version = "0.2.3" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b9434b9a5aa1450faa3f9cb14ea0e8c53bb5d2b3c1bfd1ab4fc03e9f33fbfb0" +checksum = "57cdfa5d50aad6cb4d44dcab6101a7f79925bd59d82ca42f38a9856a28865374" dependencies = [ "rustc_version", ] [[package]] name = "cc" -version = "1.0.67" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" +checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787" dependencies = [ "jobserver", ] @@ -208,9 +208,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" dependencies = [ "cfg-if", "crossbeam-utils", @@ -229,9 +229,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" dependencies = [ "cfg-if", "crossbeam-utils", @@ -242,11 +242,10 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49" +checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" dependencies = [ - "autocfg", "cfg-if", "lazy_static", ] @@ -309,9 +308,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "getrandom" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" dependencies = [ "cfg-if", "libc", @@ -320,9 +319,9 @@ dependencies = [ [[package]] name = "gif" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02efba560f227847cb41463a7395c514d127d4f74fff12ef0137fff1b84b96c4" +checksum = "5a668f699973d0f573d15749b7002a9ac9e1f9c6b220e7b165601334c173d8de" dependencies = [ "color_quant", "weezl", @@ -342,9 +341,9 @@ checksum = "62aca2aba2d62b4a7f5b33f3712cb1b0692779a56fb510499d5c0aa594daeaf3" [[package]] name = "heck" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" dependencies = [ "unicode-segmentation", ] @@ -403,9 +402,9 @@ checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" [[package]] name = "jobserver" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2" +checksum = "972f5ae5d1cb9c6ae417789196c803205313edde988685da5e3aae0827b9e7fd" dependencies = [ "libc", ] @@ -421,9 +420,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.48" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc9f84f9b115ce7843d60706df1422a916680bfdfcbdb0447c5614ff9d7e4d78" +checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062" dependencies = [ "wasm-bindgen", ] @@ -436,9 +435,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.88" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b07a082330a35e43f63177cc01689da34fbffa0105e1246cf0311472cac73a" +checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36" [[package]] name = "libwebp-sys" @@ -460,15 +459,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.3.4" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" [[package]] name = "memoffset" -version = "0.6.1" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" dependencies = [ "autocfg", ] @@ -549,11 +548,20 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + [[package]] name = "plotters" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45ca0ae5f169d0917a7c7f5a9c1a3d3d9598f18f529dd2b8373ed988efea307a" +checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a" dependencies = [ "num-traits", "plotters-backend", @@ -602,9 +610,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.24", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.62", + "syn 1.0.72", "version_check", ] @@ -614,7 +622,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.24", + "proc-macro2 1.0.27", "quote 1.0.9", "version_check", ] @@ -630,11 +638,11 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.24" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038" dependencies = [ - "unicode-xid 0.2.1", + "unicode-xid 0.2.2", ] [[package]] @@ -648,7 +656,7 @@ dependencies = [ "byteorder", "lazy_static", "num-traits", - "quick-error 2.0.0", + "quick-error 2.0.1", "rand", "rand_chacha", "rand_xorshift", @@ -665,9 +673,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quick-error" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ac73b1112776fc109b2e61909bc46c7e1bf0d7f690ffb1676553acce16d5cda" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quote" @@ -684,7 +692,7 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" dependencies = [ - "proc-macro2 1.0.24", + "proc-macro2 1.0.27", ] [[package]] @@ -738,9 +746,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" dependencies = [ "autocfg", "crossbeam-deque", @@ -750,9 +758,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -763,36 +771,33 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.5" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.4.3" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" dependencies = [ "regex-syntax", ] [[package]] name = "regex-automata" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" -dependencies = [ - "byteorder", -] +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" [[package]] name = "regex-syntax" -version = "0.6.22" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "remove_dir_all" @@ -805,9 +810,9 @@ dependencies = [ [[package]] name = "rustc_version" -version = "0.2.3" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" dependencies = [ "semver", ] @@ -875,24 +880,27 @@ dependencies = [ [[package]] name = "semver" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" dependencies = [ "semver-parser", ] [[package]] name = "semver-parser" -version = "0.7.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest", +] [[package]] name = "serde" -version = "1.0.124" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd761ff957cb2a45fbb9ab3da6512de9de55872866160b23c25f1a841e99d29f" +checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" [[package]] name = "serde_cbor" @@ -906,13 +914,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.124" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b" +checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43" dependencies = [ - "proc-macro2 1.0.24", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.62", + "syn 1.0.72", ] [[package]] @@ -964,9 +972,9 @@ checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" dependencies = [ "heck", "proc-macro-error", - "proc-macro2 1.0.24", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.62", + "syn 1.0.72", ] [[package]] @@ -982,13 +990,13 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.62" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "123a78a3596b24fee53a6464ce52d8ecbf62241e6294c7e7fe12086cd161f512" +checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" dependencies = [ - "proc-macro2 1.0.24", + "proc-macro2 1.0.27", "quote 1.0.9", - "unicode-xid 0.2.1", + "unicode-xid 0.2.2", ] [[package]] @@ -1075,6 +1083,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + [[package]] name = "unicode-segmentation" version = "1.7.1" @@ -1095,9 +1109,9 @@ checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" [[package]] name = "unicode-xid" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" [[package]] name = "vec_map" @@ -1107,9 +1121,9 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version_check" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" [[package]] name = "wait-timeout" @@ -1122,9 +1136,9 @@ dependencies = [ [[package]] name = "walkdir" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" dependencies = [ "same-file", "winapi", @@ -1139,9 +1153,9 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" [[package]] name = "wasm-bindgen" -version = "0.2.71" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee1280240b7c461d6a0071313e08f34a60b0365f14260362e5a2b17d1d31aa7" +checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1149,24 +1163,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.71" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b7d8b6942b8bb3a9b0e73fc79b98095a27de6fa247615e59d096754a3bc2aa8" +checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900" dependencies = [ "bumpalo", "lazy_static", "log", - "proc-macro2 1.0.24", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.62", + "syn 1.0.72", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.71" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ac38da8ef716661f0f36c0d8320b89028efe10c7c0afde65baffb496ce0d3b" +checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4" dependencies = [ "quote 1.0.9", "wasm-bindgen-macro-support", @@ -1174,28 +1188,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.71" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e" +checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97" dependencies = [ - "proc-macro2 1.0.24", + "proc-macro2 1.0.27", "quote 1.0.9", - "syn 1.0.62", + "syn 1.0.72", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.71" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d6f8ec44822dd71f5f221a5847fb34acd9060535c1211b70a05844c0f6383b1" +checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f" [[package]] name = "web-sys" -version = "0.3.48" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec600b26223b2948cedfde2a0aa6756dcf1fef616f43d7b3097aaf53a6c4d92b" +checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582" dependencies = [ "js-sys", "wasm-bindgen", @@ -1203,9 +1217,9 @@ dependencies = [ [[package]] name = "webp" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea1f2bd35e46165ef40a7fd74f33f64f2912ad92593fbfc5ec75eb2604cfd7b5" +checksum = "6a598dd8197b16c7569e231619b668380aefe9352daf1d503c3eea7b38fddba3" dependencies = [ "image", "libwebp-sys", @@ -1213,9 +1227,9 @@ dependencies = [ [[package]] name = "weezl" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a32b378380f4e9869b22f0b5177c68a5519f03b3454fde0b291455ddbae266c" +checksum = "d8b77fdfd5a253be4ab714e4ffa3c49caf146b4de743e97510c0656cf90f1e8e" [[package]] name = "winapi" diff --git a/Cargo.toml b/Cargo.toml index ce7b6ae..1a19f28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ test-generator = "0.3" [dependencies] anyhow = "1.0" -arrayvec = "0.5" +arrayvec = "0.7" clap = '2.33.3' csv = '1.1.5' flate2 = '1.0.19' From 5f40091d721a71c14390ed6829397b77c07fb56f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jozef=20Holl=C3=BD?= Date: Sat, 5 Jun 2021 00:11:38 +0200 Subject: [PATCH 14/20] fix tests --- src/lib.rs | 5 ----- src/main.rs | 8 +------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 64cbb67..5039114 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -440,9 +440,4 @@ mod tests { fn complete_plain(path: &str) { main(path, Palette::Default).unwrap() } - - #[test] - fn dummy() { - assert_eq!(4, 2 + 2); - } } diff --git a/src/main.rs b/src/main.rs index d4612e5..0c80f37 100644 --- a/src/main.rs +++ b/src/main.rs @@ -93,10 +93,4 @@ fn main() -> Result<()> { Ok(()) } #[cfg(test)] -mod tests { - - #[test] - fn dummy() { - assert_eq!(4, 2 + 2); - } -} +mod tests {} From 4626be1eba07b96e0790dc634d56bab1a4175ace Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jozef=20Holl=C3=BD?= Date: Sat, 5 Jun 2021 03:50:08 +0200 Subject: [PATCH 15/20] anyhow::Result with iterators --- src/lib.rs | 153 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 93 insertions(+), 60 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 5039114..80ee12f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ #![warn(clippy::unwrap_used)] +#![warn(clippy::panic)] use csv::StringRecord; use flate2::read::GzDecoder; use log::*; @@ -145,7 +146,7 @@ pub fn main>(path: P, palette: Palette) -> Result<()> { info!("Loading: {}", path.display()); //Preprocess let file = open_file(path)?; - let summary = preprocess_iter(file); + let summary = preprocess_iter(file)?; info!("Color values {} to {}", summary.min, summary.max); //Process let file = open_file(path).context("Couldn't preprocess file")?; @@ -159,18 +160,18 @@ pub fn main>(path: P, palette: Palette) -> Result<()> { Ok(()) } -pub fn preprocess(file: Box) -> Summary { +pub fn preprocess(file: Box) -> Result { let reader = read_file(file); let mut min = f32::INFINITY; let mut max = f32::NEG_INFINITY; let mut width: Option = None; let mut first_date = None; for result in reader.into_records() { - let record = { - let mut x = result.unwrap(); + let record = result.map(|mut x| { x.trim(); x - }; + })?; + let timestamp = record .get(0) .and_then(|date| record.get(1).map(|time| format!("{} {}", date, time))); @@ -208,49 +209,85 @@ pub fn preprocess(file: Box) -> Summary { } } } - Summary { + Ok(Summary { min, max, - width: width.unwrap(), - } + width: width.ok_or_else(|| { + anyhow::anyhow!("width sohuld be defined if there's at least one row of data") + })?, + }) } -pub fn preprocess_iter(file: Box) -> Summary { - read_file(file) - .into_records() - .map(|x| { - let mut x = x.unwrap(); - x.trim(); - x - }) - .group_by(|line| format!("{} {}", line.get(0).unwrap(), line.get(1).unwrap())) - .into_iter() - .map(|(_, group)| { - group - .flat_map(|line| { - let mut vals = line - .into_iter() - .skip(6) - .map(|s| { - if s == "-nan" || s == "nan" { - f32::NAN - } else { - s.parse::().unwrap_or_else(|e| { - panic!("'{}' should be a valid float: '{:?}'", s, e) - }) - } - }) - .collect::>(); - vals.pop().unwrap(); - vals - }) - .collect::>() - }) - .fold(Summary::empty(), |sum, vals| { +pub fn preprocess_iter(file: Box) -> Result { + fn trim(mut record: StringRecord) -> StringRecord { + record.trim(); + record + } + + fn get_datetime_if_not_err(res: &csv::Result) -> String { + match res { + Ok(sr) => { + format!( + "{} {}", + sr.get(0).unwrap_or("empty").to_string(), + sr.get(1).unwrap_or("empty").to_string() + ) + } + Err(_) => "err".to_string(), + } + } + + fn parse_f32(s: &str) -> Result { + if s == "-nan" || s == "nan" { + Ok(f32::NAN) + } else { + s.parse::() + .with_context(|| anyhow::anyhow!("'{}' should be a valid float", s)) + } + } + + fn get_values(record: StringRecord) -> Result> { + let mut vals = record + .into_iter() + .skip(6) + .map(parse_f32) + .collect::>>()?; + vals.pop() + .ok_or_else(||anyhow::anyhow!("there should be at least one value in a row, so we should be able to skip the last one"))?; + Ok(vals.into_iter()) + } + + fn map_group( + (_, group): (String, impl Iterator>), + ) -> Result> { + Ok(group + .map(|record| { + record + .context("failed to parse record") + .and_then(get_values) + .map(|x| x.collect_vec()) + }) + .collect::>>>()? + .into_iter() + .flat_map(|x| x.into_iter())) + } + + fn fold_vals(summary: Result, vals: impl Iterator) -> Result { + let vals = vals.collect_vec(); + summary.map(|sum| { let width = vals.len(); vals.into_iter() .fold(sum, |sum, val| Summary::update(sum, val, width)) }) + } + + read_file(file) + .into_records() + .map_ok(trim) + .group_by(get_datetime_if_not_err) + .into_iter() + .map(map_group) + .fold_ok(Ok(Summary::empty()), fold_vals)? } pub fn process( @@ -376,8 +413,8 @@ mod tests { use webp::PixelLayout; #[test] - fn preprocess_basic_result() { - let res = preprocess(open_file(Path::new("samples/46M.csv.gz")).unwrap()); + fn preprocess_basic_result() -> Result<()> { + let res = preprocess(open_file(Path::new("samples/46M.csv.gz"))?)?; assert_eq!( res, Summary { @@ -386,6 +423,7 @@ mod tests { width: 11622, } ); + Ok(()) } #[test] @@ -396,8 +434,8 @@ mod tests { } #[test] - fn preprocess_iter_result() { - let res = preprocess_iter(open_file(Path::new("samples/46M.csv.gz")).unwrap()); + fn preprocess_iter_result() -> Result<()> { + let res = preprocess_iter(open_file(Path::new("samples/46M.csv.gz"))?)?; assert_eq!( res, Summary { @@ -406,38 +444,33 @@ mod tests { width: 11622, } ); + Ok(()) } #[test_resources("samples/*.csv.gz")] - fn process_implementations_equal(path: &str) { - let sum = preprocess_iter(open_file(path).unwrap()); + fn process_implementations_equal(path: &str) -> Result<()> { + let sum = preprocess_iter(open_file(path)?)?; let basic = process( - read_file(open_file(path).unwrap()), + read_file(open_file(path)?), sum.min, sum.max, Palette::Default, - ) - .unwrap(); - let iter = process_iter( - read_file(open_file(path).unwrap()), - sum.min, - sum.max, - sum.width, - ) - .unwrap(); + )?; + let iter = process_iter(read_file(open_file(path)?), sum.min, sum.max, sum.width)?; assert!(basic.2 == iter.2, "Results differ"); assert_eq!(basic.0, iter.0, "Widths differ"); assert_eq!(basic.1, iter.1, "Heights differ"); + Ok(()) } #[test_resources("samples/*.csv.gz")] - fn complete_gzip(path: &str) { - main(path, Palette::Default).unwrap() + fn complete_gzip(path: &str) -> Result<()> { + main(path, Palette::Default) } #[test_resources("samples/*.csv")] - fn complete_plain(path: &str) { - main(path, Palette::Default).unwrap() + fn complete_plain(path: &str) -> Result<()> { + main(path, Palette::Default) } } From 8e9c947cf90133e7476a8eebe257691a533537d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jozef=20Holl=C3=BD?= Date: Sat, 5 Jun 2021 03:50:57 +0200 Subject: [PATCH 16/20] remove dummy tests --- src/palettes/default.rs | 8 +------- src/palettes/extended.rs | 8 +------- src/palettes/mod.rs | 5 ----- 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/src/palettes/default.rs b/src/palettes/default.rs index c523b98..38badcb 100644 --- a/src/palettes/default.rs +++ b/src/palettes/default.rs @@ -14,10 +14,4 @@ impl PaletteColorize for DefaultPalette { } } #[cfg(test)] -mod tests { - - #[test] - fn dummy() { - assert_eq!(4, 2 + 2); - } -} +mod tests {} diff --git a/src/palettes/extended.rs b/src/palettes/extended.rs index 70d418a..52b572a 100644 --- a/src/palettes/extended.rs +++ b/src/palettes/extended.rs @@ -22,10 +22,4 @@ impl PaletteColorize for ExtendedPalette { } } #[cfg(test)] -mod tests { - - #[test] - fn dummy() { - assert_eq!(4, 2 + 2); - } -} +mod tests {} diff --git a/src/palettes/mod.rs b/src/palettes/mod.rs index 1f4a917..c6e7cea 100644 --- a/src/palettes/mod.rs +++ b/src/palettes/mod.rs @@ -54,11 +54,6 @@ mod tests { use crate::*; use proptest::prelude::*; - #[test] - fn dummy() { - assert_eq!(4, 2 + 2); - } - #[test] fn normalize_goes_up() { assert_eq!( From 2c88a5a632bc055f881247870dfe0ffda491ab87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jozef=20Holl=C3=BD?= Date: Sat, 5 Jun 2021 14:37:50 +0200 Subject: [PATCH 17/20] fix warns --- benches/bench.rs | 6 ++-- src/lib.rs | 83 ++++++++++++++++----------------------------- src/main.rs | 1 + src/palettes/mod.rs | 7 ++-- 4 files changed, 37 insertions(+), 60 deletions(-) diff --git a/benches/bench.rs b/benches/bench.rs index 5861213..0ab269f 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -58,7 +58,7 @@ fn preprocess_bench(c: &mut Criterion) { b.iter_with_large_setup( || read_file_to_memory(file), |data| { - let summary = preprocess(data); + let summary = preprocess(data).unwrap(); black_box(summary); }, ) @@ -71,7 +71,7 @@ fn preprocess_bench(c: &mut Criterion) { b.iter_with_large_setup( || read_file_to_memory(file), |data| { - let summary = preprocess_iter(data); + let summary = preprocess_iter(data).unwrap(); black_box(summary); }, ) @@ -107,7 +107,7 @@ fn process_bench(c: &mut Criterion) { b.iter_with_large_setup( || read_csv_to_memory(file), |data| { - let summary = process_iter(data, -1000.0, 1000.0, 1); + let summary = process_iter(data, -1000.0, 1000.0, 1).unwrap(); black_box(summary); }, ) diff --git a/src/lib.rs b/src/lib.rs index 80ee12f..b5ef69b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -118,14 +118,6 @@ impl Summary { } } -fn parse_f32(s: &str) -> Result { - if s == "-nan" || s == "nan" { - Ok(f32::NAN) - } else { - Ok(s.parse::()?) - } -} - pub fn open_file>(path: P) -> Result> { let path = path.as_ref(); let file = File::open(path).context(format!("Couldn't open file '{}'", path.display()))?; @@ -176,19 +168,7 @@ pub fn preprocess(file: Box) -> Result { .get(0) .and_then(|date| record.get(1).map(|time| format!("{} {}", date, time))); - let values: Vec = record - .iter() - .skip(6) - .map(|s| { - if s == "-nan" || s == "nan" { - f32::NAN - } else { - s.trim() - .parse::() - .unwrap_or_else(|e| panic!("'{}' should be a valid float: {:?}", s, e)) - } - }) - .collect(); + let values = get_values(record)?.collect_vec(); let values_count = values.len() - 1; if first_date == None { @@ -218,6 +198,26 @@ pub fn preprocess(file: Box) -> Result { }) } +fn parse_f32(s: &str) -> Result { + if s == "-nan" || s == "nan" { + Ok(f32::NAN) + } else { + s.parse::() + .with_context(|| anyhow::anyhow!("'{}' should be a valid float", s)) + } +} + +fn get_values(record: StringRecord) -> Result> { + let mut vals = record + .into_iter() + .skip(6) + .map(parse_f32) + .collect::>>()?; + vals.pop() + .ok_or_else(||anyhow::anyhow!("there should be at least one value in a row, so we should be able to skip the last one"))?; + Ok(vals.into_iter()) +} + pub fn preprocess_iter(file: Box) -> Result { fn trim(mut record: StringRecord) -> StringRecord { record.trim(); @@ -237,26 +237,6 @@ pub fn preprocess_iter(file: Box) -> Result { } } - fn parse_f32(s: &str) -> Result { - if s == "-nan" || s == "nan" { - Ok(f32::NAN) - } else { - s.parse::() - .with_context(|| anyhow::anyhow!("'{}' should be a valid float", s)) - } - } - - fn get_values(record: StringRecord) -> Result> { - let mut vals = record - .into_iter() - .skip(6) - .map(parse_f32) - .collect::>>()?; - vals.pop() - .ok_or_else(||anyhow::anyhow!("there should be at least one value in a row, so we should be able to skip the last one"))?; - Ok(vals.into_iter()) - } - fn map_group( (_, group): (String, impl Iterator>), ) -> Result> { @@ -338,7 +318,7 @@ pub fn process_iter( max: f32, width: usize, ) -> Result<(usize, usize, std::vec::Vec)> { - let img: Vec = reader + let img = reader .into_records() .map(|res| -> anyhow::Result { let mut record = res.context("Invalid CSV record")?; @@ -346,19 +326,14 @@ pub fn process_iter( record.trim(); Measurement::new(record) }) - .map(|m| match m { - Ok(m) => Ok(m.values.into_iter().flat_map(|val| { + .map_ok(|m| { + m.values.into_iter().flat_map(|val| { ArrayVec::from(scale_tocolor(Palette::Default, val, min, max)).into_iter() - })), - Err(e) => Err(e), - }) - .flat_map(|r| match r { - Ok(inner) => inner, - Err(e) => { - panic!("{:?}", e) - } + }) }) - .collect(); + .collect::>>()?; + + let img = img.into_iter().flatten().collect_vec(); Ok((width, img.len() / 3 / width, img)) } @@ -458,7 +433,7 @@ mod tests { )?; let iter = process_iter(read_file(open_file(path)?), sum.min, sum.max, sum.width)?; - assert!(basic.2 == iter.2, "Results differ"); + assert_eq!(basic.2, iter.2, "Results differ"); assert_eq!(basic.0, iter.0, "Widths differ"); assert_eq!(basic.1, iter.1, "Heights differ"); Ok(()) diff --git a/src/main.rs b/src/main.rs index 0c80f37..9d19169 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ #![warn(clippy::unwrap_used)] +#![warn(clippy::panic)] use anyhow::Result; use anyhow::{anyhow, Context}; use log::{debug, warn}; diff --git a/src/palettes/mod.rs b/src/palettes/mod.rs index c6e7cea..f20775b 100644 --- a/src/palettes/mod.rs +++ b/src/palettes/mod.rs @@ -55,17 +55,18 @@ mod tests { use proptest::prelude::*; #[test] - fn normalize_goes_up() { + fn normalize_goes_up() -> Result<()> { assert_eq!( (0..255) .map(|v| v as f32) .map(|v| scale_tocolor(Palette::Default, v, 0.0, 255.0) .first() .cloned() - .unwrap()) - .collect::>(), + .ok_or_else(|| anyhow::anyhow!("color should be valid"))) + .collect::>>()?, (0..255).map(|v| v as u8).collect::>() ); + Ok(()) } #[test] From 980790a085463daddfc7e560783ed5890641d52a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jozef=20Holl=C3=BD?= Date: Sat, 5 Jun 2021 16:38:36 +0200 Subject: [PATCH 18/20] fix test --- src/lib.rs | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b5ef69b..6fe5db2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -159,10 +159,7 @@ pub fn preprocess(file: Box) -> Result { let mut width: Option = None; let mut first_date = None; for result in reader.into_records() { - let record = result.map(|mut x| { - x.trim(); - x - })?; + let record = result.map(trim)?; let timestamp = record .get(0) @@ -170,12 +167,11 @@ pub fn preprocess(file: Box) -> Result { let values = get_values(record)?.collect_vec(); - let values_count = values.len() - 1; if first_date == None { first_date = timestamp; - width = Some(values_count); + width = Some(values.len()); } else if first_date == timestamp { - width = width.map(|v| v + values_count); + width = width.map(|v| v + values.len()); } for value in values { @@ -218,12 +214,12 @@ fn get_values(record: StringRecord) -> Result> { Ok(vals.into_iter()) } -pub fn preprocess_iter(file: Box) -> Result { - fn trim(mut record: StringRecord) -> StringRecord { - record.trim(); - record - } +fn trim(mut record: StringRecord) -> StringRecord { + record.trim(); + record +} +pub fn preprocess_iter(file: Box) -> Result { fn get_datetime_if_not_err(res: &csv::Result) -> String { match res { Ok(sr) => { From 98a89ebf04070c17007b0a23de1602c386e6a30f Mon Sep 17 00:00:00 2001 From: J2ghz Date: Sat, 20 Nov 2021 13:12:46 +0100 Subject: [PATCH 19/20] update libs --- Cargo.lock | 6 ++++-- Cargo.toml | 18 +++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c2e972e..0ea4151 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "adler" version = "1.0.2" @@ -1215,9 +1217,9 @@ dependencies = [ [[package]] name = "webp" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a598dd8197b16c7569e231619b668380aefe9352daf1d503c3eea7b38fddba3" +checksum = "0041a9fcbdbf6402c3cb0ec7c2c1a1c240efd523519a0763ae07753bc95f7713" dependencies = [ "image", "libwebp-sys", diff --git a/Cargo.toml b/Cargo.toml index f667677..52a5ead 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,14 +31,14 @@ test-generator = "0.3" [dependencies] anyhow = "1.0" arrayvec = "0.7" -clap = '2.33.3' -csv = '1.1.6' -flate2 = '1.0.19' -image = '0.23.12' +clap = '2.33' +csv = '1.1' +flate2 = '1.0' +image = '0.23' itertools = "0.10" -log = '0.4.11' -rayon = '1.5.0' -stderrlog = '0.5.0' +log = '0.4' +rayon = '1.5' +stderrlog = '0.5' structopt = "0.3" -walkdir = '2' -webp = "0.1.2" +walkdir = '2.3' +webp = "0.2" From 25df53af25aebb2570293b130b3c252d8da7e25e Mon Sep 17 00:00:00 2001 From: J2ghz Date: Sat, 20 Nov 2021 13:19:21 +0100 Subject: [PATCH 20/20] fixes --- Cargo.toml | 2 +- benches/bench.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 52a5ead..9d63c19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ authors = ['J2ghz '] categories = ['command-line-utilities'] description = 'Generates images from rtl_power csv files' -edition = '2018' +edition = '2021' exclude = ['samples/*'] keywords = [ 'sdr', diff --git a/benches/bench.rs b/benches/bench.rs index 0ab269f..9679e92 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -49,7 +49,7 @@ fn get_test_files() -> std::vec::Vec { fn preprocess_bench(c: &mut Criterion) { let mut group = c.benchmark_group("preprocess implementations"); for file in get_test_files().iter() { - let size = get_file_size(&file); + let size = get_file_size(file); group.throughput(Throughput::Bytes(size)); group.bench_with_input( BenchmarkId::new("basic", file.display()), @@ -85,7 +85,7 @@ fn preprocess_bench(c: &mut Criterion) { fn process_bench(c: &mut Criterion) { let mut group = c.benchmark_group("process implementations"); for file in get_test_files().iter() { - let size = get_file_size(&file); + let size = get_file_size(file); group.throughput(Throughput::Bytes(size)); group.bench_with_input( BenchmarkId::new("basic", file.display()),