Merge pull request #119 from darrenldl/dev

Added --dry-run to sort, --multi-pass to decode and sort, updated decode output file name computation

CHANGELOG.md

@@ -1,3 +1,21 @@
+## 2.3.0
+
+- Changed decode mode to use only file portion of stored file name in SBX container
+
+  - In previous versions, if the stored file name contains a path, then the entire path would be used, which can lead to unexpected output locations
+
+- Added `--multi-pass` flag to decode and sort mode
+
+  - This disables truncation, and allows updating the file gradually across multiple runs
+
+- Upgraded stats tracking in sort mode
+
+  - Now it also tracks number of blocks in same order and in different order
+
+- Added `--dry-run` flag to sort mode
+
+  - Combined with the improved stats tracking allows checking if the container is sorted or not
+
 ## 2.2.0
 
 - Added `--only-pick-uid` option to show mode

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name    = "blkar"
-version = "2.2.0"
+version = "2.3.0"
 authors = ["Darren Ldl <darrenldldev@gmail.com>"]
 build   = "build.rs"
 exclude = [

src/cli_decode.rs

@@ -19,13 +19,15 @@ pub fn sub_command<'a, 'b>() -> App<'a, 'b> {
         .arg(out_arg()
              .help("Decoded file name. Supply - to use STDOUT as output. Use ./- for files named -.
 If OUT is not provided, then the original file name stored in the SBX container
-(STOREDNAME) is used if present. If OUT is provided and is a directory, then
-the output file is stored as OUT/STOREDNAME if STOREDNAME is present. If OUT is
+(STOREDNAME) is used if present (only the file part of STOREDNAME is used). If
+OUT is provided and is a directory, then the output file is stored as OUT/STOREDNAME
+if STOREDNAME is present (only the file part of STOREDNAME is used). If OUT is
 provided and is not a directory, then it is used directly."))
-        .arg(Arg::with_name("force")
-             .short("f")
-             .long("force")
+        .arg(force_arg()
              .help("Force overwrite even if OUT exists"))
+        .arg(multi_pass_arg()
+             .help("Disable truncation of OUT. This allows writing to OUT multiple
+times to update it gradually."))
         .arg(no_meta_arg())
         .arg(pr_verbosity_level_arg())
         .arg(burst_arg()
@@ -89,6 +91,7 @@ pub fn decode<'a>(matches : &ArgMatches<'a>) -> i32 {
 
     let param = Param::new(get_ref_block_choice!(matches),
                            matches.is_present("force"),
+                           matches.is_present("multi_pass"),
                            &json_printer,
                            in_file,
                            out,

src/cli_encode.rs

@@ -28,10 +28,8 @@ pub fn sub_command<'a, 'b>() -> App<'a, 'b> {
              .help("File to encode. Supply - to use STDIN as input. Use ./- for files named -."))
         .arg(out_arg()
              .help("SBX container name (defaults to INFILE.sbx). If OUT is a directory, then the
-container is stored as OUT/INFILE.sbx."))
-        .arg(Arg::with_name("force")
-             .short("f")
-             .long("force")
+container is stored as OUT/INFILE.sbx (only the file part of INFILE is used)."))
+        .arg(force_arg()
              .help("Force overwrite even if OUT exists"))
         .arg(Arg::with_name("hash_type")
              .value_name("HASH-TYPE")

src/cli_repair.rs

@@ -23,8 +23,7 @@ fails to guess correctly."))
              .short("y")
              .long("skip-warning")
              .help("Skip warning about in-place automatic repairs"))
-        .arg(Arg::with_name("dry_run")
-             .long("dry-run")
+        .arg(dry_run_arg()
              .help("Only do repairs in memory. The container will not be modified."))
         .arg(json_arg()
              .help("Output information in JSON format. Note that blkar does not

src/cli_sort.rs

@@ -14,14 +14,18 @@ pub fn sub_command<'a, 'b>() -> App<'a, 'b> {
         .about("Sort SBX blocks in container, can also readjust burst error resistance level")
         .arg(in_file_arg()
              .help("SBX container to sort"))
-        .arg(out_file_arg()
-             .required(true)
-             .help("Sorted SBX container"))
-        .arg(Arg::with_name("force")
-             .short("f")
-             .long("force")
-             .help("Force overwrite even if OUT exists"))
+        .arg(out_arg()
+             .help("Sorted SBX container (defaults to INFILE.sorted). If OUT is a directory, then the
+container is stored as OUT/INFILE.sorted (only the file part of INFILE is used).
+Ignored if --dry-run is supplied."))
+        .arg(force_arg()
+             .help("Force overwrite even if OUTFILE exists"))
+        .arg(multi_pass_arg()
+             .help("Disable truncation of OUT. This allows writing to OUT multiple
+times to update it gradually."))
         .arg(pr_verbosity_level_arg())
+        .arg(dry_run_arg()
+             .help("Only do sorting in memory, does not output the sorted container."))
         .arg(burst_arg()
              .help("Burst error resistance level to use for the output container.
 Defaults to guessing the level (guesses up to 1000) used by the
@@ -37,29 +41,46 @@ pub fn sort<'a>(matches : &ArgMatches<'a>) -> i32 {
     json_printer.print_open_bracket(None, BracketType::Curly);
 
     let in_file = get_in_file!(matches, json_printer);
-    let out_file = {
-        let out_file = matches.value_of("out_file").unwrap();
-
-        if file_utils::check_if_file_is_dir(out_file) {
-            misc_utils::make_path(&[out_file, in_file])
-        } else {
-            String::from(out_file)
-        }
+    let out = match matches.value_of("out") {
+        None    => {
+            format!("{}.sorted", in_file)
+        },
+        Some(x) => {
+            if file_utils::check_if_file_is_dir(x) {
+                let in_file = file_utils::get_file_name_part_of_path(in_file);
+                misc_utils::make_path(&[x,
+                                        &format!("{}.sorted", in_file)])
+            } else {
+                String::from(x)
+            }
+        },
     };
 
+    let force      = matches.is_present("force");
+    let multi_pass = matches.is_present("multi_pass");
+    let dry_run    = matches.is_present("dry_run");
+
     let burst = get_burst_opt!(matches, json_printer);
 
-    exit_if_file!(exists &out_file
-                  => matches.is_present("force")
+    exit_if_file!(exists &out
+                  => force || multi_pass || dry_run
                   => json_printer
-                  => "File \"{}\" already exists", out_file);
+                  => "File \"{}\" already exists", out);
 
     let pr_verbosity_level = get_pr_verbosity_level!(matches, json_printer);
 
+    let out : Option<&str> =
+        if dry_run {
+            None
+        } else {
+            Some(&out)
+        };
+
     let param = Param::new(get_ref_block_choice!(matches),
+                           multi_pass,
                            &json_printer,
                            in_file,
-                           &out_file,
+                           out,
                            matches.is_present("verbose"),
                            pr_verbosity_level,
                            burst);

src/cli_utils.rs

@@ -1,3 +1,4 @@
+#![allow(dead_code)]
 use clap::*;
 use sbx_block;
 use std::sync::Arc;
@@ -59,6 +60,17 @@ pub fn force_misalign_arg<'a, 'b>() -> Arg<'a, 'b> {
 used and is only intended for data recovery or related purposes.")
 }
 
+pub fn force_arg<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name("force")
+        .short("f")
+        .long("force")
+}
+
+pub fn multi_pass_arg<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name("multi_pass")
+        .long("multi-pass")
+}
+
 pub fn only_pick_uid_arg<'a, 'b>() -> Arg<'a, 'b> {
     Arg::with_name("uid")
         .value_name("UID-HEX")
@@ -101,6 +113,11 @@ pub fn burst_arg<'a, 'b>() -> Arg<'a, 'b> {
         .takes_value(true)
 }
 
+pub fn dry_run_arg<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name("dry_run")
+        .long("dry-run")
+}
+
 pub fn sbx_version_arg<'a, 'b>() -> Arg<'a, 'b> {
     Arg::with_name("sbx_version")
         .value_name("SBX-VERSION")

src/decode_core.rs

@@ -215,6 +215,7 @@ impl fmt::Display for Stats {
 pub struct Param {
     ref_block_choice   : RefBlockChoice,
     force_write        : bool,
+    multi_pass         : bool,
     json_printer       : Arc<JSONPrinter>,
     in_file            : String,
     out_file           : Option<String>,
@@ -226,6 +227,7 @@ pub struct Param {
 impl Param {
     pub fn new(ref_block_choice   : RefBlockChoice,
                force_write        : bool,
+               multi_pass         : bool,
                json_printer       : &Arc<JSONPrinter>,
                in_file            : &str,
                out_file           : Option<&str>,
@@ -235,6 +237,7 @@ impl Param {
         Param {
             ref_block_choice,
             force_write,
+            multi_pass,
             json_printer : Arc::clone(json_printer),
             in_file  : String::from(in_file),
             out_file : match out_file {
@@ -517,6 +520,7 @@ pub fn decode(param           : &Param,
         Some(ref out_file) => Writer::new(WriterType::File(FileWriter::new(out_file,
                                                                            FileWriterParam { read     : false,
                                                                                              append   : false,
+                                                                                             truncate : !param.multi_pass,
                                                                                              buffered : true   })?)),
         None               => Writer::new(WriterType::Stdout(std::io::stdout())),
     };
@@ -923,8 +927,8 @@ pub fn decode_file(param : &Param)
     let out_file_path : Option<String> = match param.out_file {
         None => {
             match recorded_file_name {
-                None    => { return Err(Error::with_message("No original file name was found in SBX container and no output file name/path was provided")); },
-                Some(x) => Some(x)
+                None        => { return Err(Error::with_message("No original file name was found in SBX container and no output file name/path was provided")); },
+                Some(ref x) => Some(file_utils::get_file_name_part_of_path(x)),
             }
         },
         Some(ref out) => {
@@ -946,7 +950,7 @@ pub fn decode_file(param : &Param)
 
     // check if can write out
     if let Some(ref out_file_path) = out_file_path {
-        if !param.force_write {
+        if !param.force_write && !param.multi_pass {
             if file_utils::check_if_file_exists(out_file_path) {
                 return Err(Error::with_message(&format!("File \"{}\" already exists",
                                                         out_file_path)));
@@ -962,6 +966,7 @@ pub fn decode_file(param : &Param)
     // regenerate param
     let param = Param::new(param.ref_block_choice,
                            param.force_write,
+                           param.multi_pass,
                            &param.json_printer,
                            &param.in_file,
                            out_file_path,

src/encode_core.rs

@@ -372,6 +372,7 @@ pub fn encode_file(param : &Param)
     let mut writer = FileWriter::new(&param.out_file,
                                      FileWriterParam { read     : false,
                                                        append   : false,
+                                                       truncate : true,
                                                        buffered : true   })?;
 
     let metadata = match reader.metadata() {

src/file_writer.rs

@@ -76,6 +76,7 @@ macro_rules! file_op {
 pub struct FileWriterParam {
     pub read     : bool,
     pub append   : bool,
+    pub truncate : bool,
     pub buffered : bool,
 }
 
@@ -98,7 +99,7 @@ impl FileWriter {
             OpenOptions::new()
             .append(param.append)
             .read(read_enabled)
-            .truncate(!param.append)
+            .truncate(!param.append && param.truncate)
             .write(true)
             .create(true)
             .open(&path);

src/log.rs

@@ -81,6 +81,7 @@ pub trait Log {
         let mut writer = FileWriter::new(log_file,
                                          FileWriterParam { read     : false,
                                                            append   : false,
+                                                           truncate : true,
                                                            buffered : false  })?;
         let output = self.serialize();
 

src/reader.rs

@@ -1,9 +1,11 @@
+#![allow(dead_code)]
 use file_reader::FileReader;
 use general_error::Error;
 use std::io::Read;
 use std::fs::Metadata;
 use stdin_error::{StdinError,
                   to_err};
+use std::io::SeekFrom;
 
 const READ_RETRIES : usize = 5;
 
@@ -62,4 +64,18 @@ impl Reader {
             ReaderType::Stdin(_)         => None,
         }
     }
+
+    pub fn seek(&mut self, pos : SeekFrom) -> Option<Result<u64, Error>> {
+        match self.reader {
+            ReaderType::Stdin(_)        => None,
+            ReaderType::File(ref mut f) => Some(f.seek(pos)),
+        }
+    }
+
+    pub fn cur_pos(&mut self) -> Option<Result<u64, Error>> {
+        match self.reader {
+            ReaderType::File(ref mut f) => Some(f.cur_pos()),
+            ReaderType::Stdin(_)        => None,
+        }
+    }
 }

src/rescue_core.rs

@@ -295,6 +295,7 @@ pub fn rescue_from_file(param : &Param)
         let mut writer = FileWriter::new(&path,
                                          FileWriterParam { read     : false,
                                                            append   : true,
+                                                           truncate : false,
                                                            buffered : false  })?;
 
         // use the original bytes which are still in the buffer