Skip to content

Commit

Permalink
extsort: add --tmp-dir option
Browse files Browse the repository at this point in the history
  • Loading branch information
jqnatividad committed Dec 15, 2023
1 parent b522ba4 commit ca1f461
Showing 1 changed file with 17 additions and 2 deletions.
19 changes: 17 additions & 2 deletions src/cmd/extsort.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@ Usage:
External sort option:
--memory-limit <arg> The maximum amount of memory to buffer the on-disk hash table.
This is a percentage of total memory. [default: 10]
This is a percentage of total memory.
[default: 10]
--tmp-dir <arg> The directory to use for externally sorting file segments.
[default: ./]
-j, --jobs <arg> The number of jobs to run in parallel.
When not set, the number of jobs is set to the
number of CPUs detected.
Expand Down Expand Up @@ -42,6 +45,7 @@ struct Args {
arg_output: Option<String>,
flag_jobs: Option<usize>,
flag_memory_limit: Option<u8>,
flag_tmp_dir: Option<String>,
flag_no_headers: bool,
}

Expand All @@ -51,6 +55,17 @@ const RW_BUFFER_CAPACITY: usize = 1_000_000; // 1 MB
pub fn run(argv: &[&str]) -> CliResult<()> {
let args: Args = util::get_args(USAGE, argv)?;

// check if tmp dir exists
let tmp_dir = match &args.flag_tmp_dir {
Some(tmp_dir) => {
if !path::Path::new(tmp_dir).exists() {
return fail_clierror!("tmp-dir '{tmp_dir}' does not exist");
}
tmp_dir.to_string()
},
None => "./".to_string(),
};

// memory buffer to use for external merge sort,
// if we can detect the total memory, use 10% of it by default
// and up to --memory-limit (capped at 50%),
Expand Down Expand Up @@ -101,7 +116,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> {

let sorter: ExternalSorter<String, io::Error, MemoryLimitedBufferBuilder> =
match ExternalSorterBuilder::new()
.with_tmp_dir(path::Path::new("./"))
.with_tmp_dir(path::Path::new(&tmp_dir))
.with_buffer(MemoryLimitedBufferBuilder::new(mem_limited_buffer))
.with_rw_buf_size(RW_BUFFER_CAPACITY)
.with_threads_number(util::njobs(args.flag_jobs))
Expand Down

0 comments on commit ca1f461

Please sign in to comment.