Skip to content

Commit

Permalink
count: when the polars mem-mapped CSV reader fails, fall back to re…
Browse files Browse the repository at this point in the history
…gular CSV reader
  • Loading branch information
jqnatividad committed Mar 15, 2024
1 parent 6d7c361 commit a2c0869
Showing 1 changed file with 11 additions and 3 deletions.
14 changes: 11 additions & 3 deletions src/cmd/count.rs
Expand Up @@ -193,14 +193,13 @@ pub fn polars_count_input(
None
};

let df = polars::io::csv::CsvReader::from_path(filepath.clone())?
let df_result = polars::io::csv::CsvReader::from_path(filepath.clone())?
.with_separator(conf.get_delimiter())
.with_comment_prefix(comment_prefix)
.has_header(!conf.no_headers)
.truncate_ragged_lines(conf.flexible)
.low_memory(low_memory)
.finish()?;
let count = df.height() as u64;
.finish();

// remove the temporary file we created to read from stdin
// we use the keep() method to prevent the file from being deleted
Expand All @@ -209,5 +208,14 @@ pub fn polars_count_input(
std::fs::remove_file(filepath)?;
}

let count = match df_result {
Ok(df) => df.height() as u64,
Err(_) => {
// there was a Polars error, so we fall back to the regular CSV reader
let (count_regular, _) = count_input(conf, false)?;
count_regular
},
};

Ok((count, 0))
}

0 comments on commit a2c0869

Please sign in to comment.