Skip to content

Commit

Permalink
csv: fix quoting with projection
Browse files Browse the repository at this point in the history
  • Loading branch information
ritchie46 committed Oct 30, 2021
1 parent c70e4f0 commit eb7cf5b
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 2 deletions.
28 changes: 28 additions & 0 deletions polars/polars-io/src/csv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1262,4 +1262,32 @@ linenum,last_name,first_name
assert_eq!(df.dtypes(), &[Utf8, Utf8, Utf8]);
Ok(())
}

fn test_projection_and_quoting() -> Result<()> {
let csv = "a,b,c,d
A1,'B1',C1,1
A2,\"B2\",C2,2
A3,\"B3\",C3,3
A3,\"B4_\"\"with_embedded_double_quotes\"\"\",C4,4";

let file = Cursor::new(csv);
let df = CsvReader::new(file).finish()?;
assert_eq!(df.shape(), (4, 4));

let file = Cursor::new(csv);
let df = CsvReader::new(file)
.with_n_threads(Some(1))
.with_projection(Some(vec![0, 2]))
.finish()?;
assert_eq!(df.shape(), (4, 2));

let file = Cursor::new(csv);
let df = CsvReader::new(file)
.with_n_threads(Some(1))
.with_projection(Some(vec![1]))
.finish()?;
assert_eq!(df.shape(), (4, 1));

Ok(())
}
}
3 changes: 1 addition & 2 deletions polars/polars-io/src/csv_core/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -503,13 +503,12 @@ pub(crate) fn parse_lines(
next_projected = p
}
None => {
let offset = read_sol - 1;
if let Some(b'\n') = bytes.get(0) {
bytes = &bytes[read_sol..];
read += read_sol
} else {
let (bytes_rem, len) =
skip_this_line(&bytes[offset..], quote_char, offset);
skip_this_line(bytes, quote_char, offset);
bytes = bytes_rem;
read += len;
}
Expand Down

0 comments on commit eb7cf5b

Please sign in to comment.