fix(rust, python): csv, read escaped "" as missing (#5912)

pola-rs · Dec 27, 2022 · 486c7e7 · 486c7e7
1 parent 52295bb
commit 486c7e7
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 6 deletions.
diff --git a/polars/polars-io/src/csv/buffer.rs b/polars/polars-io/src/csv/buffer.rs
@@ -95,7 +95,7 @@ where
                         let bytes = skip_whitespace(bytes);
                         return self.parse_bytes(bytes, ignore_errors, needs_escaping);
                     }
-                    if ignore_errors {
+                    if ignore_errors || bytes.is_empty() {
                         self.append_null()
                     } else {
                         return Err(PolarsError::ComputeError("".into()));
@@ -349,16 +349,17 @@ impl ParsedBuffer for BooleanChunkedBuilder {
         &mut self,
         bytes: &[u8],
         ignore_errors: bool,
-        _needs_escaping: bool,
+        needs_escaping: bool,
     ) -> PolarsResult<()> {
+        let bytes = if needs_escaping {
+            &bytes[1..bytes.len() - 1]
+        } else {
+            bytes
+        };
         if bytes.eq_ignore_ascii_case(b"false") {
             self.append_value(false);
         } else if bytes.eq_ignore_ascii_case(b"true") {
             self.append_value(true);
-        } else if bytes.eq_ignore_ascii_case(b"\"false\"") {
-            self.append_value(false);
-        } else if bytes.eq_ignore_ascii_case(b"\"true\"") {
-            self.append_value(true);
         } else if ignore_errors || bytes.is_empty() {
             self.append_null();
         } else {

diff --git a/py-polars/tests/unit/io/test_csv.py b/py-polars/tests/unit/io/test_csv.py
@@ -961,3 +961,25 @@ def test_csv_single_categorical_null() -> None:
 
     assert df.dtypes == [pl.Utf8, pl.Categorical, pl.Utf8]
     assert df.to_dict(False) == {"x": ["A"], "y": [None], "z": ["A"]}
+
+
+def test_csv_quoted_missing() -> None:
+    csv = '''"col1"|"col2"|"col3"|"col4"
+"0"|"Free text with a line
+break"|"123"|"456"
+"1"|"Free text without a linebreak"|""|"789"
+"0"|"Free text with 
+two 
+linebreaks"|"101112"|"131415"'''  # noqa: W291
+    assert pl.read_csv(csv.encode(), sep="|", dtypes={"col3": pl.Int32}).to_dict(
+        False
+    ) == {
+        "col1": [0, 1, 0],
+        "col2": [
+            "Free text with a line\nbreak",
+            "Free text without a linebreak",
+            "Free text with \ntwo \nlinebreaks",
+        ],
+        "col3": [123, None, 101112],
+        "col4": [456, 789, 131415],
+    }