first check ascii before we check utf8 in lossy encoding

pola-rs · Jan 8, 2022 · 92cf2d2 · 92cf2d2
1 parent 838f05a
commit 92cf2d2
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 5 deletions.
diff --git a/polars/polars-io/src/csv_core/buffer.rs b/polars/polars-io/src/csv_core/buffer.rs
@@ -138,7 +138,7 @@ impl ParsedBuffer<Utf8Type> for Utf8Field {
         let parse_result = if delay_utf8_validation(self.encoding, ignore_errors) {
             true
         } else {
-            simdutf8::basic::from_utf8(bytes).is_ok()
+            bytes.is_ascii() || simdutf8::basic::from_utf8(bytes).is_ok()
         };
         let data_len = self.data.len();
 

diff --git a/py-polars/tests/test_strings.py b/py-polars/tests/test_strings.py
@@ -1,6 +1,7 @@
 import polars as pl
 
-def test_extract_binary():
-    df = pl.DataFrame({'foo': ['aron', 'butler', 'charly', 'david']})
-    out = df.filter(pl.col('foo').str.extract('^(a)', 1) == 'a').to_series()
-    assert out[0] == "aron"
+
+def test_extract_binary() -> None:
+    df = pl.DataFrame({"foo": ["aron", "butler", "charly", "david"]})
+    out = df.filter(pl.col("foo").str.extract("^(a)", 1) == "a").to_series()
+    assert out[0] == "aron"