fix_lossy (#2585)

pola-rs · Feb 9, 2022 · b1de1c2 · b1de1c2
1 parent f7c5062
commit b1de1c2
Showing 1 changed file with 10 additions and 4 deletions.
diff --git a/polars/polars-io/src/csv_core/buffer.rs b/polars/polars-io/src/csv_core/buffer.rs
@@ -173,10 +173,16 @@ impl ParsedBuffer<Utf8Type> for Utf8Field {
             }
             false => {
                 if matches!(self.encoding, CsvEncoding::LossyUtf8) {
-                    let s = String::from_utf8_lossy(
-                        &self.data.as_slice()[data_len..data_len + n_written],
-                    )
-                    .into_owned();
+                    // Safety:
+                    // we extended to data_len + n_writen
+                    // so the bytes are initialized
+                    debug_assert!(self.data.capacity() >= data_len + n_written);
+                    let slice = unsafe {
+                        self.data
+                            .as_slice()
+                            .get_unchecked(data_len..data_len + n_written)
+                    };
+                    let s = String::from_utf8_lossy(slice).into_owned();
                     let b = s.as_bytes();
                     // Make sure that we extend at the proper location,
                     // otherwise we append valid bytes to invalid utf8 bytes.