Flate2 zlib ng (#1365)

* Use different variable name than "bytes" as it is reserved keyword in python.

polars/Cargo.toml

@@ -80,6 +80,7 @@ concat_str = ["polars-core/concat_str", "polars-lazy/concat_str"]
 row_hash = ["polars-core/row_hash"]
 reinterpret = ["polars-core/reinterpret", "polars-core/dtype-u64"]
 decompress = ["polars-io/decompress"]
+decompress-fast = ["polars-io/decompress-fast"]
 mode = ["polars-core/mode", "polars-lazy/mode"]
 take_opt_iter = ["polars-core/take_opt_iter"]
 extract_jsonpath = ["polars-core/extract_jsonpath", "polars-core/strings"]

polars/polars-io/Cargo.toml

@@ -22,7 +22,8 @@ dtype-date64 = ["polars-core/dtype-date64", "polars-core/temporal"]
 dtype-date32 = ["polars-core/dtype-date32"]
 csv-file = ["csv-core", "memmap", "lexical", "arrow/io_csv"]
 fmt = ["polars-core/plain_fmt"]
-decompress = ["flate2"]
+decompress = ["flate2/miniz_oxide"]
+decompress-fast = ["flate2/zlib-ng-compat"]
 # don't use this
 private = []
 
@@ -44,7 +45,7 @@ ahash = "0.7"
 num = "^0.4.0"
 dirs = "3.0"
 simdutf8 = {version="0.1", optional=true}
-flate2 = {version = "1", optional=true}
+flate2 = {version = "1", optional=true, --default-features=false}
 
 [package.metadata.docs.rs]
 all-features = true

polars/polars-io/src/csv_core/csv.rs

@@ -483,8 +483,12 @@ impl<'a> CoreReader<'a> {
 
 #[allow(clippy::too_many_arguments)]
 pub fn build_csv_reader<'a>(
-    #[cfg(feature = "decompress")] mut reader_bytes: ReaderBytes<'a>,
-    #[cfg(not(feature = "decompress"))] reader_bytes: ReaderBytes<'a>,
+    #[cfg(any(feature = "decompress", feature = "decompress-fast"))] mut reader_bytes: ReaderBytes<
+        'a,
+    >,
+    #[cfg(not(any(feature = "decompress", feature = "decompress-fast")))] reader_bytes: ReaderBytes<
+        'a,
+    >,
     n_rows: Option<usize>,
     skip_rows: usize,
     mut projection: Option<Vec<usize>>,
@@ -512,7 +516,7 @@ pub fn build_csv_reader<'a>(
     let mut schema = match schema {
         Some(schema) => Cow::Borrowed(schema),
         None => {
-            #[cfg(feature = "decompress")]
+            #[cfg(any(feature = "decompress", feature = "decompress-fast"))]
             {
                 // We keep track of the inferred schema bool
                 // In case the file is compressed this schema inference is wrong and has to be done
@@ -532,7 +536,7 @@ pub fn build_csv_reader<'a>(
                 )?;
                 Cow::Owned(inferred_schema)
             }
-            #[cfg(not(feature = "decompress"))]
+            #[cfg(not(any(feature = "decompress", feature = "decompress-fast")))]
             {
                 let (inferred_schema, _) = infer_file_schema(
                     &reader_bytes,

polars/polars-io/src/csv_core/utils.rs

@@ -248,7 +248,7 @@ pub fn infer_file_schema(
     Ok((Schema::new(fields), records_count))
 }
 
-#[cfg(feature = "decompress")]
+#[cfg(any(feature = "decompress", feature = "decompress-fast"))]
 pub(crate) fn decompress(bytes: &[u8]) -> Option<Vec<u8>> {
     // magic numbers
     let gzip: [u8; 2] = [31, 139];

py-polars/Cargo.toml

@@ -61,7 +61,7 @@ features = [
     "concat_str",
     "row_hash",
     "reinterpret",
-    "decompress",
+    "decompress-fast",
     "mode",
     "extract_jsonpath",
     "lazy_regex",

py-polars/tests/test_io.py

@@ -150,8 +150,8 @@ def test_compressed_csv():
     with gzip.GzipFile(fileobj=fout, mode="w") as f:
         f.write(csv.encode())
 
-    bytes = fout.getvalue()
-    out = pl.read_csv(bytes)
+    csv_bytes = fout.getvalue()
+    out = pl.read_csv(csv_bytes)
     expected = pl.DataFrame(
         {"a": [1, 2, 3], "b": ["a", "b", "c"], "c": [1.0, 2.0, 3.0]}
     )
@@ -162,13 +162,13 @@ def test_compressed_csv():
     assert out.frame_equal(expected)
 
     # now with column projection
-    out = pl.read_csv(bytes, columns=["a", "b"])
+    out = pl.read_csv(csv_bytes, columns=["a", "b"])
     expected = pl.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]})
     assert out.frame_equal(expected)
 
     # zlib compression
-    bytes = zlib.compress(csv.encode())
-    out = pl.read_csv(bytes)
+    csv_bytes = zlib.compress(csv.encode())
+    out = pl.read_csv(csv_bytes)
     expected = pl.DataFrame(
         {"a": [1, 2, 3], "b": ["a", "b", "c"], "c": [1.0, 2.0, 3.0]}
     )