fix parquet stats (#3378)

pola-rs · May 12, 2022 · 2d83a82 · 2d83a82
1 parent dca383f
commit 2d83a82
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 1 deletion.
diff --git a/polars/polars-io/src/parquet/predicates.rs b/polars/polars-io/src/parquet/predicates.rs
@@ -38,7 +38,12 @@ impl ColumnStats {
         let dtype = DataType::from(min_val.data_type());
         if dtype.is_numeric() || matches!(dtype, DataType::Utf8) {
             let arr = concatenate(&[min_val, max_val]).unwrap();
-            Some(Series::try_from(("", Arc::from(arr) as ArrayRef)).unwrap())
+            let s = Series::try_from(("", Arc::from(arr) as ArrayRef)).unwrap();
+            if s.null_count() > 0 {
+                None
+            } else {
+                Some(s)
+            }
         } else {
             None
         }

diff --git a/polars/polars-lazy/src/physical_plan/expressions/binary.rs b/polars/polars-lazy/src/physical_plan/expressions/binary.rs
@@ -448,6 +448,8 @@ mod stats {
                     match l.to_min_max() {
                         None => Ok(true),
                         Some(min_max_s) => {
+                            // will be incorrect if not
+                            debug_assert_eq!(min_max_s.null_count(), 0);
                             let lit_s = self.right.evaluate(&dummy, &state).unwrap();
                             Ok(apply_operator_stats_rhs_lit(&min_max_s, &lit_s, self.op))
                         }
@@ -458,6 +460,8 @@ mod stats {
                     match r.to_min_max() {
                         None => Ok(true),
                         Some(min_max_s) => {
+                            // will be incorrect if not
+                            debug_assert_eq!(min_max_s.null_count(), 0);
                             let lit_s = self.left.evaluate(&dummy, &state).unwrap();
                             Ok(apply_operator_stats_lhs_lit(&lit_s, &min_max_s, self.op))
                         }