Fix covariance of arrays with null values

polars/polars-core/src/chunked_array/mod.rs

@@ -182,8 +182,8 @@ impl<T> ChunkedArray<T> {
             Some(0)
         } else {
             let mut offset = 0;
-            for (idx, (null_count, null_bit_buffer)) in self.null_bits().iter().enumerate() {
-                if *null_count == 0 {
+            for (idx, (null_count, null_bit_buffer)) in self.null_bits().enumerate() {
+                if null_count == 0 {
                     return Some(offset);
                 } else {
                     let arr = &self.chunks[idx];
@@ -206,11 +206,8 @@ impl<T> ChunkedArray<T> {
     }
 
     /// Get the null count and the buffer of bits representing null values
-    pub fn null_bits(&self) -> Vec<(usize, Option<Buffer>)> {
-        self.chunks
-            .iter()
-            .map(|arr| get_bitmap(arr.as_ref()))
-            .collect()
+    pub fn null_bits(&self) -> impl Iterator<Item = (usize, Option<Buffer>)> + '_ {
+        self.chunks.iter().map(|arr| get_bitmap(arr.as_ref()))
     }
 
     /// Unpack a Series to the same physical type.

polars/polars-core/src/functions.rs

@@ -2,7 +2,6 @@ use crate::prelude::*;
 use num::{Float, NumCast};
 use std::ops::Div;
 
-// todo! make numerical stable from catastrophic cancellation
 pub fn cov<T>(a: &ChunkedArray<T>, b: &ChunkedArray<T>) -> Option<T::Native>
 where
     T: PolarsFloatType,
@@ -11,7 +10,9 @@ where
     if a.len() != b.len() {
         None
     } else {
-        Some((&(a - a.mean()?) * &(b - b.mean()?)).sum()? / NumCast::from(a.len() - 1).unwrap())
+        let tmp = (a - a.mean()?) * (b - b.mean()?);
+        let n = tmp.len() - tmp.null_count();
+        Some(tmp.sum()? / NumCast::from(n - 1).unwrap())
     }
 }
 pub fn pearson_corr<T>(a: &ChunkedArray<T>, b: &ChunkedArray<T>) -> Option<T::Native>

polars/polars-core/src/series/implementations/dates.rs

@@ -558,7 +558,7 @@ macro_rules! impl_dyn_series {
             }
 
             fn null_bits(&self) -> Vec<(usize, Option<Buffer>)> {
-                self.0.null_bits()
+                self.0.null_bits().collect()
             }
 
             fn reverse(&self) -> Series {

polars/polars-core/src/series/implementations/mod.rs

@@ -725,7 +725,7 @@ macro_rules! impl_dyn_series {
             }
 
             fn null_bits(&self) -> Vec<(usize, Option<Buffer>)> {
-                self.0.null_bits()
+                self.0.null_bits().collect()
             }
 
             fn reverse(&self) -> Series {

polars/polars-core/src/series/implementations/object.rs

@@ -190,7 +190,7 @@ where
     }
 
     fn null_bits(&self) -> Vec<(usize, Option<Buffer>)> {
-        ObjectChunked::null_bits(&self.0)
+        ObjectChunked::null_bits(&self.0).collect()
     }
 
     fn reverse(&self) -> Series {