Skip to content

Commit

Permalink
fix[rust]: improve numeric stability of rolling variance (#4413)
Browse files Browse the repository at this point in the history
  • Loading branch information
ritchie46 committed Aug 14, 2022
1 parent cada9de commit 83cffcc
Showing 1 changed file with 7 additions and 1 deletion.
8 changes: 7 additions & 1 deletion polars/polars-arrow/src/kernels/rolling/no_nulls/variance.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ pub(super) struct SumSquaredWindow<'a, T> {
sum_of_squares: T,
last_start: usize,
last_end: usize,
// if we don't recompute every 'n' iterations
// we get a accumulated error/drift
last_recompute: u8,
}

impl<'a, T: NativeType + IsFloat + std::iter::Sum + AddAssign + SubAssign + Mul<Output = T>>
Expand All @@ -21,15 +24,18 @@ impl<'a, T: NativeType + IsFloat + std::iter::Sum + AddAssign + SubAssign + Mul<
sum_of_squares: sum,
last_start: start,
last_end: end,
last_recompute: 0,
}
}

unsafe fn update(&mut self, start: usize, end: usize) -> T {
// if we exceed the end, we have a completely new window
// so we recompute
let recompute_sum = if start >= self.last_end {
let recompute_sum = if start >= self.last_end || self.last_recompute > 128 {
self.last_recompute = 0;
true
} else {
self.last_recompute += 1;
// remove elements that should leave the window
let mut recompute_sum = false;
for idx in self.last_start..start {
Expand Down

0 comments on commit 83cffcc

Please sign in to comment.