fix floating point errors at edge cases

#245
fslaborg · Feb 21, 2023 · 32fa0c2 · 32fa0c2
1 parent 0344f07
commit 32fa0c2
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/FSharp.Stats/Distributions/Empirical.fs b/src/FSharp.Stats/Distributions/Empirical.fs
@@ -158,7 +158,7 @@ module Empirical =
         let halfBw = bandwidth / 2.0       
         let tmp = 
             data
-            |> Seq.groupBy (fun x -> floor (x / bandwidth)) 
+            |> Seq.groupBy (fun x -> (decimal x / decBandwidth) |> float |> floor) 
             |> Seq.map (fun (k,values) -> 
                 let count = (Seq.length(values)) |> float                                        
                 if k < 0. then