Permalink
Browse files

Fixed number_with_precision rounding error [#6182 state:resolved]

  • Loading branch information...
1 parent 3b9120f commit 006ec77e60163d57d655479602230672d98e0f10 @dontangg dontangg committed with drogus Dec 16, 2010
@@ -270,12 +270,13 @@ def number_with_precision(number, options = {})
digits, rounded_number = 1, 0
else
digits = (Math.log10(number.abs) + 1).floor
- rounded_number = BigDecimal.new((number / 10 ** (digits - precision)).to_s).round.to_f * 10 ** (digits - precision)
+ rounded_number = (BigDecimal.new(number.to_s) / 10 ** (digits - precision)).round.to_f * 10 ** (digits - precision)
+ digits = (Math.log10(rounded_number.abs) + 1).floor # After rounding, the number of digits may have changed
end
precision -= digits
precision = precision > 0 ? precision : 0 #don't let it be negative
else
- rounded_number = BigDecimal.new((number * (10 ** precision)).to_s).round.to_f / 10 ** precision
+ rounded_number = BigDecimal.new(number.to_s).round(precision).to_f
end
formatted_number = number_with_delimiter("%01.#{precision}f" % rounded_number, options)
if strip_insignificant_zeros
@@ -100,6 +100,11 @@ def test_number_with_precision
assert_equal("0", number_with_precision(0, :precision => 0))
assert_equal("0.00100", number_with_precision(0.001, :precision => 5))
assert_equal("0.001", number_with_precision(0.00111, :precision => 3))
+ assert_equal("10.00", number_with_precision(9.995, :precision => 2))
+ assert_equal("11.00", number_with_precision(10.995, :precision => 2))
+ assert_equal("10.0", number_with_precision(9.995, :precision => 3, :significant => true))
+ assert_equal("9.99", number_with_precision(9.994, :precision => 3, :significant => true))
+ assert_equal("11.0", number_with_precision(10.995, :precision => 3, :significant => true))
end
def test_number_with_precision_with_custom_delimiter_and_separator

0 comments on commit 006ec77

Please sign in to comment.