Permalink
Browse files

Prevent IEEE 754 floating point errors in both decimal and numeric

  • Loading branch information...
1 parent 25e56b8 commit 1bcc77f69e300e7d3574f25b9a1b7df1aa52121d @patriksimek patriksimek committed Jun 15, 2014
Showing with 3 additions and 3 deletions.
  1. +3 −3 src/data-type.coffee
View
@@ -219,13 +219,13 @@ TYPE =
# ParamLenData (TYPE_VARBYTE)
if parameter.value?
sign = if parameter.value < 0 then 0 else 1
- value = Math.abs parameter.value * Math.pow(10, parameter.scale)
+ value = Math.round Math.abs parameter.value * Math.pow(10, parameter.scale)
if parameter.precision <= 9
buffer.writeUInt8 5
buffer.writeUInt8 sign
# Round to preven IEEE 754 floating point errors
- buffer.writeUInt32LE Math.round(value)
+ buffer.writeUInt32LE value
else if parameter.precision <= 19
buffer.writeUInt8 9
buffer.writeUInt8 sign
@@ -281,7 +281,7 @@ TYPE =
# ParamLenData (TYPE_VARBYTE)
if parameter.value?
sign = if parameter.value < 0 then 0 else 1
- value = Math.abs parameter.value * Math.pow(10, parameter.scale)
+ value = Math.round Math.abs parameter.value * Math.pow(10, parameter.scale)
if parameter.precision <= 9
buffer.writeUInt8 5

0 comments on commit 1bcc77f

Please sign in to comment.