Skip to content

Commit

Permalink
Fix toPrecition (was off by one for fractional inputs)
Browse files Browse the repository at this point in the history
  • Loading branch information
efergus committed Jan 9, 2024
1 parent c04f61c commit e00291a
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 10 deletions.
Empty file added .chat.log
Empty file.
16 changes: 6 additions & 10 deletions src/util.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ export function serializeNumber (n, {precision, unit }) {
return "none";
}

return toPrecision(n, precision) + (unit ?? "");
return toPrecision(n, precision) + (unit ?? "");
}

/**
Expand All @@ -55,16 +55,12 @@ export function skipNone (n) {
*/
export function toPrecision (n, precision) {
n = +n;
precision = +precision;
let integerLength = (Math.floor(n) + "").length;

if (precision > integerLength) {
return +n.toFixed(precision - integerLength);
}
else {
let p10 = 10 ** (integerLength - precision);
return Math.round(n / p10) * p10;
if (n === 0) {
return 0;
}
precision = +precision;
const multiplier = Math.pow(10, precision - Math.floor(Math.log10(Math.abs(n))) - 1);
return Math.round(n * multiplier) / multiplier;
}

const angleFactor = {
Expand Down

0 comments on commit e00291a

Please sign in to comment.