Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Support hexadecimal floating point literals #2460

Draft
wants to merge 5 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 68 additions & 25 deletions src/tokenizer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,10 @@ import {
isDecimal,
isOctal,
isHexBase,
isHexOrDecimal,
isHighSurrogate,
isLowSurrogate
isLowSurrogate,
parseHexFloat
} from "./util";

/** Named token types. */
Expand Down Expand Up @@ -1259,18 +1261,34 @@ export class Tokenizer extends DiagnosticEmitter {
var text = this.source.text;
var pos = this.pos;
var end = this.end;
if (pos + 1 < end && text.charCodeAt(pos) == CharCode._0) {
switch (text.charCodeAt(pos + 2) | 32) {
case CharCode.x:
var hex = false;
if (pos + 2 < end && text.charCodeAt(pos) == CharCode._0) {
switch (text.charCodeAt(pos + 1) | 32) {
case CharCode.x: {
// Don't early return for CharCode.x
// It possible a hexadecimal float.
hex = true;
pos += 2;
break;
}
case CharCode.b:
case CharCode.o: return true;
case CharCode.DOT: return false;
}
}
while (pos < end) {
let c = text.charCodeAt(pos);
if (c == CharCode.DOT || (c | 32) == CharCode.e) return false;
if (c != CharCode._ && (c < CharCode._0 || c > CharCode._9)) break;
if (c == CharCode.DOT) return false;
// does not validate separator placement (this is done in readXYInteger)
if (c != CharCode._) {
if (hex) {
if ((c | 32) == CharCode.p) return false;
if (!isHexOrDecimal(c)) break;
} else {
if ((c | 32) == CharCode.e) return false;
if (!isDecimal(c)) break;
}
}
pos++;
}
return true;
Expand Down Expand Up @@ -1389,7 +1407,7 @@ export class Tokenizer extends DiagnosticEmitter {
while (pos < end) {
let c = text.charCodeAt(pos);
if (isDecimal(c)) {
// value = value * 10 + c - CharCode._0;
// value * 10 + c - CharCode._0
nextValue = i64_add(
i64_mul(value, i64_10),
i64_new(c - CharCode._0)
Expand Down Expand Up @@ -1563,27 +1581,23 @@ export class Tokenizer extends DiagnosticEmitter {
}

readFloat(): f64 {
// var text = this.source.text;
// if (text.charCodeAt(this.pos) == CharCode._0 && this.pos + 2 < this.end) {
// switch (text.charCodeAt(this.pos + 1)) {
// case CharCode.X:
// case CharCode.x: {
// this.pos += 2;
// return this.readHexFloat();
// }
// }
// }
var text = this.source.text;
if (text.charCodeAt(this.pos) == CharCode._0 && this.pos + 2 < this.end) {
if ((text.charCodeAt(this.pos + 1) | 32) == CharCode.x) {
return this.readHexFloat();
}
}
return this.readDecimalFloat();
}

readDecimalFloat(): f64 {
var text = this.source.text;
var end = this.end;
var start = this.pos;
var sepCount = this.readDecimalFloatPartial(false);
var sepCount = this.readFloatPartial(false, false);
if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) {
++this.pos;
sepCount += this.readDecimalFloatPartial();
sepCount += this.readFloatPartial(true, false);
}
if (this.pos < end) {
let c = text.charCodeAt(this.pos);
Expand All @@ -1595,7 +1609,7 @@ export class Tokenizer extends DiagnosticEmitter {
) {
++this.pos;
}
sepCount += this.readDecimalFloatPartial();
sepCount += this.readFloatPartial(true, false);
}
}
let result = text.substring(start, this.pos);
Expand All @@ -1604,7 +1618,7 @@ export class Tokenizer extends DiagnosticEmitter {
}

/** Reads past one section of a decimal float literal. Returns the number of separators encountered. */
private readDecimalFloatPartial(allowLeadingZeroSep: bool = true): u32 {
private readFloatPartial(allowLeadingZeroSep: bool, isHexadecimal: bool): u32 {
var text = this.source.text;
var pos = this.pos;
var start = pos;
Expand All @@ -1614,7 +1628,6 @@ export class Tokenizer extends DiagnosticEmitter {

while (pos < end) {
let c = text.charCodeAt(pos);

if (c == CharCode._) {
if (sepEnd == pos) {
this.error(
Expand All @@ -1631,8 +1644,12 @@ export class Tokenizer extends DiagnosticEmitter {
}
sepEnd = pos + 1;
++sepCount;
} else if (!isDecimal(c)) {
break;
} else {
if (isHexadecimal) {
if (!isHexOrDecimal(c)) break;
} else {
if (!isDecimal(c)) break;
}
}
++pos;
}
Expand All @@ -1649,7 +1666,33 @@ export class Tokenizer extends DiagnosticEmitter {
}

readHexFloat(): f64 {
throw new Error("not implemented"); // TBD
var text = this.source.text;
var pos = this.pos;
var start = pos;
var end = this.end;

this.pos += 2; // skip 0x
var sepCount = this.readFloatPartial(false, true);
if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) {
++this.pos;
sepCount += this.readFloatPartial(true, true);
}
if (this.pos < end) {
let c = text.charCodeAt(this.pos);
if ((c | 32) == CharCode.p) {
if (
++this.pos < end &&
(c = text.charCodeAt(this.pos)) == CharCode.MINUS || c == CharCode.PLUS &&
isHexOrDecimal(text.charCodeAt(this.pos + 1))
) {
++this.pos;
}
sepCount += this.readFloatPartial(true, false);
}
}
let result = text.substring(start, this.pos);
if (sepCount) result = result.replaceAll("_", "");
return parseHexFloat(result);
}

readHexadecimalEscape(remain: i32 = 2, startIfTaggedTemplate: i32 = -1): string {
Expand Down
35 changes: 35 additions & 0 deletions src/util/math.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,38 @@ export function accuratePow64(x: f64, y: f64): f64 {
}
return Math.pow(x, y);
}

// see: https://git.musl-libc.org/cgit/musl/tree/src/math/scalbn.c
/** Equivalent of `x * (2 ** n)` */
export function scalbn(x: f64, n: i32): f64 {
const
Ox1p1023 = 8.98846567431158e+307, // 0x1p1023
Ox1p_969 = 2.00416836000897278e-292; // 0x1p-1022 * 0x1p53

var y = x;
if (n > 1023) {
y *= Ox1p1023;
n -= 1023;
if (n > 1023) {
y *= Ox1p1023;
n -= 1023;
if (n > 1023) n = 1023;
}
} else if (n < -1022) {
// make sure final n < -53 to avoid double
// rounding in the subnormal range
y *= Ox1p_969;
n += 1022 - 53;
if (n < -1022) {
y *= Ox1p_969;
n += 1022 - 53;
if (n < -1022) n = -1022;
}
}
if (!ASC_TARGET) { // ASC_TARGET == JS
return y * i64_as_f64(i64_new(0, 0x3FF + n << 20));
} else {
// @ts-ignore
return y * reinterpret<f64>(u64(0x3FF + n) << 52);
}
}
32 changes: 32 additions & 0 deletions src/util/text.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
* @license Apache-2.0
*/

import { scalbn } from "./math";

/** An enum of named character codes. */
export const enum CharCode {

Expand Down Expand Up @@ -564,3 +566,33 @@ export function escapeString(str: string, quote: CharCode): string {
if (i > off) sb.push(str.substring(off, i));
return sb.join("");
}

export function parseHexFloat(str: string): f64 {
var sign = 1, pPos = -1, dotPos = -1;
for (let i = 0, k = str.length; i < k; ++i) {
const c = str.charCodeAt(i);
if (i == 0 && c == CharCode.MINUS) {
sign = -1;
} else if ((c | 32) == CharCode.p) {
pPos = i;
} else if (c == CharCode.DOT) {
dotPos = i;
}
}
var mant: f64;
var mantissa = ~pPos ? str.substring(0, pPos) : str;
if (~dotPos) {
const integer = mantissa.substring(0, dotPos);
const fraction = mantissa.substring(dotPos + 1);
const intVal = parseInt(integer, 16);
const fracVal = fraction.length
? scalbn(parseInt(fraction, 16), -(fraction.length << 2))
: 0;
mant = intVal + sign * fracVal;
} else {
mant = parseInt(mantissa, 16);
}
return ~pPos
? scalbn(mant, parseInt(str.substring(pPos + 1)))
: mant;
}
Loading