Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(NODE-4874): support EJSON parse for BigInt from $numberLong #552

Merged
Merged
Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
75a6207
feat(NODE-4874): WIP
W-A-James Jan 5, 2023
7cd7f6c
test(NODE-4874): Start table test
W-A-James Jan 5, 2023
df97552
fix(NODE-4874): Get canonical mode parsing to work
W-A-James Jan 6, 2023
07904f9
style(NODE-4874): Style
W-A-James Jan 6, 2023
d0dd23d
style(NODE-4874): eslint
W-A-James Jan 6, 2023
5120ca5
fix(NODE-4874): Fix handling of flags
W-A-James Jan 6, 2023
d67757d
test(NODE-4874): Finish tests
W-A-James Jan 6, 2023
4f15e42
style(NODE-4874): fixup condition
W-A-James Jan 6, 2023
2f5db05
fix(NODE-4874): Remove dead code
W-A-James Jan 6, 2023
822dff8
fix(NODE-4874): Add fix and test for double bug
W-A-James Jan 11, 2023
5666255
fix(NODE-4874): Implement requested fixes
W-A-James Jan 11, 2023
79f0c81
test(NODE-4874): Add new table tests
W-A-James Jan 11, 2023
260b667
test(NODE-4874): Add test to check that legacy flag doesn't affect ou…
W-A-James Jan 11, 2023
6a5364a
test(NODE-4874): Add another assertion
W-A-James Jan 11, 2023
7eca3ea
style(NODE-4874): Update comment
W-A-James Jan 11, 2023
f6aed3d
test(NODE-4874): Remove 'only' annotation
W-A-James Jan 11, 2023
b0e67aa
fix(NODE-4874): Update Long.fromExtendedJSON
W-A-James Jan 11, 2023
4361871
test(NODE-4874): Add new tests
W-A-James Jan 11, 2023
92ae357
fix(NODE-4874): Add fixes
W-A-James Jan 12, 2023
fff3d32
test(NODE-4874): Update tests
W-A-James Jan 12, 2023
a86a942
style(NODE-4874): Change variable name
W-A-James Jan 12, 2023
d9f539f
fix(NODE-4874): Fix MAX_INT64_STRING_LENGTH constant
W-A-James Jan 13, 2023
0acf248
test(NODE-4874): Test fixups
W-A-James Jan 17, 2023
cdf0334
fix(NODE-4874): Fix fromExtendedJSON checks and errors
W-A-James Jan 17, 2023
027b5df
fix(NODE-4874): Update fromExtendedJSON
W-A-James Jan 17, 2023
a02236f
test(NODE-4874): New tests
W-A-James Jan 17, 2023
31f8bf7
test(NODE-4874): Add tests for current validation behaviour
W-A-James Jan 18, 2023
f22ce03
fix(NODE-4874): Small style fix
W-A-James Jan 18, 2023
8b0a741
test(NODE-4874): Add comments and fix test value
W-A-James Jan 18, 2023
e0049b3
fix(NODE-4874): Make error messages more verbose
W-A-James Jan 18, 2023
47cdb65
fix(NODE-4874): Fix error messages
W-A-James Jan 18, 2023
d502ddb
Merge branch 'main' into NODE-4874/support_ejson_parse_for_BigInt_fro…
W-A-James Jan 18, 2023
f80b707
fix(NODE-4874): merge conflict fix
W-A-James Jan 18, 2023
c90b783
fix(NODE-4874): Update fromExtendedJSON behaviour to be more consiste…
W-A-James Jan 19, 2023
43b39c1
test(NODE-4874): move EJSON.parse tests back under the correct descri…
W-A-James Jan 19, 2023
b99a141
test(NODE-4874): Unnest EJSON.stringify fron EJSON.parse
W-A-James Jan 19, 2023
a0b5cf0
test(NODE-4874): Remove todo
W-A-James Jan 19, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions src/extended_json.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ export type EJSONOptions = {
legacy?: boolean;
/** Enable Extended JSON's `relaxed` mode, which attempts to return native JS types where possible, rather than BSON types */
relaxed?: boolean;
/** Enable native bigint support */
useBigInt64?: boolean;
};

/** @internal */
Expand Down Expand Up @@ -70,17 +72,23 @@ const keysToCodecs = {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
function deserializeValue(value: any, options: EJSONOptions = {}) {
if (typeof value === 'number') {
const in32BitRange = value <= BSON_INT32_MAX && value >= BSON_INT32_MIN;
const in64BitRange = value <= BSON_INT64_MAX && value >= BSON_INT64_MIN;
nbbeeken marked this conversation as resolved.
Show resolved Hide resolved

if (options.relaxed || options.legacy) {
return value;
return Number.isInteger(value) && options.useBigInt64 ? BigInt(value) : value;
nbbeeken marked this conversation as resolved.
Show resolved Hide resolved
}

if (Number.isInteger(value) && !Object.is(value, -0)) {
// interpret as being of the smallest BSON integer type that can represent the number exactly
if (value >= BSON_INT32_MIN && value <= BSON_INT32_MAX) {
if (in32BitRange) {
return new Int32(value);
}
if (value >= BSON_INT64_MIN && value <= BSON_INT64_MAX) {
if (in64BitRange) {
// TODO(NODE-4377): EJSON js number handling diverges from BSON
nbbeeken marked this conversation as resolved.
Show resolved Hide resolved
if (options.useBigInt64) {
return BigInt(value);
}
return Long.fromNumber(value);
}
}
Expand Down Expand Up @@ -362,13 +370,18 @@ function serializeDocument(doc: any, options: EJSONSerializeOptions) {
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
function parse(text: string, options?: EJSONOptions): any {
const ejsonOptions = {
useBigInt64: options?.useBigInt64 ?? false,
relaxed: options?.relaxed ?? true,
legacy: options?.legacy ?? false
};
return JSON.parse(text, (key, value) => {
if (key.indexOf('\x00') !== -1) {
throw new BSONError(
`BSON Document field names cannot contain null bytes, found: ${JSON.stringify(key)}`
);
}
return deserializeValue(value, { relaxed: true, legacy: false, ...options });
return deserializeValue(value, ejsonOptions);
});
}

Expand Down
17 changes: 14 additions & 3 deletions src/long.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1021,9 +1021,20 @@ export class Long {
if (options && options.relaxed) return this.toNumber();
return { $numberLong: this.toString() };
}
static fromExtendedJSON(doc: { $numberLong: string }, options?: EJSONOptions): number | Long {
const result = Long.fromString(doc.$numberLong);
return options && options.relaxed ? result.toNumber() : result;
static fromExtendedJSON(
doc: { $numberLong: string },
options?: EJSONOptions
): number | Long | bigint {
const longResult = Long.fromString(doc.$numberLong);
const defaults = { useBigInt64: false, relaxed: true };
const ejsonOptions = { ...defaults, ...options };
nbbeeken marked this conversation as resolved.
Show resolved Hide resolved
if (ejsonOptions.useBigInt64) {
return BigInt(doc.$numberLong);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unlike in BSON and the 'number' case for EJSON, there isn't restrictions on value size or length.

Luckily BigInt casting function will throw an error for bad syntax unlike Number (yay!) (can we add a test? {$numberLong: "blah!"})

But we have added support here for syntax that should not work:

  • { $numberLong: "0xA" } should fail
  • { $numberLong: "0o7" } should fail
  • { $numberLong: "0b1" } should fail

And we've unintentionally allowed strings of any length to be provided here, I'm not sure what would be the best approach here, we can either figure out some way to assert some maximum length. Or we can construct a Long.fromString and use long.toBigInt here, asserting that the fromString logic would prevent an extremely long string from being converted.

Copy link
Contributor

@nbbeeken nbbeeken Jan 11, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From discussion: We want to assert a maximum length on the string, this isn't the same as what Long does but our newer API can be better about preventing malformed input. There should be no leading zeros permitted but a plus or minus sign needs to be supported. We can iterate the string or use regexp to assert this.

Consider making a new BSONError subclass

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There should be no leading zeros permitted

They are currently permitted when deserializing to Long, as far as I can tell. I would recommend to keep validation consistent between deserializing to Long and deserializing to bigint (and changing validation for deserializing to Long would probably be breaking, but this might be exactly the right time for that if it’s something you want to do).

Copy link
Contributor

@nbbeeken nbbeeken Jan 12, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we're all in agreement (@dariakp @baileympearson @durran @W-A-James) I think we can do the validation for both bigint and Long cases, I was aiming to only impact the bigint case here since it would be opt-in, but useBigInt64 and increasing the validation are divergent concerns so we shouldn't overload the option's behavior.

We certainly don't want to add support for the hex, oct, and bin formats just because useBigInt64 is turned on, so that assertion I think we're all on board with.

Long.fromString currently allows a number of things that probably should be errors like

  • "-----------123" - the number of minus signs corresponds to the number of recursive fromString calls (bigint will throw on this input)
  • "00000000123" - leading zeros are ignored
  • "12.23" - becomes 12
  • '8'.repeat(25) - too large, can be determined by string length (gets truncated)
  • "9223372036854775808" - too large int64.max + 1 (gets truncated)

We can decide to apply all, some, or none of the above validations, I think the most valuable and least pedantic is applying a string length check, but they're all great if we want to improve things :)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree that at the very least, we should add the string length check to inputs of fromExtendedJSON in both the bigInt and Long cases to keep with the canonical EJSON spec and to prevent hanging on strings that would be too long to be a valid 64-bit integer and may be the result of malicious user input.

Canonical EJSON representation of a BSON.Long:

{"$numberLong": <64-bit signed integer as a string>}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updating with results of Slack discussion;

Going to change bigint case to truncate rather than throw an error on encountering a bigint outside of int64 range to be consistent with behaviour of Long.toString

}
if (ejsonOptions.relaxed) {
return longResult.toNumber();
}
return longResult;
}

/** @internal */
Expand Down
125 changes: 124 additions & 1 deletion test/node/bigint.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { BSON, BSONError } from '../register-bson';
import { BSON, BSONError, EJSON } from '../register-bson';
import { bufferFromHexArray } from './tools/utils';
import { expect } from 'chai';
import { BSON_DATA_LONG } from '../../src/constants';
Expand Down Expand Up @@ -263,4 +263,127 @@ describe('BSON BigInt support', function () {
expect(serializedMap).to.deep.equal(expectedSerialization);
});
});

describe('EJSON.parse()', function () {
type ParseOptions = {
useBigInt64: boolean | undefined;
relaxed: boolean | undefined;
};
type TestTableEntry = {
options: ParseOptions;
expectedResult: BSON.Document;
};

// NOTE: legacy is not changed here as it does not affect the output of parsing a Long
const useBigInt64Values = [true, false, undefined];
const relaxedValues = [true, false, undefined];
nbbeeken marked this conversation as resolved.
Show resolved Hide resolved
const sampleCanonicalString = '{"a":{"$numberLong":"23"}}';
const sampleRelaxedIntegerString = '{"a":4294967296}';
const sampleRelaxedDoubleString = '{"a": 2147483647.9}';

function genTestTable(
useBigInt64: boolean | undefined,
relaxed: boolean | undefined,
getExpectedResult: (boolean, boolean) => BSON.Document
): [TestTableEntry] {
const useBigInt64IsSet = useBigInt64 ?? false;
const relaxedIsSet = relaxed ?? true;

const expectedResult = getExpectedResult(useBigInt64IsSet, relaxedIsSet);

return [{ options: { useBigInt64, relaxed }, expectedResult }];
}

function generateTestDescription(entry: TestTableEntry, inputString: string): string {
// TODO(NODE-4874): When NODE-4873 is merged in, replace this with EJSON.stringify
const options = entry.options;
return `parses field 'a' of '${inputString}' to '${entry.expectedResult.a.constructor.name}' when useBigInt64 is ${options.useBigInt64} and relaxed is ${options.relaxed}`;
}

function generateTest(entry: TestTableEntry, sampleString: string): () => void {
const options = entry.options;

return () => {
const parsed = EJSON.parse(sampleString, {
useBigInt64: options.useBigInt64,
relaxed: options.relaxed
});
expect(parsed).to.deep.equal(entry.expectedResult);
};
}

describe('canonical input', function () {
const canonicalInputTestTable = useBigInt64Values.flatMap(useBigInt64 => {
return relaxedValues.flatMap(relaxed => {
return genTestTable(
useBigInt64,
relaxed,
(useBigInt64IsSet: boolean, relaxedIsSet: boolean) =>
useBigInt64IsSet
? { a: 23n }
: relaxedIsSet
? { a: 23 }
: { a: BSON.Long.fromNumber(23) }
);
});
});

it('meta test: generates 9 tests', () => {
expect(canonicalInputTestTable).to.have.lengthOf(9);
});

for (const entry of canonicalInputTestTable) {
const test = generateTest(entry, sampleCanonicalString);
const description = generateTestDescription(entry, sampleCanonicalString);

it(description, test);
}
});

describe('relaxed integer input', function () {
const relaxedIntegerInputTestTable = useBigInt64Values.flatMap(useBigInt64 => {
return relaxedValues.flatMap(relaxed => {
return genTestTable(
useBigInt64,
relaxed,
(useBigInt64IsSet: boolean, relaxedIsSet: boolean) =>
useBigInt64IsSet
? { a: 4294967296n }
: relaxedIsSet
? { a: 4294967296 }
: { a: BSON.Long.fromNumber(4294967296) }
);
});
});
it('meta test: generates 9 tests', () => {
expect(relaxedIntegerInputTestTable).to.have.lengthOf(9);
});

for (const entry of relaxedIntegerInputTestTable) {
const test = generateTest(entry, sampleRelaxedIntegerString);
const description = generateTestDescription(entry, sampleRelaxedIntegerString);

it(description, test);
}
});

describe('relaxed double input where double is outside of int32 range and useBigInt64 is true', function () {
const relaxedDoubleInputTestTable = relaxedValues.flatMap(relaxed => {
return genTestTable(true, relaxed, (_, relaxedIsSet: boolean) =>
relaxedIsSet ? { a: 2147483647.9 } : { a: new BSON.Double(2147483647.9) }
);
});

it('meta test: generates 3 tests', () => {
expect(relaxedDoubleInputTestTable).to.have.lengthOf(3);
});

for (const entry of relaxedDoubleInputTestTable) {
const test = generateTest(entry, sampleRelaxedDoubleString);
const description = generateTestDescription(entry, sampleRelaxedDoubleString);

it(description, test);
}
});
});
});
20 changes: 20 additions & 0 deletions test/node/long.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { expect } from 'chai';
import { Long } from '../register-bson';

describe('Long', function () {
Expand All @@ -21,4 +22,23 @@ describe('Long', function () {
expect(new Long(13835058055282163712n).toString()).to.equal('-4611686018427387904');
expect(new Long(13835058055282163712n, true).toString()).to.equal('13835058055282163712');
});

describe('static fromExtendedJSON()', function () {
it('is not affected by the legacy flag', function () {
const ejsonDoc = { $numberLong: '123456789123456789' };
const longRelaxedLegacy = Long.fromExtendedJSON(ejsonDoc, { legacy: true, relaxed: true });
const longRelaxedNonLegacy = Long.fromExtendedJSON(ejsonDoc, {
legacy: false,
relaxed: true
});
const longCanonicalLegacy = Long.fromExtendedJSON(ejsonDoc, { legacy: true, relaxed: false });
const longCanonicalNonLegacy = Long.fromExtendedJSON(ejsonDoc, {
legacy: false,
relaxed: false
});

expect(longRelaxedLegacy).to.deep.equal(longRelaxedNonLegacy);
expect(longCanonicalLegacy).to.deep.equal(longCanonicalNonLegacy);
});
});
});