From 511efd5b27525a8424a5ca5d251427227c8ecbc0 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 4 Jun 2026 16:42:16 -0600 Subject: [PATCH 01/10] feat: bound typed-structure dictionary with maxOwnStructures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit typedStructs is append-only and pinned on the long-lived encoder, and the encoder branches per-field on value width (num8/num32/num64, float32/float64). A wide, sparsely/variably-populated schema therefore mints a distinct structure for every (key-set x width-combination), growing the dictionary + transition trie without limit — a latent unbounded-memory path. maxOwnStructures freezes dictionary growth once the cap is reached: novel shapes return 0/null from the write hooks and fall back to plain encoding, while existing structures (and any persisted on-disk) stay decodable. Applied symmetrically to both the fast (writeStructInPlace) and standalone (_encode) paths. Default is uncapped — no behavior change unless a host opts in. Co-Authored-By: Claude Opus 4.7 --- index.js | 5 ++++ struct.js | 25 +++++++++++++++++++- tests/test-cbor-x.js | 51 ++++++++++++++++++++++++++++++++++++++++ tests/test.js | 56 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 134d4a5..74c506a 100644 --- a/index.js +++ b/index.js @@ -38,6 +38,11 @@ export function createStructon(BaseClass) { constructor(options = {}) { super(options); + // Honor maxOwnStructures for the typed-struct path: bounds the per-encoder typed-structure + // dictionary (+ transition trie). Once reached, novel shapes fall back to plain encoding + // instead of growing the dictionary without limit. Default: uncapped (no behavior change). + if (options.maxOwnStructures != null) this.maxOwnStructures = options.maxOwnStructures; + // Initialise typed structures state on this instance if (!this.typedStructs) this.typedStructs = []; diff --git a/struct.js b/struct.js index 60fa50a..64b41a3 100644 --- a/struct.js +++ b/struct.js @@ -120,9 +120,20 @@ function createBlankTransition(key, parent) { }; } +// When the typed-structure dictionary reaches maxOwnStructures we stop minting new +// structures/transitions. typedStructs is append-only and pinned on the long-lived +// encoder (records reference structures by recordId), so an unbounded shape space — +// e.g. a wide, sparsely/variably-populated schema — would otherwise grow the +// dictionary + transition trie without limit. While frozen, a missing transition +// returns undefined so the caller bails and the record falls back to plain encoding. +let _frozen = false; + function createTypeTransition(transition, type, size) { const typeName = TYPE_NAMES[type] + (size << 3); - let t = transition[typeName] || (transition[typeName] = Object.create(null)); + let t = transition[typeName]; + if (t) return t; + if (_frozen) return undefined; + t = transition[typeName] = Object.create(null); t.__type = type; t.__size = size; t.__parent = transition; @@ -248,6 +259,7 @@ function _writeHeader(result, recordId, headerSize) { export function writeStructInPlace(object, target, encodingStart, position, structures, makeRoom, pack) { const packr = this; let typedStructs = packr.typedStructs || (packr.typedStructs = []); + _frozen = typedStructs.length >= (packr.maxOwnStructures ?? Infinity); let targetView = target.dataView; let refsStartPosition = (typedStructs.lastStringStart || 100) + position; let safeEnd = target.length - 10; @@ -281,6 +293,7 @@ export function writeStructInPlace(object, target, encodingStart, position, stru let value = object[key]; let nextTransition = transition[key]; if (!nextTransition) { + if (_frozen) return 0; transition[key] = nextTransition = { key, parent: transition, enumerationOffset: 0, ascii0: null, ascii8: null, num8: null, @@ -457,6 +470,7 @@ export function writeStructInPlace(object, target, encodingStart, position, stru default: queuedReferences.push(key, value, keyIndex); } + if (transition === undefined) return 0; // frozen: structure cap reached keyIndex++; } @@ -466,6 +480,7 @@ export function writeStructInPlace(object, target, encodingStart, position, stru let propertyIndex = queuedReferences[i++]; let nextTransition = transition[key]; if (!nextTransition) { + if (_frozen) return 0; transition[key] = nextTransition = { key, parent: transition, enumerationOffset: propertyIndex - keyIndex, @@ -507,11 +522,13 @@ export function writeStructInPlace(object, target, encodingStart, position, stru targetView.setInt16(position, value === null ? -10 : -9, true); position += 2; } + if (transition === undefined) return 0; // frozen: structure cap reached keyIndex++; } let recordId = transition[RECORD_SYMBOL]; if (recordId == null) { + if (_frozen) return 0; recordId = packr.typedStructs.length; const structure = []; let nextTransition = transition; @@ -618,6 +635,7 @@ export function writeStruct(object, encodeNested, packr) { function _encode(object, encodeNested, packr, work) { let typedStructs = packr.typedStructs || (packr.typedStructs = []); + _frozen = typedStructs.length >= (packr.maxOwnStructures ?? Infinity); let transition = typedStructs.transitions || (typedStructs.transitions = Object.create(null)); const nextId = typedStructs.length; @@ -639,6 +657,7 @@ function _encode(object, encodeNested, packr, work) { const value = object[key]; let nextTransition = transition[key]; if (!nextTransition) { + if (_frozen) return null; transition[key] = nextTransition = createBlankTransition(key, transition); } if (fixedPos + 8 > work.fixedBuf.length) _growFixed(work, fixedPos + 8); @@ -759,6 +778,7 @@ function _encode(object, encodeNested, packr, work) { queuedReferences.push(key, value, keyIndex); break; } + if (transition === undefined) return null; // frozen: structure cap reached keyIndex++; } @@ -771,6 +791,7 @@ function _encode(object, encodeNested, packr, work) { let nextTransition = transition[key]; if (!nextTransition) { + if (_frozen) return null; transition[key] = nextTransition = { key, parent: transition, @@ -808,12 +829,14 @@ function _encode(object, encodeNested, packr, work) { work.fixedView.setInt16(fixedPos, value === null ? -10 : -9, true); fixedPos += 2; } + if (transition === undefined) return null; // frozen: structure cap reached keyIndex++; } // Build/retrieve structure definition from the transition chain. let recordId = transition[RECORD_SYMBOL]; if (recordId == null) { + if (_frozen) return null; recordId = typedStructs.length; const structure = []; let t = transition; diff --git a/tests/test-cbor-x.js b/tests/test-cbor-x.js index c16ece1..2b18879 100644 --- a/tests/test-cbor-x.js +++ b/tests/test-cbor-x.js @@ -237,3 +237,54 @@ suite('structon (cbor-x base) – format details', function () { assert.strictEqual(msgpackResult.meta.x, 10); }); }); + +// ── maxOwnStructures cap (standalone path) ──────────────────────────────────── +// +// Same cap behavior as the msgpackr fast path, exercised through cbor-x's +// standalone encode (_encode in struct.js). Sparse, width-heterogeneous records +// would otherwise grow typedStructs without bound; maxOwnStructures freezes the +// dictionary and falls back to plain encoding once the cap is hit. + +suite('structon (cbor-x base) – maxOwnStructures cap', function () { + function plain(d) { + if (d && typeof d.toJSON === 'function') return d.toJSON(); + return { ...d }; + } + function makeGen() { + let seed = 12345; + const rnd = () => { seed = (seed * 1103515245 + 12345) & 0x7fffffff; return seed / 0x7fffffff; }; + return function makeRec() { + const o = {}; + for (let f = 0; f < 40; f++) { + if (rnd() < 0.45) continue; + const r = (rnd() * 4) | 0; + o['f' + f] = r === 0 ? ((rnd() * 200) | 0) + : r === 1 ? (((rnd() * 1e6) | 0) + 1000) + : r === 2 ? (((rnd() * 1e7) | 0) * 100000) + : (((rnd() * 8) | 0) * 0.25); + } + return o; + }; + } + function run(cap) { + const enc = new Structon({ randomAccessStructure: true, useRecords: false, maxOwnStructures: cap }); + const gen = makeGen(); + for (let i = 0; i < 4000; i++) { + const r = gen(); + assert.deepStrictEqual(plain(enc.decode(enc.encode(r))), r); + } + return enc.typedStructs.length; + } + + test('uncapped dictionary grows well past 256 for width-heterogeneous records', function () { + assert.ok(run(undefined) > 256, 'expected uncapped typedStructs to exceed 256'); + }); + + test('cap=64 bounds typedStructs and preserves round-trips', function () { + assert.ok(run(64) <= 64, 'typedStructs should not exceed the cap of 64'); + }); + + test('cap=256 bounds typedStructs and preserves round-trips', function () { + assert.ok(run(256) <= 256, 'typedStructs should not exceed the cap of 256'); + }); +}); diff --git a/tests/test.js b/tests/test.js index 10d64d4..bd9438c 100644 --- a/tests/test.js +++ b/tests/test.js @@ -520,3 +520,59 @@ suite('structon – struct-write disabled stays records-mode / v1-decodable', fu assert.ok(enc.encode(31)[0] < 0x20, 'values below the struct range stay positive fixints'); }); }); + +// ── maxOwnStructures cap (fast path) ────────────────────────────────────────── +// +// typedStructs is append-only and pinned on the long-lived encoder. The encoder +// branches per-field on value width (num8/num32/num64, float32/float64), so a +// sparse, width-heterogeneous schema mints a distinct structure for every +// (key-set × width-combination) — far more than the number of distinct key-sets. +// maxOwnStructures bounds that growth: once the cap is reached, novel shapes fall +// back to plain encoding (useRecords:false → msgpack maps here) instead of growing +// the dictionary, and everything still round-trips. + +suite('structon – maxOwnStructures cap', function () { + // Normalize a decoded record (lazy struct or plain map) to a comparable plain object. + function plain(d) { + if (d && typeof d.toJSON === 'function') return d.toJSON(); + return { ...d }; + } + // Deterministic PRNG so the generated shape space is reproducible. + function makeGen() { + let seed = 12345; + const rnd = () => { seed = (seed * 1103515245 + 12345) & 0x7fffffff; return seed / 0x7fffffff; }; + return function makeRec() { + const o = {}; + for (let f = 0; f < 40; f++) { + if (rnd() < 0.45) continue; // field absent ~45% of the time + const r = (rnd() * 4) | 0; // value width varies: u8 / u32 / large / float + o['f' + f] = r === 0 ? ((rnd() * 200) | 0) + : r === 1 ? (((rnd() * 1e6) | 0) + 1000) + : r === 2 ? (((rnd() * 1e7) | 0) * 100000) + : (((rnd() * 8) | 0) * 0.25); + } + return o; + }; + } + function run(cap) { + const enc = new Structon({ randomAccessStructure: true, useRecords: false, maxOwnStructures: cap }); + const gen = makeGen(); + for (let i = 0; i < 4000; i++) { + const r = gen(); + assert.deepStrictEqual(plain(enc.decode(enc.encode(r))), r); + } + return enc.typedStructs.length; + } + + test('uncapped dictionary grows well past 256 for width-heterogeneous records', function () { + assert.ok(run(undefined) > 256, 'expected uncapped typedStructs to exceed 256'); + }); + + test('cap=64 bounds typedStructs and preserves round-trips', function () { + assert.ok(run(64) <= 64, 'typedStructs should not exceed the cap of 64'); + }); + + test('cap=256 bounds typedStructs and preserves round-trips', function () { + assert.ok(run(256) <= 256, 'typedStructs should not exceed the cap of 256'); + }); +}); From 4cd38ee697dae8ff25da9f1111c6e3a9725987c9 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 4 Jun 2026 17:11:27 -0600 Subject: [PATCH 02/10] fix: scope structure cap to encode-time minting only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address two issues found in review of the maxOwnStructures cap: 1. onLoadedStructures rebuilds the transition trie via createTypeTransition, which returns undefined while the module-level freeze flag is set. A reader loading previously-persisted structures after any capped encoder hit its cap in the same process would throw. Clear the flag on load — replaying saved structures is never subject to the cap. 2. Re-check the cap at the record-id mint point (not just the entry-time flag): nested encodes via pack()/encodeNested() can append structures after entry, so a record could overshoot the cap by its nesting depth. The live re-check keeps typedStructs.length a hard bound. Adds regression tests for both, on the fast and standalone paths. Co-Authored-By: Claude Opus 4.7 --- struct.js | 15 +++++++++++++-- tests/test-cbor-x.js | 32 ++++++++++++++++++++++++++++++++ tests/test.js | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/struct.js b/struct.js index 64b41a3..6ab41f7 100644 --- a/struct.js +++ b/struct.js @@ -528,7 +528,10 @@ export function writeStructInPlace(object, target, encodingStart, position, stru let recordId = transition[RECORD_SYMBOL]; if (recordId == null) { - if (_frozen) return 0; + // Re-check the cap here (not just the entry-time _frozen): nested encodes via + // pack() may have appended structures since entry, so this keeps typedStructs.length + // a hard bound rather than letting a record overshoot by its nesting depth. + if (packr.typedStructs.length >= (packr.maxOwnStructures ?? Infinity)) return 0; recordId = packr.typedStructs.length; const structure = []; let nextTransition = transition; @@ -836,7 +839,10 @@ function _encode(object, encodeNested, packr, work) { // Build/retrieve structure definition from the transition chain. let recordId = transition[RECORD_SYMBOL]; if (recordId == null) { - if (_frozen) return null; + // Re-check the cap here (not just the entry-time _frozen): nested encodes via + // encodeNested() may have appended structures since entry, so this keeps + // typedStructs.length a hard bound rather than overshooting by nesting depth. + if (typedStructs.length >= (packr.maxOwnStructures ?? Infinity)) return null; recordId = typedStructs.length; const structure = []; let t = transition; @@ -1101,6 +1107,11 @@ export function readStruct(src, position, srcEnd) { * Accepts the same Map format that msgpackr's struct.js produces. */ export function onLoadedStructures(sharedData) { + // Replaying already-persisted structures must always fully rebuild the trie, + // regardless of maxOwnStructures — the cap only limits minting NEW structures + // during encode. _frozen is module-level and may be left true by a prior capped + // encode, so clear it here before the createTypeTransition rebuild below. + _frozen = false; if (!sharedData) return this.structures; let named, typed; if (sharedData instanceof Map) { diff --git a/tests/test-cbor-x.js b/tests/test-cbor-x.js index 2b18879..7c5fdbf 100644 --- a/tests/test-cbor-x.js +++ b/tests/test-cbor-x.js @@ -287,4 +287,36 @@ suite('structon (cbor-x base) – maxOwnStructures cap', function () { test('cap=256 bounds typedStructs and preserves round-trips', function () { assert.ok(run(256) <= 256, 'typedStructs should not exceed the cap of 256'); }); + + test('nested records do not overshoot the cap', function () { + // A nested object mints its own structure before the outer record is minted, so a + // stale entry-time freeze flag could let the outer record push past the cap. The mint + // guard re-checks the live length, keeping typedStructs.length a hard bound. + const enc = new Structon({ structures: [], useRecords: false, maxOwnStructures: 4 }); + let seed = 7; + const rnd = () => { seed = (seed * 1103515245 + 12345) & 0x7fffffff; return seed / 0x7fffffff; }; + for (let i = 0; i < 1000; i++) { + enc.encode({ outer: { inner: (rnd() * 1e7 | 0) * 1000 }, tag: 't' + (i % 20), n: (rnd() * 300 | 0) }); + } + assert.ok(enc.typedStructs.length <= 4, 'nested encodes must not push typedStructs past the cap, got ' + enc.typedStructs.length); + }); + + test('persisted structures still load after a capped encoder froze the dictionary', function () { + // Regression: the freeze flag is module-level. A capped encoder that hits its cap must + // not block a later reader from rebuilding previously-persisted structures on load — + // the cap governs minting NEW structures during encode, not replaying saved ones. + let saved = null; + const writer = new Structon({ structures: [], saveStructures(s) { saved = s; return true; }, getStructures() { return saved; } }); + const buf = writer.encode({ name: 'Alice', age: 30 }); + assert.ok(buf[0] >= 0x20 && buf[0] < 0x40, 'writer should produce struct bytes'); + + const capped = new Structon({ structures: [], maxOwnStructures: 1 }); + capped.encode({ x: 1 }); + capped.encode({ y: 2, z: 3 }); // cap reached → freeze flag left set on the module + + const reader = new Structon({ structures: [], getStructures() { return saved; } }); + const result = reader.decode(buf); // triggers onLoadedStructures while frozen + assert.strictEqual(result.name, 'Alice'); + assert.strictEqual(result.age, 30); + }); }); diff --git a/tests/test.js b/tests/test.js index bd9438c..f7de4f9 100644 --- a/tests/test.js +++ b/tests/test.js @@ -575,4 +575,36 @@ suite('structon – maxOwnStructures cap', function () { test('cap=256 bounds typedStructs and preserves round-trips', function () { assert.ok(run(256) <= 256, 'typedStructs should not exceed the cap of 256'); }); + + test('nested records do not overshoot the cap', function () { + // A nested object mints its own structure before the outer record is minted, so a + // stale entry-time freeze flag could let the outer record push past the cap. The mint + // guard re-checks the live length, keeping typedStructs.length a hard bound. + const enc = new Structon({ structures: [], useRecords: false, maxOwnStructures: 4 }); + let seed = 7; + const rnd = () => { seed = (seed * 1103515245 + 12345) & 0x7fffffff; return seed / 0x7fffffff; }; + for (let i = 0; i < 1000; i++) { + enc.encode({ outer: { inner: (rnd() * 1e7 | 0) * 1000 }, tag: 't' + (i % 20), n: (rnd() * 300 | 0) }); + } + assert.ok(enc.typedStructs.length <= 4, 'nested encodes must not push typedStructs past the cap, got ' + enc.typedStructs.length); + }); + + test('persisted structures still load after a capped encoder froze the dictionary', function () { + // Regression: the freeze flag is module-level. A capped encoder that hits its cap must + // not block a later reader from rebuilding previously-persisted structures on load — + // the cap governs minting NEW structures during encode, not replaying saved ones. + let saved = null; + const writer = new Structon({ structures: [], saveStructures(s) { saved = s; return true; }, getStructures() { return saved; } }); + const buf = writer.encode({ name: 'Alice', age: 30 }); + assert.ok(buf[0] >= 0x20 && buf[0] < 0x40, 'writer should produce struct bytes'); + + const capped = new Structon({ structures: [], maxOwnStructures: 1 }); + capped.encode({ x: 1 }); + capped.encode({ y: 2, z: 3 }); // cap reached → freeze flag left set on the module + + const reader = new Structon({ structures: [], getStructures() { return saved; } }); + const result = reader.decode(buf); // triggers onLoadedStructures while frozen + assert.strictEqual(result.name, 'Alice'); + assert.strictEqual(result.age, 30); + }); }); From 629fef214d6b79f64ea31c1a800feb4372ddfe88 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 4 Jun 2026 17:22:57 -0600 Subject: [PATCH 03/10] fix: bail before pack() on fast path; guard null options Two more issues from review: 1. [fast path] When frozen, a previously-learned key later carrying a non-null object reached the cap bailout only AFTER pack() had encoded the nested value and advanced the shared encoder position. Returning 0 then made msgpackr's plain-object fallback start at the wrong offset and emit garbage bytes. Bail before pack() so the fallback sees an untouched position. 2. new Structon(null) threw: the base Packr accepts null options as "use defaults", but reading options.maxOwnStructures dereferenced null. Use optional chaining. Adds regression tests (both paths): nested round-trips under cap, a known key later seen as an object, and null-options construction. Co-Authored-By: Claude Opus 4.7 --- index.js | 2 +- struct.js | 4 ++++ tests/test-cbor-x.js | 20 ++++++++++++++++++-- tests/test.js | 25 ++++++++++++++++++++++--- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index 74c506a..1efce47 100644 --- a/index.js +++ b/index.js @@ -41,7 +41,7 @@ export function createStructon(BaseClass) { // Honor maxOwnStructures for the typed-struct path: bounds the per-encoder typed-structure // dictionary (+ transition trie). Once reached, novel shapes fall back to plain encoding // instead of growing the dictionary without limit. Default: uncapped (no behavior change). - if (options.maxOwnStructures != null) this.maxOwnStructures = options.maxOwnStructures; + if (options?.maxOwnStructures != null) this.maxOwnStructures = options.maxOwnStructures; // Initialise typed structures state on this instance if (!this.typedStructs) this.typedStructs = []; diff --git a/struct.js b/struct.js index 6ab41f7..9e896b6 100644 --- a/struct.js +++ b/struct.js @@ -502,6 +502,10 @@ export function writeStructInPlace(object, target, encodingStart, position, stru transition = nextTransition.object32 || createTypeTransition(nextTransition, OBJECT_DATA, 4); size = 4; } + // Must bail BEFORE pack(): on the fast path pack() advances the shared encoder + // position, so a later `return 0` would make the plain-object fallback start at + // the wrong offset and emit garbage. Returning here leaves position untouched. + if (transition === undefined) return 0; // frozen: structure cap reached newPosition = pack(value, refPosition); if (typeof newPosition === 'object') { // re-allocated buffer — refresh local refs diff --git a/tests/test-cbor-x.js b/tests/test-cbor-x.js index 7c5fdbf..19e335d 100644 --- a/tests/test-cbor-x.js +++ b/tests/test-cbor-x.js @@ -288,7 +288,7 @@ suite('structon (cbor-x base) – maxOwnStructures cap', function () { assert.ok(run(256) <= 256, 'typedStructs should not exceed the cap of 256'); }); - test('nested records do not overshoot the cap', function () { + test('nested records do not overshoot the cap and still round-trip', function () { // A nested object mints its own structure before the outer record is minted, so a // stale entry-time freeze flag could let the outer record push past the cap. The mint // guard re-checks the live length, keeping typedStructs.length a hard bound. @@ -296,11 +296,27 @@ suite('structon (cbor-x base) – maxOwnStructures cap', function () { let seed = 7; const rnd = () => { seed = (seed * 1103515245 + 12345) & 0x7fffffff; return seed / 0x7fffffff; }; for (let i = 0; i < 1000; i++) { - enc.encode({ outer: { inner: (rnd() * 1e7 | 0) * 1000 }, tag: 't' + (i % 20), n: (rnd() * 300 | 0) }); + const r = { outer: { inner: (rnd() * 1e7 | 0) * 1000 }, tag: 't' + (i % 20), n: (rnd() * 300 | 0) }; + // JSON round-trip normalizes lazy structs / strips prototypes for deep-equality. + assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode(r)))), r); } assert.ok(enc.typedStructs.length <= 4, 'nested encodes must not push typedStructs past the cap, got ' + enc.typedStructs.length); }); + test('capped: a known key later seen as a nested object falls back cleanly', function () { + // Once frozen, a previously-learned scalar key that later carries an object must fall + // back to plain encoding cleanly (standalone path returns fresh buffers, no shared state). + const enc = new Structon({ structures: [], useRecords: false, maxOwnStructures: 1 }); + assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode({ a: 1 })))), { a: 1 }); + const r2 = { a: { x: 1 } }; + assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode(r2)))), r2); + }); + + test('new Structon(null) does not throw (null options = use defaults)', function () { + const enc = new Structon(null); + assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode({ a: 1 })))), { a: 1 }); + }); + test('persisted structures still load after a capped encoder froze the dictionary', function () { // Regression: the freeze flag is module-level. A capped encoder that hits its cap must // not block a later reader from rebuilding previously-persisted structures on load — diff --git a/tests/test.js b/tests/test.js index f7de4f9..7d65d26 100644 --- a/tests/test.js +++ b/tests/test.js @@ -576,19 +576,38 @@ suite('structon – maxOwnStructures cap', function () { assert.ok(run(256) <= 256, 'typedStructs should not exceed the cap of 256'); }); - test('nested records do not overshoot the cap', function () { + test('nested records do not overshoot the cap and still round-trip', function () { // A nested object mints its own structure before the outer record is minted, so a // stale entry-time freeze flag could let the outer record push past the cap. The mint - // guard re-checks the live length, keeping typedStructs.length a hard bound. + // guard re-checks the live length, keeping typedStructs.length a hard bound. The + // round-trip check also guards the fast-path bail-before-pack() ordering: bailing + // after pack() advances the encoder position would corrupt the fallback bytes. const enc = new Structon({ structures: [], useRecords: false, maxOwnStructures: 4 }); let seed = 7; const rnd = () => { seed = (seed * 1103515245 + 12345) & 0x7fffffff; return seed / 0x7fffffff; }; for (let i = 0; i < 1000; i++) { - enc.encode({ outer: { inner: (rnd() * 1e7 | 0) * 1000 }, tag: 't' + (i % 20), n: (rnd() * 300 | 0) }); + const r = { outer: { inner: (rnd() * 1e7 | 0) * 1000 }, tag: 't' + (i % 20), n: (rnd() * 300 | 0) }; + // JSON round-trip normalizes lazy structs / strips prototypes for deep-equality. + assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode(r)))), r); } assert.ok(enc.typedStructs.length <= 4, 'nested encodes must not push typedStructs past the cap, got ' + enc.typedStructs.length); }); + test('capped: a known key later seen as a nested object falls back cleanly', function () { + // Regression for fast-path corruption: once frozen, a previously-learned scalar key + // that later carries an object reaches the bail only after the nested pack() has + // advanced the shared encoder position. Bailing before pack() keeps the fallback valid. + const enc = new Structon({ structures: [], useRecords: false, maxOwnStructures: 1 }); + assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode({ a: 1 })))), { a: 1 }); // mints structure 0, cap hit + const r2 = { a: { x: 1 } }; + assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode(r2)))), r2); + }); + + test('new Structon(null) does not throw (null options = use defaults)', function () { + const enc = new Structon(null); + assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode({ a: 1 })))), { a: 1 }); + }); + test('persisted structures still load after a capped encoder froze the dictionary', function () { // Regression: the freeze flag is module-level. A capped encoder that hits its cap must // not block a later reader from rebuilding previously-persisted structures on load — From 223bbe8eb7384b8a6114615e8f965509ec683e63 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 4 Jun 2026 17:39:52 -0600 Subject: [PATCH 04/10] fix: never bail after a queued ref is packed (fast path) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The earlier bail-before-pack only covered the first queued reference. With multiple queued object fields, an earlier ref can pack() (advancing msgpackr's shared write position) before a later field misses its transition — bailing there still corrupts the plain-object fallback. Track whether any ref has been packed: while none has, a frozen miss bails cleanly (return 0); once a ref is packed we can no longer bail, so finish the encode via an unfrozen forceTypeTransition (bounded overshoot of a handful of structures for that one record). The cap is still enforced up front, before the first pack(). Standalone path is unaffected (it returns fresh buffers, no shared position to corrupt). Adds a multi-queued-ref regression test on both paths. Co-Authored-By: Claude Opus 4.7 --- struct.js | 54 +++++++++++++++++++++++++++++++++----------- tests/test-cbor-x.js | 9 ++++++++ tests/test.js | 12 ++++++++++ 3 files changed, 62 insertions(+), 13 deletions(-) diff --git a/struct.js b/struct.js index 9e896b6..67fafa0 100644 --- a/struct.js +++ b/struct.js @@ -140,6 +140,22 @@ function createTypeTransition(transition, type, size) { return t; } +// Unfrozen variant: always mints. Used on the fast path once a queued nested value has +// already been pack()ed — at that point pack() has advanced msgpackr's shared write +// position, so bailing with `return 0` would corrupt the fallback. We must finish the +// encode instead, even if that means minting a (bounded) handful of structures past the +// cap. The cap is still enforced up front, before the first pack(). +function forceTypeTransition(transition, type, size) { + const typeName = TYPE_NAMES[type] + (size << 3); + let t = transition[typeName]; + if (t) return t; + t = transition[typeName] = Object.create(null); + t.__type = type; + t.__size = size; + t.__parent = transition; + return t; +} + // ── Work-buffer pool (one pair per nesting depth) ───────────────────────────── // // Instead of allocating a new Uint8Array for each field value, we write @@ -474,13 +490,18 @@ export function writeStructInPlace(object, target, encodingStart, position, stru keyIndex++; } + // Once we pack() a queued nested value, msgpackr's shared write position is advanced and + // a `return 0` would corrupt the plain-object fallback. So: while no ref has been packed + // yet, a frozen miss bails cleanly (return 0). After the first pack we can no longer bail, + // so we force-mint whatever the remaining fields need (bounded overshoot — see below). + let packedRef = false; for (let i = 0, l = queuedReferences.length; i < l;) { let key = queuedReferences[i++]; let value = queuedReferences[i++]; let propertyIndex = queuedReferences[i++]; let nextTransition = transition[key]; if (!nextTransition) { - if (_frozen) return 0; + if (_frozen && !packedRef) return 0; transition[key] = nextTransition = { key, parent: transition, enumerationOffset: propertyIndex - keyIndex, @@ -497,16 +518,20 @@ export function writeStructInPlace(object, target, encodingStart, position, stru transition = nextTransition.object16; if (transition) size = 2; else if ((transition = nextTransition.object32)) size = 4; - else { transition = createTypeTransition(nextTransition, OBJECT_DATA, 2); size = 2; } + else { + if (_frozen && !packedRef) return 0; // clean bail before any pack() + transition = forceTypeTransition(nextTransition, OBJECT_DATA, 2); size = 2; + } } else { - transition = nextTransition.object32 || createTypeTransition(nextTransition, OBJECT_DATA, 4); + transition = nextTransition.object32; + if (!transition) { + if (_frozen && !packedRef) return 0; // clean bail before any pack() + transition = forceTypeTransition(nextTransition, OBJECT_DATA, 4); + } size = 4; } - // Must bail BEFORE pack(): on the fast path pack() advances the shared encoder - // position, so a later `return 0` would make the plain-object fallback start at - // the wrong offset and emit garbage. Returning here leaves position untouched. - if (transition === undefined) return 0; // frozen: structure cap reached newPosition = pack(value, refPosition); + packedRef = true; if (typeof newPosition === 'object') { // re-allocated buffer — refresh local refs refPosition = newPosition.position; @@ -522,20 +547,23 @@ export function writeStructInPlace(object, target, encodingStart, position, stru if (size === 2) { targetView.setUint16(position, refOffset, true); position += 2; } else { targetView.setUint32(position, refOffset, true); position += 4; } } else { // null or undefined - transition = nextTransition.object16 || createTypeTransition(nextTransition, OBJECT_DATA, 2); + transition = nextTransition.object16; + if (!transition) { + if (_frozen && !packedRef) return 0; // clean bail before any pack() + transition = forceTypeTransition(nextTransition, OBJECT_DATA, 2); + } targetView.setInt16(position, value === null ? -10 : -9, true); position += 2; } - if (transition === undefined) return 0; // frozen: structure cap reached keyIndex++; } let recordId = transition[RECORD_SYMBOL]; if (recordId == null) { - // Re-check the cap here (not just the entry-time _frozen): nested encodes via - // pack() may have appended structures since entry, so this keeps typedStructs.length - // a hard bound rather than letting a record overshoot by its nesting depth. - if (packr.typedStructs.length >= (packr.maxOwnStructures ?? Infinity)) return 0; + // Enforce the cap here only while nothing has been packed (flat records and the + // no-queued-refs case): bailing after a pack would corrupt the fallback, so a record + // that already packed nested refs completes and may mint a bounded few past the cap. + if (!packedRef && packr.typedStructs.length >= (packr.maxOwnStructures ?? Infinity)) return 0; recordId = packr.typedStructs.length; const structure = []; let nextTransition = transition; diff --git a/tests/test-cbor-x.js b/tests/test-cbor-x.js index 19e335d..3b7e36b 100644 --- a/tests/test-cbor-x.js +++ b/tests/test-cbor-x.js @@ -312,6 +312,15 @@ suite('structon (cbor-x base) – maxOwnStructures cap', function () { assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode(r2)))), r2); }); + test('capped: a frozen miss after an earlier nested ref still round-trips', function () { + const enc = new Structon({ structures: [], useRecords: false, maxOwnStructures: 2 }); + const norm = (r) => JSON.parse(JSON.stringify(enc.decode(enc.encode(r)))); + assert.deepStrictEqual(norm({ b: 1 }), { b: 1 }); + assert.deepStrictEqual(norm({ a: { x: 1 } }), { a: { x: 1 } }); + const r = { a: { x: 1 }, b: { y: 2 } }; + assert.deepStrictEqual(norm(r), r); + }); + test('new Structon(null) does not throw (null options = use defaults)', function () { const enc = new Structon(null); assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode({ a: 1 })))), { a: 1 }); diff --git a/tests/test.js b/tests/test.js index 7d65d26..4f11abd 100644 --- a/tests/test.js +++ b/tests/test.js @@ -603,6 +603,18 @@ suite('structon – maxOwnStructures cap', function () { assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode(r2)))), r2); }); + test('capped: a frozen miss AFTER an earlier ref was packed completes without corruption', function () { + // Fast-path hazard: the first nested ref packs (advancing the shared encoder position), + // then a later field misses its transition. Bailing there would corrupt the fallback, so + // the encode must complete instead. Round-trip verifies no garbage bytes are emitted. + const enc = new Structon({ structures: [], useRecords: false, maxOwnStructures: 2 }); + const norm = (r) => JSON.parse(JSON.stringify(enc.decode(enc.encode(r)))); + assert.deepStrictEqual(norm({ b: 1 }), { b: 1 }); // structure 0 + assert.deepStrictEqual(norm({ a: { x: 1 } }), { a: { x: 1 } }); // structure 1, cap hit + const r = { a: { x: 1 }, b: { y: 2 } }; // 'a' packs, then 'b' misses + assert.deepStrictEqual(norm(r), r); + }); + test('new Structon(null) does not throw (null options = use defaults)', function () { const enc = new Structon(null); assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode({ a: 1 })))), { a: 1 }); From 1536d9f7b9a29927018ed911a256fa752e3678fb Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 4 Jun 2026 17:51:55 -0600 Subject: [PATCH 05/10] fix: preflight queued refs so nested-object streams respect the cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bail-after-pack avoidance (force-mint once a ref is packed) prevented corruption but defeated the cap for nested-object shape streams: each variant packed its first ref then force-minted the rest, growing typedStructs without bound. Add a pre-pack preflight on the fast path: walk the queued reference chain through existing transitions first, and if the cap is reached and any field would require a new structure, fall back to plain encoding (return 0) before any pack() advances the shared position. Past the preflight the chain is known, so the queued loop completes without minting (the unfrozen forceTypeTransition only covers the rare >0xff00 offset divergence — a bounded, self-converging case). Adds a nested-object-variant-stream regression test (both paths) asserting typedStructs stays within the cap. Co-Authored-By: Claude Opus 4.7 --- struct.js | 51 +++++++++++++++++++++++++------------------- tests/test-cbor-x.js | 12 +++++++++++ tests/test.js | 15 +++++++++++++ 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/struct.js b/struct.js index 67fafa0..0c4cf80 100644 --- a/struct.js +++ b/struct.js @@ -490,10 +490,29 @@ export function writeStructInPlace(object, target, encodingStart, position, stru keyIndex++; } - // Once we pack() a queued nested value, msgpackr's shared write position is advanced and - // a `return 0` would corrupt the plain-object fallback. So: while no ref has been packed - // yet, a frozen miss bails cleanly (return 0). After the first pack we can no longer bail, - // so we force-mint whatever the remaining fields need (bounded overshoot — see below). + // Cap enforcement for queued (nested-object / null) references. pack() advances msgpackr's + // shared write position and we cannot cleanly bail afterward, so preflight the whole queued + // chain through EXISTING transitions first: if the cap is reached and any field would need a + // new structure, fall back to plain encoding now (return 0) — before touching the shared + // position. A fresh length check is used (not the entry-time _frozen, which a re-entrant + // nested pack on a prior field may have advanced). + if (queuedReferences.length > 0 && packr.typedStructs.length >= (packr.maxOwnStructures ?? Infinity)) { + let t = transition; + for (let i = 0, l = queuedReferences.length; i < l; i += 3) { + const nt = t[queuedReferences[i]]; + if (!nt) return 0; + const next = queuedReferences[i + 1] != null ? (nt.object16 || nt.object32) : nt.object16; + if (!next) return 0; + t = next; + } + if (t[RECORD_SYMBOL] == null) return 0; // exact structure not yet minted + } + + // Past the preflight the chain is known, so no minting happens — except a rare offset + // divergence (a known shape whose ref section now crosses 0xff00 and needs object32 where + // the preflight matched object16). Once a ref is packed we can no longer bail, so we finish + // via the unfrozen forceTypeTransition: a bounded, self-converging overshoot for that one + // record. packedRef keeps the record-id mint from bailing after a pack. let packedRef = false; for (let i = 0, l = queuedReferences.length; i < l;) { let key = queuedReferences[i++]; @@ -501,7 +520,6 @@ export function writeStructInPlace(object, target, encodingStart, position, stru let propertyIndex = queuedReferences[i++]; let nextTransition = transition[key]; if (!nextTransition) { - if (_frozen && !packedRef) return 0; transition[key] = nextTransition = { key, parent: transition, enumerationOffset: propertyIndex - keyIndex, @@ -518,16 +536,9 @@ export function writeStructInPlace(object, target, encodingStart, position, stru transition = nextTransition.object16; if (transition) size = 2; else if ((transition = nextTransition.object32)) size = 4; - else { - if (_frozen && !packedRef) return 0; // clean bail before any pack() - transition = forceTypeTransition(nextTransition, OBJECT_DATA, 2); size = 2; - } + else { transition = forceTypeTransition(nextTransition, OBJECT_DATA, 2); size = 2; } } else { - transition = nextTransition.object32; - if (!transition) { - if (_frozen && !packedRef) return 0; // clean bail before any pack() - transition = forceTypeTransition(nextTransition, OBJECT_DATA, 4); - } + transition = nextTransition.object32 || forceTypeTransition(nextTransition, OBJECT_DATA, 4); size = 4; } newPosition = pack(value, refPosition); @@ -547,11 +558,7 @@ export function writeStructInPlace(object, target, encodingStart, position, stru if (size === 2) { targetView.setUint16(position, refOffset, true); position += 2; } else { targetView.setUint32(position, refOffset, true); position += 4; } } else { // null or undefined - transition = nextTransition.object16; - if (!transition) { - if (_frozen && !packedRef) return 0; // clean bail before any pack() - transition = forceTypeTransition(nextTransition, OBJECT_DATA, 2); - } + transition = nextTransition.object16 || forceTypeTransition(nextTransition, OBJECT_DATA, 2); targetView.setInt16(position, value === null ? -10 : -9, true); position += 2; } @@ -560,9 +567,9 @@ export function writeStructInPlace(object, target, encodingStart, position, stru let recordId = transition[RECORD_SYMBOL]; if (recordId == null) { - // Enforce the cap here only while nothing has been packed (flat records and the - // no-queued-refs case): bailing after a pack would corrupt the fallback, so a record - // that already packed nested refs completes and may mint a bounded few past the cap. + // Flat records (no queued refs) reach here without packing, so the cap is enforced + // cleanly. Records that packed nested refs already passed the preflight (record id + // exists) or are completing a bounded overshoot; either way bailing now would corrupt. if (!packedRef && packr.typedStructs.length >= (packr.maxOwnStructures ?? Infinity)) return 0; recordId = packr.typedStructs.length; const structure = []; diff --git a/tests/test-cbor-x.js b/tests/test-cbor-x.js index 3b7e36b..ad80ce8 100644 --- a/tests/test-cbor-x.js +++ b/tests/test-cbor-x.js @@ -321,6 +321,18 @@ suite('structon (cbor-x base) – maxOwnStructures cap', function () { assert.deepStrictEqual(norm(r), r); }); + test('a stream of nested-object shape variants stays bounded by the cap', function () { + const enc = new Structon({ structures: [], useRecords: false, maxOwnStructures: 4 }); + const norm = (r) => JSON.parse(JSON.stringify(enc.decode(enc.encode(r)))); + const learn = {}; for (let k = 0; k < 10; k++) learn['k' + k] = { x: 1 }; + norm(learn); + for (let i = 2; i < 300; i++) { + const r = { k0: { x: 1 }, ['k' + i]: { x: 1 } }; + assert.deepStrictEqual(norm(r), r); + } + assert.ok(enc.typedStructs.length <= 4, 'nested-object variant stream must stay within the cap, got ' + enc.typedStructs.length); + }); + test('new Structon(null) does not throw (null options = use defaults)', function () { const enc = new Structon(null); assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode({ a: 1 })))), { a: 1 }); diff --git a/tests/test.js b/tests/test.js index 4f11abd..3e0972e 100644 --- a/tests/test.js +++ b/tests/test.js @@ -615,6 +615,21 @@ suite('structon – maxOwnStructures cap', function () { assert.deepStrictEqual(norm(r), r); }); + test('a stream of nested-object shape variants stays bounded by the cap', function () { + // Each variant carries a distinct second nested key, so the queued transition is missing + // every time. Without the pre-pack preflight, the fast path would pack the first ref then + // force-mint the rest, growing typedStructs unbounded; the preflight bails before packing. + const enc = new Structon({ structures: [], useRecords: false, maxOwnStructures: 4 }); + const norm = (r) => JSON.parse(JSON.stringify(enc.decode(enc.encode(r)))); + const learn = {}; for (let k = 0; k < 10; k++) learn['k' + k] = { x: 1 }; + norm(learn); + for (let i = 2; i < 300; i++) { + const r = { k0: { x: 1 }, ['k' + i]: { x: 1 } }; + assert.deepStrictEqual(norm(r), r); + } + assert.ok(enc.typedStructs.length <= 4, 'nested-object variant stream must stay within the cap, got ' + enc.typedStructs.length); + }); + test('new Structon(null) does not throw (null options = use defaults)', function () { const enc = new Structon(null); assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode({ a: 1 })))), { a: 1 }); From bc6918676f7fc98fec7a7fc566041cfdd0040a4e Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 4 Jun 2026 18:04:05 -0600 Subject: [PATCH 06/10] fix: make the structure cap a strict hard bound for wide queued refs The remaining gap: a second packed ref whose ref-section offset crosses 0xff00 needs an object32 structure variant; if it didn't exist, the post-pack force-mint appended it past the cap (converged at cap+1 per shape, but still over the cap). A single packed ref is always at offset 0 (object16) and cannot diverge, so the divergence requires >= 2 packed refs. Under the cap, the preflight now falls records with >= 2 packed (non-null object) refs back to plain encoding before any pack(). typedStructs is now a strict hard bound: it never exceeds maxOwnStructures. Adds a wide-ref (>0xff00) regression test on both paths. Co-Authored-By: Claude Opus 4.7 --- struct.js | 9 ++++++++- tests/test-cbor-x.js | 12 ++++++++++++ tests/test.js | 14 ++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/struct.js b/struct.js index 0c4cf80..61e3ccc 100644 --- a/struct.js +++ b/struct.js @@ -498,10 +498,17 @@ export function writeStructInPlace(object, target, encodingStart, position, stru // nested pack on a prior field may have advanced). if (queuedReferences.length > 0 && packr.typedStructs.length >= (packr.maxOwnStructures ?? Infinity)) { let t = transition; + let objectRefs = 0; for (let i = 0, l = queuedReferences.length; i < l; i += 3) { + const v = queuedReferences[i + 1]; + // A second packed ref can land at a ref-section offset >= 0xff00 and need an object32 + // structure variant that may not exist; post-pack we couldn't mint it without exceeding + // the cap. A single packed ref is always at offset 0 (object16) and can't diverge, so + // only records with >= 2 packed refs need to fall back to plain encoding under the cap. + if (v != null && ++objectRefs >= 2) return 0; const nt = t[queuedReferences[i]]; if (!nt) return 0; - const next = queuedReferences[i + 1] != null ? (nt.object16 || nt.object32) : nt.object16; + const next = v != null ? (nt.object16 || nt.object32) : nt.object16; if (!next) return 0; t = next; } diff --git a/tests/test-cbor-x.js b/tests/test-cbor-x.js index ad80ce8..007f804 100644 --- a/tests/test-cbor-x.js +++ b/tests/test-cbor-x.js @@ -333,6 +333,18 @@ suite('structon (cbor-x base) – maxOwnStructures cap', function () { assert.ok(enc.typedStructs.length <= 4, 'nested-object variant stream must stay within the cap, got ' + enc.typedStructs.length); }); + test('capped: wide nested refs (ref offset > 0xff00) stay a hard bound', function () { + const enc = new Structon({ structures: [], useRecords: false, maxOwnStructures: 1 }); + const norm = (r) => JSON.parse(JSON.stringify(enc.decode(enc.encode(r)))); + norm({ a: { x: '1' }, b: { y: 1 } }); + const big = 'z'.repeat(70000); + for (let i = 0; i < 30; i++) { + const r = { a: { x: big + i }, b: { y: i } }; + assert.deepStrictEqual(norm(r), r); + } + assert.ok(enc.typedStructs.length <= 1, 'wide multi-ref records must not exceed the cap, got ' + enc.typedStructs.length); + }); + test('new Structon(null) does not throw (null options = use defaults)', function () { const enc = new Structon(null); assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode({ a: 1 })))), { a: 1 }); diff --git a/tests/test.js b/tests/test.js index 3e0972e..060a3d1 100644 --- a/tests/test.js +++ b/tests/test.js @@ -630,6 +630,20 @@ suite('structon – maxOwnStructures cap', function () { assert.ok(enc.typedStructs.length <= 4, 'nested-object variant stream must stay within the cap, got ' + enc.typedStructs.length); }); + test('capped: wide nested refs (ref offset > 0xff00) stay a hard bound', function () { + // A second packed ref past 0xff00 needs an object32 structure variant. Records with >= 2 + // packed refs fall back to plain under the cap, so this can't mint a variant past it. + const enc = new Structon({ structures: [], useRecords: false, maxOwnStructures: 1 }); + const norm = (r) => JSON.parse(JSON.stringify(enc.decode(enc.encode(r)))); + norm({ a: { x: '1' }, b: { y: 1 } }); + const big = 'z'.repeat(70000); // > 0xff00 bytes, pushes the second ref offset past object16 + for (let i = 0; i < 30; i++) { + const r = { a: { x: big + i }, b: { y: i } }; + assert.deepStrictEqual(norm(r), r); + } + assert.ok(enc.typedStructs.length <= 1, 'wide multi-ref records must not exceed the cap, got ' + enc.typedStructs.length); + }); + test('new Structon(null) does not throw (null options = use defaults)', function () { const enc = new Structon(null); assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode({ a: 1 })))), { a: 1 }); From dc911fc36f11941d62bc3dce446b56f320b56bf3 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 4 Jun 2026 18:17:27 -0600 Subject: [PATCH 07/10] fix: strict cap under layout-retry and inline-string offsets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more fast-path edge cases: 1. Inline strings share the ref section, so even a SINGLE object ref can land past 0xff00 and need an object32 variant — my "only >=2 refs diverge" assumption was wrong. 2. The layout-retry (fixed section overflows the ref-start estimate) re-runs the encode after attempt 1 already packed+minted. If attempt 1 was unfrozen but the mint pushed length to the cap, the retry bailed under the now-frozen state after refs were packed → corrupt fallback. Simplify to a robust rule: under the cap, any record with a packing (non-null object/Date) ref falls back to plain encoding in the preflight, before any pack() — offsets can't be predicted pre-pack and we can't bail post-pack. Pass structureKnown=true on the retry so it re-encodes the already-minted structure instead of re-applying the cap. typedStructs stays a strict hard bound; flat records (the RaceEntry case) are unaffected. Adds regression tests: single object ref past 0xff00 via inline strings, and a layout-retry record with nested refs. Co-Authored-By: Claude Opus 4.7 --- struct.js | 32 +++++++++++++++++++------------- tests/test-cbor-x.js | 13 +++++++++++++ tests/test.js | 27 +++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 13 deletions(-) diff --git a/struct.js b/struct.js index 61e3ccc..026deef 100644 --- a/struct.js +++ b/struct.js @@ -272,10 +272,13 @@ function _writeHeader(result, recordId, headerSize) { * @param {function} pack - pack a nested value at a given position * @returns {number} new write position, or 0 to bail (fall back to plain object) */ -export function writeStructInPlace(object, target, encodingStart, position, structures, makeRoom, pack) { +export function writeStructInPlace(object, target, encodingStart, position, structures, makeRoom, pack, structureKnown) { const packr = this; let typedStructs = packr.typedStructs || (packr.typedStructs = []); - _frozen = typedStructs.length >= (packr.maxOwnStructures ?? Infinity); + // structureKnown is set only on the internal layout-retry below: attempt 1 already minted + // this record's structure, so the retry re-encodes a known shape and must not re-apply the + // cap (which could otherwise bail after attempt 1 already packed refs → corrupt fallback). + _frozen = !structureKnown && typedStructs.length >= (packr.maxOwnStructures ?? Infinity); let targetView = target.dataView; let refsStartPosition = (typedStructs.lastStringStart || 100) + position; let safeEnd = target.length - 10; @@ -496,19 +499,19 @@ export function writeStructInPlace(object, target, encodingStart, position, stru // new structure, fall back to plain encoding now (return 0) — before touching the shared // position. A fresh length check is used (not the entry-time _frozen, which a re-entrant // nested pack on a prior field may have advanced). - if (queuedReferences.length > 0 && packr.typedStructs.length >= (packr.maxOwnStructures ?? Infinity)) { + if (_frozen && queuedReferences.length > 0) { let t = transition; - let objectRefs = 0; for (let i = 0, l = queuedReferences.length; i < l; i += 3) { - const v = queuedReferences[i + 1]; - // A second packed ref can land at a ref-section offset >= 0xff00 and need an object32 - // structure variant that may not exist; post-pack we couldn't mint it without exceeding - // the cap. A single packed ref is always at offset 0 (object16) and can't diverge, so - // only records with >= 2 packed refs need to fall back to plain encoding under the cap. - if (v != null && ++objectRefs >= 2) return 0; + // A non-null (object/Date) ref is pack()ed into the shared buffer, advancing + // msgpackr's write position. Its structure variant (object16 vs object32) depends on + // the runtime ref-section offset (inline strings + earlier refs), which we can't know + // before packing — and we can't bail after a pack without corrupting the fallback. So + // under the cap, any record with a packing ref falls back to plain encoding now, + // before any pack(). null/undefined refs don't pack, so they're walked normally. + if (queuedReferences[i + 1] != null) return 0; const nt = t[queuedReferences[i]]; if (!nt) return 0; - const next = v != null ? (nt.object16 || nt.object32) : nt.object16; + const next = nt.object16; // null/undefined ref → OBJECT_DATA size 2 if (!next) return 0; t = next; } @@ -626,9 +629,12 @@ export function writeStructInPlace(object, target, encodingStart, position, stru typedStructs.lastStringStart = position - start; } else if (position > refsStartPosition) { if (refsStartPosition === refPosition) return position; // no refs - // fixed section overflowed our estimate — retry with the corrected size + // fixed section overflowed our estimate — retry with the corrected size. The structure + // is already minted at this point, so pass structureKnown=true to skip the cap check + // (otherwise a record that became frozen during attempt 1 would bail mid-retry, after + // refs were already packed, and corrupt the fallback). typedStructs.lastStringStart = position - start; - return writeStructInPlace.call(packr, object, target, encodingStart, start, structures, makeRoom, pack); + return writeStructInPlace.call(packr, object, target, encodingStart, start, structures, makeRoom, pack, true); } return refPosition; } diff --git a/tests/test-cbor-x.js b/tests/test-cbor-x.js index 007f804..63a64fd 100644 --- a/tests/test-cbor-x.js +++ b/tests/test-cbor-x.js @@ -345,6 +345,19 @@ suite('structon (cbor-x base) – maxOwnStructures cap', function () { assert.ok(enc.typedStructs.length <= 1, 'wide multi-ref records must not exceed the cap, got ' + enc.typedStructs.length); }); + test('capped: a single object ref past 0xff00 (via inline strings) stays bounded', function () { + const enc = new Structon({ structures: [], useRecords: false, maxOwnStructures: 1 }); + const norm = (r) => JSON.parse(JSON.stringify(enc.decode(enc.encode(r)))); + const small = 'a'.repeat(16319); + norm({ s1: small, s2: small, s3: small, s4: small, obj: { x: 1 } }); + const big = 'a'.repeat(16320); + for (let i = 0; i < 20; i++) { + const r = { s1: big, s2: big, s3: big, s4: big, obj: { x: i } }; + assert.deepStrictEqual(norm(r), r); + } + assert.ok(enc.typedStructs.length <= 1, 'wide-string single-ref records must not exceed the cap, got ' + enc.typedStructs.length); + }); + test('new Structon(null) does not throw (null options = use defaults)', function () { const enc = new Structon(null); assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode({ a: 1 })))), { a: 1 }); diff --git a/tests/test.js b/tests/test.js index 060a3d1..36329a3 100644 --- a/tests/test.js +++ b/tests/test.js @@ -644,6 +644,33 @@ suite('structon – maxOwnStructures cap', function () { assert.ok(enc.typedStructs.length <= 1, 'wide multi-ref records must not exceed the cap, got ' + enc.typedStructs.length); }); + test('capped: a single object ref past 0xff00 (via inline strings) stays bounded', function () { + // Inline strings share the ref section, so even ONE object ref can land past 0xff00 and + // need object32. Under the cap such a record falls back to plain rather than minting. + const enc = new Structon({ structures: [], useRecords: false, maxOwnStructures: 1 }); + const norm = (r) => JSON.parse(JSON.stringify(enc.decode(enc.encode(r)))); + const small = 'a'.repeat(16319); + norm({ s1: small, s2: small, s3: small, s4: small, obj: { x: 1 } }); // object16, cap hit + const big = 'a'.repeat(16320); // 4 of these push the obj ref offset past 0xff00 + for (let i = 0; i < 20; i++) { + const r = { s1: big, s2: big, s3: big, s4: big, obj: { x: i } }; + assert.deepStrictEqual(norm(r), r); + } + assert.ok(enc.typedStructs.length <= 1, 'wide-string single-ref records must not exceed the cap, got ' + enc.typedStructs.length); + }); + + test('capped: a layout-retry record (large fixed section + nested refs) does not corrupt', function () { + // A large fixed section overflows the ref-start estimate and triggers an internal retry + // after refs were packed. The retry must re-encode the already-minted structure rather + // than bailing under the now-reached cap (which would corrupt the fallback). + const enc = new Structon({ structures: [], useRecords: false, maxOwnStructures: 1 }); + const norm = (r) => JSON.parse(JSON.stringify(enc.decode(enc.encode(r)))); + const mk = (base) => { const r = {}; for (let i = 0; i < 40; i++) r['n' + i] = base + i; r.a = { x: base }; r.b = { y: base + 1 }; return r; }; + assert.deepStrictEqual(norm(mk(1000000)), mk(1000000)); + assert.deepStrictEqual(norm(mk(2000000)), mk(2000000)); + assert.ok(enc.typedStructs.length <= 1, 'retry-path records must not exceed the cap, got ' + enc.typedStructs.length); + }); + test('new Structon(null) does not throw (null options = use defaults)', function () { const enc = new Structon(null); assert.deepStrictEqual(JSON.parse(JSON.stringify(enc.decode(enc.encode({ a: 1 })))), { a: 1 }); From f75a81488e39810c3b3c4c96608a0dfb9e912f8b Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 4 Jun 2026 18:28:11 -0600 Subject: [PATCH 08/10] fix: preserve persisted typed structures on capped fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the standalone path, a capped miss falls back to superEncode (plain base encoding). For bases that persist named structures (e.g. cbor-x records), that fallback can call the user's saveStructures with only the base named array, overwriting the combined {named, typed} payload — stranding previously written struct data so a fresh reader can't decode it. Re-save the combined structures after the capped fallback so the typed structures survive (this.structures also carries any base record added). The fast path is unaffected (its hook-based persistence already combines them). Adds a regression test on both paths. Co-Authored-By: Claude Opus 4.7 --- index.js | 7 +++++++ tests/test-cbor-x.js | 14 ++++++++++++++ tests/test.js | 11 +++++++++++ 3 files changed, 32 insertions(+) diff --git a/index.js b/index.js index 1efce47..e89aa7c 100644 --- a/index.js +++ b/index.js @@ -80,6 +80,13 @@ export function createStructon(BaseClass) { } return encoded; } + // Capped miss: fall back to plain base encoding. The base may persist its own + // named structures via saveStructures, overwriting our combined {named, typed} + // payload and stranding previously written struct data. Re-save afterward so the + // typed structures survive (this.structures now also holds any base record added). + const result = superEncode(value, encodeOptions); + if (this.typedStructs && this.typedStructs.length > 0) this._saveTypedStructures(); + return result; } finally { this._onStructureAdded = null; } diff --git a/tests/test-cbor-x.js b/tests/test-cbor-x.js index 63a64fd..4149e69 100644 --- a/tests/test-cbor-x.js +++ b/tests/test-cbor-x.js @@ -381,4 +381,18 @@ suite('structon (cbor-x base) – maxOwnStructures cap', function () { assert.strictEqual(result.name, 'Alice'); assert.strictEqual(result.age, 30); }); + + test('a capped plain-fallback does not strand previously-persisted typed structures', function () { + // Regression: on a capped miss the record falls back to plain base encoding, which may + // persist only the base named structures via saveStructures — overwriting the saved + // {named, typed} payload. A fresh reader must still decode the earlier struct buffer. + let saved = null; + const writer = new Structon({ structures: [], maxOwnStructures: 1, saveStructures(s) { saved = s; return true; }, getStructures() { return saved; } }); + const buf1 = writer.encode({ x: 1 }); // mints typed structure 0 + const buf2 = writer.encode({ y: 2, z: 3 }); // capped miss → plain fallback + + const reader = new Structon({ structures: [], getStructures() { return saved; } }); + assert.deepStrictEqual(JSON.parse(JSON.stringify(reader.decode(buf1))), { x: 1 }); + assert.deepStrictEqual(JSON.parse(JSON.stringify(reader.decode(buf2))), { y: 2, z: 3 }); + }); }); diff --git a/tests/test.js b/tests/test.js index 36329a3..ee28e2c 100644 --- a/tests/test.js +++ b/tests/test.js @@ -694,4 +694,15 @@ suite('structon – maxOwnStructures cap', function () { assert.strictEqual(result.name, 'Alice'); assert.strictEqual(result.age, 30); }); + + test('a capped plain-fallback does not strand previously-persisted typed structures', function () { + let saved = null; + const writer = new Structon({ structures: [], maxOwnStructures: 1, saveStructures(s) { saved = s; return true; }, getStructures() { return saved; } }); + const buf1 = writer.encode({ x: 1 }); // mints typed structure 0 + const buf2 = writer.encode({ y: 2, z: 3 }); // capped miss → plain fallback + + const reader = new Structon({ structures: [], getStructures() { return saved; } }); + assert.deepStrictEqual(JSON.parse(JSON.stringify(reader.decode(buf1))), { x: 1 }); + assert.deepStrictEqual(JSON.parse(JSON.stringify(reader.decode(buf2))), { y: 2, z: 3 }); + }); }); From c6c9bcd916bdfd91fab9e9d13af0c3e6c82cedbb Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 4 Jun 2026 18:44:27 -0600 Subject: [PATCH 09/10] fix: scope the freeze decision per-instance, not a shared global MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The freeze flag was module-scoped, so a re-entrant encode on another Structon instance (e.g. an enumerable getter that encodes with an uncapped sibling during iteration) could flip it and let a capped encoder mint past its cap — even maxOwnStructures:0 could produce typed structs. Derive the freeze state from the encoding instance's own typedStructs.length: a local `frozen` in writeStructInPlace/_encode (passed explicitly to createTypeTransition), recomputed in the standalone path after each encodeNested (which can self-mint). No shared mutable state, so cross-instance re-entrancy can't lift the cap. onLoadedStructures rebuilds with frozen=false (replaying persisted structures is never capped). Adds per-instance regression tests on both paths. Co-Authored-By: Claude Opus 4.7 --- struct.js | 98 +++++++++++++++++++++++--------------------- tests/test-cbor-x.js | 15 +++++++ tests/test.js | 17 ++++++++ 3 files changed, 83 insertions(+), 47 deletions(-) diff --git a/struct.js b/struct.js index 026deef..e312737 100644 --- a/struct.js +++ b/struct.js @@ -124,15 +124,15 @@ function createBlankTransition(key, parent) { // structures/transitions. typedStructs is append-only and pinned on the long-lived // encoder (records reference structures by recordId), so an unbounded shape space — // e.g. a wide, sparsely/variably-populated schema — would otherwise grow the -// dictionary + transition trie without limit. While frozen, a missing transition -// returns undefined so the caller bails and the record falls back to plain encoding. -let _frozen = false; - -function createTypeTransition(transition, type, size) { +// dictionary + transition trie without limit. `frozen` is passed in (derived from the +// encoding instance's own typedStructs.length, never a shared global) so a re-entrant +// encode on another instance can't flip it; while frozen, a missing transition returns +// undefined so the caller bails and the record falls back to plain encoding. +function createTypeTransition(transition, type, size, frozen) { const typeName = TYPE_NAMES[type] + (size << 3); let t = transition[typeName]; if (t) return t; - if (_frozen) return undefined; + if (frozen) return undefined; t = transition[typeName] = Object.create(null); t.__type = type; t.__size = size; @@ -278,7 +278,9 @@ export function writeStructInPlace(object, target, encodingStart, position, stru // structureKnown is set only on the internal layout-retry below: attempt 1 already minted // this record's structure, so the retry re-encodes a known shape and must not re-apply the // cap (which could otherwise bail after attempt 1 already packed refs → corrupt fallback). - _frozen = !structureKnown && typedStructs.length >= (packr.maxOwnStructures ?? Infinity); + // `frozen` is a local (from this instance's typedStructs) — never a shared global — so a + // re-entrant encode on another instance (e.g. via an enumerable getter) can't flip it. + const frozen = !structureKnown && typedStructs.length >= (packr.maxOwnStructures ?? Infinity); let targetView = target.dataView; let refsStartPosition = (typedStructs.lastStringStart || 100) + position; let safeEnd = target.length - 10; @@ -312,7 +314,7 @@ export function writeStructInPlace(object, target, encodingStart, position, stru let value = object[key]; let nextTransition = transition[key]; if (!nextTransition) { - if (_frozen) return 0; + if (frozen) return 0; transition[key] = nextTransition = { key, parent: transition, enumerationOffset: 0, ascii0: null, ascii8: null, num8: null, @@ -340,10 +342,10 @@ export function writeStructInPlace(object, target, encodingStart, position, stru (nextTransition.num8 && !(nextId > 200 && nextTransition.num32) || number < 0x20 && !nextTransition.num32) ) { - transition = nextTransition.num8 || createTypeTransition(nextTransition, NUMBER, 1); + transition = nextTransition.num8 || createTypeTransition(nextTransition, NUMBER, 1, frozen); target[position++] = number; } else { - transition = nextTransition.num32 || createTypeTransition(nextTransition, NUMBER, 4); + transition = nextTransition.num32 || createTypeTransition(nextTransition, NUMBER, 4, frozen); targetView.setUint32(position, number, true); position += 4; } @@ -353,14 +355,14 @@ export function writeStructInPlace(object, target, encodingStart, position, stru if (float32Headers[target[position + 3] >>> 5]) { let xShifted; if (((xShifted = number * mult10[((target[position + 3] & 0x7f) << 1) | (target[position + 2] >> 7)]) >> 0) === xShifted) { - transition = nextTransition.num32 || createTypeTransition(nextTransition, NUMBER, 4); + transition = nextTransition.num32 || createTypeTransition(nextTransition, NUMBER, 4, frozen); position += 4; break; } } } } - transition = nextTransition.num64 || createTypeTransition(nextTransition, NUMBER, 8); + transition = nextTransition.num64 || createTypeTransition(nextTransition, NUMBER, 8, frozen); targetView.setFloat64(position, number, true); position += 8; break; @@ -433,20 +435,20 @@ export function writeStructInPlace(object, target, encodingStart, position, stru nextTransition.string8 = transition; pack(null, 0, true); // notify structure update } else { - transition = createTypeTransition(nextTransition, UTF8, 1); + transition = createTypeTransition(nextTransition, UTF8, 1, frozen); } } } else if (refOffset === 0 && !usedAscii0) { usedAscii0 = true; - transition = nextTransition.ascii0 || createTypeTransition(nextTransition, ASCII, 0); + transition = nextTransition.ascii0 || createTypeTransition(nextTransition, ASCII, 0, frozen); break; // size=0: don't increment position } else if (!(transition = nextTransition.ascii8) && !(typedStructs.length > 10 && (transition = nextTransition.string8))) { - transition = createTypeTransition(nextTransition, ASCII, 1); + transition = createTypeTransition(nextTransition, ASCII, 1, frozen); } target[position++] = refOffset; } else { - transition = nextTransition.string16 || createTypeTransition(nextTransition, UTF8, 2); + transition = nextTransition.string16 || createTypeTransition(nextTransition, UTF8, 2, frozen); targetView.setUint16(position, refOffset, true); position += 2; } @@ -455,7 +457,7 @@ export function writeStructInPlace(object, target, encodingStart, position, stru case 'object': { if (value) { if (value.constructor === Date) { - transition = nextTransition.date64 || createTypeTransition(nextTransition, DATE, 8); + transition = nextTransition.date64 || createTypeTransition(nextTransition, DATE, 8, frozen); targetView.setFloat64(position, value.getTime(), true); position += 8; } else { @@ -473,7 +475,7 @@ export function writeStructInPlace(object, target, encodingStart, position, stru break; } case 'boolean': - transition = nextTransition.num8 || nextTransition.ascii8 || createTypeTransition(nextTransition, NUMBER, 1); + transition = nextTransition.num8 || nextTransition.ascii8 || createTypeTransition(nextTransition, NUMBER, 1, frozen); target[position++] = value ? 0xf9 : 0xf8; break; case 'undefined': { @@ -497,9 +499,8 @@ export function writeStructInPlace(object, target, encodingStart, position, stru // shared write position and we cannot cleanly bail afterward, so preflight the whole queued // chain through EXISTING transitions first: if the cap is reached and any field would need a // new structure, fall back to plain encoding now (return 0) — before touching the shared - // position. A fresh length check is used (not the entry-time _frozen, which a re-entrant - // nested pack on a prior field may have advanced). - if (_frozen && queuedReferences.length > 0) { + // position. Uses the local `frozen` (this instance's state), immune to cross-instance clobber. + if (frozen && queuedReferences.length > 0) { let t = transition; for (let i = 0, l = queuedReferences.length; i < l; i += 3) { // A non-null (object/Date) ref is pack()ed into the shared buffer, advancing @@ -690,7 +691,10 @@ export function writeStruct(object, encodeNested, packr) { function _encode(object, encodeNested, packr, work) { let typedStructs = packr.typedStructs || (packr.typedStructs = []); - _frozen = typedStructs.length >= (packr.maxOwnStructures ?? Infinity); + const cap = packr.maxOwnStructures ?? Infinity; + // Local (not a shared global), recomputed after each encodeNested below since a nested + // encode on this same instance can mint and grow typedStructs. + let frozen = typedStructs.length >= cap; let transition = typedStructs.transitions || (typedStructs.transitions = Object.create(null)); const nextId = typedStructs.length; @@ -712,7 +716,7 @@ function _encode(object, encodeNested, packr, work) { const value = object[key]; let nextTransition = transition[key]; if (!nextTransition) { - if (_frozen) return null; + if (frozen) return null; transition[key] = nextTransition = createBlankTransition(key, transition); } if (fixedPos + 8 > work.fixedBuf.length) _growFixed(work, fixedPos + 8); @@ -727,10 +731,10 @@ function _encode(object, encodeNested, packr, work) { (nextTransition.num8 && !(nextId > 200 && nextTransition.num32) || number < 0x20 && !nextTransition.num32) ) { - transition = nextTransition.num8 || createTypeTransition(nextTransition, NUMBER, 1); + transition = nextTransition.num8 || createTypeTransition(nextTransition, NUMBER, 1, frozen); work.fixedBuf[fixedPos++] = number; } else { - transition = nextTransition.num32 || createTypeTransition(nextTransition, NUMBER, 4); + transition = nextTransition.num32 || createTypeTransition(nextTransition, NUMBER, 4, frozen); work.fixedView.setUint32(fixedPos, number, true); fixedPos += 4; } @@ -740,14 +744,14 @@ function _encode(object, encodeNested, packr, work) { if (float32Headers[work.fixedBuf[fixedPos + 3] >>> 5]) { let xShifted; if (((xShifted = number * mult10[((work.fixedBuf[fixedPos + 3] & 0x7f) << 1) | (work.fixedBuf[fixedPos + 2] >> 7)]) >> 0) === xShifted) { - transition = nextTransition.num32 || createTypeTransition(nextTransition, NUMBER, 4); + transition = nextTransition.num32 || createTypeTransition(nextTransition, NUMBER, 4, frozen); fixedPos += 4; break; } } } } - transition = nextTransition.num64 || createTypeTransition(nextTransition, NUMBER, 8); + transition = nextTransition.num64 || createTypeTransition(nextTransition, NUMBER, 8, frozen); work.fixedView.setFloat64(fixedPos, number, true); fixedPos += 8; break; @@ -770,22 +774,22 @@ function _encode(object, encodeNested, packr, work) { nextTransition.string8 = transition; structureUpdated = true; } else { - transition = createTypeTransition(nextTransition, UTF8, 1); + transition = createTypeTransition(nextTransition, UTF8, 1, frozen); } } work.fixedBuf[fixedPos++] = curOffset; } else if (curOffset === 0 && !usedAscii0) { usedAscii0 = true; - transition = nextTransition.ascii0 || createTypeTransition(nextTransition, ASCII, 0); + transition = nextTransition.ascii0 || createTypeTransition(nextTransition, ASCII, 0, frozen); // size=0: no fixed byte written } else { if (!(transition = nextTransition.ascii8) && !(typedStructs.length > 10 && (transition = nextTransition.string8))) - transition = createTypeTransition(nextTransition, ASCII, 1); + transition = createTypeTransition(nextTransition, ASCII, 1, frozen); work.fixedBuf[fixedPos++] = curOffset; } } else { - transition = nextTransition.string16 || createTypeTransition(nextTransition, UTF8, 2); + transition = nextTransition.string16 || createTypeTransition(nextTransition, UTF8, 2, frozen); work.fixedView.setUint16(fixedPos, curOffset, true); fixedPos += 2; } @@ -794,7 +798,7 @@ function _encode(object, encodeNested, packr, work) { case 'object': { if (value && value.constructor === Date) { - transition = nextTransition.date64 || createTypeTransition(nextTransition, DATE, 8); + transition = nextTransition.date64 || createTypeTransition(nextTransition, DATE, 8, frozen); work.fixedView.setFloat64(fixedPos, value.getTime(), true); fixedPos += 8; } else if (value) { @@ -814,7 +818,7 @@ function _encode(object, encodeNested, packr, work) { case 'boolean': transition = nextTransition.num8 || nextTransition.ascii8 || - createTypeTransition(nextTransition, NUMBER, 1); + createTypeTransition(nextTransition, NUMBER, 1, frozen); work.fixedBuf[fixedPos++] = value ? 0xf9 : 0xf8; break; @@ -846,7 +850,7 @@ function _encode(object, encodeNested, packr, work) { let nextTransition = transition[key]; if (!nextTransition) { - if (_frozen) return null; + if (frozen) return null; transition[key] = nextTransition = { key, parent: transition, @@ -861,6 +865,9 @@ function _encode(object, encodeNested, packr, work) { if (value != null) { const encoded = encodeNested(value); + // encodeNested may have minted on this same instance — refresh the cap state so a + // later missing transition still bails instead of minting past the cap. + frozen = typedStructs.length >= cap; const curOffset = refsPos; if (refsPos + encoded.length > work.refsBuf.length) _growRefs(work, encoded.length); work.refsBuf.set(encoded, refsPos); @@ -871,16 +878,16 @@ function _encode(object, encodeNested, packr, work) { transition = nextTransition.object16; if (transition) size = 2; else if ((transition = nextTransition.object32)) size = 4; - else { transition = createTypeTransition(nextTransition, OBJECT_DATA, 2); size = 2; } + else { transition = createTypeTransition(nextTransition, OBJECT_DATA, 2, frozen); size = 2; } } else { - transition = nextTransition.object32 || createTypeTransition(nextTransition, OBJECT_DATA, 4); + transition = nextTransition.object32 || createTypeTransition(nextTransition, OBJECT_DATA, 4, frozen); size = 4; } if (size === 2) { work.fixedView.setUint16(fixedPos, curOffset, true); fixedPos += 2; } else { work.fixedView.setUint32(fixedPos, curOffset, true); fixedPos += 4; } } else { // null or undefined sentinel - transition = nextTransition.object16 || createTypeTransition(nextTransition, OBJECT_DATA, 2); + transition = nextTransition.object16 || createTypeTransition(nextTransition, OBJECT_DATA, 2, frozen); work.fixedView.setInt16(fixedPos, value === null ? -10 : -9, true); fixedPos += 2; } @@ -891,10 +898,9 @@ function _encode(object, encodeNested, packr, work) { // Build/retrieve structure definition from the transition chain. let recordId = transition[RECORD_SYMBOL]; if (recordId == null) { - // Re-check the cap here (not just the entry-time _frozen): nested encodes via - // encodeNested() may have appended structures since entry, so this keeps - // typedStructs.length a hard bound rather than overshooting by nesting depth. - if (typedStructs.length >= (packr.maxOwnStructures ?? Infinity)) return null; + // Re-check the cap with a fresh length read: nested encodeNested() calls may have + // appended structures since entry, so this keeps typedStructs.length a hard bound. + if (typedStructs.length >= cap) return null; recordId = typedStructs.length; const structure = []; let t = transition; @@ -1159,11 +1165,9 @@ export function readStruct(src, position, srcEnd) { * Accepts the same Map format that msgpackr's struct.js produces. */ export function onLoadedStructures(sharedData) { - // Replaying already-persisted structures must always fully rebuild the trie, - // regardless of maxOwnStructures — the cap only limits minting NEW structures - // during encode. _frozen is module-level and may be left true by a prior capped - // encode, so clear it here before the createTypeTransition rebuild below. - _frozen = false; + // Replaying already-persisted structures must always fully rebuild the trie, regardless of + // maxOwnStructures — the cap only limits minting NEW structures during encode. So the + // createTypeTransition rebuild below is called with frozen=false. if (!sharedData) return this.structures; let named, typed; if (sharedData instanceof Map) { @@ -1207,7 +1211,7 @@ export function onLoadedStructures(sharedData) { float64: null, date64: null, }; } - t = createTypeTransition(next, type, size); + t = createTypeTransition(next, type, size, false); } t[RECORD_SYMBOL] = i; } diff --git a/tests/test-cbor-x.js b/tests/test-cbor-x.js index 4149e69..e8700b6 100644 --- a/tests/test-cbor-x.js +++ b/tests/test-cbor-x.js @@ -395,4 +395,19 @@ suite('structon (cbor-x base) – maxOwnStructures cap', function () { assert.deepStrictEqual(JSON.parse(JSON.stringify(reader.decode(buf1))), { x: 1 }); assert.deepStrictEqual(JSON.parse(JSON.stringify(reader.decode(buf2))), { y: 2, z: 3 }); }); + + test('the cap is per-instance: a re-entrant encode on another instance cannot lift it', function () { + // Regression: the freeze decision must be derived from THIS instance's typedStructs, not a + // shared module flag. An enumerable getter that encodes with an uncapped instance during + // iteration must not flip the capped instance's freeze state and let it mint past the cap. + const uncapped = new Structon({ structures: [], useRecords: false }); + const capped = new Structon({ structures: [], useRecords: false, maxOwnStructures: 0 }); + for (let i = 0; i < 20; i++) { + const obj = { a: i }; + Object.defineProperty(obj, 'g', { enumerable: true, get() { uncapped.encode({ deep: { n: i }, ['k' + i]: i }); return { x: i }; } }); + assert.deepStrictEqual(JSON.parse(JSON.stringify(capped.decode(capped.encode(obj)))), { a: i, g: { x: i } }); + } + assert.strictEqual(capped.typedStructs.length, 0, 'maxOwnStructures:0 must mint no typed structures'); + assert.ok(uncapped.typedStructs.length > 0, 'the uncapped instance should still grow'); + }); }); diff --git a/tests/test.js b/tests/test.js index ee28e2c..ffc8280 100644 --- a/tests/test.js +++ b/tests/test.js @@ -705,4 +705,21 @@ suite('structon – maxOwnStructures cap', function () { assert.deepStrictEqual(JSON.parse(JSON.stringify(reader.decode(buf1))), { x: 1 }); assert.deepStrictEqual(JSON.parse(JSON.stringify(reader.decode(buf2))), { y: 2, z: 3 }); }); + + test('the cap is per-instance: another encoder growing does not lift this one', function () { + // The freeze decision is derived from THIS instance's typedStructs, not a shared module + // flag — so an uncapped sibling encoder churning out structures can't lift the cap here. + const uncapped = new Structon({ structures: [], useRecords: false }); + const capped = new Structon({ structures: [], useRecords: false, maxOwnStructures: 2 }); + let seed = 1; + const rnd = () => { seed = (seed * 1103515245 + 12345) & 0x7fffffff; return seed / 0x7fffffff; }; + const mk = () => { const o = {}; for (let f = 0; f < 20; f++) if (rnd() < 0.5) o['f' + f] = (rnd() * 1e7 | 0); return o; }; + for (let i = 0; i < 500; i++) { + uncapped.encode(mk()); // grows the sibling's dictionary freely + const r = mk(); + assert.deepStrictEqual(JSON.parse(JSON.stringify(capped.decode(capped.encode(r)))), r); + } + assert.ok(capped.typedStructs.length <= 2, 'capped must stay bounded regardless of the sibling, got ' + capped.typedStructs.length); + assert.ok(uncapped.typedStructs.length > 2, 'the uncapped sibling should grow past 2'); + }); }); From 6653b5b2078d3c8b207b88655025af18f3eefcad Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 4 Jun 2026 18:56:14 -0600 Subject: [PATCH 10/10] fix: fresh cap re-check vs same-encoder getters; avoid accessor double-read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more enumerable-getter edge cases: 1. A getter that mints on the SAME encoder grows typedStructs after the entry-time freeze state was captured; the queued-ref preflight used that stale value and let the record mint past the cap. The preflight now uses a fresh typedStructs.length read (it runs after values, hence getters, are read). 2. A capped miss on a new key read the property value (invoking the getter) during the failed struct attempt, then the plain fallback read it again — double-running a side-effecting accessor. Resolve the key transition and bail on a frozen miss BEFORE reading the value, on both paths. Adds regression tests: accessor single-read, and a same-encoder getter mint staying within the cap. Co-Authored-By: Claude Opus 4.7 --- struct.js | 17 ++++++++++++----- tests/test-cbor-x.js | 24 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/struct.js b/struct.js index e312737..57a71ba 100644 --- a/struct.js +++ b/struct.js @@ -280,7 +280,8 @@ export function writeStructInPlace(object, target, encodingStart, position, stru // cap (which could otherwise bail after attempt 1 already packed refs → corrupt fallback). // `frozen` is a local (from this instance's typedStructs) — never a shared global — so a // re-entrant encode on another instance (e.g. via an enumerable getter) can't flip it. - const frozen = !structureKnown && typedStructs.length >= (packr.maxOwnStructures ?? Infinity); + const cap = packr.maxOwnStructures ?? Infinity; + const frozen = !structureKnown && typedStructs.length >= cap; let targetView = target.dataView; let refsStartPosition = (typedStructs.lastStringStart || 100) + position; let safeEnd = target.length - 10; @@ -311,8 +312,10 @@ export function writeStructInPlace(object, target, encodingStart, position, stru let keyIndex = 0; for (let key in object) { - let value = object[key]; let nextTransition = transition[key]; + // Resolve the key transition BEFORE reading the value: when frozen and the key is new we + // bail here, so an enumerable getter isn't invoked during this (failed) struct attempt and + // then again by the plain fallback (which would double-read a side-effecting accessor). if (!nextTransition) { if (frozen) return 0; transition[key] = nextTransition = { @@ -322,6 +325,7 @@ export function writeStructInPlace(object, target, encodingStart, position, stru float64: null, date64: null, }; } + let value = object[key]; if (position > safeEnd) { target = makeRoom(position); targetView = target.dataView; @@ -499,8 +503,9 @@ export function writeStructInPlace(object, target, encodingStart, position, stru // shared write position and we cannot cleanly bail afterward, so preflight the whole queued // chain through EXISTING transitions first: if the cap is reached and any field would need a // new structure, fall back to plain encoding now (return 0) — before touching the shared - // position. Uses the local `frozen` (this instance's state), immune to cross-instance clobber. - if (frozen && queuedReferences.length > 0) { + // position. Uses a FRESH length read (not the entry-time `frozen`): a getter invoked while + // reading values above may have minted on this same instance since entry. + if (!structureKnown && queuedReferences.length > 0 && typedStructs.length >= cap) { let t = transition; for (let i = 0, l = queuedReferences.length; i < l; i += 3) { // A non-null (object/Date) ref is pack()ed into the shared buffer, advancing @@ -713,12 +718,14 @@ function _encode(object, encodeNested, packr, work) { let structureUpdated = false; for (const key in object) { - const value = object[key]; let nextTransition = transition[key]; + // Resolve the key transition before reading the value, so a frozen miss on a new key bails + // without invoking an enumerable getter that the plain fallback would then read again. if (!nextTransition) { if (frozen) return null; transition[key] = nextTransition = createBlankTransition(key, transition); } + const value = object[key]; if (fixedPos + 8 > work.fixedBuf.length) _growFixed(work, fixedPos + 8); switch (typeof value) { diff --git a/tests/test-cbor-x.js b/tests/test-cbor-x.js index e8700b6..c71e81b 100644 --- a/tests/test-cbor-x.js +++ b/tests/test-cbor-x.js @@ -410,4 +410,28 @@ suite('structon (cbor-x base) – maxOwnStructures cap', function () { assert.strictEqual(capped.typedStructs.length, 0, 'maxOwnStructures:0 must mint no typed structures'); assert.ok(uncapped.typedStructs.length > 0, 'the uncapped instance should still grow'); }); + + test('a capped miss does not double-read a side-effecting accessor', function () { + // A frozen miss on a new key must bail before the value is read, so the getter runs once + // (here) — not twice (failed struct attempt + plain fallback). + const enc = new Structon({ structures: [], useRecords: false, maxOwnStructures: 0 }); + let reads = 0; + const obj = {}; + Object.defineProperty(obj, 'a', { enumerable: true, get() { return ++reads; } }); + const out = enc.decode(enc.encode(obj)); + assert.strictEqual(reads, 1, 'accessor should be read exactly once'); + assert.strictEqual((out.toJSON ? out.toJSON() : out).a, 1); + }); + + test('a getter that mints on the same encoder cannot push it past the cap', function () { + // The getter mints (via a re-entrant encode) after the entry-time freeze state is captured; + // the fresh cap re-check (preflight / record-id mint) must still bound typedStructs. + const enc = new Structon({ structures: [], useRecords: false, maxOwnStructures: 1 }); + for (let k = 0; k < 10; k++) { + const obj = {}; + Object.defineProperty(obj, 'a', { enumerable: true, get() { enc.encode({ ['z' + k]: k }); return { nested: k }; } }); + enc.encode(obj); + } + assert.ok(enc.typedStructs.length <= 1, 'same-encoder getter mints must not exceed the cap, got ' + enc.typedStructs.length); + }); });