From dbe70c1215314385160d3e208f029c948b545f20 Mon Sep 17 00:00:00 2001 From: Krmjn09 Date: Sun, 15 Jun 2025 19:46:20 +0530 Subject: [PATCH 1/5] Added Field Descriptor logic --- demo/node/rntuple.js | 2 +- demo/node/rntuple_test.js | 20 ++++-- modules/rntuple.mjs | 142 ++++++++++++++++++++++++++++++-------- 3 files changed, 128 insertions(+), 36 deletions(-) diff --git a/demo/node/rntuple.js b/demo/node/rntuple.js index 5dc470ec2..fe2d0f3f7 100644 --- a/demo/node/rntuple.js +++ b/demo/node/rntuple.js @@ -61,4 +61,4 @@ else console.log('test 3 - readString passed'); else console.error('FAILURE: test 3 - readString does not match'); -} +} \ No newline at end of file diff --git a/demo/node/rntuple_test.js b/demo/node/rntuple_test.js index 3712eb199..a364efb77 100644 --- a/demo/node/rntuple_test.js +++ b/demo/node/rntuple_test.js @@ -18,11 +18,6 @@ if (rntuple.builder?.name !== 'Staff') else console.log('OK: name is', rntuple.builder?.name); -if (typeof rntuple.builder?.version !== 'number') - console.error('FAILURE: version is missing or invalid'); -else - console.log('OK: version is', rntuple.builder.version); - if (!rntuple.builder?.description) console.error('FAILURE: description is missing'); else @@ -33,3 +28,18 @@ if (rntuple.builder?.xxhash3 === undefined || rntuple.builder.xxhash3 === null) else console.log('OK: xxhash3 is', '0x' + rntuple.builder.xxhash3.toString(16).padStart(16, '0')); +// Fields Check + +if (!rntuple.builder?.fieldDescriptors?.length) + console.error('FAILURE: No fields deserialized'); +else { + console.log(`OK: ${rntuple.builder.fieldDescriptors.length} field(s) deserialized`); + for (let i = 0; i < rntuple.builder.fieldDescriptors.length; ++i) { + const field = rntuple.builder.fieldDescriptors[i]; + if (!field.fieldName || !field.typeName) + console.error(`FAILURE: Field ${i} is missing name or type`); + else + console.log(`OK: Field ${i}: ${field.fieldName} (${field.typeName})`); + } +} + diff --git a/modules/rntuple.mjs b/modules/rntuple.mjs index d2cde094c..680ead8e7 100644 --- a/modules/rntuple.mjs +++ b/modules/rntuple.mjs @@ -3,15 +3,17 @@ const LITTLE_ENDIAN = true; class RBufferReader { constructor(buffer) { - if (buffer instanceof ArrayBuffer) - this.buffer = buffer; - else if (ArrayBuffer.isView(buffer)) { - const bytes = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); - this.buffer = bytes.slice().buffer; + if (buffer instanceof ArrayBuffer) { + this.buffer = buffer; + this.byteOffset = 0; + this.byteLength = buffer.byteLength; + } else if (ArrayBuffer.isView(buffer)) { + this.buffer = buffer.buffer; + this.byteOffset = buffer.byteOffset; + this.byteLength = buffer.byteLength; } else throw new TypeError('Invalid buffer type'); - this.view = new DataView(this.buffer); this.offset = 0; } @@ -23,7 +25,7 @@ class RBufferReader { // Read unsigned 8-bit integer (1 BYTE) readU8() { - const val = this.view.getUint8(this.offset, LITTLE_ENDIAN); + const val = this.view.getUint8(this.offset); this.offset += 1; return val; } @@ -44,7 +46,7 @@ class RBufferReader { // Read signed 8-bit integer (1 BYTE) readS8() { - const val = this.view.getInt8(this.offset, LITTLE_ENDIAN); + const val = this.view.getInt8(this.offset); this.offset += 1; return val; } @@ -70,6 +72,13 @@ class RBufferReader { return val; } + // Read 64-bit float (8 BYTES) + readF64() { + const val = this.view.getFloat64(this.offset, LITTLE_ENDIAN); + this.offset += 8; + return val; + } + // Read a string with 32-bit length prefix readString() { const length = this.readU32(); @@ -98,36 +107,33 @@ class RBufferReader { class RNTupleDescriptorBuilder { - deserializeHeader(header_blob) { +deserializeHeader(header_blob) { if (!header_blob) return; - const reader = new RBufferReader(header_blob); + const reader = new RBufferReader(header_blob); + // Read the envelope metadata + this._readEnvelopeMetadata(reader); - // 1. Read header version (4 bytes) - this.version = reader.readU32(); + // TODO: Validate the envelope checksum at the end of deserialization + // const payloadStart = reader.offset; - // 2. Read feature flags (4 bytes) - this.headerFeatureFlags = reader.readU32(); + // Read feature flags list (may span multiple 64-bit words) + this._readFeatureFlags(reader); - // 3. Read xxhash3 (64-bit, 8 bytes) - this.xxhash3 = reader.readU64(); + // Read metadata strings + this.name = reader.readString(); + this.description = reader.readString(); + this.writer = reader.readString(); + // TODO: Remove debug logs before finalizing + console.log('Name:', this.name); + console.log('Description:', this.description); + console.log('Writer:', this.writer); - // 4. Read name (length-prefixed string) - this.name = reader.readString(); - - // 5. Read description (length-prefixed string) - this.description = reader.readString(); - - - // Console output to verify deserialization results - console.log('Version:', this.version); - console.log('Header Feature Flags:', this.headerFeatureFlags); - console.log('xxhash3:', '0x' + this.xxhash3.toString(16).padStart(16, '0')); - console.log('Name:', this.name); - console.log('Description:', this.description); + // List frame: list of field record frames + // TODO: Uncomment this section once the offset out of bounds issue is resolved + // this._readFieldDescriptors(reader); } - deserializeFooter(footer_blob) { if (!footer_blob) return; @@ -141,6 +147,82 @@ deserializeFooter(footer_blob) { } +_readEnvelopeMetadata(reader) { + const typeAndLength = reader.readU64(), + + // Envelope metadata + // The 16 bits are the envelope type ID, and the 48 bits are the envelope length + envelopeType = Number(typeAndLength & 0xFFFFn), + envelopeLength = Number((typeAndLength >> 16n) & 0xFFFFFFFFFFFFn); + + console.log('Envelope Type ID:', envelopeType); + console.log('Envelope Length:', envelopeLength); + return { envelopeType, envelopeLength }; +} + +_readFeatureFlags(reader) { + this.featureFlags = []; + while (true) { + const val = reader.readU64(); + this.featureFlags.push(val); + if ((val & 0x8000000000000000n) === 0n) break; // MSB not set: end of list + } + + // verify all feature flags are zero + if (this.featureFlags.some(v => v !== 0n)) + throw new Error('Unexpected non-zero feature flags: ' + this.featureFlags); +} + +_readFieldDescriptors(reader) { +const fieldListSize = reader.readS64(), // signed 64-bit +fieldListIsList = fieldListSize < 0; + + + if (!fieldListIsList) + throw new Error('Field list frame is not a list frame, which is required.'); + + const fieldListCount = reader.readU32(); // number of field entries + console.log('Field List Count:', fieldListCount); + + // List frame: list of field record frames + + const fieldDescriptors = []; + for (let i = 0; i < fieldListCount; ++i) { + const fieldVersion = reader.readU32(), + typeVersion = reader.readU32(), + parentFieldId = reader.readU32(), + structRole = reader.readU16(), + flags = reader.readU16(), + + fieldName = reader.readString(), + typeName = reader.readString(), + typeAlias = reader.readString(), + description = reader.readString(); + let arraySize = null, sourceFieldId = null, checksum = null; + + if (flags & 0x1) arraySize = reader.readU32(); + if (flags & 0x2) sourceFieldId = reader.readU32(); + if (flags & 0x4) checksum = reader.readU32(); + + fieldDescriptors.push({ + fieldVersion, + typeVersion, + parentFieldId, + structRole, + flags, + fieldName, + typeName, + typeAlias, + description, + arraySize, + sourceFieldId, + checksum + }); + console.log(`Field ${i + 1}:`, fieldName, '&&', typeName); +} + this.fieldDescriptors = fieldDescriptors; +} + } /** @summary Very preliminary function to read header/footer from RNTuple From 80d750ef5f5a149d837573019c72528e77a6102c Mon Sep 17 00:00:00 2001 From: Krmjn09 Date: Mon, 16 Jun 2025 15:41:53 +0530 Subject: [PATCH 2/5] completed fieldDescriptor Deserialisation --- demo/node/rntuple_test.js | 9 ++++++--- modules/rntuple.mjs | 12 ++++-------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/demo/node/rntuple_test.js b/demo/node/rntuple_test.js index a364efb77..be94c0261 100644 --- a/demo/node/rntuple_test.js +++ b/demo/node/rntuple_test.js @@ -18,8 +18,12 @@ if (rntuple.builder?.name !== 'Staff') else console.log('OK: name is', rntuple.builder?.name); -if (!rntuple.builder?.description) - console.error('FAILURE: description is missing'); +if (!rntuple.builder?.description){ + if (rntuple.builder?.description === '') + console.warn('WARNING: description is empty'); + else + console.error('FAILURE: description is missing'); +} else console.log('OK: description is', rntuple.builder.description); @@ -42,4 +46,3 @@ else { console.log(`OK: Field ${i}: ${field.fieldName} (${field.typeName})`); } } - diff --git a/modules/rntuple.mjs b/modules/rntuple.mjs index 680ead8e7..61193827f 100644 --- a/modules/rntuple.mjs +++ b/modules/rntuple.mjs @@ -124,14 +124,9 @@ deserializeHeader(header_blob) { this.name = reader.readString(); this.description = reader.readString(); this.writer = reader.readString(); - // TODO: Remove debug logs before finalizing - console.log('Name:', this.name); - console.log('Description:', this.description); - console.log('Writer:', this.writer); // List frame: list of field record frames - // TODO: Uncomment this section once the offset out of bounds issue is resolved - // this._readFieldDescriptors(reader); + this._readFieldDescriptors(reader); } deserializeFooter(footer_blob) { @@ -188,7 +183,8 @@ fieldListIsList = fieldListSize < 0; const fieldDescriptors = []; for (let i = 0; i < fieldListCount; ++i) { - const fieldVersion = reader.readU32(), + const fieldRecordSize = reader.readS64(), + fieldVersion = reader.readU32(), typeVersion = reader.readU32(), parentFieldId = reader.readU32(), structRole = reader.readU16(), @@ -205,6 +201,7 @@ fieldListIsList = fieldListSize < 0; if (flags & 0x4) checksum = reader.readU32(); fieldDescriptors.push({ + fieldRecordSize, fieldVersion, typeVersion, parentFieldId, @@ -218,7 +215,6 @@ fieldListIsList = fieldListSize < 0; sourceFieldId, checksum }); - console.log(`Field ${i + 1}:`, fieldName, '&&', typeName); } this.fieldDescriptors = fieldDescriptors; } From 9df256e12d5b37335623f9882dd443495b20618c Mon Sep 17 00:00:00 2001 From: Krmjn09 Date: Mon, 16 Jun 2025 15:53:17 +0530 Subject: [PATCH 3/5] suggested changes commited , added a console.log for recordframesize to avoid ESlint error --- demo/node/rntuple_test.js | 6 ++---- modules/rntuple.mjs | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/demo/node/rntuple_test.js b/demo/node/rntuple_test.js index be94c0261..49a1523fe 100644 --- a/demo/node/rntuple_test.js +++ b/demo/node/rntuple_test.js @@ -19,10 +19,8 @@ else console.log('OK: name is', rntuple.builder?.name); if (!rntuple.builder?.description){ - if (rntuple.builder?.description === '') - console.warn('WARNING: description is empty'); - else - console.error('FAILURE: description is missing'); + if (rntuple.builder?.description !== '') + console.error('FAILURE: description should be the empty string'); } else console.log('OK: description is', rntuple.builder.description); diff --git a/modules/rntuple.mjs b/modules/rntuple.mjs index 61193827f..eccf780ce 100644 --- a/modules/rntuple.mjs +++ b/modules/rntuple.mjs @@ -194,6 +194,7 @@ fieldListIsList = fieldListSize < 0; typeName = reader.readString(), typeAlias = reader.readString(), description = reader.readString(); + console.log(`Field Record Size: ${fieldRecordSize}`); let arraySize = null, sourceFieldId = null, checksum = null; if (flags & 0x1) arraySize = reader.readU32(); @@ -201,7 +202,6 @@ fieldListIsList = fieldListSize < 0; if (flags & 0x4) checksum = reader.readU32(); fieldDescriptors.push({ - fieldRecordSize, fieldVersion, typeVersion, parentFieldId, From 823ab26e2b0209425313b475a2da47ae98599c26 Mon Sep 17 00:00:00 2001 From: Krmjn09 Date: Mon, 16 Jun 2025 15:55:21 +0530 Subject: [PATCH 4/5] arraysize changed to 64 bits --- modules/rntuple.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/rntuple.mjs b/modules/rntuple.mjs index eccf780ce..abf87a004 100644 --- a/modules/rntuple.mjs +++ b/modules/rntuple.mjs @@ -197,7 +197,7 @@ fieldListIsList = fieldListSize < 0; console.log(`Field Record Size: ${fieldRecordSize}`); let arraySize = null, sourceFieldId = null, checksum = null; - if (flags & 0x1) arraySize = reader.readU32(); + if (flags & 0x1) arraySize = reader.readU64(); if (flags & 0x2) sourceFieldId = reader.readU32(); if (flags & 0x4) checksum = reader.readU32(); From 46a1d3efa22fccca68d4e2ee6484721411ce22e7 Mon Sep 17 00:00:00 2001 From: Krmjn09 Date: Mon, 16 Jun 2025 17:06:10 +0530 Subject: [PATCH 5/5] Added changes --- demo/node/rntuple_test.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/demo/node/rntuple_test.js b/demo/node/rntuple_test.js index 49a1523fe..01993a23d 100644 --- a/demo/node/rntuple_test.js +++ b/demo/node/rntuple_test.js @@ -18,10 +18,9 @@ if (rntuple.builder?.name !== 'Staff') else console.log('OK: name is', rntuple.builder?.name); -if (!rntuple.builder?.description){ - if (rntuple.builder?.description !== '') + +if (rntuple.builder?.description !== '') console.error('FAILURE: description should be the empty string'); -} else console.log('OK: description is', rntuple.builder.description);