From e72124873009711ef48400de4adfacaafa2be8d9 Mon Sep 17 00:00:00 2001 From: Sergey Linev Date: Mon, 24 Nov 2025 12:29:52 +0100 Subject: [PATCH 1/6] Introduce reorder handle for http requests Idea to resort and merge ranges before submitting to http server --- modules/io.mjs | 59 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/modules/io.mjs b/modules/io.mjs index df02485d1..1fe51276d 100644 --- a/modules/io.mjs +++ b/modules/io.mjs @@ -46,7 +46,10 @@ const clTStreamerElement = 'TStreamerElement', clTStreamerObject = 'TStreamerObj StlNames = ['', 'vector', 'list', 'deque', 'map', 'multimap', 'set', 'multiset', 'bitset'], // TObject bits - kIsReferenced = BIT(4), kHasUUID = BIT(5); + kIsReferenced = BIT(4), kHasUUID = BIT(5), + + // gap in http which can be merged into single http request + kMinimalHttpGap = 128; /** @summary Custom streamers for root classes @@ -3039,6 +3042,55 @@ class TFile { * @private */ async _open() { return this.readKeys(); } + /** @summary check if requested segments can be reordered or merged + * @private */ + #checkNeedReorder(place) { + let res = false, resort = false; + for (let n = 0; n < place.length - 2; n += 2) { + if (place[n] > place[n + 2]) { + res = resort = true; + break; + } + if (place[n] + place[n + 1] > place[n + 2] - kMinimalHttpGap) { + res = true; + break; + } + } + if (!res) + return false; + + res = { place, reorder: [], place_new: [], blobs: [] }; + + for (let n = 0; n < place.length; n += 2) + res.reorder.push({ pos: place[n], len: place[n + 1], indx: [n] }); + + if (resort) + res.reorder.sort((a, b) => a.pos > b.pos); + + for(let n = 0; n < res.reorder.length - 1; n++) { + const curr = res.reorder[n], + next = res.reorder[n + 1]; + if (curr.pos + curr.len + kMinimalHttpGap > next.pos) { + curr.indx.push(...next.indx); + curr.len = next.pos + next.len - curr.pos; + res.reorder.splice(n + 1, 1); // remove segment + n--; + } + } + + res.reorder.forEach(elem => res.place_new.push(elem.pos, elem.len)); + + res.addBuffer = function(indx, buf, o) { + const elem = this.reorder[indx / 2], + pos0 = elem.pos; + elem.indx.forEach(indx0 => { + this.blobs[indx0/2] = new DataView(buf, o + this.place[indx0] - pos0, this.place[indx0 + 1]); + }); + }; + + return res; + } + /** @summary read buffer(s) from the file * @return {Promise} with read buffers * @private */ @@ -3046,6 +3098,11 @@ class TFile { if ((this.fFileContent !== null) && !filename && (!this.fAcceptRanges || this.fFileContent.canExtract(place))) return this.fFileContent.extract(place); + const need_reorder = this.#checkNeedReorder(place); + if (need_reorder) + console.log('!!!!! One can reorder/merge', place, need_reorder.place_new); + + let resolveFunc, rejectFunc; const file = this, first_block = (place[0] === 0) && (place.length === 2), From efb6b3ec5fb111a065332bc5784f8aa14cf5918f Mon Sep 17 00:00:00 2001 From: Sergey Linev Date: Mon, 24 Nov 2025 12:33:19 +0100 Subject: [PATCH 2/6] Also support plain handle without reordering --- modules/io.mjs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/io.mjs b/modules/io.mjs index 1fe51276d..4a156a940 100644 --- a/modules/io.mjs +++ b/modules/io.mjs @@ -3056,8 +3056,11 @@ class TFile { break; } } - if (!res) - return false; + if (!res) { + return { place, blobs: [], addBuffer: function(indx, buf, o) { + this.blobs[indx/2] = new DataView(buf, o, this.place[indx + 1]); + }} + } res = { place, reorder: [], place_new: [], blobs: [] }; @@ -3099,7 +3102,7 @@ class TFile { return this.fFileContent.extract(place); const need_reorder = this.#checkNeedReorder(place); - if (need_reorder) + if (need_reorder?.place_new) console.log('!!!!! One can reorder/merge', place, need_reorder.place_new); From 19e2ff8f2ec5cebb28bb846213f54877b1fb272c Mon Sep 17 00:00:00 2001 From: Sergey Linev Date: Mon, 24 Nov 2025 14:36:51 +0100 Subject: [PATCH 3/6] Fix comparasion --- modules/io.mjs | 110 +++++++++++-------------------------------------- 1 file changed, 25 insertions(+), 85 deletions(-) diff --git a/modules/io.mjs b/modules/io.mjs index 4a156a940..5284ce262 100644 --- a/modules/io.mjs +++ b/modules/io.mjs @@ -3049,11 +3049,9 @@ class TFile { for (let n = 0; n < place.length - 2; n += 2) { if (place[n] > place[n + 2]) { res = resort = true; - break; } if (place[n] + place[n + 1] > place[n + 2] - kMinimalHttpGap) { res = true; - break; } } if (!res) { @@ -3068,7 +3066,7 @@ class TFile { res.reorder.push({ pos: place[n], len: place[n + 1], indx: [n] }); if (resort) - res.reorder.sort((a, b) => a.pos > b.pos); + res.reorder.sort((a, b) => { return a.pos - b.pos; }); for(let n = 0; n < res.reorder.length - 1; n++) { const curr = res.reorder[n], @@ -3101,15 +3099,15 @@ class TFile { if ((this.fFileContent !== null) && !filename && (!this.fAcceptRanges || this.fFileContent.canExtract(place))) return this.fFileContent.extract(place); - const need_reorder = this.#checkNeedReorder(place); - if (need_reorder?.place_new) - console.log('!!!!! One can reorder/merge', place, need_reorder.place_new); - + const reorder = this.#checkNeedReorder(place); + if (reorder?.place_new) { + console.log('apply reorder', place, reorder?.place_new); + place = reorder?.place_new; + } let resolveFunc, rejectFunc; const file = this, first_block = (place[0] === 0) && (place.length === 2), - blobs = [], // array of requested segments promise = new Promise((resolve, reject) => { resolveFunc = resolve; rejectFunc = reject; @@ -3140,7 +3138,7 @@ class TFile { first = last; last = Math.min(first + file.fMaxRanges * 2, place.length); if (first >= place.length) - return resolveFunc(blobs); + return resolveFunc(reorder.blobs.length === 1 ? reorder.blobs[0] : reorder.blobs); } let fullurl = fileurl, ranges = 'bytes', totalsz = 0; @@ -3157,7 +3155,7 @@ class TFile { // when read first block, allow to read more - maybe ranges are not supported and full file content will be returned if (file.fAcceptRanges && first_block) - totalsz = Math.max(totalsz, 1e7); + totalsz = Math.max(totalsz, 1e5); return createHttpRequest(fullurl, 'buf', read_callback, undefined, true).then(xhr => { if (file.fAcceptRanges) { @@ -3275,10 +3273,7 @@ class TFile { // if only single segment requested, return result as is if (last - first === 2) { - const b = new DataView(res); - if (place.length === 2) - return resolveFunc(b); - blobs.push(b); + reorder.addBuffer(first, res, 0); return send_new_request(true); } @@ -3288,55 +3283,26 @@ class TFile { view = new DataView(res); if (!ismulti) { - // server may returns simple buffer, which combines all segments together - - const hdr_range = this.getResponseHeader('Content-Range'); - let segm_start = 0, segm_last = -1; - - if (isStr(hdr_range) && hdr_range.indexOf('bytes') >= 0) { - const parts = hdr_range.slice(hdr_range.indexOf('bytes') + 6).split(/[\s-/]+/); - if (parts.length === 3) { - segm_start = Number.parseInt(parts[0]); - segm_last = Number.parseInt(parts[1]); - if (!Number.isInteger(segm_start) || !Number.isInteger(segm_last) || (segm_start > segm_last)) { - segm_start = 0; - segm_last = -1; - } - } - } - - let canbe_single_segment = (segm_start <= segm_last); - for (let n = first; n < last; n += 2) { - if ((place[n] < segm_start) || (place[n] + place[n + 1] - 1 > segm_last)) - canbe_single_segment = false; - } - - if (canbe_single_segment) { - for (let n = first; n < last; n += 2) - blobs.push(new DataView(res, place[n] - segm_start, place[n + 1])); - return send_new_request(true); - } - - if ((file.fMaxRanges === 1) || first) - return rejectFunc(Error('Server returns normal response when multipart was requested, disable multirange support')); - file.fMaxRanges = 1; - last = Math.min(last, file.fMaxRanges * 2); - + last = Math.min(last, first + file.fMaxRanges * 2); return send_new_request(); } // multipart messages requires special handling const indx = hdr.indexOf('boundary='); - let boundary = '', n = first, o = 0, normal_order = true; + let boundary = '', n = first, o = 0; if (indx > 0) { boundary = hdr.slice(indx + 9); if ((boundary[0] === '"') && (boundary.at(-1) === '"')) boundary = boundary.slice(1, boundary.length - 1); boundary = '--' + boundary; - } else + } else { console.error('Did not found boundary id in the response header'); + file.fMaxRanges = 1; + last = Math.min(last, first + file.fMaxRanges * 2); + return send_new_request(); + } while (n < last) { let code1, code2 = view.getUint8(o), nline = 0, line = '', @@ -3357,6 +3323,7 @@ class TFile { if (parts.length === 3) { segm_start = Number.parseInt(parts[0]); segm_last = Number.parseInt(parts[1]); + // TODO: check for consistency if (!Number.isInteger(segm_start) || !Number.isInteger(segm_last) || (segm_start > segm_last)) { segm_start = 0; segm_last = -1; @@ -3379,44 +3346,17 @@ class TFile { o++; } - if (!finish_header) - return rejectFunc(Error('Cannot decode header in multipart message')); - - if (segm_start > segm_last) { - // fall-back solution, believe that segments same as requested - blobs.push(new DataView(res, o, place[n + 1])); - o += place[n + 1]; - n += 2; - } else if (normal_order) { - const n0 = n; - while ((n < last) && (place[n] >= segm_start) && (place[n] + place[n + 1] - 1 <= segm_last)) { - blobs.push(new DataView(res, o + place[n] - segm_start, place[n + 1])); - n += 2; - } - - if (n > n0) - o += (segm_last - segm_start + 1); - else - normal_order = false; + if (!finish_header || (segm_start > segm_last)) { + console.error('Failure decoding multirange header'); + file.fMaxRanges = 1; + last = Math.min(last, first + file.fMaxRanges * 2); + return send_new_request(); } - if (!normal_order) { - // special situation when server reorder segments in the reply - let isany = false; - for (let n1 = n; n1 < last; n1 += 2) { - if ((place[n1] >= segm_start) && (place[n1] + place[n1 + 1] - 1 <= segm_last)) { - blobs[n1 / 2] = new DataView(res, o + place[n1] - segm_start, place[n1 + 1]); - isany = true; - } - } - if (!isany) - return rejectFunc(Error(`Provided fragment ${segm_start} - ${segm_last} out of requested multi-range request`)); - - while (blobs[n / 2]) - n += 2; + reorder.addBuffer(n, res, o); - o += (segm_last - segm_start + 1); - } + n += 2; + o += (segm_last - segm_start + 1); } send_new_request(true); From 6f7ae658fe78ef39f7f75360953e5e9c36b96d12 Mon Sep 17 00:00:00 2001 From: Sergey Linev Date: Mon, 24 Nov 2025 14:51:47 +0100 Subject: [PATCH 4/6] Simplify fall-back to single range request While such fallback appears in several places - implement it in send new request --- modules/io.mjs | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/modules/io.mjs b/modules/io.mjs index 5284ce262..7939eb50f 100644 --- a/modules/io.mjs +++ b/modules/io.mjs @@ -3100,10 +3100,8 @@ class TFile { return this.fFileContent.extract(place); const reorder = this.#checkNeedReorder(place); - if (reorder?.place_new) { - console.log('apply reorder', place, reorder?.place_new); + if (reorder?.place_new) place = reorder?.place_new; - } let resolveFunc, rejectFunc; @@ -3133,8 +3131,11 @@ class TFile { } } - function send_new_request(increment) { - if (increment) { + function send_new_request(arg) { + if (arg === 'noranges') { + file.fMaxRanges = 1; + last = Math.min(last, first + file.fMaxRanges * 2); + } else if (arg) { first = last; last = Math.min(first + file.fMaxRanges * 2, place.length); if (first >= place.length) @@ -3278,19 +3279,16 @@ class TFile { } // object to access response data - const hdr = this.getResponseHeader('Content-Type'), - ismulti = isStr(hdr) && (hdr.indexOf('multipart') >= 0), - view = new DataView(res); + const hdr = this.getResponseHeader('Content-Type'); - if (!ismulti) { - file.fMaxRanges = 1; - last = Math.min(last, first + file.fMaxRanges * 2); - return send_new_request(); + if (!isStr(hdr) || (hdr.indexOf('multipart') < 0)) { + console.error('Did not found multipart in content-type - fallback to single range request'); + return send_new_request('noranges'); } // multipart messages requires special handling - const indx = hdr.indexOf('boundary='); + const indx = hdr.indexOf('boundary='), view = new DataView(res); let boundary = '', n = first, o = 0; if (indx > 0) { boundary = hdr.slice(indx + 9); @@ -3298,10 +3296,8 @@ class TFile { boundary = boundary.slice(1, boundary.length - 1); boundary = '--' + boundary; } else { - console.error('Did not found boundary id in the response header'); - file.fMaxRanges = 1; - last = Math.min(last, first + file.fMaxRanges * 2); - return send_new_request(); + console.error('Did not found boundary id in the response header - fallback to single range request'); + return send_new_request('noranges'); } while (n < last) { @@ -3347,10 +3343,8 @@ class TFile { } if (!finish_header || (segm_start > segm_last)) { - console.error('Failure decoding multirange header'); - file.fMaxRanges = 1; - last = Math.min(last, first + file.fMaxRanges * 2); - return send_new_request(); + console.error('Failure decoding multirange header - fallback to single range request'); + return send_new_request('noranges'); } reorder.addBuffer(n, res, o); From a30a16f63526d1271c1194fb3d6ea87f1bb7e523 Mon Sep 17 00:00:00 2001 From: Sergey Linev Date: Mon, 24 Nov 2025 14:54:49 +0100 Subject: [PATCH 5/6] Improve looping --- modules/io.mjs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/io.mjs b/modules/io.mjs index 7939eb50f..9bd81af75 100644 --- a/modules/io.mjs +++ b/modules/io.mjs @@ -3289,7 +3289,7 @@ class TFile { // multipart messages requires special handling const indx = hdr.indexOf('boundary='), view = new DataView(res); - let boundary = '', n = first, o = 0; + let boundary = ''; if (indx > 0) { boundary = hdr.slice(indx + 9); if ((boundary[0] === '"') && (boundary.at(-1) === '"')) @@ -3300,7 +3300,7 @@ class TFile { return send_new_request('noranges'); } - while (n < last) { + for(let n = first, o = 0; n < last; n += 2) { let code1, code2 = view.getUint8(o), nline = 0, line = '', finish_header = false, segm_start = 0, segm_last = -1; @@ -3349,7 +3349,6 @@ class TFile { reorder.addBuffer(n, res, o); - n += 2; o += (segm_last - segm_start + 1); } From 97f68442ccab5435bfada5b655f9526779863116 Mon Sep 17 00:00:00 2001 From: Sergey Linev Date: Mon, 24 Nov 2025 15:40:40 +0100 Subject: [PATCH 6/6] Check decoded segment size to avoid failures --- modules/io.mjs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/modules/io.mjs b/modules/io.mjs index 9bd81af75..43a0df620 100644 --- a/modules/io.mjs +++ b/modules/io.mjs @@ -3055,9 +3055,16 @@ class TFile { } } if (!res) { - return { place, blobs: [], addBuffer: function(indx, buf, o) { - this.blobs[indx/2] = new DataView(buf, o, this.place[indx + 1]); - }} + return { + place, + blobs: [], + expectedSize: function(indx) { + return this.place[indx + 1]; + }, + addBuffer: function(indx, buf, o) { + this.blobs[indx/2] = new DataView(buf, o, this.place[indx + 1]); + } + }; } res = { place, reorder: [], place_new: [], blobs: [] }; @@ -3081,6 +3088,10 @@ class TFile { res.reorder.forEach(elem => res.place_new.push(elem.pos, elem.len)); + res.expectedSize = function(indx) { + return this.reorder[indx / 2].len; + }; + res.addBuffer = function(indx, buf, o) { const elem = this.reorder[indx / 2], pos0 = elem.pos; @@ -3342,14 +3353,16 @@ class TFile { o++; } - if (!finish_header || (segm_start > segm_last)) { + const segm_size = segm_last - segm_start + 1; + + if (!finish_header || (segm_size <= 0) || (reorder.expectedSize(n) !== segm_size)) { console.error('Failure decoding multirange header - fallback to single range request'); return send_new_request('noranges'); } reorder.addBuffer(n, res, o); - o += (segm_last - segm_start + 1); + o += segm_size; } send_new_request(true);