Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Full implementation of xhr.responseType, rigorous test and a perf imp… #16

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Full implementation of xhr.responseType, rigorous test and a perf imp… #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d3af58a
Full implementation of xhr.responseType, rigorous test and a perf imp…
Oct 9, 2023
5733915
Minor typos...
Oct 9, 2023
1a5edb1
Modified test-utf8-tearing.js to incorporate xhr.responseType changes…
Oct 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
274 changes: 244 additions & 30 deletions lib/XMLHttpRequest.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,17 @@ function XMLHttpRequest(opts) {
this.status = null;
this.statusText = null;

// xhr.responseType is supported:
// When responseType is 'text' or '', self.responseText will be utf8 decoded text.
// When responseType is 'json', self.responseText initially will be utf8 decoded text,
// which is then JSON parsed into self.response.
// When responseType is 'document', self.responseText initially will be utf8 decoded text,
// which is then parsed by npmjs package '@xmldom/xmldom' into a DOM object self.responseXML.
// When responseType is 'arraybuffer', self.response is an ArrayBuffer.
// When responseType is 'blob', self.response is a Blob.
// cf. section 3.6, subsections 8,9,10,11 of https://xhr.spec.whatwg.org/#the-response-attribute
this.responseType = ""; /* 'arraybuffer' or 'text' or '' or 'json' or 'blob' or 'document' */

/**
* Private methods
*/
Expand All @@ -158,6 +169,89 @@ function XMLHttpRequest(opts) {
return (method && forbiddenRequestMethods.indexOf(method) === -1);
};

/**
* When xhr.responseType === 'arraybuffer', xhr.response must have type ArrayBuffer according
* to section 3.6.9 of https://xhr.spec.whatwg.org/#the-response-attribute .
* However, bufTotal = Buffer.concat(...) often has byteOffset > 0, so bufTotal.buffer is larger
* than the useable region in bufTotal. This means that a new copy of bufTotal would need to be
* created to get the correct ArrayBuffer. Instead, do the concat by hand to create the right
* sized ArrayBuffer in the first place.
*
* The return type is Uint8Array,
* because often Buffer will have Buffer.length < Buffer.buffer.byteLength.
*
* @param {Array<Buffer>} bufferArray
* @returns {Uint8Array}
*/
var concat = function(bufferArray) {
let length = 0, offset = 0;
for (let k = 0; k < bufferArray.length; k++)
length += bufferArray[k].length;
const result = new Uint8Array(length);
for (let k = 0; k < bufferArray.length; k++)
{
result.set(bufferArray[k], offset);
offset += bufferArray[k].length;
}
return result;
};

/**
* When xhr.responseType === 'arraybuffer', xhr.response must have type ArrayBuffer according
* to section 3.6.9 of https://xhr.spec.whatwg.org/#the-response-attribute .
* However, buf = Buffer.from(str) often has byteOffset > 0, so buf.buffer is larger than the
* usable region in buf. This means that a new copy of buf would need to be created to get the
* correct arrayBuffer. Instead, do it by hand to create the right sized ArrayBuffer in the
* first place.
*
* @param {string} str
* @returns {Buffer}
*/
var stringToBuffer = function(str) {
const ab = new ArrayBuffer(str.length)
const buf = Buffer.from(ab);
for (let k = 0; k < str.length; k++)
buf[k] = Number(str.charCodeAt(k));
return buf;
}

/**
* Given a Buffer buf, check whether buf.buffer.byteLength > buf.length and if so,
* create a new ArrayBuffer whose byteLength is buf.length, containing the bytes.
* of buf. This function shouldn't usually be needed, unless there's a future
* behavior change where buf.buffer.byteLength > buf.length unexpectedly.
*
* @param {Buffer} buf
* @returns {ArrayBuffer}
*/
var checkAndShrinkBuffer = function(buf) {
if (buf.length === buf.buffer.byteLength)
return buf.buffer;
const ab = new ArrayBuffer(buf.length);
const result = Buffer.from(ab);
for (let k = 0; k < buf.length; k++)
result[k] = buf[k];
return ab;
}

/**
* Parse HTML string and return a DOM.
* When self.responseType is 'document', users are reuired to install the npmjs module '@xmldom/xmldom'.
* cf. 3.6.11 of https://xhr.spec.whatwg.org/#the-response-attribute .
*
* @param {string} - HTML to be parsed.
* @return {string} - DOM representation.
*/
var html2dom = function(html) {
const DOMParser = require('@xmldom/xmldom').DOMParser;
try {
return new DOMParser().parseFromString(html, 'text/xml');
} catch(err) {
/** @todo throw appropriate DOMException. */
}
return '';
};

/**
* Public methods
*/
Expand Down Expand Up @@ -328,16 +422,17 @@ function XMLHttpRequest(opts) {
self.handleError(error, error.errno || -1);
} else {
self.status = 200;
self.responseText = data.toString('utf8');
self.response = data;
// Use self.responseType to create the correct self.responseType, self.response, self.XMLDocument.
self.createFileOrSyncResponse(data);
setState(self.DONE);
}
});
} else {
try {
this.response = fs.readFileSync(unescape(url.pathname));
this.responseText = this.response.toString('utf8');
this.status = 200;
const syncData = fs.readFileSync(unescape(url.pathname));
// Use self.responseType to create the correct self.responseType, self.response, self.XMLDocument.
this.createFileOrSyncResponse(syncData);
setState(self.DONE);
} catch(e) {
this.handleError(e, e.errno || -1);
Expand Down Expand Up @@ -421,6 +516,8 @@ function XMLHttpRequest(opts) {
// Set response var to the response we got back
// This is so it remains accessable outside this scope
response = resp;
// Collect buffers and concatenate once.
const buffers = [];
// Check for redirect
// @TODO Prevent looped redirects
if (response.statusCode === 302 || response.statusCode === 303 || response.statusCode === 307) {
Expand Down Expand Up @@ -456,13 +553,34 @@ function XMLHttpRequest(opts) {
}

setState(self.HEADERS_RECEIVED);

// When responseType is 'text' or '', self.responseText will be utf8 decoded text.
// When responseType is 'json', self.responseText initially will be utf8 decoded text,
// which is then JSON parsed into self.response.
// When responseType is 'document', self.responseText initially will be utf8 decoded text,
// which is then parsed by npmjs package '@xmldom/xmldom'into a DOM object self.responseXML.
// When responseType is 'arraybuffer', self.response is an ArrayBuffer.
// When responseType is 'blob', self.response is a Blob.
// cf. section 3.6, subsections 8,9,10,11 of https://xhr.spec.whatwg.org/#the-response-attribute
const isUtf8 = self.responseType === "" || self.responseType === "text"
|| self.responseType === "json" || self.responseType === "document";
if (isUtf8 && response.setEncoding) {
response.setEncoding("utf8");
}

self.status = response.statusCode;

response.on('data', function(chunk) {
// Make sure there's some data
if (chunk) {
var data = Buffer.from(chunk);
self.response = Buffer.concat([self.response, data]);
if (isUtf8) {
// When responseType is 'text', '', 'json' or 'document',
// then each chunk is already utf8 decoded.
self.responseText += chunk;
} else {
// Otherwise collect the chunk buffers.
buffers.push(chunk);
}
}
// Don't emit state changes if the connection has been aborted.
if (sendFlag) {
Expand All @@ -475,10 +593,10 @@ function XMLHttpRequest(opts) {
// The sendFlag needs to be set before setState is called. Otherwise if we are chaining callbacks
// there can be a timing issue (the callback is called and a new call is made before the flag is reset).
sendFlag = false;
// Create the correct response for responseType.
self.createResponse(buffers);
// Discard the 'end' event if the connection has been aborted
setState(self.DONE);
// Construct responseText from response
self.responseText = self.response.toString('utf8');
}
});

Expand Down Expand Up @@ -516,33 +634,43 @@ function XMLHttpRequest(opts) {
fs.writeFileSync(syncFile, "", "utf8");
// The async request the other Node process executes
var execString = "var http = require('http'), https = require('https'), fs = require('fs');"
+ "function concat(bufferArray) {"
+ " let length = 0, offset = 0;"
+ " for (let k = 0; k < bufferArray.length; k++)"
+ " length += bufferArray[k].length;"
+ " const result = Buffer.alloc(length);"
+ " for (let k = 0; k < bufferArray.length; k++)"
+ " {"
+ " result.set(bufferArray[k], offset);"
+ " offset += bufferArray[k].length;"
+ " }"
+ " return result;"
+ "};"
+ "var doRequest = http" + (ssl ? "s" : "") + ".request;"
+ "var options = " + JSON.stringify(options) + ";"
+ "var responseText = '';"
+ "var responseData = Buffer.alloc(0);"
+ "var buffers = [];"
+ "var req = doRequest(options, function(response) {"
+ "response.on('data', function(chunk) {"
+ " var data = Buffer.from(chunk);"
+ " responseText += data.toString('utf8');"
+ " responseData = Buffer.concat([responseData, data]);"
+ "});"
+ "response.on('end', function() {"
+ "fs.writeFileSync('" + contentFile + "', JSON.stringify({err: null, data: {statusCode: response.statusCode, headers: response.headers, text: responseText, data: responseData.toString('base64')}}), 'utf8');"
+ "fs.unlinkSync('" + syncFile + "');"
+ "});"
+ "response.on('error', function(error) {"
+ "fs.writeFileSync('" + contentFile + "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');"
+ "fs.unlinkSync('" + syncFile + "');"
+ "});"
+ " response.on('data', function(chunk) {"
+ " buffers.push(chunk);"
+ " });"
+ " response.on('end', function() {"
+ " responseData = concat(buffers);"
+ " fs.writeFileSync('" + contentFile + "', JSON.stringify({err: null, data: {statusCode: response.statusCode, headers: response.headers, data: responseData.toString('utf8')}}), 'utf8');"
+ " fs.unlinkSync('" + syncFile + "');"
+ " });"
+ " response.on('error', function(error) {"
+ " fs.writeFileSync('" + contentFile + "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');"
+ " fs.unlinkSync('" + syncFile + "');"
+ " });"
+ "}).on('error', function(error) {"
+ "fs.writeFileSync('" + contentFile + "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');"
+ "fs.unlinkSync('" + syncFile + "');"
+ " fs.writeFileSync('" + contentFile + "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');"
+ " fs.unlinkSync('" + syncFile + "');"
+ "});"
+ (data ? "req.write('" + JSON.stringify(data).slice(1,-1).replace(/'/g, "\\'") + "');":"")
+ "req.end();";
// Start the other Node Process, executing this string
var syncProc = spawn(process.argv[0], ["-e", execString]);
var statusText;
while(fs.existsSync(syncFile)) {
// Wait while the sync file is empty
}
Expand All @@ -557,16 +685,19 @@ function XMLHttpRequest(opts) {
self.handleError(errorObj, 503);
} else {
// If the file returned okay, parse its data and move to the DONE state
self.status = self.responseText.replace(/^NODE-XMLHTTPREQUEST-STATUS:([0-9]*),.*/, "$1");
var resp = JSON.parse(self.responseText.replace(/^NODE-XMLHTTPREQUEST-STATUS:[0-9]*,(.*)/, "$1"));
const resp = JSON.parse(self.responseText);
self.status = resp.data.statusCode;
self.response = stringToBuffer(resp.data.data);
// Use self.responseType to create the correct self.responseType, self.response, self.responseXML.
self.createFileOrSyncResponse(self.response);
// Set up response correctly.
response = {
statusCode: self.status,
headers: resp.data.headers
};
self.responseText = resp.data.text;
self.response = Buffer.from(resp.data.data, 'base64');
setState(self.DONE, true);
setState(self.DONE);
}

}
};

Expand All @@ -578,6 +709,8 @@ function XMLHttpRequest(opts) {
this.status = status || 0;
this.statusText = error;
this.responseText = error.stack;
this.responseXML = "";
this.response = Buffer.alloc(0);
errorFlag = true;
setState(this.DONE);
};
Expand Down Expand Up @@ -650,6 +783,87 @@ function XMLHttpRequest(opts) {
}
};

/**
* Construct the correct form of response, given responseType when in non-file based, asynchronous mode.
*
* When self.responseType is "", "text", "json", "document", self.responseText is a utf8 string.
* When self.responseType is "arraybuffer", "blob", the response is in the buffers parameter,
* an Array of Buffers. Then concat(buffers) is Uint8Array, from which checkAndShrinkBuffer
* extracts the correct sized ArrayBuffer.
*
* @param {Array<Buffer>} buffers
*/
this.createResponse = function(buffers) {
self.responseXML = '';
switch (self.responseType) {
case "":
case "text":
self.response = Buffer.alloc(0);
break;
case 'json':
self.response = JSON.parse(self.responseText);
self.responseText = '';
break;
case "document":
const xml = JSON.parse(self.responseText);
self.responseXML = html2dom(xml);
self.responseText = '';
self.response = Buffer.alloc(0);
break;
default:
self.responseText = '';
const totalResponse = concat(buffers);
// When self.responseType === 'arraybuffer', self.response is an ArrayBuffer.
// Get the correct sized ArrayBuffer.
self.response = checkAndShrinkBuffer(totalResponse);
if (self.responseType === 'blob' && typeof Blob === 'function') {
// Construct the Blob object that contains response.
self.response = new Blob([self.response]);
}
break;
}
}

/**
* Construct the correct form of response, given responseType when in synchronous mode or file based.
*
* The input is the response parameter which is a Buffer.
* When self.responseType is "", "text", "json", "document",
* the input is further refined to be: response.toString('utf8').
* When self.responseType is "arraybuffer", "blob",
* the input is further refined to be: checkAndShrinkBuffer(response).
*
* @param {Buffer} response
*/
this.createFileOrSyncResponse = function(response) {
self.responseText = '';
self.responseXML = '';
switch (self.responseType) {
case "":
case "text":
self.responseText = response.toString('utf8');
self.response = Buffer.alloc(0);
break;
case 'json':
self.response = JSON.parse(response.toString('utf8'));
break;
case "document":
const xml = JSON.parse(response.toString('utf8'));
self.responseXML = html2dom(xml);
self.response = Buffer.alloc(0);
break;
default:
// When self.responseType === 'arraybuffer', self.response is an ArrayBuffer.
// Get the correct sized ArrayBuffer.
self.response = checkAndShrinkBuffer(response);
if (self.responseType === 'blob' && typeof Blob === 'function') {
// Construct the Blob object that contains response.
self.response = new Blob([self.response]);
}
break;
}
}

/**
* Changes readyState and calls onreadystatechange.
*
Expand Down
11 changes: 10 additions & 1 deletion tests/server.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@

var http = require("http");

var server = http.createServer(function (req, res) {
Expand All @@ -14,10 +15,18 @@ var server = http.createServer(function (req, res) {
return res
.writeHead(200, {"Content-Type": "application/json"})
.end(JSON.stringify({ foo: "bar" }));
case "/binary":
case "/binary1":
return res
.writeHead(200, {"Content-Type": "application/octet-stream"})
.end(Buffer.from("Hello world!"));
case "/binary2": {
const ta = new Float32Array([1, 5, 6, 7]);
const buf = Buffer.from(ta.buffer);
const str = buf.toString('binary');
return res
.writeHead(200, {"Content-Type": "application/octet-stream"})
.end(str);
}
default:
return res
.writeHead(404, {"Content-Type": "text/plain"})
Expand Down