From 276a594771329da8334984771cb536de7322d5b4 Mon Sep 17 00:00:00 2001 From: dcodeIO Date: Fri, 27 Jan 2017 01:31:45 +0100 Subject: [PATCH] New: Initial attempt on a backwards compatible fetch implementation with binary support, see #661 --- lib/aspromise/.gitignore | 3 -- lib/base64/.gitignore | 3 -- lib/codegen/.gitignore | 3 -- lib/eventemitter/.gitignore | 3 -- lib/fetch/.gitignore | 3 -- lib/fetch/index.js | 95 ++++++++++++++++++++++++++++++------- lib/fetch/package.json | 2 +- lib/fetch/tests/index.js | 2 +- lib/inquire/.gitignore | 3 -- lib/path/.gitignore | 3 -- lib/pool/.gitignore | 3 -- lib/utf8/.gitignore | 3 -- scripts/bundle.js | 5 +- tests/node/lib_fetch.js | 1 + 14 files changed, 83 insertions(+), 49 deletions(-) delete mode 100644 lib/aspromise/.gitignore delete mode 100644 lib/base64/.gitignore delete mode 100644 lib/codegen/.gitignore delete mode 100644 lib/eventemitter/.gitignore delete mode 100644 lib/fetch/.gitignore delete mode 100644 lib/inquire/.gitignore delete mode 100644 lib/path/.gitignore delete mode 100644 lib/pool/.gitignore delete mode 100644 lib/utf8/.gitignore create mode 100644 tests/node/lib_fetch.js diff --git a/lib/aspromise/.gitignore b/lib/aspromise/.gitignore deleted file mode 100644 index c3fc82ecb37d..000000000000 --- a/lib/aspromise/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -npm-debug.* -node_modules/ -coverage/ diff --git a/lib/base64/.gitignore b/lib/base64/.gitignore deleted file mode 100644 index c3fc82ecb37d..000000000000 --- a/lib/base64/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -npm-debug.* -node_modules/ -coverage/ diff --git a/lib/codegen/.gitignore b/lib/codegen/.gitignore deleted file mode 100644 index c3fc82ecb37d..000000000000 --- a/lib/codegen/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -npm-debug.* -node_modules/ -coverage/ diff --git a/lib/eventemitter/.gitignore b/lib/eventemitter/.gitignore deleted file mode 100644 index c3fc82ecb37d..000000000000 --- a/lib/eventemitter/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -npm-debug.* -node_modules/ -coverage/ diff --git a/lib/fetch/.gitignore b/lib/fetch/.gitignore deleted file mode 100644 index c3fc82ecb37d..000000000000 --- a/lib/fetch/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -npm-debug.* -node_modules/ -coverage/ diff --git a/lib/fetch/index.js b/lib/fetch/index.js index e4714d093ffe..7b59e3a8fac4 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -15,38 +15,99 @@ var fs = inquire("fs"); * @returns {undefined} */ +/** + * Options as used by {@link util.fetch}. + * @typedef {Object} + * @property {boolean} [binary=false] Whether expecting a binary response or not + */ + /** * Fetches the contents of a file. * @memberof util - * @param {string} path File path or url - * @param {FetchCallback} [callback] Callback function - * @returns {Promise} A Promise if `callback` has been omitted, otherwise `undefined` - * @property {function(string, FetchCallback=):Promise} xhr XHR/browser fetch with an identical signature + * @param {string} filename File path or url + * @param {FetchOptions} options Fetch options + * @param {FetchCallback} callback Callback function + * @returns {undefined} */ -function fetch(path, callback) { +function fetch(filename, options, callback) { + if (typeof options === "function") { + callback = options; + options = {}; + } else if (!options) + options = {}; + if (!callback) - return asPromise(fetch, this, path); // eslint-disable-line no-invalid-this + return asPromise(fetch, this, filename, options); // eslint-disable-line no-invalid-this + + // if a node-like filesystem is present, try it first but fall back to XHR if nothing is found. if (fs && fs.readFile) - return fs.readFile(path, "utf8", function fetchReadFileCallback(err, contents) { + return fs.readFile(filename, function fetchReadFileCallback(err, contents) { return err && typeof XMLHttpRequest !== "undefined" - ? fetch.xhr(path, callback) - : callback(err, contents); + ? fetch.xhr(filename, options, callback) + : err + ? callback(err) + : callback(null, options.binary ? contents : contents.toString("utf8")); }); - return fetch.xhr(path, callback); + + // use the XHR version otherwise. + return fetch.xhr(filename, options, callback); } -fetch.xhr = function fetch_xhr(path, callback) { +/** + * Fetches the contents of a file. + * @name util.fetch + * @function + * @param {string} path File path or url + * @param {FetchCallback} callback Callback function + * @returns {undefined} + * @variation 2 + */ + +/** + * Fetches the contents of a file. + * @name util.fetch + * @function + * @param {string} path File path or url + * @param {FetchOptions} [options] Fetch options + * @returns {Promise} Promise + * @variation 3 + */ + +/**/ +fetch.xhr = function fetch_xhr(filename, options, callback) { var xhr = new XMLHttpRequest(); xhr.onreadystatechange /* works everywhere */ = function fetchOnReadyStateChange() { - return xhr.readyState === 4 - ? xhr.status === 0 || xhr.status === 200 - ? callback(null, xhr.responseText) - : callback(Error("status " + xhr.status)) - : undefined; + + if (xhr.readyState !== 4) + return undefined; + // local cors security errors return status 0 / empty string, too. afaik this cannot be // reliably distinguished from an actually empty file for security reasons. feel free // to send a pull request if you are aware of a solution. + if (xhr.status !== 0 && xhr.status !== 200) + return callback(Error("status " + xhr.status)); + + // if binary data is expected, make sure that some sort of array is returned, even if + // ArrayBuffers are not supported. the binary string fallback, however, is unsafe. + if (options.binary) { + var buffer = xhr.response; + if (!buffer) { + buffer = []; + for (var i = 0; i < xhr.responseText.length; ++i) + buffer.push(xhr.responseText.charCodeAt(i) & 255); + } + return callback(null, typeof Uint8Array !== "undefined" ? new Uint8Array(buffer) : buffer); + } + return callback(null, xhr.responseText); }; - xhr.open("GET", path); + + if (options.binary) { + // ref: https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/Sending_and_Receiving_Binary_Data#Receiving_binary_data_in_older_browsers + if ("overrideMimeType" in xhr) + xhr.overrideMimeType("text/plain; charset=x-user-defined"); + xhr.responseType = "arraybuffer"; + } + + xhr.open("GET", filename); xhr.send(); }; diff --git a/lib/fetch/package.json b/lib/fetch/package.json index d59b75d92c8e..e01b8c18b31f 100644 --- a/lib/fetch/package.json +++ b/lib/fetch/package.json @@ -1,7 +1,7 @@ { "name": "@protobufjs/fetch", "description": "Fetches the contents of a file accross node and browsers.", - "version": "1.0.4", + "version": "1.0.5", "author": "Daniel Wirtz ", "repository": { "type": "git", diff --git a/lib/fetch/tests/index.js b/lib/fetch/tests/index.js index 5be4add82d12..3f5249833139 100644 --- a/lib/fetch/tests/index.js +++ b/lib/fetch/tests/index.js @@ -10,7 +10,7 @@ tape.test("fetch", function(test) { test.ok(promise instanceof Promise, "should return a promise if callback has been omitted"); } - // TODO + // TODO - some what to test this properly? test.end(); }); diff --git a/lib/inquire/.gitignore b/lib/inquire/.gitignore deleted file mode 100644 index c3fc82ecb37d..000000000000 --- a/lib/inquire/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -npm-debug.* -node_modules/ -coverage/ diff --git a/lib/path/.gitignore b/lib/path/.gitignore deleted file mode 100644 index c3fc82ecb37d..000000000000 --- a/lib/path/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -npm-debug.* -node_modules/ -coverage/ diff --git a/lib/pool/.gitignore b/lib/pool/.gitignore deleted file mode 100644 index c3fc82ecb37d..000000000000 --- a/lib/pool/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -npm-debug.* -node_modules/ -coverage/ diff --git a/lib/utf8/.gitignore b/lib/utf8/.gitignore deleted file mode 100644 index c3fc82ecb37d..000000000000 --- a/lib/utf8/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -npm-debug.* -node_modules/ -coverage/ diff --git a/scripts/bundle.js b/scripts/bundle.js index 41bed2bd749d..02ca30ffddb4 100644 --- a/scripts/bundle.js +++ b/scripts/bundle.js @@ -13,6 +13,8 @@ var buffer = require("vinyl-buffer"); var vinylfs = require("vinyl-fs"); var source = require("vinyl-source-stream"); +var zopfli = require("node-zopfli"); + var pkg = require(__dirname + "/../package.json"); var license = [ @@ -86,9 +88,6 @@ function bundle(options) { .on("error", gutil.log); } -var fs = require("fs"); -var zopfli = require("node-zopfli"); - /** * Compresses a file using zopfli gzip. * @param {string} sourceFile Source file diff --git a/tests/node/lib_fetch.js b/tests/node/lib_fetch.js new file mode 100644 index 000000000000..75fd66d36f3d --- /dev/null +++ b/tests/node/lib_fetch.js @@ -0,0 +1 @@ +require("../../lib/fetch/tests");