From ba47ae7a927ff2927f09652c4f95308bb103f252 Mon Sep 17 00:00:00 2001 From: Nolan Lawson Date: Sun, 24 May 2015 17:02:46 -0400 Subject: [PATCH] (#2832) - implement multipart/related If a document contains non-stub attachments, then upload them as multipart-related instead of application/json. --- lib/adapters/http/http.js | 95 +++++++++++++++++---------- lib/deps/multipart.js | 93 ++++++++++++++++++++++++++ tests/integration/test.attachments.js | 4 +- 3 files changed, 153 insertions(+), 39 deletions(-) create mode 100644 lib/deps/multipart.js diff --git a/lib/adapters/http/http.js b/lib/adapters/http/http.js index 7100c1dd06..c3998bd2fd 100644 --- a/lib/adapters/http/http.js +++ b/lib/adapters/http/http.js @@ -11,6 +11,7 @@ var MAX_URL_LENGTH = 1800; var utils = require('../../utils'); var Promise = utils.Promise; +var clone = utils.clone; var base64 = require('../../deps/base64'); var btoa = base64.btoa; var atob = base64.atob; @@ -18,6 +19,7 @@ var errors = require('../../deps/errors'); var log = require('debug')('pouchdb:http'); var isBrowser = typeof process === 'undefined' || process.browser; var buffer = require('../../deps/buffer'); +var createMultipart = require('../../deps/multipart'); function blobToBase64(blobOrBuffer) { if (!isBrowser) { @@ -83,7 +85,7 @@ function getHost(name, opts) { // except for the database name) with '/'s uri.path = parts.join('/'); opts = opts || {}; - opts = utils.clone(opts); + opts = clone(opts); uri.headers = opts.headers || (opts.ajax && opts.ajax.headers) || {}; if (opts.auth || uri.auth) { @@ -139,12 +141,12 @@ function HttpPouch(opts, callback) { var dbUrl = genDBUrl(host, ''); api.getUrl = function () {return dbUrl; }; - api.getHeaders = function () {return utils.clone(host.headers); }; + api.getHeaders = function () {return clone(host.headers); }; var ajaxOpts = opts.ajax || {}; - opts = utils.clone(opts); + opts = clone(opts); function ajax(options, callback) { - var reqOpts = utils.extend(true, utils.clone(ajaxOpts), options); + var reqOpts = utils.extend(true, clone(ajaxOpts), options); log(reqOpts.method + ' ' + reqOpts.url); return utils.ajax(reqOpts, callback); } @@ -162,11 +164,15 @@ function HttpPouch(opts, callback) { // Create a new CouchDB database based on the given opts var createDB = function () { - ajax({headers: host.headers, method: 'PUT', url: dbUrl}, function (err) { + ajax({ + headers: clone(host.headers), + method: 'PUT', + url: dbUrl + }, function (err) { // If we get an "Unauthorized" error if (err && err.status === 401) { // Test if the database already exists - ajax({headers: host.headers, method: 'HEAD', url: dbUrl}, + ajax({headers: clone(host.headers), method: 'HEAD', url: dbUrl}, function (err) { // If there is still an error if (err) { @@ -190,7 +196,11 @@ function HttpPouch(opts, callback) { }; if (!opts.skipSetup) { - ajax({headers: host.headers, method: 'GET', url: dbUrl}, function (err) { + ajax({ + headers: clone(host.headers), + method: 'GET', + url: dbUrl + }, function (err) { //check if the db exists if (err) { if (err.status === 404) { @@ -214,7 +224,7 @@ function HttpPouch(opts, callback) { api.id = utils.adapterFun('id', function (callback) { ajax({ - headers: host.headers, + headers: clone(host.headers), method: 'GET', url: genUrl(host, '') }, function (err, result) { @@ -240,9 +250,9 @@ function HttpPouch(opts, callback) { callback = opts; opts = {}; } - opts = utils.clone(opts); + opts = clone(opts); ajax({ - headers: host.headers, + headers: clone(host.headers), url: genDBUrl(host, '_compact'), method: 'POST' }, function () { @@ -267,7 +277,7 @@ function HttpPouch(opts, callback) { // version: The version of CouchDB it is running api._info = function (callback) { ajax({ - headers: host.headers, + headers: clone(host.headers), method: 'GET', url: genDBUrl(host, '') }, function (err, res) { @@ -289,7 +299,7 @@ function HttpPouch(opts, callback) { callback = opts; opts = {}; } - opts = utils.clone(opts); + opts = clone(opts); // List of parameters to add to the GET request var params = []; @@ -344,7 +354,7 @@ function HttpPouch(opts, callback) { // Set the options for the ajax call var options = { - headers: host.headers, + headers: clone(host.headers), method: 'GET', url: genDBUrl(host, id + params) }; @@ -366,7 +376,7 @@ function HttpPouch(opts, callback) { var path = encodeDocId(doc._id) + '/' + encodeAttachmentId(filename) + '?rev=' + doc._rev; return ajaxPromise({ - headers: host.headers, + headers: clone(host.headers), method: 'GET', url: genDBUrl(host, path), binary: true @@ -430,7 +440,7 @@ function HttpPouch(opts, callback) { // Delete the document ajax({ - headers: host.headers, + headers: clone(host.headers), method: 'DELETE', url: genDBUrl(host, encodeDocId(doc._id)) + '?rev=' + rev }, callback); @@ -452,7 +462,7 @@ function HttpPouch(opts, callback) { var url = genDBUrl(host, encodeDocId(docId)) + '/' + encodeAttachmentId(attachmentId) + params; ajax({ - headers: host.headers, + headers: clone(host.headers), method: 'GET', url: url, binary: true @@ -468,7 +478,7 @@ function HttpPouch(opts, callback) { encodeAttachmentId(attachmentId)) + '?rev=' + rev; ajax({ - headers: host.headers, + headers: clone(host.headers), method: 'DELETE', url: url }, callback); @@ -514,7 +524,7 @@ function HttpPouch(opts, callback) { } var opts = { - headers: utils.clone(host.headers), + headers: clone(host.headers), method: 'PUT', url: url, processData: false, @@ -537,7 +547,7 @@ function HttpPouch(opts, callback) { return callback(errors.error(errors.NOT_AN_OBJECT)); } - doc = utils.clone(doc); + doc = clone(doc); preprocessAttachments(doc).then(function () { while (true) { @@ -549,7 +559,7 @@ function HttpPouch(opts, callback) { } else if (temptype === "string" && id && !('_rev' in doc)) { doc._rev = temp; } else if (temptype === "object") { - opts = utils.clone(temp); + opts = clone(temp); } if (!args.length) { break; @@ -577,21 +587,34 @@ function HttpPouch(opts, callback) { params = '?' + params; } - // Add the document - ajax({ - headers: host.headers, + var ajaxOpts = { + headers: clone(host.headers), method: 'PUT', url: genDBUrl(host, encodeDocId(doc._id)) + params, body: doc - }, function (err, res) { - if (err) { - return callback(err); + }; + + return Promise.resolve().then(function () { + var hasNonStubAttachments = doc._attachments && + Object.keys(doc._attachments).filter(function (att) { + return !doc._attachments[att].stub; + }).length; + if (hasNonStubAttachments) { + // use multipart/related for more efficient attachment uploading + var multipart = createMultipart(doc); + ajaxOpts.body = multipart.body; + ajaxOpts.processData = false; + ajaxOpts.headers = utils.extend(ajaxOpts.headers, multipart.headers); } - res.ok = true; + }).catch(function () { + throw new Error('Did you forget to base64-encode an attachment?'); + }).then(function () { + return ajaxPromise(ajaxOpts); + }).then(function (res) { + res.ok = true; // smooths out cloudant not doing this callback(null, res); }); }).catch(callback); - })); // Add the document given by doc (in JSON string format) to the database @@ -603,7 +626,7 @@ function HttpPouch(opts, callback) { callback = opts; opts = {}; } - opts = utils.clone(opts); + opts = clone(opts); if (typeof doc !== 'object') { return callback(errors.error(errors.NOT_AN_OBJECT)); } @@ -634,7 +657,7 @@ function HttpPouch(opts, callback) { Promise.all(req.docs.map(preprocessAttachments)).then(function () { // Update/create the documents ajax({ - headers: host.headers, + headers: clone(host.headers), method: 'POST', url: genDBUrl(host, '_bulk_docs'), body: req @@ -657,7 +680,7 @@ function HttpPouch(opts, callback) { callback = opts; opts = {}; } - opts = utils.clone(opts); + opts = clone(opts); // List of parameters to add to the GET request var params = []; var body; @@ -744,7 +767,7 @@ function HttpPouch(opts, callback) { // Get the document listing ajax({ - headers: host.headers, + headers: clone(host.headers), method: method, url: genDBUrl(host, '_all_docs' + params), body: body @@ -761,7 +784,7 @@ function HttpPouch(opts, callback) { // set of changes to return and attempting to process them at once var batchSize = 'batch_size' in opts ? opts.batch_size : CHANGES_BATCH_SIZE; - opts = utils.clone(opts); + opts = clone(opts); opts.timeout = opts.timeout || 30 * 1000; // We give a 5 second buffer for CouchDB changes to respond with @@ -875,7 +898,7 @@ function HttpPouch(opts, callback) { // Set the options for the ajax call var xhrOpts = { - headers: host.headers, + headers: clone(host.headers), method: method, url: genDBUrl(host, '_changes' + paramStr), // _changes can take a long time to generate, especially when filtered @@ -1047,7 +1070,7 @@ function HttpPouch(opts, callback) { // Get the missing document/revision IDs ajax({ - headers: host.headers, + headers: clone(host.headers), method: 'POST', url: genDBUrl(host, '_revs_diff'), body: JSON.stringify(req) @@ -1062,7 +1085,7 @@ function HttpPouch(opts, callback) { ajax({ url: genDBUrl(host, ''), method: 'DELETE', - headers: host.headers + headers: clone(host.headers) }, function (err, resp) { if (err) { api.emit('error', err); diff --git a/lib/deps/multipart.js b/lib/deps/multipart.js new file mode 100644 index 0000000000..a00e47dae9 --- /dev/null +++ b/lib/deps/multipart.js @@ -0,0 +1,93 @@ +'use strict'; + +// Create a multipart/related stream out of a document, +// so we can upload documents in that format when +// attachments are large. This is shamefully stolen from +// https://github.com/sballesteros/couch-multipart-stream/ +// and https://github.com/npm/npm-fullfat-registry + +var base64 = require('./base64'); +var uuid = require('./uuid'); +var utils = require('../utils'); +var isBrowser = typeof process === 'undefined' || process.browser; +var buffer = require('./buffer'); + +function BinaryBuilder() { + this.parts = []; +} +BinaryBuilder.prototype.append = function add(part) { + this.parts.push(part); + return this; +}; +BinaryBuilder.prototype.build = function build(type) { + if (isBrowser) { + return utils.createBlob(this.parts, {type: type}); + } + return buffer.concat(this.parts.map(function (part) { + return new buffer(part, 'binary'); + })); +}; + +module.exports = function createMultipart(doc) { + doc = utils.clone(doc); + + var boundary = uuid(); + + var attachments = {}; + + // according to npm/npm-fullfat-registry, these need to be sorted + // or else CouchDB has a fit + var filenames = Object.keys(doc._attachments).sort(); + filenames.forEach(function (filename) { + var att = doc._attachments[filename]; + if (att.stub) { + return; + } + var binData = base64.atob(att.data); + attachments[filename] = {type: att.content_type, data: binData}; + att.length = binData.length; + att.follows = true; + delete att.data; + }); + + var preamble = '--' + boundary + + '\r\nContent-Type: application/json\r\n\r\n'; + + var origAttachments = doc._attachments; + doc._attachments = {}; + var origKeys = Object.keys(origAttachments).sort(); + origKeys.forEach(function (key) { + doc._attachments[key] = origAttachments[key]; + }); + + var docJson = JSON.stringify(doc); + + var body = new BinaryBuilder() + .append(preamble) + .append(docJson); + + var attsToEncode = Object.keys(attachments).sort(); + attsToEncode.forEach(function (filename) { + var att = attachments[filename]; + var binString = att.data; + var type = att.type; + var preamble = '\r\n--' + boundary + + '\r\nContent-Disposition: attachment; filename=' + + JSON.stringify(filename) + '' + + '\r\nContent-Type: ' + type + + '\r\nContent-Length: ' + binString.length + + '\r\n\r\n'; + body.append(preamble).append(utils.fixBinary(binString)); + }); + + var ending = '\r\n--' + boundary + '--'; + body.append(ending); + + var type = 'multipart/related; boundary=' + boundary; + return { + headers: { + 'Content-Type': type + }, + body: body.build(type) + }; +}; \ No newline at end of file diff --git a/tests/integration/test.attachments.js b/tests/integration/test.attachments.js index a228c27737..49d5bc392a 100644 --- a/tests/integration/test.attachments.js +++ b/tests/integration/test.attachments.js @@ -1388,10 +1388,8 @@ adapters.forEach(function (adapter) { } } }; - db.put(doc, function (err, res) { + db.put(doc, function (err) { should.exist(err); - err.status.should.equal(500, 'correct error'); - err.name.should.equal('badarg', 'correct error'); done(); }); });