Skip to content

Commit

Permalink
(#2832) - implement multipart/related
Browse files Browse the repository at this point in the history
If a document contains non-stub attachments,
then upload them as multipart-related instead
of application/json.
  • Loading branch information
nolanlawson committed May 24, 2015
1 parent bc2cef2 commit ba47ae7
Show file tree
Hide file tree
Showing 3 changed files with 153 additions and 39 deletions.
95 changes: 59 additions & 36 deletions lib/adapters/http/http.js
Expand Up @@ -11,13 +11,15 @@ var MAX_URL_LENGTH = 1800;

var utils = require('../../utils');
var Promise = utils.Promise;
var clone = utils.clone;
var base64 = require('../../deps/base64');
var btoa = base64.btoa;
var atob = base64.atob;
var errors = require('../../deps/errors');
var log = require('debug')('pouchdb:http');
var isBrowser = typeof process === 'undefined' || process.browser;
var buffer = require('../../deps/buffer');
var createMultipart = require('../../deps/multipart');

function blobToBase64(blobOrBuffer) {
if (!isBrowser) {
Expand Down Expand Up @@ -83,7 +85,7 @@ function getHost(name, opts) {
// except for the database name) with '/'s
uri.path = parts.join('/');
opts = opts || {};
opts = utils.clone(opts);
opts = clone(opts);
uri.headers = opts.headers || (opts.ajax && opts.ajax.headers) || {};

if (opts.auth || uri.auth) {
Expand Down Expand Up @@ -139,12 +141,12 @@ function HttpPouch(opts, callback) {
var dbUrl = genDBUrl(host, '');

api.getUrl = function () {return dbUrl; };
api.getHeaders = function () {return utils.clone(host.headers); };
api.getHeaders = function () {return clone(host.headers); };

var ajaxOpts = opts.ajax || {};
opts = utils.clone(opts);
opts = clone(opts);
function ajax(options, callback) {
var reqOpts = utils.extend(true, utils.clone(ajaxOpts), options);
var reqOpts = utils.extend(true, clone(ajaxOpts), options);
log(reqOpts.method + ' ' + reqOpts.url);
return utils.ajax(reqOpts, callback);
}
Expand All @@ -162,11 +164,15 @@ function HttpPouch(opts, callback) {

// Create a new CouchDB database based on the given opts
var createDB = function () {
ajax({headers: host.headers, method: 'PUT', url: dbUrl}, function (err) {
ajax({
headers: clone(host.headers),
method: 'PUT',
url: dbUrl
}, function (err) {
// If we get an "Unauthorized" error
if (err && err.status === 401) {
// Test if the database already exists
ajax({headers: host.headers, method: 'HEAD', url: dbUrl},
ajax({headers: clone(host.headers), method: 'HEAD', url: dbUrl},
function (err) {
// If there is still an error
if (err) {
Expand All @@ -190,7 +196,11 @@ function HttpPouch(opts, callback) {
};

if (!opts.skipSetup) {
ajax({headers: host.headers, method: 'GET', url: dbUrl}, function (err) {
ajax({
headers: clone(host.headers),
method: 'GET',
url: dbUrl
}, function (err) {
//check if the db exists
if (err) {
if (err.status === 404) {
Expand All @@ -214,7 +224,7 @@ function HttpPouch(opts, callback) {

api.id = utils.adapterFun('id', function (callback) {
ajax({
headers: host.headers,
headers: clone(host.headers),
method: 'GET',
url: genUrl(host, '')
}, function (err, result) {
Expand All @@ -240,9 +250,9 @@ function HttpPouch(opts, callback) {
callback = opts;
opts = {};
}
opts = utils.clone(opts);
opts = clone(opts);
ajax({
headers: host.headers,
headers: clone(host.headers),
url: genDBUrl(host, '_compact'),
method: 'POST'
}, function () {
Expand All @@ -267,7 +277,7 @@ function HttpPouch(opts, callback) {
// version: The version of CouchDB it is running
api._info = function (callback) {
ajax({
headers: host.headers,
headers: clone(host.headers),
method: 'GET',
url: genDBUrl(host, '')
}, function (err, res) {
Expand All @@ -289,7 +299,7 @@ function HttpPouch(opts, callback) {
callback = opts;
opts = {};
}
opts = utils.clone(opts);
opts = clone(opts);

// List of parameters to add to the GET request
var params = [];
Expand Down Expand Up @@ -344,7 +354,7 @@ function HttpPouch(opts, callback) {

// Set the options for the ajax call
var options = {
headers: host.headers,
headers: clone(host.headers),
method: 'GET',
url: genDBUrl(host, id + params)
};
Expand All @@ -366,7 +376,7 @@ function HttpPouch(opts, callback) {
var path = encodeDocId(doc._id) + '/' + encodeAttachmentId(filename) +
'?rev=' + doc._rev;
return ajaxPromise({
headers: host.headers,
headers: clone(host.headers),
method: 'GET',
url: genDBUrl(host, path),
binary: true
Expand Down Expand Up @@ -430,7 +440,7 @@ function HttpPouch(opts, callback) {

// Delete the document
ajax({
headers: host.headers,
headers: clone(host.headers),
method: 'DELETE',
url: genDBUrl(host, encodeDocId(doc._id)) + '?rev=' + rev
}, callback);
Expand All @@ -452,7 +462,7 @@ function HttpPouch(opts, callback) {
var url = genDBUrl(host, encodeDocId(docId)) + '/' +
encodeAttachmentId(attachmentId) + params;
ajax({
headers: host.headers,
headers: clone(host.headers),
method: 'GET',
url: url,
binary: true
Expand All @@ -468,7 +478,7 @@ function HttpPouch(opts, callback) {
encodeAttachmentId(attachmentId)) + '?rev=' + rev;

ajax({
headers: host.headers,
headers: clone(host.headers),
method: 'DELETE',
url: url
}, callback);
Expand Down Expand Up @@ -514,7 +524,7 @@ function HttpPouch(opts, callback) {
}

var opts = {
headers: utils.clone(host.headers),
headers: clone(host.headers),
method: 'PUT',
url: url,
processData: false,
Expand All @@ -537,7 +547,7 @@ function HttpPouch(opts, callback) {
return callback(errors.error(errors.NOT_AN_OBJECT));
}

doc = utils.clone(doc);
doc = clone(doc);

preprocessAttachments(doc).then(function () {
while (true) {
Expand All @@ -549,7 +559,7 @@ function HttpPouch(opts, callback) {
} else if (temptype === "string" && id && !('_rev' in doc)) {
doc._rev = temp;
} else if (temptype === "object") {
opts = utils.clone(temp);
opts = clone(temp);
}
if (!args.length) {
break;
Expand Down Expand Up @@ -577,21 +587,34 @@ function HttpPouch(opts, callback) {
params = '?' + params;
}

// Add the document
ajax({
headers: host.headers,
var ajaxOpts = {
headers: clone(host.headers),
method: 'PUT',
url: genDBUrl(host, encodeDocId(doc._id)) + params,
body: doc
}, function (err, res) {
if (err) {
return callback(err);
};

return Promise.resolve().then(function () {
var hasNonStubAttachments = doc._attachments &&
Object.keys(doc._attachments).filter(function (att) {
return !doc._attachments[att].stub;
}).length;
if (hasNonStubAttachments) {
// use multipart/related for more efficient attachment uploading
var multipart = createMultipart(doc);
ajaxOpts.body = multipart.body;
ajaxOpts.processData = false;
ajaxOpts.headers = utils.extend(ajaxOpts.headers, multipart.headers);
}
res.ok = true;
}).catch(function () {
throw new Error('Did you forget to base64-encode an attachment?');
}).then(function () {
return ajaxPromise(ajaxOpts);
}).then(function (res) {
res.ok = true; // smooths out cloudant not doing this
callback(null, res);
});
}).catch(callback);

}));

// Add the document given by doc (in JSON string format) to the database
Expand All @@ -603,7 +626,7 @@ function HttpPouch(opts, callback) {
callback = opts;
opts = {};
}
opts = utils.clone(opts);
opts = clone(opts);
if (typeof doc !== 'object') {
return callback(errors.error(errors.NOT_AN_OBJECT));
}
Expand Down Expand Up @@ -634,7 +657,7 @@ function HttpPouch(opts, callback) {
Promise.all(req.docs.map(preprocessAttachments)).then(function () {
// Update/create the documents
ajax({
headers: host.headers,
headers: clone(host.headers),
method: 'POST',
url: genDBUrl(host, '_bulk_docs'),
body: req
Expand All @@ -657,7 +680,7 @@ function HttpPouch(opts, callback) {
callback = opts;
opts = {};
}
opts = utils.clone(opts);
opts = clone(opts);
// List of parameters to add to the GET request
var params = [];
var body;
Expand Down Expand Up @@ -744,7 +767,7 @@ function HttpPouch(opts, callback) {

// Get the document listing
ajax({
headers: host.headers,
headers: clone(host.headers),
method: method,
url: genDBUrl(host, '_all_docs' + params),
body: body
Expand All @@ -761,7 +784,7 @@ function HttpPouch(opts, callback) {
// set of changes to return and attempting to process them at once
var batchSize = 'batch_size' in opts ? opts.batch_size : CHANGES_BATCH_SIZE;

opts = utils.clone(opts);
opts = clone(opts);
opts.timeout = opts.timeout || 30 * 1000;

// We give a 5 second buffer for CouchDB changes to respond with
Expand Down Expand Up @@ -875,7 +898,7 @@ function HttpPouch(opts, callback) {

// Set the options for the ajax call
var xhrOpts = {
headers: host.headers,
headers: clone(host.headers),
method: method,
url: genDBUrl(host, '_changes' + paramStr),
// _changes can take a long time to generate, especially when filtered
Expand Down Expand Up @@ -1047,7 +1070,7 @@ function HttpPouch(opts, callback) {

// Get the missing document/revision IDs
ajax({
headers: host.headers,
headers: clone(host.headers),
method: 'POST',
url: genDBUrl(host, '_revs_diff'),
body: JSON.stringify(req)
Expand All @@ -1062,7 +1085,7 @@ function HttpPouch(opts, callback) {
ajax({
url: genDBUrl(host, ''),
method: 'DELETE',
headers: host.headers
headers: clone(host.headers)
}, function (err, resp) {
if (err) {
api.emit('error', err);
Expand Down
93 changes: 93 additions & 0 deletions lib/deps/multipart.js
@@ -0,0 +1,93 @@
'use strict';

// Create a multipart/related stream out of a document,
// so we can upload documents in that format when
// attachments are large. This is shamefully stolen from
// https://github.com/sballesteros/couch-multipart-stream/
// and https://github.com/npm/npm-fullfat-registry

var base64 = require('./base64');
var uuid = require('./uuid');
var utils = require('../utils');
var isBrowser = typeof process === 'undefined' || process.browser;
var buffer = require('./buffer');

function BinaryBuilder() {
this.parts = [];
}
BinaryBuilder.prototype.append = function add(part) {
this.parts.push(part);
return this;
};
BinaryBuilder.prototype.build = function build(type) {
if (isBrowser) {
return utils.createBlob(this.parts, {type: type});
}
return buffer.concat(this.parts.map(function (part) {
return new buffer(part, 'binary');
}));
};

module.exports = function createMultipart(doc) {
doc = utils.clone(doc);

var boundary = uuid();

var attachments = {};

// according to npm/npm-fullfat-registry, these need to be sorted
// or else CouchDB has a fit
var filenames = Object.keys(doc._attachments).sort();
filenames.forEach(function (filename) {
var att = doc._attachments[filename];
if (att.stub) {
return;
}
var binData = base64.atob(att.data);
attachments[filename] = {type: att.content_type, data: binData};
att.length = binData.length;
att.follows = true;
delete att.data;
});

var preamble = '--' + boundary +
'\r\nContent-Type: application/json\r\n\r\n';

var origAttachments = doc._attachments;
doc._attachments = {};
var origKeys = Object.keys(origAttachments).sort();
origKeys.forEach(function (key) {
doc._attachments[key] = origAttachments[key];
});

var docJson = JSON.stringify(doc);

var body = new BinaryBuilder()
.append(preamble)
.append(docJson);

var attsToEncode = Object.keys(attachments).sort();
attsToEncode.forEach(function (filename) {
var att = attachments[filename];
var binString = att.data;
var type = att.type;
var preamble = '\r\n--' + boundary +
'\r\nContent-Disposition: attachment; filename=' +
JSON.stringify(filename) + '' +
'\r\nContent-Type: ' + type +
'\r\nContent-Length: ' + binString.length +
'\r\n\r\n';
body.append(preamble).append(utils.fixBinary(binString));
});

var ending = '\r\n--' + boundary + '--';
body.append(ending);

var type = 'multipart/related; boundary=' + boundary;
return {
headers: {
'Content-Type': type
},
body: body.build(type)
};
};

0 comments on commit ba47ae7

Please sign in to comment.