diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js index ebc4d00e81..cffc7b1b58 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js @@ -1,5 +1,24 @@ 'use strict'; -import { changesHandler } from 'pouchdb-utils'; +import { + adapterFun, + changesHandler, + processAttachments, + processRevsRevsInfo, +} from 'pouchdb-utils'; + +import { + collectLeaves, + collectConflicts, + isDeleted, + latest as getLatest, +} from 'pouchdb-merge'; + +import { + INVALID_ID, + UNKNOWN_ERROR, + INVALID_REV, + createError +} from 'pouchdb-errors'; import setup from './setup'; @@ -16,7 +35,7 @@ import destroy from './destroy'; import {query, viewCleanup} from './find'; import purge from './purge'; -import { DOC_STORE } from './util'; +import { DOC_STORE, idbError } from './util'; var ADAPTER_NAME = 'indexeddb'; @@ -26,6 +45,10 @@ var idbChanges = new changesHandler(); // A shared list of database handles var openDatabases = {}; +// Current implementation of _getAttachment() means we DO NOT need a database +// transaction, so the "ctx" does not need to be passed to processAttachments(). +const NO_CTX_REQUIRED = null; + function IndexeddbPouch(dbOpts, callback) { if (dbOpts.view_adapter) { @@ -153,6 +176,130 @@ function IndexeddbPouch(dbOpts, callback) { api._purge = $t(purge, [DOC_STORE], 'readwrite'); + api.get = adapterFun('get', function (id, opts, cb) { + if (typeof opts === 'function') { + cb = opts; + opts = {}; + } + opts = opts || {}; + if (typeof id !== 'string') { + return cb(createError(INVALID_ID)); + } + + if (opts.open_revs) { + let leaves; + let metadata; + + const finishOpenRevs = async () => { + var result = []; + /* istanbul ignore if */ + if (!leaves.length) { + return cb(null, result); + } + + const processing = []; + const seen = new Map(); + for (let i=leaves.length-1; i>=0; --i) { + const leaf = leaves[i]; + + const rev = opts.latest ? getLatest(leaf, metadata) : leaf; + if (seen.has(rev)) { + continue; + } + seen.set(rev); + + if (!(rev in metadata.revs)) { + result.push({ missing: leaf }); + } else { + const doc = metadata.revs[rev].data; + doc._id = metadata.id; + doc._rev = rev; + + if (isDeleted(metadata, rev)) { + doc._deleted = true; + } + + if (opts.revs) { + processRevsRevsInfo(id, doc, metadata.rev_tree, opts); + } + + processing.push(new Promise((resolve, reject) => { + const cb = (err, res) => err ? reject(err) : resolve(res); + processAttachments(api, metadata, doc, opts, NO_CTX_REQUIRED, cb); + })); + + result.push({ ok: doc }); + } + } + try { + await Promise.all(processing); + cb(null, result); + } catch (err) { + cb(err); + } + }; + + api._openTransactionSafely([DOC_STORE], 'readonly', function (err, txn) { + txn.onabort = function () { + cb(createError(UNKNOWN_ERROR, 'transaction was aborted')); + }; + txn.ontimeout = idbError(cb); + var req = txn.objectStore(DOC_STORE).get(id); + req.onsuccess = function (e) { + metadata = e.target.result; + if (opts.open_revs === "all") { + leaves = collectLeaves(metadata.rev_tree).map(function (leaf) { + return leaf.rev; + }); + finishOpenRevs(); + } else { + if (Array.isArray(opts.open_revs)) { + leaves = opts.open_revs; + for (var i = 0; i < leaves.length; i++) { + var l = leaves[i]; + // looks like it's the only thing couchdb checks + // TODO replace with !isValidRev(l); + if (!(typeof (l) === "string" && /^\d+-/.test(l))) { + return cb(createError(INVALID_REV)); + } + } + finishOpenRevs(); + } else { + return cb(createError(UNKNOWN_ERROR, 'function_clause')); + } + } + }; + }); + return; // open_revs does not like other options + } + + return api._get(id, opts, (err, result) => { + if (err) { + err.docId = id; + return cb(err); + } + + var doc = result.doc; + var metadata = result.metadata; + var ctx = result.ctx; + + if (opts.conflicts) { + var conflicts = collectConflicts(metadata); + if (conflicts.length) { + doc._conflicts = conflicts; + } + } + + if (isDeleted(metadata, doc._rev)) { + doc._deleted = true; + } + + processRevsRevsInfo(id, doc, metadata.rev_tree, opts); + + processAttachments(api, metadata, doc, opts, ctx, cb); + }); + }).bind(api); + // TODO: this setTimeout seems nasty, if its needed lets // figure out / explain why setTimeout(function () { diff --git a/packages/node_modules/pouchdb-core/src/adapter.js b/packages/node_modules/pouchdb-core/src/adapter.js index 41a634171b..17aaa7d0da 100644 --- a/packages/node_modules/pouchdb-core/src/adapter.js +++ b/packages/node_modules/pouchdb-core/src/adapter.js @@ -1,7 +1,9 @@ import { rev, guardedConsole, - isRemote + isRemote, + processAttachments, + processRevsRevsInfo, } from 'pouchdb-utils'; import EventEmitter from 'events'; import Changes from './changes'; @@ -17,7 +19,6 @@ import { import { traverseRevTree, collectLeaves, - rootToLeaf, collectConflicts, isDeleted, isLocalId, @@ -555,6 +556,7 @@ class AbstractPouchDB extends EventEmitter { for (var i = 0; i < leaves.length; i++) { var l = leaves[i]; // looks like it's the only thing couchdb checks + // TODO replace with !isValidRev(l); if (!(typeof (l) === "string" && /^\d+-/.test(l))) { return cb(createError(INVALID_REV)); } @@ -589,89 +591,14 @@ class AbstractPouchDB extends EventEmitter { } if (opts.revs || opts.revs_info) { - var splittedRev = doc._rev.split('-'); - var revNo = parseInt(splittedRev[0], 10); - var revHash = splittedRev[1]; - - var paths = rootToLeaf(metadata.rev_tree); - var path = null; - - for (var i = 0; i < paths.length; i++) { - var currentPath = paths[i]; - var hashIndex = currentPath.ids.map(function (x) { return x.id; }) - .indexOf(revHash); - var hashFoundAtRevPos = hashIndex === (revNo - 1); - - if (hashFoundAtRevPos || (!path && hashIndex !== -1)) { - path = currentPath; - } - } - - /* istanbul ignore if */ - if (!path) { - err = new Error('invalid rev tree'); - err.docId = id; + try { + processRevsRevsInfo(id, doc, metadata.rev_tree, opts); + } catch (err) { return cb(err); } - - var indexOfRev = path.ids.map(function (x) { return x.id; }) - .indexOf(doc._rev.split('-')[1]) + 1; - var howMany = path.ids.length - indexOfRev; - path.ids.splice(indexOfRev, howMany); - path.ids.reverse(); - - if (opts.revs) { - doc._revisions = { - start: (path.pos + path.ids.length) - 1, - ids: path.ids.map(function (rev) { - return rev.id; - }) - }; - } - if (opts.revs_info) { - var pos = path.pos + path.ids.length; - doc._revs_info = path.ids.map(function (rev) { - pos--; - return { - rev: pos + '-' + rev.id, - status: rev.opts.status - }; - }); - } } - if (opts.attachments && doc._attachments) { - var attachments = doc._attachments; - var count = Object.keys(attachments).length; - if (count === 0) { - return cb(null, doc); - } - Object.keys(attachments).forEach((key) => { - this._getAttachment(doc._id, key, attachments[key], { - binary: opts.binary, - metadata: metadata, - ctx: ctx - }, function (err, data) { - var att = doc._attachments[key]; - att.data = data; - delete att.stub; - delete att.length; - if (!--count) { - cb(null, doc); - } - }); - }); - } else { - if (doc._attachments) { - for (var key in doc._attachments) { - /* istanbul ignore else */ - if (Object.prototype.hasOwnProperty.call(doc._attachments, key)) { - doc._attachments[key].stub = true; - } - } - } - cb(null, doc); - } + processAttachments(this, metadata, doc, opts, ctx, cb); }); }).bind(this); diff --git a/packages/node_modules/pouchdb-utils/src/index.js b/packages/node_modules/pouchdb-utils/src/index.js index 60c928908b..aed9905879 100644 --- a/packages/node_modules/pouchdb-utils/src/index.js +++ b/packages/node_modules/pouchdb-utils/src/index.js @@ -21,6 +21,8 @@ import once from './once'; import parseDdocFunctionName from './parseDdocFunctionName'; import parseUri from './parseUri'; import pick from './pick'; +import processAttachments from './processAttachments'; +import processRevsRevsInfo from './processRevsRevsInfo'; import scopeEval from './scopeEval'; import toPromise from './toPromise'; import upsert from './upsert'; @@ -50,6 +52,8 @@ export { parseDdocFunctionName, parseUri, pick, + processAttachments, + processRevsRevsInfo, rev, scopeEval, toPromise, diff --git a/packages/node_modules/pouchdb-utils/src/processAttachments.js b/packages/node_modules/pouchdb-utils/src/processAttachments.js new file mode 100644 index 0000000000..592ec8e231 --- /dev/null +++ b/packages/node_modules/pouchdb-utils/src/processAttachments.js @@ -0,0 +1,41 @@ + +/** + * Process doc._attachments as requested in opts. + */ + +function processAttachments(api, metadata, doc, opts, ctx, cb) { + if (opts.attachments && doc._attachments) { + const attachments = doc._attachments; + let count = Object.keys(attachments).length; + if (count === 0) { + return cb(null, doc); + } + Object.keys(attachments).forEach((key) => { + api._getAttachment(doc._id, key, attachments[key], { + binary: opts.binary, + metadata: metadata, + ctx: ctx, + }, function (err, data) { + const att = doc._attachments[key]; + att.data = data; + delete att.stub; + delete att.length; + if (!--count) { + cb(null, doc); + } + }); + }); + } else { + if (doc._attachments) { + for (const key in doc._attachments) { + /* istanbul ignore else */ + if (Object.prototype.hasOwnProperty.call(doc._attachments, key)) { + doc._attachments[key].stub = true; + } + } + } + cb(null, doc); + } +} + +export default processAttachments; diff --git a/packages/node_modules/pouchdb-utils/src/processRevsRevsInfo.js b/packages/node_modules/pouchdb-utils/src/processRevsRevsInfo.js new file mode 100644 index 0000000000..c69a432fa1 --- /dev/null +++ b/packages/node_modules/pouchdb-utils/src/processRevsRevsInfo.js @@ -0,0 +1,58 @@ +import { rootToLeaf } from 'pouchdb-merge'; + +/** + * Adds _revisions to a doc. + */ +function processRevsRevsInfo(id, doc, rev_tree, opts) { + const splittedRev = doc._rev.split('-'); + const revNo = parseInt(splittedRev[0], 10); + const revHash = splittedRev[1]; + + const paths = rootToLeaf(rev_tree); + let path; + + for (let i = 0; i < paths.length; i++) { + const currentPath = paths[i]; + const hashIndex = currentPath.ids.map(function (x) { return x.id; }) + .indexOf(revHash); + const hashFoundAtRevPos = hashIndex === (revNo - 1); + + if (hashFoundAtRevPos || (!path && hashIndex !== -1)) { + path = currentPath; + } + } + + /* istanbul ignore if */ + if (!path) { + const err = new Error('invalid rev tree'); + err.docId = id; + throw err; + } + + const indexOfRev = path.ids.map(function (x) { return x.id; }) + .indexOf(doc._rev.split('-')[1]) + 1; + const howMany = path.ids.length - indexOfRev; + path.ids.splice(indexOfRev, howMany); + path.ids.reverse(); + + if (opts.revs) { + doc._revisions = { + start: (path.pos + path.ids.length) - 1, + ids: path.ids.map(function (rev) { + return rev.id; + }) + }; + } + if (opts.revs_info) { + let pos = path.pos + path.ids.length; + doc._revs_info = path.ids.map(function (rev) { + pos--; + return { + rev: pos + '-' + rev.id, + status: rev.opts.status + }; + }); + } +} + +export default processRevsRevsInfo; diff --git a/tests/common-utils.js b/tests/common-utils.js index 955932d98c..222eafd0d1 100644 --- a/tests/common-utils.js +++ b/tests/common-utils.js @@ -177,5 +177,6 @@ commonUtils.createDocId = function (i) { var PouchForCoverage = require('../packages/node_modules/pouchdb-for-coverage'); var pouchUtils = PouchForCoverage.utils; commonUtils.Promise = pouchUtils.Promise; +commonUtils.rev = pouchUtils.rev; module.exports = commonUtils; diff --git a/tests/performance/perf.basics.js b/tests/performance/perf.basics.js index b2db2f7ad5..5dcf2d9cf9 100644 --- a/tests/performance/perf.basics.js +++ b/tests/performance/perf.basics.js @@ -135,6 +135,27 @@ module.exports = function (PouchDB, callback) { db.get(commonUtils.createDocId(itr), done); } }, + { + name: 'gets-open-revs', + assertions: 1, + iterations: 1, + setup: function (db, callback) { + var docs = []; + for (var i = 0; i < this.iterations; i++) { + const id = commonUtils.createDocId(i); + const numRevs = 400; //200; // repro "url too long" error with open_revs + for (var j = 0; j < numRevs; j++) { + const rev = '1-' + commonUtils.rev(); + docs.push({ _id: id, _rev: rev }); + } + } + db.bulkDocs(docs, {new_edits: false}, callback); + }, + test: function (db, itr, docs, done) { + const id = commonUtils.createDocId(itr); + db.get(id, { open_revs: 'all', revs_info: true, include_docs: true }, done); + } + }, { name: 'all-docs-skip-limit', assertions: 1, @@ -303,7 +324,15 @@ module.exports = function (PouchDB, callback) { name: 'pull-replication-one-generation', assertions: 1, iterations: 1, - setup: oneGen.setup(1, 1), + setup: oneGen.setup({ itr:1, gens:1 }), + test: oneGen.test(), + tearDown: oneGen.tearDown() + }, + { + name: 'pull-replication-one-generation-reverse', + assertions: 1, + iterations: 1, + setup: oneGen.setup({ itr:1, gens:1, reverse:true }), test: oneGen.test(), tearDown: oneGen.tearDown() }, @@ -311,10 +340,50 @@ module.exports = function (PouchDB, callback) { name: 'pull-replication-two-generation', assertions: 1, iterations: 1, - setup: twoGen.setup(1, 2), + setup: twoGen.setup({ itr:1, gens:2 }), + test: twoGen.test(), + tearDown: twoGen.tearDown() + }, + { + name: 'pull-replication-two-generation-reverse', + assertions: 1, + iterations: 1, + setup: twoGen.setup({ itr:1, gens:2, reverse:true }), + test: twoGen.test(), + tearDown: twoGen.tearDown() + }, + { + name: 'pull-replication-one-generation-open-revs', + assertions: 1, + iterations: 1, + setup: oneGen.setup({ itr:1, gens:1, openRevs:20 }), + test: oneGen.test(), + tearDown: oneGen.tearDown() + }, + { + name: 'pull-replication-one-generation-open-revs-reverse', + assertions: 1, + iterations: 1, + setup: oneGen.setup({ itr:1, gens:1, openRevs:20, reverse:true }), + test: oneGen.test(), + tearDown: oneGen.tearDown() + }, + { + name: 'pull-replication-two-generation-open-revs', + assertions: 1, + iterations: 1, + setup: twoGen.setup({ itr:1, gens:2, openRevs:20 }), + test: twoGen.test(), + tearDown: twoGen.tearDown() + }, + { + name: 'pull-replication-two-generation-open-revs-reverse', + assertions: 1, + iterations: 1, + setup: twoGen.setup({ itr:1, gens:2, openRevs:20, reverse:true }), test: twoGen.test(), tearDown: twoGen.tearDown() - } + }, ]; utils.runTests(PouchDB, 'basics', testCases, callback); diff --git a/tests/performance/replication-test.js b/tests/performance/replication-test.js index 8d5dbe4879..11bba623b7 100644 --- a/tests/performance/replication-test.js +++ b/tests/performance/replication-test.js @@ -9,65 +9,71 @@ module.exports = function (PouchDB, Promise) { var commonUtils = require('../common-utils.js'); var log = require('debug')('pouchdb:tests:performance'); - var PullRequestTestObject = function () { - this.localPouches = []; - }; + var PullRequestTestObject = function () {}; - PullRequestTestObject.prototype.setup = function (itr, gens) { + PullRequestTestObject.prototype.setup = function ({ itr, gens, openRevs=1, reverse=false }) { var self = this; - return function (localDB, callback) { - var remoteDBOpts = {ajax: {pool: {maxSockets: MAX_SOCKETS}}}; - var remoteCouchUrl = commonUtils.couchHost() + "/" + - commonUtils.safeRandomDBName(); - - self.remoteDB = new PouchDB(remoteCouchUrl, remoteDBOpts); - + return function (_, callback) { var i; var docs = []; - var localOpts = {ajax: {timeout: 60 * 1000}}; + var bulkDocsOpts = {ajax: {timeout: 60 * 1000}, new_edits: false}; + var remoteDbOpts = {ajax: {pool: {maxSockets: MAX_SOCKETS}}}; + + const localPouches = []; + const remoteDbs = []; + self.sourceDbs = reverse ? localPouches : remoteDbs; + self.targetDbs = reverse ? remoteDbs : localPouches; for (i = 0; i < itr; ++i) { - self.localPouches[i] = new PouchDB(commonUtils.safeRandomDBName()); + localPouches[i] = new PouchDB(commonUtils.safeRandomDBName()); + + const remoteCouchUrl = commonUtils.couchHost() + "/" + + commonUtils.safeRandomDBName(); + remoteDbs[i] = new PouchDB(remoteCouchUrl, remoteDbOpts); } for (i = 0; i < NUM_DOCS; i++) { - docs.push({ - _id: commonUtils.createDocId(i), - foo: Math.random(), - bar: Math.random() - }); + for (let j = 0; j < openRevs; j++) { + docs.push({ + _id: commonUtils.createDocId(i), + foo: Math.random(), + bar: Math.random(), + _rev: '1-' + commonUtils.rev(), + }); + } } - var addGeneration = function (generationCount, docs) { - return self.remoteDB.bulkDocs({docs: docs}, localOpts) - .then(function (bulkDocsResponse) { - --generationCount; - if (generationCount <= 0) { - return {}; - } - var updatedDocs = bulkDocsResponse.map(function (doc) { - return { - _id: doc.id, - _rev: doc.rev, - foo: Math.random(), - bar: Math.random() - }; + Promise.all(self.sourceDbs.map(sourceDb => new Promise((resolve, reject) => { + var addGeneration = function (generationCount, docs) { + return sourceDb.bulkDocs({docs: docs}, bulkDocsOpts) + .then(function (bulkDocsResponse) { + --generationCount; + if (generationCount <= 0) { + return {}; + } + var updatedDocs = bulkDocsResponse.map(function (doc) { + return { + _id: doc.id, + _rev: doc.rev, + foo: Math.random(), + bar: Math.random() + }; + }); + return addGeneration(generationCount, updatedDocs); }); - return addGeneration(generationCount, updatedDocs); - }); - }; + }; - return addGeneration(gens, docs).then(function (/* result */) { - callback(); - }).catch(callback); + return addGeneration(gens, docs).then(resolve).catch(reject); + }))).then(() => callback()).catch(callback); }; }; PullRequestTestObject.prototype.test = function () { var self = this; return function (ignoreDB, itr, ignoreContext, done) { - var localDB = self.localPouches[itr]; - PouchDB.replicate(self.remoteDB, localDB, + const sourceDb = self.sourceDbs[itr]; + const targetDb = self.targetDbs[itr]; + PouchDB.replicate(sourceDb, targetDb, {live: false, batch_size: BATCH_SIZE}) .on('change', function (info) { log("replication - info - " + JSON.stringify(info)); @@ -86,12 +92,8 @@ module.exports = function (PouchDB, Promise) { PullRequestTestObject.prototype.tearDown = function () { var self = this; return function () { - return self.remoteDB.destroy().then(function () { - return Promise.all( - self.localPouches.map(function (localPouch) { - return localPouch.destroy(); - })); - }); + return Promise.all([ ...self.sourceDbs, ...self.targetDbs ] + .map(db => db.destroy)); }; };