indexeddb: improve allDocs() perf with skip & key ranges (#8603)

This improves `pouchdb-adapter-indexeddb` performance for `allDocs()` with key ranges by:

* adding a new index (`'deleted,id' / [deleted, id]`) to allow for skipping deleted docs by default
* moving `_local/` docs to the `META_STORE` to simplify `allDocs()` implementation

Also:

* removes `'seq'` index on Map/Reduce views (db name: `.*-mrview-.*`)
* changes `getAttachment()` behaviour for local docs for `pouchdb-adapter-indexeddb`

Author: alxndrsn

packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js

@@ -1,6 +1,5 @@
 'use strict';
 
-import { isLocalId } from 'pouchdb-adapter-utils';
 import { createError, IDB_ERROR } from 'pouchdb-errors';
 import { collectConflicts } from 'pouchdb-merge';
 
@@ -27,33 +26,18 @@ function allDocsKeys(keys, docStore, allDocsInner) {
   });
 }
 
-function createKeyRange(start, end, inclusiveEnd, key, descending) {
+function createKeyRange(start, end, inclusiveStart, inclusiveEnd, key, descending) {
   try {
-    if (start && end) {
-      if (descending) {
-        return IDBKeyRange.bound(end, start, !inclusiveEnd, false);
-      } else {
-        return IDBKeyRange.bound(start, end, false, !inclusiveEnd);
-      }
-    } else if (start) {
-      if (descending) {
-        return IDBKeyRange.upperBound(start);
-      } else {
-        return IDBKeyRange.lowerBound(start);
-      }
-    } else if (end) {
-      if (descending) {
-        return IDBKeyRange.lowerBound(end, !inclusiveEnd);
-      } else {
-        return IDBKeyRange.upperBound(end, !inclusiveEnd);
-      }
-    } else if (key) {
-      return IDBKeyRange.only(key);
+    if (key) {
+      return IDBKeyRange.only([0, key]);
+    } else if (descending) {
+      return IDBKeyRange.bound(end, start, !inclusiveEnd, !inclusiveStart);
+    } else {
+      return IDBKeyRange.bound(start, end, !inclusiveStart, !inclusiveEnd);
     }
   } catch (e) {
     return {error: e};
   }
-  return null;
 }
 
 function handleKeyRangeError(opts, metadata, err, callback) {
@@ -96,31 +80,46 @@ export default function (txn, metadata, opts, callback) {
   var results = [];
   var processing = [];
 
-  var start = 'startkey' in opts ? opts.startkey : false;
-  var end = 'endkey' in opts ? opts.endkey : false;
   var key = 'key' in opts ? opts.key : false;
   var keys = 'keys' in opts ? opts.keys : false;
   var skip = opts.skip || 0;
-  var limit = typeof opts.limit === 'number' ? opts.limit : -1;
+  var limit = typeof opts.limit === 'number' ? opts.limit : undefined;
   var inclusiveEnd = opts.inclusive_end !== false;
   var descending = 'descending' in opts && opts.descending ? 'prev' : null;
-
-  var keyRange;
-  if (!keys) {
-    keyRange = createKeyRange(start, end, inclusiveEnd, key, descending);
-    if (keyRange && keyRange.error) {
-      return handleKeyRangeError(opts, metadata, keyRange.error, callback);
-    }
-  }
+  var start = 'startkey' in opts ? opts.startkey : (descending ?  '\uffff' : '');
+  var end   = 'endkey'   in opts ? opts.endkey   : (descending ? '' :  '\uffff');
 
   var docStore = txn.txn.objectStore(DOC_STORE);
 
-  txn.txn.oncomplete = onTxnComplete;
-
   if (keys) {
-    return allDocsKeys(opts.keys, docStore, allDocsInner);
+    txn.txn.oncomplete = onTxnComplete;
+    const allDocsInner = doc => {
+      if (doc.error) {
+        return results.push(doc);
+      }
+
+      const row = { id:doc.id, key:doc.id, value:{ rev:doc.rev } };
+
+      if (doc.deleted) {
+        row.value.deleted = true;
+        row.doc = null;
+      } else if (opts.include_docs) {
+        include_doc(row, doc);
+      }
+
+      results.push(row);
+    };
+    return allDocsKeys(keys, docStore, allDocsInner);
+  }
+
+  let keyRange = createKeyRange([0, start], [0, end], true, inclusiveEnd, key, descending);
+  if (keyRange.error) {
+    return handleKeyRangeError(opts, metadata, keyRange.error, callback);
   }
 
+  // txn.oncomplete must be set AFTER key-range-error is generated
+  txn.txn.oncomplete = onTxnComplete;
+
   function include_doc(row, doc) {
     var docData = doc.revs[doc.rev].data;
 
@@ -141,79 +140,120 @@ export default function (txn, metadata, opts, callback) {
     }
   }
 
-  function allDocsInner(doc) {
-    if (doc.error && keys) {
-      // key was not found with "keys" requests
-      results.push(doc);
-      return true;
+  function onTxnComplete() {
+    const returnVal = {
+      total_rows: metadata.doc_count,
+      offset: 0,
+      rows: results
+    };
+    /* istanbul ignore if */
+    if (opts.update_seq) {
+      returnVal.update_seq = metadata.seq;
     }
 
-    var row = {
-      id: doc.id,
-      key: doc.id,
-      value: {
-        rev: doc.rev
-      }
-    };
+    if (processing.length) {
+      Promise.all(processing).then(function () {
+        callback(null, returnVal);
+      });
+    } else {
+      callback(null, returnVal);
+    }
+  }
 
-    var deleted = doc.deleted;
-    if (deleted) {
-      if (keys) {
-        results.push(row);
-        row.value.deleted = true;
-        row.doc = null;
-      }
-    } else if (skip-- <= 0) {
-      results.push(row);
-      if (opts.include_docs) {
-        include_doc(row, doc);
+  const dbIndex = docStore.index('deleted,id');
+
+  if (!skip && !limit) {
+    fetchResults();
+  } else {
+    let firstKey;
+    let limitKey = limit > 0;
+
+    dbIndex.openKeyCursor(keyRange, descending || 'next').onsuccess = (e) => {
+      const cursor = e.target.result;
+
+      if (skip) {
+        if (!cursor) { return txn.txn.commit(); }
+        cursor.advance(skip);
+        skip = 0;
+        return;
       }
-      if (--limit === 0) {
-        return false;
+
+      if (firstKey === undefined) {
+        firstKey = cursor && cursor.key;
+        if (!firstKey) { return txn.txn.commit(); }
       }
-    }
-    return true;
-  }
 
-  function onTxnComplete() {
-    Promise.all(processing).then(function () {
-      var returnVal = {
-        total_rows: metadata.doc_count,
-        offset: 0,
-        rows: results
-      };
-
-      /* istanbul ignore if */
-      if (opts.update_seq) {
-        returnVal.update_seq = metadata.seq;
+      if (limit) {
+        if (limit > 1 && cursor) {
+          cursor.advance(limit - 1);
+          limit = undefined;
+          return;
+        }
+        limit = undefined;
       }
-      callback(null, returnVal);
-    });
-  }
 
-  var cursor = descending ?
-    docStore.openCursor(keyRange, descending) :
-    docStore.openCursor(keyRange);
 
-  cursor.onsuccess = function (e) {
+      if (limitKey) {
+        limitKey = cursor && cursor.key;
+      }
+      if (!limitKey) {
+        limitKey = descending ? keyRange.lower : keyRange.upper;
+      }
+
+      keyRange = createKeyRange(firstKey, limitKey, true, inclusiveEnd, key, descending);
+      if (keyRange.error) {
+        txn.txn.abort();
+        return handleKeyRangeError(opts, metadata, keyRange.error, callback);
+      }
 
-    var doc = e.target.result && e.target.result.value;
+      fetchResults();
+    };
+  }
 
-    // Happens if opts does not have limit,
-    // because cursor will end normally then,
-    // when all docs are retrieved.
-    // Would not be needed, if getAll() optimization was used like in #6059
-    if (!doc) { return; }
+  async function fetchResults() {
+    // There is a risk here with getting all results into memory - if they have multiple
+    // revs, then we risk loading loads of extra data which is then discarded.  This is
+    // reduced by batching.  This also loads unused data when include_docs is false.
+    //
+    // Current batch size is quite arbitrary, but seems like (1) more than a typical
+    // result size, and (2) not so big it's likely to cause issues.
+    const batchSize = 100;
 
-    // Skip local docs
-    if (isLocalId(doc.id)) {
-      return e.target.result.continue();
+    let kr = keyRange;
+    do {
+      kr = await fetchNextBatch(kr);
+    } while (kr);
+    if (descending) {
+      results.reverse();
     }
+    return txn.txn.commit();
 
-    var continueCursor = allDocsInner(doc);
-    if (continueCursor) {
-      e.target.result.continue();
-    }
-  };
+    function fetchNextBatch(kr) {
+      return new Promise((resolve) => {
+        dbIndex.getAll(kr, batchSize).onsuccess = (e) => {
+          const batch = e.target.result;
+          for (let i=0; i<batch.length; ++i) {
+            const doc = batch[i];
+            const row = { id:doc.id, key:doc.id, value:{ rev:doc.rev } };
+            if (opts.include_docs) {
+              include_doc(row, doc);
+            }
+            results.push(row);
+          }
 
+          if (batch.length >= batchSize) {
+            const lastSeenKey = [ 0, batch[batch.length-1].id ];
+            const startKey = descending ? kr.upper : lastSeenKey;
+            const endKey = descending ? lastSeenKey : kr.upper;
+            if (startKey[1] !== endKey[1]) {
+              const incEnd = descending ? false : inclusiveEnd;
+              const incStart = descending ? true : false;
+              return resolve(createKeyRange(startKey, endKey, incStart, incEnd, key, descending));
+            }
+          }
+          return resolve();
+        };
+      });
+    }
+  }
 }

packages/node_modules/pouchdb-adapter-indexeddb/src/bulkDocs.js

@@ -19,7 +19,7 @@ import { isLocalId, parseDoc } from 'pouchdb-adapter-utils';
 import { binaryMd5 as md5 } from 'pouchdb-md5';
 import { winningRev as calculateWinningRev, merge, compactTree } from 'pouchdb-merge';
 
-import { DOC_STORE, META_STORE, idbError } from './util';
+import { DOC_STORE, META_LOCAL_STORE, idbError } from './util';
 
 import { rewrite } from './rewrite';
 
@@ -62,7 +62,8 @@ export default function (api, req, opts, metadata, dbOpts, idbChanges, callback)
     }
 
     docs.forEach(function (doc) {
-      txn.objectStore(DOC_STORE).get(doc.id).onsuccess = readDone;
+      const docStore = isLocalId(doc.id) ? META_LOCAL_STORE : DOC_STORE;
+      txn.objectStore(docStore).get(doc.id).onsuccess = readDone;
     });
   }
 
@@ -212,8 +213,6 @@ export default function (api, req, opts, metadata, dbOpts, idbChanges, callback)
     doc.deleted = doc.revs[winningRev].deleted ? 1 : 0;
 
     // Bump the seq for every new (non local) revision written
-    // TODO: index expects a unique seq, not sure if ignoring local will
-    // work
     if (!isLocal) {
       doc.seq = ++metadata.seq;
 
@@ -285,7 +284,7 @@ export default function (api, req, opts, metadata, dbOpts, idbChanges, callback)
 
     // Local documents have different revision handling
     if (isLocal && doc.deleted) {
-      txn.objectStore(DOC_STORE).delete(doc.id).onsuccess = function () {
+      txn.objectStore(META_LOCAL_STORE).delete(doc.id).onsuccess = function () {
         results[i] = {
           ok: true,
           id: doc.id,
@@ -296,7 +295,8 @@ export default function (api, req, opts, metadata, dbOpts, idbChanges, callback)
       return;
     }
 
-    txn.objectStore(DOC_STORE).put(doc).onsuccess = function () {
+    const docStore = isLocal ? META_LOCAL_STORE : DOC_STORE;
+    txn.objectStore(docStore).put(doc).onsuccess = function () {
       results[i] = {
         ok: true,
         id: doc.id,
@@ -308,7 +308,7 @@ export default function (api, req, opts, metadata, dbOpts, idbChanges, callback)
 
   function updateSeq(i) {
     if (i === lastWriteIndex) {
-      txn.objectStore(META_STORE).put(metadata);
+      txn.objectStore(META_LOCAL_STORE).put(metadata);
     }
   }
 
@@ -396,7 +396,10 @@ export default function (api, req, opts, metadata, dbOpts, idbChanges, callback)
   }
 
   preProcessAttachments().then(function () {
-    api._openTransactionSafely([DOC_STORE, META_STORE], 'readwrite', function (err, _txn) {
+    // We _could_ check doc ids here, and skip opening DOC_STORE if all docs are local.
+    // This may marginally slow things down for local docs.  It seems pragmatic to keep
+    // the code simple and optimise for calls to bulkDocs() which include non-local docs.
+    api._openTransactionSafely([DOC_STORE, META_LOCAL_STORE], 'readwrite', function (err, _txn) {
       if (err) {
         return callback(err);
       }

packages/node_modules/pouchdb-adapter-indexeddb/src/getAttachment.js

@@ -1,12 +1,19 @@
 'use strict';
 
+import { isLocalId } from 'pouchdb-adapter-utils';
+import { createError, MISSING_DOC } from 'pouchdb-errors';
 import {
   base64StringToBlobOrBuffer as b64StringToBluffer,
   btoa,
   readAsBinaryString,
 } from 'pouchdb-binary-utils';
 
 export default function getAttachment(docId, attachId, attachment, opts, cb) {
+  if (isLocalId(docId)) {
+    cb(createError(MISSING_DOC, 'missing'));
+    return;
+  }
+
   const doc = opts.metadata;
   const data = doc.attachments[attachment.digest].data;