INCOMPLETE: Retest

- dist: Remove old Node minified files - Enhancement: Add `useSQLiteIndexes` option to add (and drop) SQLite indexes where utilized (indexeddbshim#210) - Fix: Ensure prevunique cursor presorts key in ascending order so will skip over later keys - Docs (README): Add link to IDB spec - Testing (QUnit, Fake, Mock, W3C Old, Mocha, W3C): Utilize `useSQLiteIndexes` config in tests (more complex) - Testing (Qunit): Remove references to old Node minified files
brettz9 · May 11, 2017 · b31b864 · b31b864
1 parent dbf1b2f
commit b31b864
Show file tree

Hide file tree

Showing 36 changed files with 1,847 additions and 1,322 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -132,7 +132,10 @@ they were actually changes since a more recent version on `master`.
     repeat `IDBFactory.open` call to the same name and version (assuming
     no deletes or aborts causing rollbacks) will reuse the same SQLite
     `openDatabase` instance
+- Enhancement: Add `useSQLiteIndexes` option to add (and drop) SQLite
+    indexes where utilized (#210)
 - Enhancement: Provide `avoidAutoShim` option (#270)
+- Enhancement: Provide `shimIndexedDB.__setConnectionQueueOrigin()` utility
 - Add missing API: Add `IDBCursor.continuePrimaryKey`
 - Add missing API: Implement `IDBObjectStore.getKey`
 - Add missing APIs: Implement `IDBIndex.getAll/getAllKeys`

diff --git a/README.md b/README.md
@@ -29,8 +29,9 @@ to the data format, thereby breaking old data.***
 __Use a single, indexable, offline storage API across all desktop and mobile
 browsers and Node.js.__
 
-Even if a browser natively supports IndexedDB, you may still want to use this
-shim.  Some native IndexedDB implementations are [very buggy](http://www.raymondcamden.com/2014/9/25/IndexedDB-on-iOS-8--Broken-Bad).
+Even if a browser natively supports [IndexedDB](http://w3c.github.io/IndexedDB/),
+you may still want to use this shim.  Some native IndexedDB implementations are
+[very buggy](http://www.raymondcamden.com/2014/9/25/IndexedDB-on-iOS-8--Broken-Bad).
 Others are [missing certain features](http://codepen.io/cemerick/pen/Itymi).
 There are also many minor inconsistencies between different browser
 implementations of IndexedDB, such as how errors are handled, how transaction
@@ -307,6 +308,8 @@ The available properties relevant to browser or Node are:
     browser may use this information to suggest the use of this quota to the
     user rather than prompting the user regularly for say incremental 5MB
     permissions).
+- __useSQLiteIndexes__ - Whether to create indexes on SQLite tables (and also
+    whether to try dropping). Defaults to `false`.
 - __avoidAutoShim__ - Where WebSQL is detected but where `indexedDB` is
     missing or poor support is known (non-Chrome Android or
     non-Safari iOS9), the shim will be auto-applied without

diff --git a/dist/indexeddbshim-UnicodeIdentifiers-node.js b/dist/indexeddbshim-UnicodeIdentifiers-node.js
diff --git a/dist/indexeddbshim-UnicodeIdentifiers-node.js.map b/dist/indexeddbshim-UnicodeIdentifiers-node.js.map
diff --git a/dist/indexeddbshim-UnicodeIdentifiers-node.min.js b/dist/indexeddbshim-UnicodeIdentifiers-node.min.js
diff --git a/dist/indexeddbshim-UnicodeIdentifiers-node.min.js.map b/dist/indexeddbshim-UnicodeIdentifiers-node.min.js.map
diff --git a/dist/indexeddbshim-UnicodeIdentifiers.js b/dist/indexeddbshim-UnicodeIdentifiers.js
diff --git a/dist/indexeddbshim-UnicodeIdentifiers.js.map b/dist/indexeddbshim-UnicodeIdentifiers.js.map
diff --git a/dist/indexeddbshim-UnicodeIdentifiers.min.js b/dist/indexeddbshim-UnicodeIdentifiers.min.js
diff --git a/dist/indexeddbshim-UnicodeIdentifiers.min.js.map b/dist/indexeddbshim-UnicodeIdentifiers.min.js.map
diff --git a/dist/indexeddbshim-node.js b/dist/indexeddbshim-node.js
diff --git a/dist/indexeddbshim-node.js.map b/dist/indexeddbshim-node.js.map
diff --git a/dist/indexeddbshim-node.min.js b/dist/indexeddbshim-node.min.js
diff --git a/dist/indexeddbshim-node.min.js.map b/dist/indexeddbshim-node.min.js.map
diff --git a/dist/indexeddbshim-noninvasive.js b/dist/indexeddbshim-noninvasive.js
diff --git a/dist/indexeddbshim-noninvasive.js.map b/dist/indexeddbshim-noninvasive.js.map
diff --git a/dist/indexeddbshim-noninvasive.min.js b/dist/indexeddbshim-noninvasive.min.js
diff --git a/dist/indexeddbshim-noninvasive.min.js.map b/dist/indexeddbshim-noninvasive.min.js.map
diff --git a/dist/indexeddbshim.js b/dist/indexeddbshim.js
diff --git a/dist/indexeddbshim.js.map b/dist/indexeddbshim.js.map
diff --git a/dist/indexeddbshim.min.js b/dist/indexeddbshim.min.js
diff --git a/dist/indexeddbshim.min.js.map b/dist/indexeddbshim.min.js.map
diff --git a/src/CFG.js b/src/CFG.js
@@ -54,6 +54,8 @@ const CFG = {};
     //  of the user agent prompting the user for permission to increase the
     //  quota every five megabytes."
     'DEFAULT_DB_SIZE', // Defaults to (4 * 1024 * 1024) or (25 * 1024 * 1024) in Safari
+    // Whether to create indexes on SQLite tables (and also whether to try dropping)
+    'useSQLiteIndexes', // Effectively defaults to `false` (ignored unless `true`)
 
     // NODE-IMPINGING SETTINGS (created for sake of limitations in Node or desktop file
     //    system implementation but applied by default in browser for parity)

diff --git a/src/IDBCursor.js b/src/IDBCursor.js
@@ -119,13 +119,17 @@ IDBCursor.prototype.__findBasic = function (key, primaryKey, tx, success, error,
         // 1. Sort by key
         sql.push('ORDER BY', quotedKeyColumnName, direction);
 
-        // 2. Sort by primaryKey (if defined and not unique)
-        if (!me.__unique && me.__keyColumnName !== 'key') { // Avoid adding 'key' twice
-            sql.push(',', quotedKey, direction);
+        if (me.__keyColumnName !== 'key') { // Avoid adding 'key' twice
+            if (!me.__unique) {
+                // 2. Sort by primaryKey (if defined and not unique)
+                // 3. Sort by position (if defined)
+                sql.push(',', quotedKey, direction);
+            } else if (me.direction === 'prevunique') {
+                // Sort by first record with key matching
+                sql.push(',', quotedKey, 'ASC');
+            }
         }
 
-        // 3. Sort by position (if defined)
-
         if (!me.__unique && me.__indexSource) {
             // 4. Sort by object store position (if defined and not unique)
             sql.push(',', util.sqlQuote(me.__valueColumnName), direction);

diff --git a/src/IDBFactory.js b/src/IDBFactory.js
diff --git a/src/IDBIndex.js b/src/IDBIndex.js
@@ -7,6 +7,7 @@ import IDBTransaction from './IDBTransaction';
 import * as Sca from './Sca';
 import CFG from './CFG';
 import IDBObjectStore from './IDBObjectStore';
+import SyncPromise from 'sync-promise';
 
 const readonlyProperties = ['objectStore', 'keyPath', 'multiEntry', 'unique'];
 
@@ -222,14 +223,37 @@ IDBIndex.__createIndex = function (store, index) {
             }, error);
         }
 
+        const escapedStoreNameSQL = util.escapeStoreNameForSQL(storeName);
+        const escapedIndexNameSQL = util.escapeIndexNameForSQL(index.name);
+
+        function addIndexSQL (tx) {
+            if (!CFG.useSQLiteIndexes) {
+                applyIndex(tx);
+                return;
+            }
+            tx.executeSql(
+                'CREATE INDEX IF NOT EXISTS "' +
+                    // The escaped index name must be unique among indexes in the whole database;
+                    //    so we prefix with store name; as prefixed, will also not conflict with
+                    //    index on `key`
+                    // Avoid quotes and separate with special escape sequence
+                    escapedStoreNameSQL.slice(1, -1) + '^5' + escapedIndexNameSQL.slice(1, -1) +
+                    '" ON ' + escapedStoreNameSQL + '(' + escapedIndexNameSQL + ')',
+                [],
+                applyIndex,
+                error
+            );
+        }
+
         if (columnExists) {
-            // For a previously existing index, just update the index entries in the existing column
+            // For a previously existing index, just update the index entries in the existing column;
+            //   no need to add SQLite index to it either as should already exist
             applyIndex(tx);
         } else {
             // For a new index, add a new column to the object store, then apply the index
-            const sql = ['ALTER TABLE', util.escapeStoreNameForSQL(storeName), 'ADD', util.escapeIndexNameForSQL(index.name), 'BLOB'].join(' ');
+            const sql = ['ALTER TABLE', escapedStoreNameSQL, 'ADD', escapedIndexNameSQL, 'BLOB'].join(' ');
             CFG.DEBUG && console.log(sql);
-            tx.executeSql(sql, [], applyIndex, error);
+            tx.executeSql(sql, [], addIndexSQL, error);
         }
     }, undefined, store);
 };
@@ -257,17 +281,34 @@ IDBIndex.__deleteIndex = function (store, index) {
             failure(createDOMException('UnknownError', 'Could not delete index "' + index.name + '"', err));
         }
 
-        // Update the object store's index list
-        IDBIndex.__updateIndexList(store, tx, function (store) {
-            delete index.__pendingDelete;
-            delete index.__recreated;
-            index.__deleted = true;
-            if (indexHandle) {
-                indexHandle.__deleted = true;
-                delete indexHandle.__pendingDelete;
-            }
-            success(store);
-        }, error);
+        function finishDeleteIndex () {
+            // Update the object store's index list
+            IDBIndex.__updateIndexList(store, tx, function (store) {
+                delete index.__pendingDelete;
+                delete index.__recreated;
+                index.__deleted = true;
+                if (indexHandle) {
+                    indexHandle.__deleted = true;
+                    delete indexHandle.__pendingDelete;
+                }
+                success(store);
+            }, error);
+        }
+
+        if (!CFG.useSQLiteIndexes) {
+            finishDeleteIndex();
+            return;
+        }
+        tx.executeSql(
+            'DROP INDEX IF EXISTS ' +
+                util.sqlQuote(
+                    util.escapeStoreNameForSQL(store.name).slice(1, -1) + '^5' +
+                    util.escapeIndexNameForSQL(index.name).slice(1, -1)
+                ),
+            [],
+            finishDeleteIndex,
+            error
+        );
     }, undefined, store);
 };
 
@@ -403,6 +444,7 @@ IDBIndex.prototype.__renameIndex = function (store, oldName, newName, colInfoToP
     const newNameType = 'BLOB';
     const storeName = store.__currentName;
     const escapedStoreNameSQL = util.escapeStoreNameForSQL(storeName);
+    const escapedNewIndexNameSQL = util.escapeIndexNameForSQL(newName);
     const escapedTmpStoreNameSQL = util.escapeStoreNameForSQL('tmp_' + storeName);
     const colNamesToPreserve = colInfoToPreserveArr.map((colInfo) => colInfo[0]);
     const colInfoToPreserve = colInfoToPreserveArr.map((colInfo) => colInfo.join(' '));
@@ -412,32 +454,67 @@ IDBIndex.prototype.__renameIndex = function (store, oldName, newName, colInfoToP
     // We could adapt the approach at http://stackoverflow.com/a/8430746/271577
     //    to make the approach reusable without passing column names, but it is a bit fragile
     store.transaction.__addNonRequestToTransactionQueue(function renameIndex (tx, args, success, error) {
-        const sql = 'ALTER TABLE ' + escapedStoreNameSQL + ' RENAME TO ' + escapedTmpStoreNameSQL;
-        tx.executeSql(sql, [], function (tx, data) {
-            const sql = 'CREATE TABLE ' + escapedStoreNameSQL + '(' + listColInfoToPreserve + util.escapeIndexNameForSQL(newName) + ' ' + newNameType + ')';
-            tx.executeSql(sql, [], function (tx, data) {
-                const sql = 'INSERT INTO ' + escapedStoreNameSQL + '(' +
-                    listColsToPreserve +
-                    util.escapeIndexNameForSQL(newName) +
-                    ') SELECT ' + listColsToPreserve + util.escapeIndexNameForSQL(oldName) + ' FROM ' + escapedTmpStoreNameSQL;
-                tx.executeSql(sql, [], function (tx, data) {
-                    const sql = 'DROP TABLE ' + escapedTmpStoreNameSQL;
+        function sqlError (tx, err) {
+            error(err);
+        }
+        function finish () {
+            if (cb) {
+                cb(tx, success);
+                return;
+            }
+            success();
+        }
+        // See https://www.sqlite.org/lang_altertable.html#otheralter
+        // We don't query for indexes as we already have the info
+        // This approach has the advantage of auto-deleting indexes via the DROP TABLE
+        const sql = 'CREATE TABLE ' + escapedTmpStoreNameSQL +
+            '(' + listColInfoToPreserve + escapedNewIndexNameSQL + ' ' + newNameType + ')';
+        CFG.DEBUG && console.log(sql);
+        tx.executeSql(sql, [], function () {
+            const sql = 'INSERT INTO ' + escapedTmpStoreNameSQL + '(' +
+                listColsToPreserve + escapedNewIndexNameSQL +
+                ') SELECT ' + listColsToPreserve + util.escapeIndexNameForSQL(oldName) + ' FROM ' + escapedStoreNameSQL;
+            CFG.DEBUG && console.log(sql);
+            tx.executeSql(sql, [], function () {
+                const sql = 'DROP TABLE ' + escapedStoreNameSQL;
+                CFG.DEBUG && console.log(sql);
+                tx.executeSql(sql, [], function () {
+                    const sql = 'ALTER TABLE ' + escapedTmpStoreNameSQL + ' RENAME TO ' + escapedStoreNameSQL;
+                    CFG.DEBUG && console.log(sql);
                     tx.executeSql(sql, [], function (tx, data) {
-                        if (cb) {
-                            cb(tx, success);
+                        if (!CFG.useSQLiteIndexes) {
+                            finish();
                             return;
                         }
-                        success();
-                    }, function (tx, err) {
-                        error(err);
-                    });
-                }, function (tx, err) {
-                    error(err);
-                });
-            });
-        }, function (tx, err) {
-            error(err);
-        });
+                        const indexCreations = colNamesToPreserve
+                            .slice(2) // Doing `key` separately and no need for index on `value`
+                            .map((escapedIndexNameSQL) => new SyncPromise(function (resolve, reject) {
+                                const sql = 'CREATE INDEX ' +
+                                    util.sqlQuote(
+                                        escapedStoreNameSQL.slice(1, -1) + '^5' + escapedIndexNameSQL.slice(1, -1)
+                                    ) + ' ON ' + escapedStoreNameSQL + '(' + escapedIndexNameSQL + ')';
+                                CFG.DEBUG && console.log(sql);
+                                tx.executeSql(sql, [], resolve, function (tx, err) {
+                                    reject(err);
+                                });
+                            }
+                        ));
+                        indexCreations.push(
+                            new SyncPromise(function (resolve, reject) {
+                                const sql = 'CREATE INDEX ' +
+                                    util.sqlQuote('sk_' + escapedStoreNameSQL.slice(1, -1)) +
+                                    ' ON ' + escapedStoreNameSQL + '("key")';
+                                CFG.DEBUG && console.log(sql);
+                                tx.executeSql(sql, [], resolve, function (tx, err) {
+                                    reject(err);
+                                });
+                            })
+                        );
+                        SyncPromise.all(indexCreations).then(finish, error);
+                    }, sqlError);
+                }, sqlError);
+            }, sqlError);
+        }, sqlError);
     });
 };
 

diff --git a/src/IDBObjectStore.js b/src/IDBObjectStore.js
@@ -96,6 +96,7 @@ IDBObjectStore.__createInstance = function (storeProperties, transaction) {
                 CFG.DEBUG && console.log(sql, sqlValues);
                 me.transaction.__addNonRequestToTransactionQueue(function objectStoreClear (tx, args, success, error) {
                     tx.executeSql(sql, sqlValues, function (tx, data) {
+                        // This SQL preserves indexes per https://www.sqlite.org/lang_altertable.html
                         const sql = 'ALTER TABLE ' + util.escapeStoreNameForSQL(oldName) + ' RENAME TO ' + util.escapeStoreNameForSQL(name);
                         CFG.DEBUG && console.log(sql);
                         tx.executeSql(sql, [], function (tx, data) {
@@ -172,17 +173,27 @@ IDBObjectStore.__createObjectStore = function (db, store) {
             failure(createDOMException('UnknownError', 'Could not create object store "' + storeName + '"', err));
         }
 
+        const escapedStoreNameSQL = util.escapeStoreNameForSQL(storeName);
         // key INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL UNIQUE
-        const sql = ['CREATE TABLE', util.escapeStoreNameForSQL(storeName), '(key BLOB', store.autoIncrement ? 'UNIQUE, inc INTEGER PRIMARY KEY AUTOINCREMENT' : 'PRIMARY KEY', ', value BLOB)'].join(' ');
+        const sql = ['CREATE TABLE', escapedStoreNameSQL, '(key BLOB', store.autoIncrement ? 'UNIQUE, inc INTEGER PRIMARY KEY AUTOINCREMENT' : 'PRIMARY KEY', ', value BLOB)'].join(' ');
         CFG.DEBUG && console.log(sql);
         tx.executeSql(sql, [], function (tx, data) {
-            Sca.encode(store.keyPath, function (encodedKeyPath) {
-                tx.executeSql('INSERT INTO __sys__ VALUES (?,?,?,?,?)', [util.escapeSQLiteStatement(storeName), encodedKeyPath, store.autoIncrement, '{}', 1], function () {
-                    delete store.__pendingCreate;
-                    delete store.__deleted;
-                    success(store);
-                }, error);
-            });
+            function insertStoreInfo () {
+                Sca.encode(store.keyPath, function (encodedKeyPath) {
+                    tx.executeSql('INSERT INTO __sys__ VALUES (?,?,?,?,?)', [util.escapeSQLiteStatement(storeName), encodedKeyPath, store.autoIncrement, '{}', 1], function () {
+                        delete store.__pendingCreate;
+                        delete store.__deleted;
+                        success(store);
+                    }, error);
+                });
+            }
+            if (!CFG.useSQLiteIndexes) {
+                insertStoreInfo();
+                return;
+            }
+            tx.executeSql('CREATE INDEX IF NOT EXISTS ' + (
+                util.sqlQuote('sk_' + escapedStoreNameSQL.slice(1, -1))
+            ) + ' ON ' + escapedStoreNameSQL + '("key")', [], insertStoreInfo, error);
         }, error);
     });
     return storeHandles[storeName];

diff --git a/test-support/node-idb-test.js b/test-support/node-idb-test.js
@@ -258,7 +258,13 @@ function readAndEvaluate (jsFiles, initial = '', ending = '', workers = false, i
                             shimNS
                         };
 
-                        const baseCfg = {replaceNonIDBGlobals: true, checkOrigin: false, databaseNameLengthLimit: 1000, DEBUG};
+                        const baseCfg = {
+                            replaceNonIDBGlobals: true,
+                            checkOrigin: false,
+                            databaseNameLengthLimit: 1000,
+                            useSQLiteIndexes: true,
+                            DEBUG
+                        };
                         if (['idbfactory-open-opaque-origin.js', 'idbfactory-deleteDatabase-opaque-origin.js'].includes(
                             shimNS.fileName
                         )) {

diff --git a/test-support/wrap-w3c-tests.js b/test-support/wrap-w3c-tests.js
@@ -82,7 +82,8 @@ loaderWin.addEventListener('DOMContentLoaded', function () {
 
     loaderWin.setGlobalVars(testWin, {
         fullIDLSupport: true,
-        replaceNonIDBGlobals: true
+        replaceNonIDBGlobals: true,
+        useSQLiteIndexes: true
     });
     testWin.shimIndexedDB.__useShim();
     /*
@@ -123,7 +124,8 @@ loaderWin.addEventListener('DOMContentLoaded', function () {
     };
     setGlobalVars(null, {
         fullIDLSupport: true,
-        replaceNonIDBGlobals: true
+        replaceNonIDBGlobals: true,
+        useSQLiteIndexes: true
     });
     shimIndexedDB.__useShim();
 </script>

diff --git a/tests-mocha/test-environment.js b/tests-mocha/test-environment.js
@@ -98,6 +98,7 @@
         if (useShim || !window.indexedDB || window.indexedDB === window.shimIndexedDB) {
             // Replace the browser's native IndexedDB with the shim
             shimIndexedDB.__useShim();
+            shimIndexedDB.__setConfig('useSQLiteIndexes', true);
             shimIndexedDB.__debug(true);
             env.isNative = false;
             if (IDBFactory === shimIndexedDB.modules.IDBFactory) {

diff --git a/tests-polyfill/fakeIndexedDB/fakeIndexedDB.js b/tests-polyfill/fakeIndexedDB/fakeIndexedDB.js
@@ -1,6 +1,7 @@
 function getFakeTestName () {
     return 'test' + Math.random();
 }
+shimIndexedDB.__setConfig('useSQLiteIndexes', true);
 
 describe('indexedDB Tests', function () {
     describe('Transaction Lifetime', function () {

diff --git a/tests-polyfill/indexedDBmock/setup.js b/tests-polyfill/indexedDBmock/setup.js
@@ -20,6 +20,8 @@ global.addData9 = { test: "addData9", name: "name9", id: 9 };
 global.addData10 = { test: "addData10", name: "name10", id: 10 };
 global.msgCreatingInitialSituationFailed = "Creating initial situation failed";
 
+shimIndexedDB.__setConfig('useSQLiteIndexes', true);
+
 global.initionalSituation = function(callBack, done, assert) {
     var request = indexedDB.deleteDatabase(dbName);
 

diff --git a/tests-polyfill/w3c/index.html b/tests-polyfill/w3c/index.html
@@ -68,6 +68,7 @@ <h2>
 
     <script src="support.js"></script>
     <script>
+    shimIndexedDB.__setConfig('useSQLiteIndexes', true);
     // mocha.setup({ui: 'tdd'});
     mocha.setup({globals: ['done']}); // Set in support.js
     var assert = chai.assert;

diff --git a/tests-polyfill/w3c/test-node.js b/tests-polyfill/w3c/test-node.js
@@ -74,6 +74,7 @@ window.onerror = function () {
     //    var base = '../test-helper';
     tests.forEach(function (path) {
         var indexedDB = require(base);
+        indexedDB.__setConfig('useSQLiteIndexes', true);
         require('./' + path);
     });
 }());
diff --git a/tests-qunit/nodeTest.js b/tests-qunit/nodeTest.js
@@ -2,7 +2,7 @@
 /* global testFiles, addTestSuite, startTests */
 require('source-map-support').install();
 
-const setGlobalVars = require('../dist/indexeddbshim-node.min');
+const setGlobalVars = require('../dist/indexeddbshim-node');
 setGlobalVars(global.window);
 
 function addTest (i) { // eslint-disable-line no-unused-vars

diff --git a/tests-qunit/startTests.js b/tests-qunit/startTests.js
@@ -41,6 +41,7 @@ function startTests () { // eslint-disable-line no-unused-vars
     } else {
         window.indexedDB = window.shimIndexedDB;
         window.shimIndexedDB.__useShim();
+        window.shimIndexedDB.__setConfig('useSQLiteIndexes', true);
         window.shimIndexedDB.__debug(true);
         window.shimIndexedDB.__setConfig({checkOrigin: false});
         console.log('Starting Tests with shimIndexedDB');