Save and Load Analysis from IndexedDB (#57)

* feat: handler for managing indexed db operations

* feat: ui  changes to save and load from indexeddb

* Promisify the Kana code.

* Added reference counting for files.

* Use separate store for the file reference count.

* Shared files across multiple analyses in KanaDb.

* Added MD5 sum to the file IDs to further guarantee uniqueness.

* Asyncify a few lambdas for nicer viewing.

* feat: db handler return records after save/deletion

* feat: can now delete indexed db saved analysis

* feat: set name after upload, fix issue when new dataset is chosen

Co-authored-by: LTLA <infinite.monkeys.with.keyboards@gmail.com>

public/scran/KanaDBHandler.js

@@ -0,0 +1,240 @@
+var kana_db = {};
+
+(function (x) {
+    /** Private members **/
+    var kanaDB;
+    var init = null;
+
+    x.initialize = function () {
+        init = new Promise(resolve => {
+            // initialize database on worker creation
+            kanaDB = indexedDB.open("KanaDB");
+
+            kanaDB.onupgradeneeded = (e) => {
+                var kanaDBClient = e.target.result;
+                kanaDBClient.createObjectStore("analysis", { keyPath: 'id' });
+                kanaDBClient.createObjectStore("analysis_files", { keyPath: 'id' });
+                kanaDBClient.createObjectStore("file", { keyPath: 'id' });
+                kanaDBClient.createObjectStore("file_ref_count", { keyPath: 'id' });
+            };
+
+            // Send existing stored analyses, if available.
+            kanaDB.onsuccess = () => {
+                getRecords(resolve);
+            };
+
+            kanaDB.onerror = () => {
+                resolve(null);
+            };
+        });
+
+        return init;
+    };
+
+    function getRecords(resolve) {
+        var allAnalysis = kanaDB.result
+            .transaction(["analysis"], "readonly")
+            .objectStore("analysis").getAllKeys();
+
+        allAnalysis.onsuccess = function () {
+            resolve(allAnalysis.result);
+        };
+        allAnalysis.onerror = function () {
+            resolve(null);
+        };
+    }
+
+    /** Helper functions **/
+    async function loadContent(id, store) {
+        return new Promise(resolve => {
+            let request = store.get(id);
+            request.onsuccess = function () {
+                if (request.result !== undefined) {
+                    resolve(request.result.payload);
+                } else {
+                    resolve(null);
+                }
+            };
+            request.onerror = function () {
+                resolve(null);
+            };
+        });
+    }
+
+    function allOK(promises) {
+        return Promise.allSettled(promises)
+            .then(vals => {
+                for (const x of vals) {
+                    if (!x) {
+                        return false;
+                    }
+                }
+                return true;
+            });
+    }
+
+    /** Functions to save content **/
+    x.saveFile = async function (id, buffer) {
+        await init;
+        let trans = kanaDB.result.transaction(["file", "file_ref_count"], "readwrite");
+        let file_store = trans.objectStore("file");
+        let ref_store = trans.objectStore("file_ref_count");
+
+        var refcount = await loadContent(id, ref_store);
+        if (refcount === null) {
+            refcount = 0;
+        }
+        refcount++;
+
+        var data_saving = new Promise(resolve => {
+            var putrequest = file_store.put({ "id": id, "payload": buffer });
+            putrequest.onsuccess = function (event) {
+                resolve(true);
+            };
+            putrequest.onerror = function (event) {
+                resolve(false);
+            };
+        });
+
+        var ref_saving = new Promise(resolve => {
+            var putrequest = ref_store.put({ "id": id, "payload": refcount });
+            putrequest.onsuccess = function (event) {
+                resolve(true);
+            };
+            putrequest.onerror = function (event) {
+                resolve(false);
+            };
+        });
+
+        return allOK([data_saving, ref_saving])
+    };
+
+    x.saveAnalysis = async function (id, state, files) {
+        await init;
+        let trans = kanaDB.result.transaction(["analysis", "analysis_files"], "readwrite")
+        let analysis_store = trans.objectStore("analysis");
+        let file_id_store = trans.objectStore("analysis_files");
+
+        var data_saving = new Promise(resolve => {
+            var putrequest = analysis_store.put({ "id": id, "payload": state });
+            putrequest.onsuccess = function (event) {
+                getRecords(resolve);
+            };
+            putrequest.onerror = function (event) {
+                resolve(false);
+            };
+        });
+
+        var id_saving = new Promise(resolve => {
+            var putrequest = file_id_store.put({ "id": id, "payload": files });
+            putrequest.onsuccess = function (event) {
+                resolve(true);
+            };
+            putrequest.onerror = function (event) {
+                resolve(false);
+            };
+        });
+
+        return allOK([data_saving, id_saving])
+    };
+
+    /** Functions to load content **/
+    x.loadFile = async function (id) {
+        await init;
+        let file_store = kanaDB.result
+            .transaction(["file"], "readonly")
+            .objectStore("file");
+        return loadContent(id, file_store);
+    };
+
+    x.loadAnalysis = async function (id) {
+        await init;
+        let analysis_store = kanaDB.result
+            .transaction(["analysis"], "readonly")
+            .objectStore("analysis");
+        return loadContent(id, analysis_store);
+    };
+
+    /** Functions to load content **/
+    x.removeFile = async function removeFile(id) {
+        await init;
+        let trans = kanaDB.result.transaction(["file", "file_ref_count"], "readwrite");
+        let file_store = trans.objectStore("file");
+        let ref_store = trans.objectStore("file_ref_count");
+
+        var promises = [];
+        var refcount = await loadContent(id, file_ref_count);
+        refcount--;
+
+        if (refcount == 0) {
+            promises.push(new Promise(resolve => {
+                let request = file_store.remove(id);
+                request.onerror = function (event) {
+                    resolve(false);
+                };
+                request.onsuccess = function (event) {
+                    resolve(true);
+                };
+            }));
+            promises.push(new Promise(resolve => {
+                let request = ref_store.delete(id);
+                request.onerror = function (event) {
+                    resolve(false);
+                };
+                request.onsuccess = function (event) {
+                    resolve(true);
+                };
+            }))
+        } else {
+            promises.push(new Promise(resolve => {
+                let request = ref_store.put({ "id": id, "payload": refcount })
+                request.onsuccess = function (event) {
+                    resolve(true);
+                };
+                request.onerror = function (event) {
+                    resolve(false);
+                };
+            }));
+        }
+
+        return allOK(promises);
+    };
+
+    x.removeAnalysis = async function removeFile(id) {
+        await init;
+        let trans = kanaDB.result.transaction(["analysis", "analysis_files"], "readwrite")
+        let analysis_store = trans.objectStore("analysis");
+        let file_id_store = trans.objectStore("analysis_files");
+
+        var promises = [];
+
+        promises.push(new Promise(resolve => {
+            let request = analysis_store.delete(id);
+            request.onsuccess = function (event) {
+                getRecords(resolve);
+            };
+            request.onerror = function (event) {
+                resolve(false);
+            };
+        }));
+
+        // Removing all files as well.
+        var files = await loadContent(id, file_id_store);
+        for (const f of files) {
+            promises.push(x.removeFile(f));
+        }
+
+        promises.push(new Promise(resolve => {
+            let request = file_id_store.delete(id);
+            request.onsuccess = function (event) {
+                resolve(true);
+            };
+            request.onerror = function (event) {
+                resolve(false);
+            };
+        }));
+
+        return allOK(promises);
+    };
+
+})(kana_db);

public/scran/_utils_serialize.js

@@ -4,8 +4,8 @@ const scran_utils_serialize = {};
   /* Private members */
 
   // Must be integers!
-  const FORMAT_WITH_FILES = 0;
-  const FORMAT_WITHOUT_FILES = 1;
+  const FORMAT_EMBEDDED_FILES = 0;
+  const FORMAT_EXTERNAL_KANADB = 1;
   const FORMAT_VERSION = 0;
 
   /* Private functions */
@@ -112,18 +112,40 @@ const scran_utils_serialize = {};
   }
 
   /* Public functions */
-  x.save = function(contents) {
+  x.save = async function(contents, mode = "full") {
     // Extract out the file buffers.
     var buffered = contents.inputs.parameters.files;
     var all_buffers = [];
     var total_len = 0;
+    var format_type;
+
+    if (mode == "full") {
+      format_type = FORMAT_EMBEDDED_FILES;
+      buffered.forEach((x, i) => {
+        var val = x.buffer;
+        all_buffers.push(val);
+        buffered[i].buffer = { "offset": total_len, "size": val.byteLength };
+        total_len += val.byteLength;
+      });
+
+    } else if (mode == "KanaDB") {
+      // Saving the files to IndexedDB instead. 'all_buffers' now holds a promise
+      // indicating whether all of these things were saved properly.
+      format_type = FORMAT_EXTERNAL_KANADB;
+      for (const x of buffered) {
+        var md5 = await hashwasm.md5(new Uint8Array(x.buffer));
+        var id = x.type + "_" + x.name + "_" + x.buffer.byteLength + "_" + md5;
+        var ok = await kana_db.saveFile(id, x.buffer);
+        if (!ok) {
+          throw "failed to save file '" + id + "' to KanaDB";
+        }
+        x.buffer = id;
+        all_buffers.push(id);
+      }
 
-    buffered.forEach((x, i) => {
-      var val = buffered[i].buffer;
-      all_buffers.push(val);
-      buffered[i].buffer = { "offset": total_len, "size": val.byteLength };
-      total_len += val.byteLength;
-    });
+    } else {
+        throw "unsupported mode " + mode;
+    }
 
     // Converting all other TypedArrays to normal arrays.
     contents = normalizeTypedArrays(contents);
@@ -137,7 +159,7 @@ const scran_utils_serialize = {};
     var combined_arr = new Uint8Array(combined);
     var offset = 0;
 
-    let format = numberToBuffer(FORMAT_WITH_FILES);
+    let format = numberToBuffer(format_type);
     combined_arr.set(format, offset); 
     offset += format.length;
 
@@ -156,16 +178,23 @@ const scran_utils_serialize = {};
     combined_arr.set(json_view, offset);
     offset += json_view.length;
 
-    for (const buf of all_buffers) {
-      const tmp = new Uint8Array(buf);
-      combined_arr.set(tmp, offset);
-      offset += tmp.length;
+    if (mode == "full") {
+      for (const buf of all_buffers) {
+        const tmp = new Uint8Array(buf);
+        combined_arr.set(tmp, offset);
+        offset += tmp.length;
+      }
+      return combined;
+
+    } else if (mode == "KanaDB") {
+      return { "file_ids": all_buffers, "state": combined };
+
+    } else {
+      throw "unsupported mode " + mode;
     }
-
-    return combined;
   };
 
-  x.load = function(buffer) {
+  x.load = async function(buffer) {
     var offset = 0;
     var format = bufferToNumber(new Uint8Array(buffer, offset, 8));
     offset += 8;
@@ -182,13 +211,33 @@ const scran_utils_serialize = {};
     offset += json_len;
 
     var buffered = contents.inputs.parameters.files;
-    buffered.forEach((x, i) => {
-      var details = buffered[i].buffer;
-      var target = new Uint8Array(buffer, offset + details.offset, details.size);
-      var tmp = new ArrayBuffer(details.size);
-      (new Uint8Array(tmp)).set(target);
-      buffered[i].buffer = tmp;
-    });
+    if (format == FORMAT_EMBEDDED_FILES) {
+      buffered.forEach((x, i) => {
+        var details = x.buffer;
+        var target = new Uint8Array(buffer, offset + details.offset, details.size);
+        var tmp = new ArrayBuffer(details.size);
+        (new Uint8Array(tmp)).set(target);
+        buffered[i].buffer = tmp;
+      });
+
+    } else if (format == FORMAT_EXTERNAL_KANADB) {
+      var collected = [];
+      buffered.forEach((x, i) => {
+        var id = x.buffer;
+        collected.push(kana_db.loadFile(id));
+      });
+
+      var resolved = await Promise.all(collected);
+      buffered.forEach((x, i) => {
+        if (resolved[i] === null) {
+          throw "KanaDB loading failed for file ID '" + x.buffer + "'";
+        }
+        x.buffer = resolved[i];
+      });
+
+    } else {
+        throw "unsupported format type";
+    }
 
     return contents;
   };