Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

chrome-storage-sync adapter (work in progress) #150

Open
wants to merge 20 commits into from

2 participants

@hayksaakian

I wrote a lawnchair adapter for chrome.storage.sync (only for chrome apps and extensions at the moment).

Advantages:

  • automatically syncs data between a user's chrome browsers
  • relatively simple
  • like localStorage but better:
  • you can do mass reads, mass writes and mass deletes

Disadvantages:

  • severe data caps (MAX_WRITE_OPERATIONS_PER_HOUR, MAX_ITEMS, QUOTA_BYTES, MAX_SUSTAINED_WRITE_OPERATIONS_PER_MINUTE)
  • most can't be worked around (only QUOTA_BYTES_PER_ITEM really can)
  • needs a work around for individual objects (linked-lists maybe?)
  • "chrome.storage is not a big truck. It's a series of tubes."

see https://developer.chrome.com/apps/storage.html#sync-properties
for details

@brianleroux
Owner

ack! I want to merge but many of these commits are complete rewrites (reformats?) of all the files---- can you cherry pick to just the files for this feature?

@hayksaakian

sorry... I'm a relative git newbie. the only file that matters is chrome-storage-sync.js

im not sure how to cherry pick just that one.

the rest of the commits are from when i was trying to figure out how to test it.

@brianleroux
Owner

No worries, my git-fu is super google dependent too. Found this trick: http://jasonrudolph.com/blog/2009/02/25/git-tip-how-to-merge-specific-files-from-another-branch/ which should work nicely.

@hayksaakian

nice find. i just wrote and posted a stackoverflow question as this update showed up on my end.

@hayksaakian

one other issue is in using multiple lawnchair instances. chrome.storage does not provide a 'table/collection' level of abstraction. a work around could be to use an indexing system, but given how little space is provided by chrome.storage.sync, i'm not sure if this is the best way.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Jan 21, 2013
  1. @hayksaakian

    chrome storage sync

    hayksaakian authored
  2. @hayksaakian
  3. @hayksaakian

    all

    hayksaakian authored
  4. @hayksaakian

    cleanup

    hayksaakian authored
  5. @hayksaakian

    cleanup

    hayksaakian authored
  6. @hayksaakian

    cleanup

    hayksaakian authored
  7. @hayksaakian

    cleanup

    hayksaakian authored
  8. @hayksaakian
  9. @hayksaakian
  10. typo

    hayksaakian authored
  11. minor fix

    hayksaakian authored
  12. remember to init the indexer

    hayksaakian authored
  13. navigating through callback hell

    hayksaakian authored
Commits on Jan 22, 2013
  1. mostly working, need todo .keys

    hayksaakian authored
  2. batch works

    hayksaakian authored
  3. batch works

    hayksaakian authored
  4. seems stable

    hayksaakian authored
  5. keys fallback

    hayksaakian authored
  6. @hayksaakian

    fixed indexer.del

    hayksaakian authored
This page is out of date. Refresh to see the latest.
Showing with 485 additions and 0 deletions.
  1. +280 −0 test/lib/adapters/chrome-storage-sync.js
  2. +205 −0 test/lib/adapters/dom.js
View
280 test/lib/adapters/chrome-storage-sync.js
@@ -0,0 +1,280 @@
+/**
+ * chrome storage adapter
+ * syncs a user's data on google's cloud
+ * only works in a chrome app or extension with a
+ * manifest that has the 'storage' permission
+ * see https://developer.chrome.com/apps/storage.html
+ * ===
+ * - based on dom.js
+ * - dom.js originally authored by Joseph Pecoraro
+ *
+ */
+//
+// decision: use an indexer to make .keys faster?
+// // .exists would be just as fast (you're getting one entry)
+//
+Lawnchair.adapter('chrome-storage-sync', (function() {
+ var storage = chrome.storage.sync
+ // the indexer is an encapsulation of the helpers needed to keep an ordered index of the keys
+ // the only real reason to use an index here is to make .keys faster
+
+ var indexer = function(name) {
+ return {
+ // the key
+ key: name + '._index_',
+ // returns the index, an array of keys
+ idx: function(that, callback) {
+ // var that = this;
+ var self = this;
+ storage.get(self.key, function(data){
+ // could be optimized
+ var t_index = [];
+ // in case there is no index
+ if(data[self.key]){
+ t_index = t_index.concat(data[self.key]);
+ }
+ callback.call(this, t_index)
+ // apply the callback to the index array
+ });
+ },
+ all: function(that, callback) {
+ // var that = this;
+ var self = this;
+ storage.get(null, function(everything){
+ //you probably don't also want the index
+ delete everything[self.key]
+ //exlusion is faster for a db with > 2 keys
+ //now we want it to be an array because that's the spec
+ callback.call(this, everything);
+ });
+ },
+ // adds a key to the index
+ add: function (that, keyOrArray) {
+ var self = this;
+ this.idx(that, function(a){
+ if(Array.isArray(keyOrArray)){
+ a = a.concat(keyOrArray)
+ }else{
+ a.push(keyOrArray);
+ }
+ var l = a.length
+ for(var i=0; i<l; ++i) {
+ for(var j=i+1; j<l; ++j) {
+ if(a[i] === a[j])
+ a.splice(j, 1);
+ }
+ }
+ var tosave = {}
+ tosave[self.key] = a;
+ storage.set(tosave, function() {
+ if(chrome.runtime.lastError){
+ console.log(chrome.runtime.lastError);
+ }else{
+ // console.log('updated the index!')
+ }
+ });
+ });
+ },
+ // deletes a key from the index
+ del: function (that, keyOrArray) {
+ var self = this;
+ this.idx(that, function(the_index){
+ var tosave = {}
+
+ if(Array.isArray(keyOrArray)){
+ the_index = the_index.filter(function(item) {
+ return keyOrArray.indexOf(item) === -1;
+ });
+ }else{
+ the_index.splice(the_index.indexOf(keyOrArray), 1);
+ }
+
+ tosave[self.key] = the_index
+ storage.set(tosave, function() {
+ if(chrome.runtime.lastError){
+ console.log(chrome.runtime.lastError);
+ }else{
+ // console.log('updated the index!')
+ }
+ });
+ });
+ },
+ find: function (that, key, callback){
+ var self = this;
+ this.idx(self, function(the_index){
+ var exists = the_index.indexOf(key) > -1
+ callback.call(this, exists)
+ });
+ }
+ }
+ }
+
+ // adapter api
+ return {
+
+ // ensure we are in an env with localStorage
+ valid: function () {
+ return !!storage
+ },
+
+ init: function (options, callback) { // done
+ // consider making the indexer optional
+ this.indexer = indexer(this.name)
+ if (callback) this.fn(this.name, callback).call(this, this)
+ },
+
+ save: function (obj, callback) { // done
+ var that = this;
+ var key = obj.key ? obj.key : this.uuid()
+ // now we kil the key and use it in the store colleciton
+ delete obj.key;
+ var tosave = {}
+ tosave[key] = obj
+
+ storage.set(tosave, function() {
+ if(chrome.runtime.lastError){
+ console.log(chrome.runtime.lastError);
+ }else{
+ that.indexer.add(that, key)
+ }
+ // checking for existence and THEN writing is slower
+ // than just writing; Unless you keep the index in memory instead
+ // the indexer rejects dupes
+ if (callback) {
+ that.lambda(callback).call(that, obj)
+ }
+ });
+ return this
+ },
+
+ batch: function (arr, callback) { // done
+ var that = this;
+ var keys_to_index = [];
+ var n_arr = [];
+ var tosave = {}
+ for (var i = 0, l = arr.length; i < l; i++) {
+ var key = arr[i].key ? arr[i].key : that.uuid()
+ keys_to_index.push(key);
+ tosave[key] = arr[i];
+ n_arr.push({key:key, value:arr[i]})
+ }
+ console.log(tosave);
+ storage.set(tosave, function(){
+ if(chrome.runtime.lastError){
+ console.log(chrome.runtime.lastError);
+ }else{
+ // success!
+ that.indexer.add(that, keys_to_index);
+ }
+ if (callback) that.lambda(callback).call(that, n_arr);
+ });
+ return this
+ },
+
+ // accepts [options], callback
+ keys: function(callback) { // done
+ if (callback) {
+ var that = this;
+ //with indexer
+ that.indexer.idx(that, function(the_index){
+ if(the_index.length > 0){
+ that.lambda(callback).call(that, the_index)
+ }else{
+ //in case the index was borked
+ chrome.storage.sync.get(null, function(objs){
+ var keys = Object.keys(objs);
+ that.lambda(callback).call(that, keys)
+ });
+ }
+ });
+ //without indexer
+ // chrome.storage.sync.get(null, function(objs){
+ // var keys = Object.keys(objs);
+ // that.fn('keys', callback).call(that, keys)
+ // });
+ }
+ return this // TODO options for limit/offset, return promise
+ },
+
+ get: function (keys, callback) { // done
+ if(callback){
+ var that = this;
+ if (this.isArray(keys)) {
+ storage.get(keys, function(items){
+ var results = [];
+ var rs_keys = Object.keys(items);
+ for (var i = rs_keys.length - 1; i >= 0; i--) {
+ results.push({key:rs_keys[i], value:items[rs_keys[i]] })
+ };
+ that.lambda(callback).call(that, results)
+ });
+ } else {
+ var key = keys;
+ storage.get(key, function(item){
+ var result = {}
+ result.key = key;
+ result.value = item;
+ that.lambda(callback).call(that, result)
+ });
+ }
+ }
+ return this
+ },
+
+ exists: function (key, cb) { // done
+ var that = this;
+ that.indexer.find(that, key, function(bool){
+ that.lambda(cb).call(that, bool);
+ });
+ // without an indexer
+ // storage.get(key, function(obj){
+ // var exists = Object.keys(obj).length === 0
+ // that.lambda(cb).call(that, exists);
+ // });
+ return this;
+ },
+ // NOTE adapters cannot set this.__results but plugins do
+ // this probably should be reviewed
+ all: function (callback) { // done
+ var that = this;
+ if (callback) {
+ storage.get(null, function(everything){
+ //you probably don't also want the index
+ delete everything[that.indexer.key]
+ //exlusion is faster for a db with > 2 keys
+ //now we want it to be an array because that's the spec
+ // TODO: Optimize this
+ var results = [];
+ var rs_keys = Object.keys(everything);
+ for (var i = rs_keys.length - 1; i >= 0; i--) {
+ results.push({key:rs_keys[i], value:everything[rs_keys[i]]})
+ };
+ that.lambda(callback.call(this, results));
+ });
+ }
+ return this
+ },
+
+ remove: function (keyOrArray, callback) { // done
+ var that = this;
+ storage.remove(keyOrArray, function(){
+ if(chrome.runtime.lastError){
+ console.log(chrome.runtime.lastError);
+ }else{
+ // console.log('updated the index!')
+ that.indexer.del(that, keyOrArray);
+ }
+ if (callback) that.lambda(callback).call(that)
+ });
+ return this
+ },
+
+ nuke: function (callback) { // done
+ var that = this;
+ storage.clear(function(){
+ // wohoo! end of the world!
+ if (callback) that.lambda(callback).call(that)
+ });
+ return this
+ }
+}})());
View
205 test/lib/adapters/dom.js
@@ -0,0 +1,205 @@
+/**
+ * dom storage adapter
+ * ===
+ * - originally authored by Joseph Pecoraro
+ *
+ */
+//
+// TODO does it make sense to be chainable all over the place?
+// chainable: nuke, remove, all, get, save, all
+// not chainable: valid, keys
+//
+Lawnchair.adapter('dom', (function() {
+ var storage = window.localStorage
+ // the indexer is an encapsulation of the helpers needed to keep an ordered index of the keys
+ var indexer = function(name) {
+ return {
+ // the key
+ key: name + '._index_',
+ // returns the index
+ all: function() {
+ var a = storage.getItem(this.key)
+ if (a) {
+ a = JSON.parse(a)
+ }
+ if (a === null) storage.setItem(this.key, JSON.stringify([])) // lazy init
+ return JSON.parse(storage.getItem(this.key))
+ },
+ // adds a key to the index
+ add: function (key) {
+ var a = this.all()
+ a.push(key)
+ storage.setItem(this.key, JSON.stringify(a))
+ },
+ // deletes a key from the index
+ del: function (key) {
+ var a = this.all(), r = []
+ // FIXME this is crazy inefficient but I'm in a strata meeting and half concentrating
+ for (var i = 0, l = a.length; i < l; i++) {
+ if (a[i] != key) r.push(a[i])
+ }
+ storage.setItem(this.key, JSON.stringify(r))
+ },
+ // returns index for a key
+ find: function (key) {
+ var a = this.all()
+ for (var i = 0, l = a.length; i < l; i++) {
+ if (key === a[i]) return i
+ }
+ return false
+ }
+ }
+ }
+
+ // adapter api
+ return {
+
+ // ensure we are in an env with localStorage
+ valid: function () {
+ return !!storage && function() {
+ // in mobile safari if safe browsing is enabled, window.storage
+ // is defined but setItem calls throw exceptions.
+ var success = true
+ var value = Math.random()
+ try {
+ storage.setItem(value, value)
+ } catch (e) {
+ success = false
+ }
+ storage.removeItem(value)
+ return success
+ }()
+ },
+
+ init: function (options, callback) {
+ this.indexer = indexer(this.name)
+ if (callback) this.fn(this.name, callback).call(this, this)
+ },
+
+ save: function (obj, callback) {
+ var key = obj.key ? this.name + '.' + obj.key : this.name + '.' + this.uuid()
+ // now we kil the key and use it in the store colleciton
+ delete obj.key;
+ storage.setItem(key, JSON.stringify(obj))
+ // if the key is not in the index push it on
+ if (this.indexer.find(key) === false) this.indexer.add(key)
+ obj.key = key.slice(this.name.length + 1)
+ if (callback) {
+ this.lambda(callback).call(this, obj)
+ }
+ return this
+ },
+
+ batch: function (ary, callback) {
+ var saved = []
+ // not particularily efficient but this is more for sqlite situations
+ for (var i = 0, l = ary.length; i < l; i++) {
+ this.save(ary[i], function(r){
+ saved.push(r)
+ })
+ }
+ if (callback) this.lambda(callback).call(this, saved)
+ return this
+ },
+
+ // accepts [options], callback
+ keys: function(callback) {
+ if (callback) {
+ var name = this.name
+ var indices = this.indexer.all();
+ var keys = [];
+ //Checking for the support of map.
+ if(Array.prototype.map) {
+ keys = indices.map(function(r){ return r.replace(name + '.', '') })
+ } else {
+ for (var key in indices) {
+ keys.push(key.replace(name + '.', ''));
+ }
+ }
+ this.fn('keys', callback).call(this, keys)
+ }
+ return this // TODO options for limit/offset, return promise
+ },
+
+ get: function (key, callback) {
+ if (this.isArray(key)) {
+ var r = []
+ for (var i = 0, l = key.length; i < l; i++) {
+ var k = this.name + '.' + key[i]
+ var obj = storage.getItem(k)
+ if (obj) {
+ obj = JSON.parse(obj)
+ obj.key = key[i]
+ }
+ r.push(obj)
+ }
+ if (callback) this.lambda(callback).call(this, r)
+ } else {
+ var k = this.name + '.' + key
+ var obj = storage.getItem(k)
+ if (obj) {
+ obj = JSON.parse(obj)
+ obj.key = key
+ }
+ if (callback) this.lambda(callback).call(this, obj)
+ }
+ return this
+ },
+
+ exists: function (key, cb) {
+ var exists = this.indexer.find(this.name+'.'+key) === false ? false : true ;
+ this.lambda(cb).call(this, exists);
+ return this;
+ },
+ // NOTE adapters cannot set this.__results but plugins do
+ // this probably should be reviewed
+ all: function (callback) {
+ var idx = this.indexer.all()
+ , r = []
+ , o
+ , k
+ for (var i = 0, l = idx.length; i < l; i++) {
+ k = idx[i] //v
+ o = JSON.parse(storage.getItem(k))
+ o.key = k.replace(this.name + '.', '')
+ r.push(o)
+ }
+ if (callback) this.fn(this.name, callback).call(this, r)
+ return this
+ },
+
+ remove: function (keyOrArray, callback) {
+ var self = this;
+ if (this.isArray(keyOrArray)) {
+ // batch remove
+ var i, done = keyOrArray.length;
+ var removeOne = function(i) {
+ self.remove(keyOrArray[i], function() {
+ if ((--done) > 0) { return; }
+ if (callback) {
+ self.lambda(callback).call(self);
+ }
+ });
+ };
+ for (i=0; i < keyOrArray.length; i++)
+ removeOne(i);
+ return this;
+ }
+ var key = this.name + '.' +
+ ((keyOrArray.key) ? keyOrArray.key : keyOrArray)
+ this.indexer.del(key)
+ storage.removeItem(key)
+ if (callback) this.lambda(callback).call(this)
+ return this
+ },
+
+ nuke: function (callback) {
+ this.all(function(r) {
+ for (var i = 0, l = r.length; i < l; i++) {
+ this.remove(r[i]);
+ }
+ if (callback) this.lambda(callback).call(this)
+ })
+ return this
+ }
+}})());
Something went wrong with that request. Please try again.