diff --git a/README.apply-after.dot b/README.apply-after.dot index fe77791..77d9484 100644 --- a/README.apply-after.dot +++ b/README.apply-after.dot @@ -2,7 +2,7 @@ digraph structs { node [shape=record]; subgraph clusterqueue { label = "Queue"; - structqueue [shape=record,label="{ Positon | | 0 | 1 | 2 | 3 } | { Dataset | | Car | Person | Car | Car }|{ Datakey | | Ford | Simon | Ford | Subaru }|{ Version | | 2 | 6 | 3 | 4 }|{ Operation | | Update | Set | Remove | Update }|{ Update | | \{...\} | \{...\} | \{...\} | \{ \"$inc \{ \"Likes\": 1 \}\} }"]; + structqueue [shape=record,label="{ Positon | | 0 | 1 | 2 | 3 } | { Dataset | | Car | Person | Car | Car }|{ Datakey | | Ford | Simon | Ford | Subaru }|{ Operation | | Update | Set | Remove | Update }|{ Update | | \{...\} | \{...\} | \{...\} | \{ \"$inc \{ \"Likes\": 1 \}\} }"]; } subgraph clusterstore { label = "Store"; diff --git a/README.download_updates_from_server.dot b/README.download_updates_from_server.dot index bc353ef..d17289a 100644 --- a/README.download_updates_from_server.dot +++ b/README.download_updates_from_server.dot @@ -15,7 +15,7 @@ digraph a { JLib -> Server [style=dashed, label="XHR? (2 & 3)"]; Server -> JLib [style=dashed]; JLib -> JLibCallback [label="(4)"]; - JLibCallback -> SyncIt [fontcolor=white, color=white]; + JLibCallback -> SyncIt [label="syncIt.feed()", fontcolor=white, color=white]; SyncIt -> SyncItCallback [fontcolor=white, color=white]; SyncIt -> SyncItEventHandler [fontcolor=white, color=white]; SyncIt -> SyncItEventHandler [fontcolor=white, color=white]; diff --git a/README.feed_into_syncit.dot b/README.feed_into_syncit.dot index 35bfbf7..636fc5a 100644 --- a/README.feed_into_syncit.dot +++ b/README.feed_into_syncit.dot @@ -15,7 +15,7 @@ digraph a { JLib -> Server [fontcolor=white, color=white, label="XHR? (2 & 3)"]; Server -> JLib [fontcolor=white, color=white]; JLib -> JLibCallback [fontcolor=white, color=white]; - JLibCallback -> SyncIt; + JLibCallback -> SyncIt [label="syncIt.feed()"]; SyncIt -> SyncItCallback; SyncIt -> SyncItEventHandler; SyncIt -> SyncItEventHandler; diff --git a/README.getfirst.dot b/README.getfirst.dot new file mode 100644 index 0000000..32ffb70 --- /dev/null +++ b/README.getfirst.dot @@ -0,0 +1,11 @@ +digraph structs { + subgraph x { + Server [rank=min]; + node [shape=record]; + subgraph clusterqueue { + label = "Queue"; + structqueue [shape=record,label="{ Position | 0 | 1 | 2 | 3 | 4 } | { Dataset | Car | Car | Person | Car | Car }|{ Datakey | Subaru | Ford | Simon | Ford | Subaru }|{ Operation | Set | Update | Set | Remove | Update }|{ Update | \{\"Seats\": \"Leather\"\} | \{...\} | \{...\} | \{...\} | \{ \"$inc \{ \"Likes\": 1 \}\} }"]; + } + structqueue:f3 -> Server [color="red",style="bold"]; + } +} diff --git a/README.md b/README.md index 8f10ced..be1d320 100644 --- a/README.md +++ b/README.md @@ -164,10 +164,9 @@ The set operation will add a Queueitem to the end of the Queue. Because SyncIt does not know anything about the implementation details of the server pushing changes it a two stage process. These are: - 1. Your App should request the next Queueitem that needs uploading to the Server from SyncIt then begin communicating that to the server ( SyncIt.getFirst() ). - 2. The server will either Accept or Reject your Queueitem, assuming it is accepted your App should notify SyncIt it has been uploaded ( SyncIt.apply() ). +Your App should request the next Queueitem that needs uploading to the Server from SyncIt then begin communicating that to the server ( SyncIt.getFirst() ). -![Uploading to server and applying accepted changes](bin/README/img/getfirst_and_apply.png) +##### Code ```javascript syncIt.getFirst(function(err,queueitem) { @@ -184,10 +183,16 @@ syncIt.getFirst(function(err,queueitem) { }); ``` +##### Diagram + +![SyncIt.getFirst()](bin/README/img/getfirst.png) + Note: This will not change the Queue in any way. Assuming the Queueitem is accepted by the Server, the first local Queueitem should then be applied to the local Store so that Dataset/Datakey matches the state on the Server in the Store and the Queueitem should then be removed. +##### Code + ```javascript jamesSyncIt.apply( function(err, appliedQueueitem, storedrecord ) { if (err) { @@ -210,6 +215,10 @@ jamesSyncIt.apply( function(err, appliedQueueitem, storedrecord ) { ![What happens during a syncIt.apply() - After](bin/README/img/apply-after.png) +##### Full overall diagram + +![Uploading to server and applying accepted changes](bin/README/img/getfirst_and_apply.png) + ### What happens if the data is modified by two different users / devices? At some point, you will find that data has been modified by two different users or devices. The first thing to note about this is that, just like Subversion, Git or Mercurial SyncIt does not dictate how conflicts should be resolved but instead exposes a callback which exposes all required information for doing so. This is the middle parameter to the `SyncIt.feed()` function: @@ -278,12 +287,14 @@ There is a (reasonably) complete set of [API Docs](http://forbesmyester.github.i User James is sat on the the underground using an application developed using SyncIt. He is trying to decide what car to buy and the App performs the following change while out of mobile coverage. - jamesSyncIt.set( - 'cars', - 'Subaru', - { color: 'blue' } - function(err) { if (err === SyncIt_Constant.Error.OK) { success(); } } - ); +```javascript +jamesSyncIt.set( + 'cars', + 'Subaru', + { color: 'blue' } + function(err) { if (err === SyncIt_Constant.Error.OK) { success(); } } +); +``` User | Dataset | Datakey | Store | Queueitem Update ------|---------|---------|-------|----------------- @@ -297,28 +308,30 @@ James is happy because he is making progress on deciding on his next car. Later, when James exits the underground the App detects that it can connect and makes the following API call: - syncIt.getFirst(function(err,queueitem) { - if (err !== SyncIt_Constant.Error.OK) { - // throw? +```javascript +syncIt.getFirst(function(err,queueitem) { + if (err !== SyncIt_Constant.Error.OK) { + // throw? + } + xhr( + 'http://server/' + queueitem.s + '/' + queueitem.k, + { + method: 'PATCH', + ... } - xhr( - 'http://server/' + queueitem.s + '/' + queueitem.k, - { - method: 'PATCH', + ).then( + function() { + // data now stored on server + jamesSyncIt.apply(function(err) { ... - } - ).then( - function() { - // data now stored on server - jamesSyncIt.apply(function(err) { - ... - }); - }, - function(err) { - // something went wrong... throw err? - } - ); - }); + }); + }, + function(err) { + // something went wrong... throw err? + } + ); +}); +``` User | Dataset | Datakey | Store | Queueitem ------|---------|---------|----------------------------------------|----------- @@ -332,11 +345,13 @@ The reason this is two steps as apposed to the one commit step for Subversion is His wife, Emily is using the same App and either through a push notification or polling gets James's update from the server which calls: - emilySyncIt.feed( - [Queueitem], // The update from James - function( ... ) { ... }, // Conflict Resolution - We'll get to this soon - function(err) { ... } - ); +```javascript +emilySyncIt.feed( + [Queueitem], // The update from James + function( ... ) { ... }, // Conflict Resolution - We'll get to this soon + function(err) { ... } +); +``` User | Dataset | Datakey | Store | Queueitem ------|---------|---------|----------------------------------------|----------- @@ -347,11 +362,13 @@ This will add the change to the local store, assuming that there are no local ch Emily does not like the idea of thier car being a Subaru and makes the following changes: - emilySyncIt.set( - 'cars', - 'Subaru', - { color: 'blue', style: 'a bit too boy racer for Emily' } - ); +```javascript +emilySyncIt.set( + 'cars', + 'Subaru', + { color: 'blue', style: 'a bit too boy racer for Emily' } +); +``` User | Dataset | Datakey | Store | Queueitem ------|---------|---------|----------------------------------------|----------- @@ -364,12 +381,14 @@ Because she is still in the park and has good mobile coverage that change is upl James is again out of mobile coverage and is completely unaware of Emily's change but has discovered that Subaru's are four wheel drive... - jamesSyncIt.update( - 'cars', - 'Subaru', - { $set: { pluspoints: ['has 4WD'] }, $inc { votes: 1 } }, - function(err) { if (err === SyncIt_Constant.Error.OK) { success(); } } - ); +```javascript +jamesSyncIt.update( + 'cars', + 'Subaru', + { $set: { pluspoints: ['has 4WD'] }, $inc { votes: 1 } }, + function(err) { if (err === SyncIt_Constant.Error.OK) { success(); } } +); +``` User | Dataset | Datakey | Store | Queueitem | Reads ------|---------|---------|----------------------------------------|---------------------------------------------------------------|------------ @@ -384,18 +403,20 @@ What is the correct course of action that James's App should take in this situat Internally SyncIt stores everything including a Modifier and a Version. The data structure looks something like the following: - { - s: "cars", // dataset - k: "Subaru", // datakey - b: 1, // what version this Queueitem is based on (so this is version 2) - m: "james", // the user/device that made the change - o: "update", // the operation that was performed - t: 1369345483365, // timestamp when the operation was performed - u: { // the data for the operation - "$set": { "pluspoints": ["has 4WD"] }, - "$inc" { "votes": 1 } - } - } +```javascript +{ + s: "cars", // dataset + k: "Subaru", // datakey + b: 1, // what version this Queueitem is based on (so this is version 2) + m: "james", // the user/device that made the change + o: "update", // the operation that was performed + t: 1369345483365, // timestamp when the operation was performed + u: { // the data for the operation + "$set": { "pluspoints": ["has 4WD"] }, + "$inc" { "votes": 1 } + } +} +``` So conflicts are possible to detect by comparing versions. @@ -405,43 +426,45 @@ The second part of the solution is that `SyncIt.feed()` includes a callback para The reason for conflict resolution being a callback function I feel it would be impossible for SyncIt to dictate and could be part of your core application logic. So the code/data for feeding data into SyncIt could end up looking something like the following: - SyncIt.feed( - [{ - // The data which has been recieved from other parties via a server - "s": "cars", - "k": "Subaru", - "u": { color:'blue', style:'a bit too boy racer for Emily' }, - "o": "set", "b": 1, "m": "emily", "t": 1369345483321 - }], - function(dataset, datakey, stored, localChanges, remoteChanges, resolved) { - - // This is a super basic, perhaps too basic, example of a conflict - // resolution function that will apply the local update on top of - // the remote update if it has a later timestamp - - if ( - localChanges[localChanges.length - 1].t > - remoteChanges[remoteChanges.length - 1].t - ) { - // James made the last change, so blindly take it! - return resolved( - true, - [remoteChanges[remoteChanges.length - 1]] - ); - } - - // Emily made the last change, so we will throw away our changes - return resolved(true,[]); - }, - function(err,remoteUpdatesNotFed) { - if (err === SyncIt_Constant.Error.OK) { - return success(); - } - // err explains the reason for the error - // remoteUpdatesNotFed includes the update that are on the server - // which we could not feed due to the error. +```javascript +SyncIt.feed( + [{ + // The data which has been recieved from other parties via a server + "s": "cars", + "k": "Subaru", + "u": { color:'blue', style:'a bit too boy racer for Emily' }, + "o": "set", "b": 1, "m": "emily", "t": 1369345483321 + }], + function(dataset, datakey, stored, localChanges, remoteChanges, resolved) { + + // This is a super basic, perhaps too basic, example of a conflict + // resolution function that will apply the local update on top of + // the remote update if it has a later timestamp + + if ( + localChanges[localChanges.length - 1].t > + remoteChanges[remoteChanges.length - 1].t + ) { + // James made the last change, so blindly take it! + return resolved( + true, + [localChanges[localChanges.length - 1]] + ); } - ); + + // Emily made the last change, so we will throw away our changes + return resolved(true,[]); + }, + function(err,remoteUpdatesNotFed) { + if (err === SyncIt_Constant.Error.OK) { + return success(); + } + // err explains the reason for the error + // remoteUpdatesNotFed includes the update that are on the server + // which we could not feed due to the error. + } +); +``` User | Dataset | Datakey | Store | Queueitem | Reads ------|---------|---------|----------------------------------------|---------------------------------------------------------------|------------ @@ -458,8 +481,8 @@ I need to do the following: * Add a license to all files (It'll be MIT/BSD) * Test demo (and everythign else!) in IE * Client SyncIt - * localStorage for SyncIt (Store) - * Add Async wrappers for Store & Persist + * Add persistance to `allLocalToApplyAfterwards` in SyncIt.feed() (Important) + * Add Async wrappers for IndexedDb Store and Persist... * Server SyncIt * Create a real SyncItServer based on SyncItTestServer, it should be pretty easy, because SyncItTestServer is pretty abstracted. * I want to make ServerPersist for both MongoDB and DynamoDB. diff --git a/README.work_locally_set.dot b/README.work_locally_set.dot index c2f7b26..c93d252 100644 --- a/README.work_locally_set.dot +++ b/README.work_locally_set.dot @@ -2,7 +2,7 @@ digraph a { subgraph clusteryourcode { label = "Your Code"; - App [label="Request Data", rank=1]; + App [label="Change Data", rank=1]; SyncItCallback [label=Callback]; SyncItEventHandler [label="Added to Queue listener"]; } diff --git a/bin/README/img/apply-after.png b/bin/README/img/apply-after.png index c4c845f..e37d5ac 100644 Binary files a/bin/README/img/apply-after.png and b/bin/README/img/apply-after.png differ diff --git a/bin/README/img/download_updates_from_server.png b/bin/README/img/download_updates_from_server.png index 3a0ec40..f405cef 100644 Binary files a/bin/README/img/download_updates_from_server.png and b/bin/README/img/download_updates_from_server.png differ diff --git a/bin/README/img/feed_into_syncit.png b/bin/README/img/feed_into_syncit.png index b834c59..2715901 100644 Binary files a/bin/README/img/feed_into_syncit.png and b/bin/README/img/feed_into_syncit.png differ diff --git a/bin/README/img/getfirst.png b/bin/README/img/getfirst.png new file mode 100644 index 0000000..726756a Binary files /dev/null and b/bin/README/img/getfirst.png differ diff --git a/bin/README/img/work_locally_set.png b/bin/README/img/work_locally_set.png index bfa4a8b..de2c1dd 100644 Binary files a/bin/README/img/work_locally_set.png and b/bin/README/img/work_locally_set.png differ diff --git a/build-docs b/build-docs index 8936972..35af04c 100755 --- a/build-docs +++ b/build-docs @@ -13,6 +13,7 @@ cat README.download_updates_from_server.dot | dot -Gdpi=64 -Tpng:cairo:cairo > b cat README.feed_into_syncit.dot | dot -Gdpi=64 -Tpng:cairo:cairo > bin/README/img/feed_into_syncit.png cat README.work_locally_get.dot| dot -Gdpi=64 -Tpng:cairo:cairo > bin/README/img/work_locally_get.png cat README.work_locally_set.dot| dot -Gdpi=64 -Tpng:cairo:cairo > bin/README/img/work_locally_set.png +cat README.getfirst.dot | dot -Gdpi=64 -Tpng:cairo:cairo > bin/README/img/getfirst.png cat README.getfirst_and_apply.dot | dot -Gdpi=64 -Tpng:cairo:cairo > bin/README/img/getfirst_and_apply.png dia -e bin/README/img/process.png -O /tmp README.process.dia marked -i README.md -o README.html diff --git a/demo/public/index.html b/demo/public/index.html index a68948e..7ad040a 100644 --- a/demo/public/index.html +++ b/demo/public/index.html @@ -106,12 +106,12 @@

API

    diff --git a/js/Queue/LocalStorage.js b/js/Queue/LocalStorage.js index 1643fd6..7ed34d1 100644 --- a/js/Queue/LocalStorage.js +++ b/js/Queue/LocalStorage.js @@ -1,3 +1,4 @@ +/* jshint strict: true, smarttabs: true, es3: true, forin: true, immed: true, latedef: true, newcap: true, noarg: true, undef: true, unused: true, es3: true, bitwise: false, curly: true, latedef: true, newcap: true, noarg: true, noempty: true */ (function (root, factory) { // UMD from https://github.com/umdjs/umd/blob/master/returnExports.js if (typeof exports === 'object') { module.exports = factory( @@ -15,7 +16,6 @@ } })(this, function (SyncIt_Constant) { -/* jshint strict: true, smarttabs: true, es3: true, forin: true, immed: true, latedef: true, newcap: true, noarg: true, undef: true, unused: true, es3: true, bitwise: false, curly: true, latedef: true, newcap: true, noarg: true, noempty: true */ "use strict"; // Author: Matthew Forrester @@ -27,7 +27,6 @@ var Lsq = function(namespace,localStorage,stringifyFunc,parseFunc,maxOrderDigitLength) { this._ns = namespace; this._ls = localStorage; - this._listening = false; this._stringifyFunc = stringifyFunc ? stringifyFunc : function(ob) { diff --git a/js/Store/LocalStorage.js b/js/Store/LocalStorage.js new file mode 100644 index 0000000..5fc31f4 --- /dev/null +++ b/js/Store/LocalStorage.js @@ -0,0 +1,149 @@ +/* jshint strict: true, smarttabs: true, es3: true, forin: true, immed: true, latedef: true, newcap: true, noarg: true, undef: true, unused: true, es3: true, bitwise: false, curly: true, latedef: true, newcap: true, noarg: true, noempty: true */ +(function (root, factory) { // UMD from https://github.com/umdjs/umd/blob/master/returnExports.js + if (typeof exports === 'object') { + module.exports = factory( + require('../Constant.js') + ); + } else if (typeof define === 'function' && define.amd) { + define( + ['syncit/Constant'], + factory + ); + } else { + root.SyncIt_Store_LocalStorage = factory( + root.SyncIt_Constant + ); + } +})(this, function (SyncIt_Constant) { + +"use strict"; + +// Author: Matthew Forrester +// Copyright: Matthew Forrester +// License: MIT/BSD-style + +// For all docs, please see Queue/Persist at present + +var Store = function(namespace,localStorage,stringifyFunc,parseFunc) { + this._ns = namespace; + this._ls = localStorage; + this._stringifyFunc = stringifyFunc ? + stringifyFunc : + function(ob) { + return JSON.stringify(ob); + }; + this._parseFunc = parseFunc ? + parseFunc : + function(str) { + return JSON.parse(str); + }; +}; + +Store.prototype._keyCheck = function(dataset, datakey) { + var checkRe = /(\.)|(^[0-9])|(^_)/, + toCheck = {Dataset: dataset, Datakey: datakey}, + msg = '', + k=0; + + for (k in toCheck) { + if (toCheck.hasOwnProperty(k)) { + if (checkRe.test(toCheck[k])) { + msg = 'SyncIt.Store does not support '+k+' '+ + 'which includes a "." or starts with a "_" or number'; + } + if (toCheck[k].length < 2) { + msg = 'SyncIt.Store does not support '+k+' '+ + 'with a length less than 2'; + } + if (msg) { + throw 'SyncIt.Store.Invalid'+k+': '+msg; + } + } + } +}; + +Store.prototype.get = function(dataset, datakey, whenRetrieved) { + this._keyCheck(dataset,datakey); + var item = this._parseFunc(this._ls.getItem( + this._ns + '.' + dataset + '.' + datakey + )); + if (!item) { + return whenRetrieved(SyncIt_Constant.Error.NO_DATA_FOUND,null); + } + item.s = dataset; + item.k = datakey; + whenRetrieved(SyncIt_Constant.Error.OK,item); +}; + +Store.prototype.set = function(dataset, datakey, value, whenSet) { + this._keyCheck(dataset,datakey); + this._ls.setItem( + this._ns + '.' + dataset + '.' + datakey, + this._stringifyFunc({ + i: value.i, + t: value.t, + m: value.m, + v: value.v, + o: value.o + }) + ); + whenSet(SyncIt_Constant.Error.OK); +}; + +Store.prototype._extractFromKey = function(key) { + var keyInfo = key.split('.'); + if (keyInfo.length != 3) { + return false; + } + if (keyInfo[0] != this._ns) { + return false; + } + keyInfo = { + s: keyInfo[1], + k: keyInfo[2] + }; + try { + this._keyCheck(keyInfo.s,keyInfo.k) + } catch(e) { + return false; + } + return keyInfo; +}; + +Store.prototype.getDatasetNames = function(setsRetrieved) { + var r = [], + i = 0, + l = 0, + keyInfo = {}; + + for ( i=0, l=this._ls.length; i + + -