From ee96345618e8a33a9daeaf6bcecdb03b05f406da Mon Sep 17 00:00:00 2001 From: Tristan Cavelier Date: Fri, 22 Mar 2013 17:29:25 +0100 Subject: [PATCH] adding some storage libraries --- lib/jio/davstorage.js | 766 ++++++++++++++++++++ lib/jio/indexstorage.js | 960 +++++++++++++++++++++++++ lib/jio/localstorage.js | 394 ++++++++++ lib/jio/replicaterevisionstorage.js | 677 ++++++++++++++++++ lib/jio/revisionstorage.js | 1033 +++++++++++++++++++++++++++ lib/md5/md5.js | 379 ++++++++++ 6 files changed, 4209 insertions(+) create mode 100644 lib/jio/davstorage.js create mode 100644 lib/jio/indexstorage.js create mode 100644 lib/jio/localstorage.js create mode 100644 lib/jio/replicaterevisionstorage.js create mode 100644 lib/jio/revisionstorage.js create mode 100644 lib/md5/md5.js diff --git a/lib/jio/davstorage.js b/lib/jio/davstorage.js new file mode 100644 index 0000000..0e23526 --- /dev/null +++ b/lib/jio/davstorage.js @@ -0,0 +1,766 @@ +/* +* Copyright 2013, Nexedi SA +* Released under the LGPL license. +* http://www.gnu.org/licenses/lgpl.html +*/ +/*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true */ +/*global jIO: true, $: true, btoa: true */ +jIO.addStorageType('dav', function (spec, my) { + + spec = spec || {}; + var that, priv, super_serialized; + that = my.basicStorage(spec, my); + priv = {}; + super_serialized = that.serialized; + + priv.secureDocId = function (string) { + var split = string.split('/'), + i; + if (split[0] === '') { + split = split.slice(1); + } + for (i = 0; i < split.length; i += 1) { + if (split[i] === '') { + return ''; + } + } + return split.join('%2F'); + }; + priv.convertSlashes = function (string) { + return string.split('/').join('%2F'); + }; + + priv.restoreSlashes = function (string) { + return string.split('%2F').join('/'); + }; + + /** + * Checks if an object has no enumerable keys + * @method objectIsEmpty + * @param {object} obj The object + * @return {boolean} true if empty, else false + */ + priv.objectIsEmpty = function (obj) { + var k; + for (k in obj) { + if (obj.hasOwnProperty(k)) { + return false; + } + } + return true; + }; + + priv.username = spec.username || ''; + priv.secured_username = priv.convertSlashes(priv.username); + priv.password = spec.password || ''; + priv.url = spec.url || ''; + + that.serialized = function () { + var o = super_serialized(); + o.username = priv.username; + o.url = priv.url; + o.password = priv.password; + return o; + }; + + priv.newAsyncModule = function () { + var async = {}; + async.call = function (obj, function_name, arglist) { + obj._wait = obj._wait || {}; + if (obj._wait[function_name]) { + obj._wait[function_name] -= 1; + return function () {}; + } + // ok if undef or 0 + arglist = arglist || []; + return obj[function_name].apply(obj[function_name], arglist); + }; + async.neverCall = function (obj, function_name) { + obj._wait = obj._wait || {}; + obj._wait[function_name] = -1; + }; + async.wait = function (obj, function_name, times) { + obj._wait = obj._wait || {}; + obj._wait[function_name] = times; + }; + async.end = function () { + async.call = function () {}; + }; + return async; + }; + + /** + * Checks if a browser supports cors (cross domain ajax requests) + * @method checkCors + * @return {boolean} true if supported, else false + */ + priv.checkCors = function () { + return $.support.cors; + }; + + /** + * Replaces last "." with "_." in document filenames + * @method underscoreFileExtenisons + * @param {string} url url to clean up + * @return {string} clean_url cleaned up URL + */ + priv.underscoreFileExtenisons = function (url) { + var clean_url = url.replace(/,\s(\w+)$/, "_.$1"); + return clean_url; + }; + + /** + * Replaces "_." with "." in document filenames + * @method restoreDots + * @param {string} url url to clean up + * @return {string} clean_url cleaned up URL + */ + priv.restoreDots = function (url) { + var clean_url = url.replace(/_\./g, '.'); + return clean_url; + }; + + /** + * Runs all ajax requests for davStorage + * @method ajax + * @param {object} ajax_object The request parameters + */ + priv.ajax = function (ajax_object) { + $.ajax({ + url: ajax_object.url, + type: ajax_object.type, + async: true, + dataType: ajax_object.dataType || null, + data: ajax_object.data || null, + crossdomain : true, + headers : { + Authorization: 'Basic ' + btoa( + priv.username + ':' + priv.password + ), + Depth: ajax_object.headers === undefined ? null : + ajax_object.headers.depth + }, + // xhrFields: {withCredentials: 'true'}, + success: ajax_object.success, + error: ajax_object.error + }); + }; + + /** + * Creates error objects for this storage + * @method createError + * @param {string} url url to clean up + * @return {object} error The error object + */ + priv.createError = function (status, message, reason) { + var error = {}; + + switch (status) { + case 404: + error.status = status; + error.statusText = "Not found"; + error.error = "not_found"; + error.message = message; + error.reason = reason; + break; + + case 405: + error.status = status; + error.statusText = "Method Not Allowed"; + error.error = "method_not_allowed"; + error.message = message; + error.reason = reason; + break; + + case 409: + error.status = status; + error.statusText = "Conflicts"; + error.error = "conflicts"; + error.message = message; + error.reason = reason; + break; + } + return error; + }; + + /** + * Check if method can be run on browser + * @method support + */ + priv.support = function (docid) { + // no docId + if (!(typeof docid === "string" && docid !== "")) { + that.error(priv.createError(405, "Can't create document without id", + "Document id is undefined" + )); + return true; + } + // no cross domain ajax + if (priv.checkCors === false) { + that.error(priv.createError(405, + "Browser does not support cross domain ajax", "CORS is undefined" + )); + return true; + } + }; + + // wedDav methods rfc4918 (short summary) + // COPY Reproduces single resources (files) and collections (directory + // trees). Will overwrite files (if specified by request) but will + // respond 209 (Conflict) if it would overwrite a tree + // DELETE deletes files and directory trees + // GET just the vanilla HTTP/1.1 behaviour + // HEAD ditto + // LOCK locks a resources + // MKCOL creates a directory + // MOVE Moves (rename or copy) a file or a directory tree. Will + // 'overwrite' files (if specified by the request) but will respond + // 209 (Conflict) if it would overwrite a tree. + // OPTIONS If WebDAV is enabled and available for the path this reports the + // WebDAV extension methods + // PROPFIND Retrieves the requested file characteristics, DAV lock status + // and 'dead' properties for individual files, a directory and its + // child files, or a directory tree + // PROPPATCHset and remove 'dead' meta-data properties + // PUT Update or create resource or collections + // UNLOCK unlocks a resource + + // Notes: all Ajax requests should be CORS (cross-domain) + // adding custom headers triggers preflight OPTIONS request! + // http://remysharp.com/2011/04/21/getting-cors-working/ + + + priv.putOrPost = function (command, type) { + var docid = command.getDocId(), secured_docid, url, ajax_object; + + if (priv.support(docid)) { + return; + } + + secured_docid = priv.secureDocId(command.getDocId()); + url = priv.url + '/' + priv.underscoreFileExtenisons(secured_docid); + + ajax_object = { + url: url + '?_=' + Date.now(), + type: "GET", + dataType: "text", + success: function () { + if (type === 'POST') { + // POST the document already exists + that.error(priv.createError(409, + "Cannot create a new document", "Document already exists" + )); + return; + } + ajax_object = { + url: url, + type: type, + data: JSON.stringify(command.getDoc()), + success: function () { + that.success({ + ok: true, + id: command.getDocId() + }); + }, + error: function () { + that.error(priv.createError(409, "Cannot modify document", + "Error writing to remote storage" + )); + } + }; + priv.ajax(ajax_object); + }, + error: function (err) { + // Firefox returns 0 instead of 404 on CORS? + if (err.status === 404 || err.status === 0) { + ajax_object = { + url: url, + type: "PUT", + data: JSON.stringify(command.getDoc()), + success: function () { + that.success({ + ok: true, + id: command.getDocId() + }); + }, + error: function () { + that.error(priv.createError(409, + "Cannot modify document", "Error writing to remote storage" + )); + } + }; + priv.ajax(ajax_object); + } else { + // error accessing remote storage + that.error({ + "status": err.status, + "statusText": err.statusText, + "error": "error", + "message": err.message, + "reason": "Failed to access remote storage" + }); + } + } + }; + priv.ajax(ajax_object); + }; + + /** + * Creates a new document + * @method post + * @param {object} command The JIO command + */ + that.post = function (command) { + priv.putOrPost(command, 'POST'); + }; + + /** + * Creates or updates a document + * @method put + * @param {object} command The JIO command + */ + that.put = function (command) { + priv.putOrPost(command, 'PUT'); + }; + + /** + * Add an attachment to a document + * @method putAttachment + * @param {object} command The JIO command + */ + that.putAttachment = function (command) { + var docid = command.getDocId(), + doc, + url, + secured_docid, + secured_attachmentid, + attachment_url, + ajax_object; + + priv.support(docid); + + secured_docid = priv.secureDocId(docid); + url = priv.url + '/' + priv.underscoreFileExtenisons(secured_docid); + + ajax_object = { + url: url + '?_=' + Date.now(), + type: 'GET', + dataType: 'text', + success: function (response) { + doc = JSON.parse(response); + + // the document exists - update document + doc._attachments = doc._attachments || {}; + doc._attachments[command.getAttachmentId()] = { + "content_type": command.getAttachmentMimeType(), + "digest": "md5-" + command.md5SumAttachmentData(), + "length": command.getAttachmentLength() + }; + // put updated document data + ajax_object = { + url: url + '?_=' + Date.now(), + type: 'PUT', + data: JSON.stringify(doc), + success: function () { + secured_attachmentid = priv.secureDocId(command.getAttachmentId()); + attachment_url = url + '.' + + priv.underscoreFileExtenisons(secured_attachmentid); + ajax_object = { + url: attachment_url + '?_=' + Date.now(), + type: 'PUT', + data: JSON.stringify(command.getDoc()), + success: function () { + that.success({ + ok: true, + id: command.getDocId() + '/' + command.getAttachmentId() + }); + }, + error: function () { + that.error(priv.createError(409, + "Cannot modify document", "Error when saving attachment" + )); + return; + } + }; + priv.ajax(ajax_object); + }, + error: function () { + that.error(priv.createError(409, + "Cannot modify document", "Error writing to remote storage" + )); + return; + } + }; + priv.ajax(ajax_object); + }, + error: function () { + // the document does not exist + that.error(priv.createError(404, + "Impossible to add attachment", "Document not found" + )); + return; + } + }; + // see if the underlying document exists + priv.ajax(ajax_object); + }; + + /** + * Get a document or attachment from distant storage + * Options: + * - {boolean} revs Add simple revision history (false by default). + * - {boolean} revs_info Add revs info (false by default). + * - {boolean} conflicts Add conflict object (false by default). + * @method get + * @param {object} command The JIO command + */ + that.get = function (command) { + var docid = command.getDocId(), + doc, + url, + secured_docid, + secured_attachmentid, + attachment_url, + ajax_object; + + if (priv.support(docid)) { + return; + } + + secured_docid = priv.secureDocId(command.getDocId()); + url = priv.url + '/' + priv.underscoreFileExtenisons(secured_docid); + + if (typeof command.getAttachmentId() === "string") { + secured_attachmentid = priv.secureDocId(command.getAttachmentId()); + attachment_url = url + '.' + priv.underscoreFileExtenisons( + secured_attachmentid + ); + // get attachment + ajax_object = { + url: attachment_url + '?_=' + Date.now(), + type: "GET", + dataType: "text", + success: function (response) { + doc = JSON.parse(response); + that.success(doc); + }, + error: function () { + that.error(priv.createError(404, + "Cannot find the attachment", "Attachment does not exist" + )); + } + }; + priv.ajax(ajax_object); + } else { + // get document + ajax_object = { + url: url + '?_=' + Date.now(), + type: "GET", + dataType: "text", + success: function (response) { + // metadata_only should not be handled by jIO, as it is a + // webDav only option, shouldn't it? + // ditto for content_only + doc = JSON.parse(response); + that.success(doc); + }, + error: function () { + that.error(priv.createError(404, + "Cannot find the document", "Document does not exist" + )); + } + }; + priv.ajax(ajax_object); + } + }; + + /** + * Remove a document or attachment + * @method remove + * @param {object} command The JIO command + */ + that.remove = function (command) { + var docid = command.getDocId(), doc, url, + secured_docid, secured_attachmentid, attachment_url, + attachment_list = [], i, j, k = 1, deleteAttachment, ajax_object; + + if (priv.support(docid)) { + return; + } + + secured_docid = priv.secureDocId(command.getDocId()); + url = priv.url + '/' + priv.underscoreFileExtenisons(secured_docid); + + // remove attachment + if (typeof command.getAttachmentId() === "string") { + secured_attachmentid = priv.secureDocId(command.getAttachmentId()); + attachment_url = url + '.' + priv.underscoreFileExtenisons( + secured_attachmentid + ); + ajax_object = { + url: attachment_url + '?_=' + Date.now(), + type: "DELETE", + success: function () { + // retrieve underlying document + ajax_object = { + url: url + '?_=' + Date.now(), + type: "GET", + dataType: "text", + success: function (response) { + // underlying document + doc = JSON.parse(response); + + // update doc._attachments + if (typeof doc._attachments === "object") { + if (typeof doc._attachments[command.getAttachmentId()] === + "object") { + delete doc._attachments[command.getAttachmentId()]; + if (priv.objectIsEmpty(doc._attachments)) { + delete doc._attachments; + } + // PUT back to server + ajax_object = { + url: url + '?_=' + Date.now(), + type: 'PUT', + data: JSON.stringify(doc), + success: function () { + that.success({ + "ok": true, + "id": command.getDocId() + '/' + + command.getAttachmentId() + }); + }, + error: function () { + that.error(priv.createError(409, + "Cannot modify document", "Error saving attachment" + )); + } + }; + priv.ajax(ajax_object); + } else { + // sure this if-else is needed? + that.error(priv.createError(404, + "Cannot find document", "Error updating attachment" + )); + } + } else { + // no attachments, we are done + that.success({ + "ok": true, + "id": command.getDocId() + '/' + command.getAttachmentId() + }); + } + }, + error: function () { + that.error(priv.createError(404, + "Cannot find the document", "Document does not exist" + )); + } + }; + priv.ajax(ajax_object); + }, + error: function () { + that.error(priv.createError(404, + "Cannot find the attachment", "Error removing attachment" + )); + } + }; + priv.ajax(ajax_object); + // remove document incl. all attachments + } else { + ajax_object = { + url: url + '?_=' + Date.now(), + type: 'GET', + dataType: 'text', + success: function (response) { + var x; + doc = JSON.parse(response); + // prepare attachment loop + if (typeof doc._attachments === "object") { + // prepare list of attachments + for (x in doc._attachments) { + if (doc._attachments.hasOwnProperty(x)) { + attachment_list.push(x); + } + } + } + // delete document + ajax_object = { + url: url + '?_=' + Date.now(), + type: 'DELETE', + success: function () { + j = attachment_list.length; + // no attachments, done + if (j === 0) { + that.success({ + "ok": true, + "id": command.getDocId() + }); + } else { + deleteAttachment = function (attachment_url, j, k) { + ajax_object = { + url: attachment_url + '?_=' + Date.now(), + type: 'DELETE', + success: function () { + // all deleted, return response, need k as async couter + if (j === k) { + that.success({ + "ok": true, + "id": command.getDocId() + }); + } else { + k += 1; + } + }, + error: function () { + that.error(priv.createError(404, + "Cannot find attachment", "Error removing attachment" + )); + } + }; + priv.ajax(ajax_object); + }; + for (i = 0; i < j; i += 1) { + secured_attachmentid = priv.secureDocId(attachment_list[i]); + attachment_url = url + '.' + priv.underscoreFileExtenisons( + secured_attachmentid + ); + deleteAttachment(attachment_url, j, k); + } + } + }, + error: function () { + that.error(priv.createError(404, + "Cannot find the document", "Error removing document" + )); + } + }; + priv.ajax(ajax_object); + }, + error: function () { + that.error(priv.createError(404, + "Cannot find the document", "Document does not exist" + )); + } + }; + priv.ajax(ajax_object); + } + }; + + /** + * Gets a document list from a distant dav storage + * Options: + * - {boolean} include_docs Also retrieve the actual document content. + * @method allDocs + * @param {object} command The JIO command + */ + //{ + // "total_rows": 4, + // "rows": [ + // { + // "id": "otherdoc", + // "key": "otherdoc", + // "value": { + // "rev": "1-3753476B70A49EA4D8C9039E7B04254C" + // } + // },{...} + // ] + //} + that.allDocs = function (command) { + var rows = [], url, + am = priv.newAsyncModule(), + o = {}, + ajax_object; + + o.getContent = function (file) { + var docid = priv.secureDocId(file.id), + url = priv.url + '/' + docid; + ajax_object = { + url: url + '?_=' + Date.now(), + type: 'GET', + dataType: 'text', + success: function (content) { + file.doc = JSON.parse(content); + rows.push(file); + am.call(o, 'success'); + }, + error: function (type) { + that.error(priv.createError(404, + "Cannot find the document", "Can't get document from storage" + )); + am.call(o, 'error', [type]); + } + }; + priv.ajax(ajax_object); + }; + o.getDocumentList = function () { + url = priv.url + '/'; + ajax_object = { + url: url + '?_=' + Date.now(), + type: "PROPFIND", + dataType: "xml", + headers : { depth: '1' }, + success: function (xml) { + var response = $(xml).find('D\\:response, response'), + len = response.length; + + if (len === 1) { + return am.call(o, 'success'); + } + am.wait(o, 'success', len - 2); + response.each(function (i, data) { + if (i > 0) { // exclude parent folder + var file = { + value: {} + }; + $(data).find('D\\:href, href').each(function () { + var split = $(this).text().split('/'); + file.id = split[split.length - 1]; + file.id = priv.restoreSlashes(file.id); + file.key = file.id; + }); + if (command.getOption('include_docs')) { + am.call(o, 'getContent', [file]); + } else { + rows.push(file); + am.call(o, 'success'); + } + } + }); + }, + error: function (type) { + that.error(priv.createError(404, + "Cannot find the document", "Can't get document list" + )); + am.call(o, 'retry', [type]); + } + }; + priv.ajax(ajax_object); + }; + o.retry = function (error) { + am.neverCall(o, 'retry'); + am.neverCall(o, 'success'); + am.neverCall(o, 'error'); + that.retry(error); + }; + o.error = function (error) { + am.neverCall(o, 'retry'); + am.neverCall(o, 'success'); + am.neverCall(o, 'error'); + that.error(error); + }; + o.success = function () { + am.neverCall(o, 'retry'); + am.neverCall(o, 'success'); + am.neverCall(o, 'error'); + that.success({ + total_rows: rows.length, + rows: rows + }); + }; + // first get the XML list + am.call(o, 'getDocumentList'); + }; + + return that; +}); \ No newline at end of file diff --git a/lib/jio/indexstorage.js b/lib/jio/indexstorage.js new file mode 100644 index 0000000..6a5ecb8 --- /dev/null +++ b/lib/jio/indexstorage.js @@ -0,0 +1,960 @@ +/* +* Copyright 2013, Nexedi SA +* Released under the LGPL license. +* http://www.gnu.org/licenses/lgpl.html +*/ +/*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true */ +/*global jIO: true, localStorage: true, setTimeout: true */ +/** + * JIO Index Storage. + * Manages indexes for specified storages. + * Description: + * { + * "type": "index", + * "indices": [ + * {"indexA",["field_A"]}, + * {"indexAB",["field_A","field_B"]} + * ], + * "field_types": { + * "field_A": "dateTime", + * "field_B": "string" + * }, + * "storage": [ + * , + * ... + * ] + * } + * Index file will contain + * { + * "_id": "app-name_indices.json", + * "indexA": + * "fieldA": { + * "keyword_abc": ["some_id","some_other_id",...] + * } + * }, + * "indexAB": { + * "fieldA": { + * "keyword_abc": ["some_id"] + * }, + * "fieldB": { + * "keyword_def": ["some_id"] +* } + * } + * } + * NOTES: + * It may be difficult to "un-sort" multi-field indices, like + * indexAB, because all keywords will be listed regrardless + * of underlying field, so an index on author and year would produce + * two entries per record like: + * + * "William Shakespeare":["id_Romeo_and_Juliet", "id_Othello"], + * "1591":["id_Romeo_and_Juliet"], + * "1603":["id_Othello"] + * + * So for direct lookups, this should be convient, but for other types + * of queries, it depends + */ +jIO.addStorageType('indexed', function (spec, my) { + + "use strict"; + var that, priv = {}; + + spec = spec || {}; + that = my.basicStorage(spec, my); + + priv.indices = spec.indices; + priv.field_types = spec.field_types; + priv.substorage_key = "sub_storage"; + priv.substorage = spec[priv.substorage_key]; + priv.index_indicator = spec.sub_storage.application_name || "index"; + priv.index_suffix = priv.index_indicator + "_indices.json"; + + my.env = my.env || spec.env || {}; + + that.specToStore = function () { + var o = {}; + o[priv.substorage_key] = priv.substorage; + o.env = my.env; + return o; + }; + + /** + * Generate a new uuid + * @method generateUuid + * @return {string} The new uuid + */ + priv.generateUuid = function () { + var S4 = function () { + var i, string = Math.floor( + Math.random() * 0x10000 /* 65536 */ + ).toString(16); + for (i = string.length; i < 4; i += 1) { + string = "0" + string; + } + return string; + }; + return S4() + S4() + "-" + + S4() + "-" + + S4() + "-" + + S4() + "-" + + S4() + S4() + S4(); + }; + + /** + * Get number of elements in object + * @method getObjectSize + * @param {object} obj The object to check + * @return {number} size The amount of elements in the object + */ + priv.getObjectSize = function (obj) { + var size = 0, key; + for (key in obj) { + if (obj.hasOwnProperty(key)) { + size += 1; + } + } + return size; + }; + + /** + * Creates an empty indices array + * @method createEmptyIndexArray + * @param {array} indices An array of indices (optional) + * @return {object} The new index array + */ + priv.createEmptyIndexArray = function (indices) { + var i, k, j = priv.indices.length, new_index, + new_index_object = {}, new_index_name, new_index_fields; + + if (indices === undefined) { + for (i = 0; i < j; i += 1) { + new_index = priv.indices[i]; + new_index_name = new_index.name; + new_index_fields = new_index.fields; + new_index_object[new_index_name] = {}; + + // loop index fields and add objects to hold value/id pairs + for (k = 0; k < new_index_fields.length; k += 1) { + new_index_object[new_index_name][new_index_fields[k]] = {}; + } + } + } + return new_index_object; + }; + + /** + * Determine if a key/value pair exists in an object by VALUE + * @method searchObjectByValue + * @param {object} indexToSearch The index to search + * @param {string} docid The document id to find + * @param {string} passback The value that should be returned + * @return {boolean} true/false + */ + priv.searchIndexByValue = function (indexToSearch, docid, passback) { + var key, obj, prop; + + for (key in indexToSearch) { + if (indexToSearch.hasOwnProperty(key)) { + obj = indexToSearch[key]; + for (prop in obj) { + if (obj[prop] === docid) { + return passback === "bool" ? true : key; + } + } + } + } + return false; + }; + + /** + * Get element position in array + * @method getPositionInArray + * @param {object} indices The index file + * @param {object} indices The index file + * @returns {number} i Position of element in array + */ + priv.getPositionInArray = function (element, array) { + var i, l = array.length; + for (i = 0; i < l; i += 1) { + if (array[i] === element) { + return i; + } + } + return null; + }; + + /** + * Find id in indices + * @method isDocidInIndex + * @param {object} indices The file containing the indeces + * @param {object} doc The document which should be added to the index + * @return {boolean} true/false + */ + priv.isDocidInIndex = function (indices, doc) { + var index, i, j, label, l = priv.indices.length; + + // loop indices + for (i = 0; i < l; i += 1) { + index = {}; + index.reference = priv.indices[i]; + index.reference_size = index.reference.fields.length; + index.current = indices[index.reference.name]; + + for (j = 0; j < index.reference_size; j += 1) { + label = index.reference.fields[j]; + index.current_size = priv.getObjectSize(index.current[label]); + + // check for existing entries to remove (put-update) + if (index.current_size > 0) { + if (priv.searchIndexByValue(index.current[label], doc._id, "bool")) { + return true; + } + } + } + } + return false; + }; + + /** + * Clean up indexes when removing a file + * @method cleanIndices + * @param {object} indices The file containing the indeces + * @param {object} doc The document which should be added to the index + * @return {object} indices The cleaned up file + */ + priv.cleanIndices = function (indices, doc) { + var i, j, k, index, key, label, l = priv.indices.length; + + // loop indices (indexA, indexAB...) + for (i = 0; i < l; i += 1) { + index = {}; + index.reference = priv.indices[i]; + index.reference_size = index.reference.fields.length; + index.current = indices[index.reference.name]; + + // loop index fields + for (j = 0; j < index.reference_size; j += 1) { + label = index.reference.fields[j]; + index.current_size = priv.getObjectSize(index.current[label]); + + // loop field entries + for (k = 0; k < index.current_size; k += 1) { + key = priv.searchIndexByValue(index.current[label], doc._id, "key"); + index.result_array = index.current[label][key]; + if (!!key) { + // if there is more than one docid in the result array, + // just remove this one and not the whole array + if (index.result_array.length > 1) { + index.result_array.splice(k, 1); + } else { + delete index.current[label][key]; + } + } + } + } + } + return indices; + }; + /** + * Adds entries to indices + * @method createEmptyIndexArray + * @param {object} indices The file containing the indeces + * @param {object} doc The document which should be added to the index + */ + priv.updateIndices = function (indices, doc) { + var i, j, index, value, label, key, l = priv.indices.length; + + // loop indices + for (i = 0; i < l; i += 1) { + index = {}; + index.reference = priv.indices[i]; + index.reference_size = index.reference.fields.length; + index.current = indices[index.reference.name]; + + // build array of values to create entries in index + for (j = 0; j < index.reference_size; j += 1) { + label = index.reference.fields[j]; + value = doc[label]; + if (value !== undefined) { + index.current_size = priv.getObjectSize(index.current[label]); + + // check for existing entries to remove (put-update) + if (index.current_size > 0) { + key = priv.searchIndexByValue( + index.current[label], + doc._id, + "key" + ); + if (!!key) { + delete index.current[label][key]; + } + } + if (index.current[label][value] === undefined) { + index.current[label][value] = []; + } + // add a new entry + index.current[label][value].push(doc._id); + } + } + } + return indices; + }; + + /** + * Check available indices to find the best one. + * TODOS: NOT NICE, redo + * @method findBestIndexForQuery + * @param {object} syntax of query + * @returns {object} response The query object constructed from Index file + */ + priv.findBestIndexForQuery = function (syntax) { + var i, j, k, l, n, p, o, element, key, block, + search_ids, use_index = [], select_ids = {}, index, query_param, + current_query, current_query_size; + + // try to parse into object + if (syntax.query !== undefined) { + current_query = jIO.ComplexQueries.parse(syntax.query); + } else { + current_query = {}; + current_query_size = 0; + } + + // loop indices + for (i = 0; i < priv.indices.length; i += 1) { + search_ids = []; + block = false; + index = {}; + index.reference = priv.indices[i]; + index.reference_size = index.reference.fields.length; + + if (current_query_size !== 0) { + // rebuild search_ids for iteration + if (current_query.query_list === undefined) { + search_ids.push(current_query.id); + } else { + for (j = 0; j < current_query.query_list.length; j += 1) { + if (priv.getPositionInArray(current_query.query_list[j].id, + search_ids) === null) { + search_ids.push(current_query.query_list[j].id); + } + } + } + + // loop search ids and find matches in index + for (k = 0; k < search_ids.length; k += 1) { + query_param = search_ids[0]; + for (l = 0; l < index.reference_size; l += 1) { + if (query_param === index.reference.fields[l]) { + search_ids.splice( + priv.getPositionInArray(query_param, search_ids), + 1 + ); + } + } + } + } + + // rebuild select_ids + for (o = 0; o < syntax.filter.select_list.length; o += 1) { + element = syntax.filter.select_list[o]; + select_ids[element] = true; + } + + // search_ids empty = all needed search fields found on index + if (search_ids.length === 0) { + p = priv.getObjectSize(select_ids); + if (p === 0) { + use_index.push({ + "name": index.reference.name, + "search": true, + "results": false + }); + } else { + for (n = 0; n < index.reference_size; n += 1) { + delete select_ids[index.reference.fields[n]]; + } + for (key in select_ids) { + if (select_ids.hasOwnProperty(key)) { + use_index.push({ + "name": index.reference.name, + "search": true, + "results": false + }); + block = true; + } + } + if (block === false) { + use_index.push({ + "name": index.reference.name, + "search": true, + "results": true + }); + } + } + } + } + return use_index; + }; + + /** + * Converts the indices file into an object usable by complex queries + * @method constructQueryObject + * @param {object} indices The index file + * @returns {object} response The query object constructed from Index file + */ + priv.constructQueryObject = function (indices, query_syntax) { + var j, k, l, m, n, use_index, index, + index_name, field_names, field, key, element, + query_index, query_object = [], field_name, + entry; + + // returns index-to-use|can-do-query|can-do-query-and-results + use_index = priv.findBestIndexForQuery(query_syntax); + + if (use_index.length > 0) { + for (j = 0; j < use_index.length; j += 1) { + index = use_index[j]; + + // NOTED: the index could be used to: + // (a) get all document ids matching query + // (b) get all document ids and results (= run complex query on index) + // right now, only (b) is supported, because the complex query is + // a single step process. If it was possible to first get the + // relevant document ids, then get the results, the index could be + // used to do the first step plus use GET on the returned documents + if (index.search && index.results) { + index_name = use_index[j].name; + query_index = indices[index_name]; + + // get fieldnames from this index + for (k = 0; k < priv.indices.length; k += 1) { + if (priv.indices[k].name === use_index[j].name) { + field_names = priv.indices[k].fields; + } + } + for (l = 0; l < field_names.length; l += 1) { + field_name = field_names[l]; + // loop entries for this field name + field = query_index[field_name]; + for (key in field) { + if (field.hasOwnProperty(key)) { + element = field[key]; + // key can be "string" or "number" right now + if (priv.field_types[field_name] === "number") { + key = +key; + } + for (m = 0; m < element.length; m += 1) { + if (priv.searchIndexByValue( + query_object, + element[m], + "bool" + )) { + // loop object + for (n = 0; n < query_object.length; n += 1) { + entry = query_object[n]; + if (entry.id === element[m]) { + entry[field_name] = key; + } + } + } else { + entry = {}; + entry.id = element[m]; + entry[field_name] = key; + query_object.push(entry); + } + } + } + } + } + } + } + } + return query_object; + }; + /** + * Build the alldocs response from the index file (overriding substorage) + * @method allDocsResponseFromIndex + * @param {object} command The JIO command + * @param {boolean} include_docs Whether to also supply the document + * @param {object} option The options set for this method + * @returns {object} response The allDocs response + */ + priv.allDocsResponseFromIndex = function (indices, include_docs, option) { + var i, j, k, m, n = 0, l = priv.indices.length, + index, key, obj, prop, found, file, label, + unique_count = 0, unique_docids = [], all_doc_response = {}, + success = function (content) { + file = { value: {} }; + file.id = unique_docids[n]; + file.key = unique_docids[n]; + file.doc = content; + all_doc_response.rows.push(file); + // async counter, must be in callback + n += 1; + if (n === unique_count) { + that.success(all_doc_response); + } + }, + error = function () { + that.error({ + "status": 404, + "statusText": "Not Found", + "error": "not_found", + "message": "Cannot find the document", + "reason": "Cannot get a document from substorage" + }); + return; + }; + + // loop indices + for (i = 0; i < l; i += 1) { + index = {}; + index.reference = priv.indices[i]; + index.reference_size = index.reference.fields.length; + index.current = indices[index.reference.name]; + + // a lot of loops, not sure this is the fastest way + // loop index fields + for (j = 0; j < index.reference_size; j += 1) { + label = index.reference.fields[j]; + index.current_field = index.current[label]; + index.current_size = priv.getObjectSize(index.current_field); + + // loop field id array + for (j = 0; j < index.current_size; j += 1) { + for (key in index.current_field) { + if (index.current_field.hasOwnProperty(key)) { + obj = index.current_field[key]; + for (prop in obj) { + if (obj.hasOwnProperty(prop)) { + for (k = 0; k < unique_docids.length; k += 1) { + if (obj[prop] === unique_docids[k]) { + found = true; + break; + } + } + if (!found) { + unique_docids.push(obj[prop]); + unique_count += 1; + } + } + } + } + } + } + } + } + + // construct allDocs response + all_doc_response.total_rows = unique_count; + all_doc_response.rows = []; + for (m = 0; m < unique_count; m += 1) { + // include_docs + if (include_docs) { + that.addJob( + "get", + priv.substorage, + unique_docids[m], + option, + success, + error + ); + } else { + file = { value: {} }; + file.id = unique_docids[m]; + file.key = unique_docids[m]; + all_doc_response.rows.push(file); + if (m === (unique_count - 1)) { + return all_doc_response; + } + } + } + }; + + /** + * Post document to substorage and create/update index file(s) + * @method post + * @param {object} command The JIO command + * @param {string} source The source of the function call + */ + priv.postOrPut = function (command, source) { + var f = {}, indices, doc; + doc = command.cloneDoc(); + if (typeof doc._id !== "string") { + doc._id = priv.generateUuid(); + } + f.getIndices = function () { + var option = command.cloneOption(); + that.addJob( + "get", + priv.substorage, + {"_id": priv.index_suffix}, + option, + function (response) { + indices = response; + f.postDocument("put"); + }, + function (err) { + switch (err.status) { + case 404: + if (source !== 'PUTATTACHMENT') { + indices = priv.createEmptyIndexArray(); + f.postDocument("post"); + } else { + that.error({ + "status": 404, + "statusText": "Not Found", + "error": "not found", + "message": "Document not found", + "reason": "Document not found" + }); + return; + } + break; + default: + err.message = "Cannot retrieve index array"; + that.error(err); + break; + } + } + ); + }; + f.postDocument = function (index_update_method) { + if (priv.isDocidInIndex(indices, doc) && source === 'POST') { + // POST the document already exists + that.error({ + "status": 409, + "statusText": "Conflicts", + "error": "conflicts", + "message": "Cannot create a new document", + "reason": "Document already exists" + }); + return; + } + if (source !== 'PUTATTACHMENT') { + indices = priv.updateIndices(indices, doc); + } + that.addJob( + source === 'PUTATTACHMENT' ? "putAttachment" : "post", + priv.substorage, + doc, + command.cloneOption(), + function () { + if (source !== 'PUTATTACHMENT') { + f.sendIndices(index_update_method); + } else { + that.success({ + "ok": true, + "id": doc._id, + "attachment": doc._attachment + }); + } + }, + function (err) { + switch (err.status) { + case 409: + // file already exists + if (source !== 'PUTATTACHMENT') { + f.sendIndices(index_update_method); + } else { + that.success({ + "ok": true, + "id": doc._id + }); + } + break; + default: + err.message = "Cannot upload document"; + that.error(err); + break; + } + } + ); + }; + f.sendIndices = function (method) { + indices._id = priv.index_suffix; + that.addJob( + method, + priv.substorage, + indices, + command.cloneOption(), + function () { + that.success({ + "ok": true, + "id": doc._id + }); + }, + function (err) { + // xxx do we try to delete the posted document ? + err.message = "Cannot save index file"; + that.error(err); + } + ); + }; + f.getIndices(); + }; + + /** + * Update the document metadata and update the index + * @method put + * @param {object} command The JIO command + */ + that.post = function (command) { + priv.postOrPut(command, 'POST'); + }; + + /** + * Update the document metadata and update the index + * @method put + * @param {object} command The JIO command + */ + that.put = function (command) { + priv.postOrPut(command, 'PUT'); + }; + + /** + * Add an attachment to a document (no index modification) + * @method putAttachment + * @param {object} command The JIO command + */ + that.putAttachment = function (command) { + priv.postOrPut(command, 'PUTATTACHMENT'); + }; + + /** + * Get the document metadata + * @method get + * @param {object} command The JIO command + */ + that.get = function (command) { + that.addJob( + "get", + priv.substorage, + command.cloneDoc(), + command.cloneOption(), + function (response) { + that.success(response); + }, + function (err) { + that.error(err); + } + ); + }; + + /** + * Get the attachment. + * @method getAttachment + * @param {object} command The JIO command + */ + that.getAttachment = function (command) { + that.addJob( + "getAttachment", + priv.substorage, + command.cloneDoc(), + command.cloneOption(), + function (response) { + that.success(response); + }, + function (err) { + that.error(err); + } + ); + }; + + /** + * Remove document - removing documents updates index!. + * @method remove + * @param {object} command The JIO command + */ + that.remove = function (command) { + var f = {}, indices, doc, docid, option; + + doc = command.cloneDoc(); + option = command.cloneOption(); + + f.removeDocument = function (type) { + that.addJob( + "remove", + priv.substorage, + doc, + option, + function (response) { + that.success(response); + }, + function () { + that.error({ + "status": 409, + "statusText": "Conflict", + "error": "conflict", + "message": "Document Update Conflict", + "reason": "Could not delete document or attachment" + }); + } + ); + }; + f.getIndices = function () { + that.addJob( + "get", + priv.substorage, + {"_id": priv.index_suffix}, + option, + function (response) { + // if deleting an attachment + if (typeof command.getAttachmentId() === 'string') { + f.removeDocument('attachment'); + } else { + indices = priv.cleanIndices(response, doc); + // store update index file + that.addJob( + "put", + priv.substorage, + indices, + command.cloneOption(), + function () { + // remove actual document + f.removeDocument('doc'); + }, + function (err) { + err.message = "Cannot save index file"; + that.error(err); + } + ); + } + }, + function () { + that.error({ + "status": 404, + "statusText": "Not Found", + "error": "not_found", + "message": "Document index not found, please check document ID", + "reason": "Incorrect document ID" + }); + return; + } + ); + }; + f.getIndices(); + }; + + /** + * Remove document - removing documents updates index!. + * @method remove + * @param {object} command The JIO command + */ + that.removeAttachment = function (command) { + var f = {}, indices, doc, docid, option; + doc = command.cloneDoc(); + option = command.cloneOption(); + f.removeDocument = function (type) { + that.addJob( + "removeAttachment", + priv.substorage, + doc, + option, + that.success, + that.error + ); + }; + f.getIndices = function () { + that.addJob( + "get", + priv.substorage, + {"_id": priv.index_suffix}, + option, + function (response) { + // if deleting an attachment + if (typeof command.getAttachmentId() === 'string') { + f.removeDocument('attachment'); + } else { + indices = priv.cleanIndices(response, doc); + // store update index file + that.addJob( + "put", + priv.substorage, + indices, + command.cloneOption(), + function () { + // remove actual document + f.removeDocument('doc'); + }, + function (err) { + err.message = "Cannot save index file"; + that.error(err); + } + ); + } + }, + function (err) { + that.error(err); + } + ); + }; + f.getIndices(); + }; + + /** + * Gets a document list from the substorage + * Options: + * - {boolean} include_docs Also retrieve the actual document content. + * @method allDocs + * @param {object} command The JIO command + */ + //{ + // "total_rows": 4, + // "rows": [ + // { + // "id": "otherdoc", + // "key": "otherdoc", + // "value": { + // "rev": "1-3753476B70A49EA4D8C9039E7B04254C" + // } + // },{...} + // ] + //} + that.allDocs = function (command) { + var f = {}, option, all_docs_response, query_object, query_syntax, + query_response; + option = command.cloneOption(); + + f.getIndices = function () { + that.addJob( + "get", + priv.substorage, + {"_id": priv.index_suffix}, + option, + function (response) { + query_syntax = command.getOption('query'); + if (query_syntax !== undefined) { + // build complex query object + query_object = priv.constructQueryObject(response, query_syntax); + if (query_object.length === 0) { + that.addJob( + "allDocs", + priv.substorage, + undefined, + option, + that.success, + that.error + ); + } else { + // we can use index, run query on index + query_response = + jIO.ComplexQueries.query(query_syntax, query_object); + that.success(query_response); + } + } else if (command.getOption('include_docs')) { + priv.allDocsResponseFromIndex(response, true, option); + } else { + all_docs_response = + priv.allDocsResponseFromIndex(response, false, option); + that.success(all_docs_response); + } + }, + that.error + ); + }; + f.getIndices(); + }; + return that; +}); diff --git a/lib/jio/localstorage.js b/lib/jio/localstorage.js new file mode 100644 index 0000000..08e6b5b --- /dev/null +++ b/lib/jio/localstorage.js @@ -0,0 +1,394 @@ +/* +* Copyright 2013, Nexedi SA +* Released under the LGPL license. +* http://www.gnu.org/licenses/lgpl.html +*/ +/*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true */ +/*global jIO: true, localStorage: true, setTimeout: true */ +/** + * JIO Local Storage. Type = 'local'. + * Local browser "database" storage. + */ +jIO.addStorageType('local', function (spec, my) { + + spec = spec || {}; + var that, priv, localstorage; + that = my.basicStorage(spec, my); + priv = {}; + + /* + * Wrapper for the localStorage used to simplify instion of any kind of + * values + */ + localstorage = { + getItem: function (item) { + var value = localStorage.getItem(item); + return value === null ? null : JSON.parse(value); + }, + setItem: function (item, value) { + return localStorage.setItem(item, JSON.stringify(value)); + }, + removeItem: function (item) { + return localStorage.removeItem(item); + } + }; + + // attributes + priv.username = spec.username || ''; + priv.application_name = spec.application_name || 'untitled'; + + priv.localpath = 'jio/localstorage/' + priv.username + '/' + + priv.application_name; + + // ==================== Tools ==================== + /** + * Update [doc] the document object and remove [doc] keys + * which are not in [new_doc]. It only changes [doc] keys not starting + * with an underscore. + * ex: doc: {key:value1,_key:value2} with + * new_doc: {key:value3,_key:value4} updates + * doc: {key:value3,_key:value2}. + * @param {object} doc The original document object. + * @param {object} new_doc The new document object + */ + priv.documentObjectUpdate = function (doc, new_doc) { + var k; + for (k in doc) { + if (doc.hasOwnProperty(k)) { + if (k[0] !== '_') { + delete doc[k]; + } + } + } + for (k in new_doc) { + if (new_doc.hasOwnProperty(k)) { + if (k[0] !== '_') { + doc[k] = new_doc[k]; + } + } + } + }; + + /** + * Checks if an object has no enumerable keys + * @method objectIsEmpty + * @param {object} obj The object + * @return {boolean} true if empty, else false + */ + priv.objectIsEmpty = function (obj) { + var k; + for (k in obj) { + if (obj.hasOwnProperty(k)) { + return false; + } + } + return true; + }; + + // ===================== overrides ====================== + that.specToStore = function () { + return { + "application_name": priv.application_name, + "username": priv.username + }; + }; + + that.validateState = function () { + if (typeof priv.username === "string" && priv.username !== '') { + return ''; + } + return 'Need at least one parameter: "username".'; + }; + + // ==================== commands ==================== + /** + * Create a document in local storage. + * @method post + * @param {object} command The JIO command + */ + that.post = function (command) { + setTimeout(function () { + var doc = command.getDocId(); + if (!(typeof doc === "string" && doc !== "")) { + that.error({ + "status": 405, + "statusText": "Method Not Allowed", + "error": "method_not_allowed", + "message": "Cannot create document which id is undefined", + "reason": "Document id is undefined" + }); + return; + } + doc = localstorage.getItem(priv.localpath + "/" + doc); + if (doc === null) { + // the document does not exist + localstorage.setItem(priv.localpath + "/" + command.getDocId(), + command.cloneDoc()); + that.success({ + "ok": true, + "id": command.getDocId() + }); + } else { + // the document already exists + that.error({ + "status": 409, + "statusText": "Conflicts", + "error": "conflicts", + "message": "Cannot create a new document", + "reason": "Document already exists" + }); + } + }); + }; + + /** + * Create or update a document in local storage. + * @method put + * @param {object} command The JIO command + */ + that.put = function (command) { + setTimeout(function () { + var doc; + doc = localstorage.getItem(priv.localpath + "/" + command.getDocId()); + if (doc === null) { + // the document does not exist + doc = command.cloneDoc(); + } else { + // the document already exists + priv.documentObjectUpdate(doc, command.cloneDoc()); + } + // write + localstorage.setItem(priv.localpath + "/" + command.getDocId(), doc); + that.success({ + "ok": true, + "id": command.getDocId() + }); + }); + }; + + /** + * Add an attachment to a document + * @method putAttachment + * @param {object} command The JIO command + */ + that.putAttachment = function (command) { + setTimeout(function () { + var doc; + doc = localstorage.getItem(priv.localpath + "/" + command.getDocId()); + if (doc === null) { + // the document does not exist + that.error({ + "status": 404, + "statusText": "Not Found", + "error": "not_found", + "message": "Impossible to add attachment", + "reason": "Document not found" + }); + return; + } + + // the document already exists + doc._attachments = doc._attachments || {}; + doc._attachments[command.getAttachmentId()] = { + "content_type": command.getAttachmentMimeType(), + "digest": "md5-" + command.md5SumAttachmentData(), + "length": command.getAttachmentLength() + }; + + // upload data + localstorage.setItem(priv.localpath + "/" + command.getDocId() + "/" + + command.getAttachmentId(), + command.getAttachmentData()); + // write document + localstorage.setItem(priv.localpath + "/" + command.getDocId(), doc); + that.success({ + "ok": true, + "id": command.getDocId(), + "attachment": command.getAttachmentId() + }); + }); + }; + + /** + * Get a document + * @method get + * @param {object} command The JIO command + */ + that.get = function (command) { + setTimeout(function () { + var doc = localstorage.getItem(priv.localpath + "/" + command.getDocId()); + if (doc !== null) { + that.success(doc); + } else { + that.error({ + "status": 404, + "statusText": "Not Found", + "error": "not_found", + "message": "Cannot find the document", + "reason": "Document does not exist" + }); + } + }); + }; + + /** + * Get a attachment + * @method getAttachment + * @param {object} command The JIO command + */ + that.getAttachment = function (command) { + setTimeout(function () { + var doc = localstorage.getItem(priv.localpath + "/" + command.getDocId() + + "/" + command.getAttachmentId()); + if (doc !== null) { + that.success(doc); + } else { + that.error({ + "status": 404, + "statusText": "Not Found", + "error": "not_found", + "message": "Cannot find the attachment", + "reason": "Attachment does not exist" + }); + } + }); + }; + + /** + * Remove a document + * @method remove + * @param {object} command The JIO command + */ + that.remove = function (command) { + setTimeout(function () { + var doc, i, attachment_list; + doc = localstorage.getItem(priv.localpath + "/" + command.getDocId()); + attachment_list = []; + if (doc !== null && typeof doc === "object") { + if (typeof doc._attachments === "object") { + // prepare list of attachments + for (i in doc._attachments) { + if (doc._attachments.hasOwnProperty(i)) { + attachment_list.push(i); + } + } + } + } else { + return that.error({ + "status": 404, + "statusText": "Not Found", + "error": "not_found", + "message": "Document not found", + "reason": "missing" + }); + } + localstorage.removeItem(priv.localpath + "/" + command.getDocId()); + // delete all attachments + for (i = 0; i < attachment_list.length; i += 1) { + localstorage.removeItem(priv.localpath + "/" + command.getDocId() + + "/" + attachment_list[i]); + } + that.success({ + "ok": true, + "id": command.getDocId() + }); + }); + }; + + /** + * Remove an attachment + * @method removeAttachment + * @param {object} command The JIO command + */ + that.removeAttachment = function (command) { + setTimeout(function () { + var doc, error, i, attachment_list; + error = function (word) { + that.error({ + "status": 404, + "statusText": "Not Found", + "error": "not_found", + "message": word + " not found", + "reason": "missing" + }); + }; + doc = localstorage.getItem(priv.localpath + "/" + command.getDocId()); + // remove attachment from document + if (doc !== null && typeof doc === "object" && + typeof doc._attachments === "object") { + if (typeof doc._attachments[command.getAttachmentId()] === + "object") { + delete doc._attachments[command.getAttachmentId()]; + if (priv.objectIsEmpty(doc._attachments)) { + delete doc._attachments; + } + localstorage.setItem(priv.localpath + "/" + command.getDocId(), + doc); + localstorage.removeItem(priv.localpath + "/" + command.getDocId() + + "/" + command.getAttachmentId()); + that.success({ + "ok": true, + "id": command.getDocId(), + "attachment": command.getAttachmentId() + }); + } else { + error("Attachment"); + } + } else { + error("Document"); + } + }); + }; + + /** + * Get all filenames belonging to a user from the document index + * @method allDocs + * @param {object} command The JIO command + */ + that.allDocs = function (command) { + var i, j, file, items = 0, + s = new RegExp("^" + priv.localpath + "\\/[^/]+$"), + all_doc_response = {}, + query_object = [], query_syntax, query_response = []; + + query_syntax = command.getOption('query'); + if (query_syntax === undefined) { + all_doc_response.rows = []; + + for (i in localStorage) { + if (localStorage.hasOwnProperty(i)) { + // filter non-documents + if (s.test(i)) { + items += 1; + j = i.split('/').slice(-1)[0]; + + file = { value: {} }; + file.id = j; + file.key = j; + if (command.getOption('include_docs')) { + file.doc = JSON.parse(localStorage.getItem(i)); + } + all_doc_response.rows.push(file); + } + } + } + all_doc_response.total_rows = items; + that.success(all_doc_response); + } else { + // create complex query object from returned results + for (i in localStorage) { + if (localStorage.hasOwnProperty(i)) { + if (s.test(i)) { + items += 1; + j = i.split('/').slice(-1)[0]; + query_object.push(localstorage.getItem(i)); + } + } + } + query_response = jIO.ComplexQueries.query(query_syntax, query_object); + that.success(query_response); + } + }; + + return that; +}); diff --git a/lib/jio/replicaterevisionstorage.js b/lib/jio/replicaterevisionstorage.js new file mode 100644 index 0000000..965d007 --- /dev/null +++ b/lib/jio/replicaterevisionstorage.js @@ -0,0 +1,677 @@ +/*jslint indent: 2, maxlen: 80, nomen: true */ +/*global jIO: true */ +/** + * JIO Replicate Revision Storage. + * It manages storages that manage revisions and conflicts. + * Description: + * { + * "type": "replicaterevision", + * "storage_list": [ + * , + * ... + * ] + * } + */ +jIO.addStorageType('replicaterevision', function (spec, my) { + "use strict"; + var that, priv = {}; + spec = spec || {}; + that = my.basicStorage(spec, my); + + priv.storage_list_key = "storage_list"; + priv.storage_list = spec[priv.storage_list_key]; + priv.emptyFunction = function () {}; + + that.specToStore = function () { + var o = {}; + o[priv.storage_list_key] = priv.storage_list; + return o; + }; + + /** + * Generate a new uuid + * @method generateUuid + * @return {string} The new uuid + */ + priv.generateUuid = function () { + var S4 = function () { + var i, string = Math.floor( + Math.random() * 0x10000 /* 65536 */ + ).toString(16); + for (i = string.length; i < 4; i += 1) { + string = "0" + string; + } + return string; + }; + return S4() + S4() + "-" + + S4() + "-" + + S4() + "-" + + S4() + "-" + + S4() + S4() + S4(); + }; + + /** + * Create an array containing dictionnary keys + * @method dictKeys2Array + * @param {object} dict The object to convert + * @return {array} The array of keys + */ + priv.dictKeys2Array = function (dict) { + var k, newlist = []; + for (k in dict) { + if (dict.hasOwnProperty(k)) { + newlist.push(k); + } + } + return newlist; + }; + + /** + * Checks a revision format + * @method checkRevisionFormat + * @param {string} revision The revision string + * @return {boolean} True if ok, else false + */ + priv.checkRevisionFormat = function (revision) { + return (/^[0-9]+-[0-9a-zA-Z_]+$/.test(revision)); + }; + + /** + * Clones an object in deep (without functions) + * @method clone + * @param {any} object The object to clone + * @return {any} The cloned object + */ + priv.clone = function (object) { + var tmp = JSON.stringify(object); + if (tmp === undefined) { + return undefined; + } + return JSON.parse(tmp); + }; + + /** + * Like addJob but also return the method and the index of the storage + * @method send + * @param {string} method The request method + * @param {number} index The storage index + * @param {object} doc The document object + * @param {object} option The request object + * @param {function} callback The callback. Parameters: + * - {string} The request method + * - {number} The storage index + * - {object} The error object + * - {object} The response object + */ + priv.send = function (method, index, doc, option, callback) { + var wrapped_callback_success, wrapped_callback_error; + callback = callback || priv.emptyFunction; + wrapped_callback_success = function (response) { + callback(method, index, undefined, response); + }; + wrapped_callback_error = function (err) { + callback(method, index, err, undefined); + }; + that.addJob( + method, + priv.storage_list[index], + doc, + option, + wrapped_callback_success, + wrapped_callback_error + ); + }; + + /** + * Use "send" method to all sub storages. + * Calling "callback" for each storage response. + * @method sendToAll + * @param {string} method The request method + * @param {object} doc The document object + * @param {object} option The request option + * @param {function} callback The callback. Parameters: + * - {string} The request method + * - {number} The storage index + * - {object} The error object + * - {object} The response object + */ + priv.sendToAll = function (method, doc, option, callback) { + var i; + for (i = 0; i < priv.storage_list.length; i += 1) { + priv.send(method, i, doc, option, callback); + } + }; + + /** + * Use "send" method to all sub storages. + * Calling "callback" only with the first response + * @method sendToAllFastestResponseOnly + * @param {string} method The request method + * @param {object} doc The document object + * @param {object} option The request option + * @param {function} callback The callback. Parameters: + * - {string} The request method + * - {object} The error object + * - {object} The response object + */ + priv.sendToAllFastestResponseOnly = function (method, doc, option, callback) { + var i, callbackWrapper, error_count, last_error; + error_count = 0; + callbackWrapper = function (method, index, err, response) { + if (err) { + error_count += 1; + last_error = err; + if (error_count === priv.storage_list.length) { + return callback(method, err, response); + } + } + callback(method, err, response); + }; + for (i = 0; i < priv.storage_list.length; i += 1) { + priv.send(method, i, doc, option, callbackWrapper); + } + }; + + /** + * Use "sendToAll" method, calling "callback" at the last response with + * the response list + * @method sendToAllGetResponseList + * @param {string} method The request method + * @param {object} doc The document object + * @param {object} option The request option + * @return {function} callback The callback. Parameters: + * - {string} The request method + * - {object} The error object + * - {object} The response object + */ + priv.sendToAllGetResponseList = function (method, doc, option, callback) { + var wrapper, callback_count = 0, response_list = [], error_list = []; + response_list.length = priv.storage_list.length; + wrapper = function (method, index, err, response) { + error_list[index] = err; + response_list[index] = response; + callback_count += 1; + if (callback_count === priv.storage_list.length) { + callback(error_list, response_list); + } + }; + priv.sendToAll(method, doc, option, wrapper); + }; + + /** + * Checks if the sub storage are identical + * @method check + * @param {object} command The JIO command + */ + that.check = function (command) { + function callback(err, response) { + if (err) { + return that.error(err); + } + that.success(response); + } + priv.check( + command.cloneDoc(), + command.cloneOption(), + callback + ); + }; + + /** + * Repair the sub storages to make them identical + * @method repair + * @param {object} command The JIO command + */ + that.repair = function (command) { + function callback(err, response) { + if (err) { + return that.error(err); + } + that.success(response); + } + priv.repair( + command.cloneDoc(), + command.cloneOption(), + true, + callback + ); + }; + + priv.check = function (doc, option, success, error) { + priv.repair(doc, option, false, success, error); + }; + + priv.repair = function (doc, option, repair, callback) { + var functions = {}; + callback = callback || priv.emptyFunction; + option = option || {}; + functions.begin = function () { + // }; + // functions.repairAllSubStorages = function () { + var i; + for (i = 0; i < priv.storage_list.length; i += 1) { + priv.send( + repair ? "repair" : "check", + i, + doc, + option, + functions.repairAllSubStoragesCallback + ); + } + }; + functions.repair_sub_storages_count = 0; + functions.repairAllSubStoragesCallback = function (method, + index, err, response) { + if (err) { + return that.error(err); + } + functions.repair_sub_storages_count += 1; + if (functions.repair_sub_storages_count === priv.storage_list.length) { + functions.getAllDocuments(functions.newParam( + doc, + option, + repair + )); + } + }; + functions.newParam = function (doc, option, repair) { + var param = { + "doc": doc, // the document to repair + "option": option, + "repair": repair, + "responses": { + "count": 0, + "list": [ + // 0: response0 + // 1: response1 + // 2: response2 + ], + "stats": { + // responseA: [0, 1] + // responseB: [2] + }, + "stats_items": [ + // 0: [responseA, [0, 1]] + // 1: [responseB, [2]] + ], + "attachments": { + // attachmentA : {_id: attachmentA, _revs_info, _mimetype: ..} + // attachmentB : {_id: attachmentB, _revs_info, _mimetype: ..} + } + }, + "conflicts": { + // revC: true + // revD: true + }, + "deal_result_state": "ok", + "my_rev": undefined + }; + param.responses.list.length = priv.storage_list.length; + return param; + }; + functions.getAllDocuments = function (param) { + var i, doc = priv.clone(param.doc), option = priv.clone(param.option); + option.conflicts = true; + option.revs = true; + option.revs_info = true; + for (i = 0; i < priv.storage_list.length; i += 1) { + // if the document is not loaded + priv.send("get", i, doc, option, functions.dealResults(param)); + } + functions.finished_count += 1; + }; + functions.dealResults = function (param) { + return function (method, index, err, response) { + var response_object = {}; + if (param.deal_result_state !== "ok") { + // deal result is in a wrong state, exit + return; + } + if (err) { + if (err.status !== 404) { + // get document failed, exit + param.deal_result_state = "error"; + callback({ + "status": 40, + "statusText": "Check Failed", + "error": "check_failed", + "message": "An error occured on the sub storage", + "reason": err.reason + }, undefined); + return; + } + } + // success to get the document + // add the response in memory + param.responses.count += 1; + param.responses.list[index] = response; + + // add the conflicting revision for other synchronizations + functions.addConflicts(param, (response || {})._conflicts); + if (param.responses.count !== param.responses.list.length) { + // this is not the last response, wait for the next response + return; + } + + // this is now the last response + functions.makeResponsesStats(param.responses); + if (param.responses.stats_items.length === 1) { + // the responses are equals! + response_object.ok = true; + response_object.id = param.doc._id; + if (doc._rev) { + response_object.rev = doc._rev; + // "rev": (typeof param.responses.list[0] === "object" ? + // param.responses.list[0]._rev : undefined) + } + callback(undefined, response_object); + return; + } + // the responses are different + if (param.repair === false) { + // do not repair + callback({ + "status": 41, + "statusText": "Check Not Ok", + "error": "check_not_ok", + "message": "Some documents are different in the sub storages", + "reason": "Storage contents differ" + }, undefined); + return; + } + // repair + functions.getAttachments(param); + }; + }; + functions.addConflicts = function (param, list) { + var i; + list = list || []; + for (i = 0; i < list.length; i += 1) { + param.conflicts[list[i]] = true; + } + }; + functions.makeResponsesStats = function (responses) { + var i, str_response; + for (i = 0; i < responses.count; i += 1) { + str_response = JSON.stringify(responses.list[i]); + if (responses.stats[str_response] === undefined) { + responses.stats[str_response] = []; + responses.stats_items.push([ + str_response, + responses.stats[str_response] + ]); + } + responses.stats[str_response].push(i); + } + }; + functions.getAttachments = function (param) { + var response, parsed_response, attachment; + for (response in param.responses.stats) { + if (param.responses.stats.hasOwnProperty(response)) { + parsed_response = JSON.parse(response); + for (attachment in parsed_response._attachments) { + if ((parsed_response._attachments).hasOwnProperty(attachment)) { + functions.get_attachment_count += 1; + priv.send( + "getAttachment", + param.responses.stats[response][0], + { + "_id": param.doc._id, + "_attachment": attachment, + "_rev": JSON.parse(response)._rev + }, + param.option, + functions.getAttachmentsCallback( + param, + attachment, + param.responses.stats[response] + ) + ); + } + } + } + } + }; + functions.get_attachment_count = 0; + functions.getAttachmentsCallback = function ( + param, + attachment_id, + index_list + ) { + return function (method, index, err, response) { + if (err) { + callback({ + "status": 40, + "statusText": "Check Failed", + "error": "check_failed", + "message": "Unable to retreive attachments", + "reason": err.reason + }, undefined); + return; + } + functions.get_attachment_count -= 1; + param.responses.attachments[attachment_id] = response; + if (functions.get_attachment_count === 0) { + functions.synchronizeAllSubStorage(param); + if (param.option.synchronize_conflicts !== false) { + functions.synchronizeConflicts(param); + } + } + }; + }; + functions.synchronizeAllSubStorage = function (param) { + var i, j, len = param.responses.stats_items.length; + for (i = 0; i < len; i += 1) { + // browsing responses + for (j = 0; j < len; j += 1) { + // browsing storage list + if (i !== j) { + functions.synchronizeResponseToSubStorage( + param, + param.responses.stats_items[i][0], + param.responses.stats_items[j][1] + ); + } + } + } + functions.finished_count -= 1; + }; + functions.synchronizeResponseToSubStorage = function ( + param, + response, + storage_list + ) { + var i, new_doc, attachment_to_put = []; + if (response === undefined) { + // no response to sync + return; + } + new_doc = JSON.parse(response); + new_doc._revs = new_doc._revisions; + delete new_doc._rev; + delete new_doc._revisions; + delete new_doc._conflicts; + for (i in new_doc._attachments) { + if (new_doc._attachments.hasOwnProperty(i)) { + attachment_to_put.push({ + "_id": i, + "_mimetype": new_doc._attachments[i].content_type, + "_revs_info": new_doc._revs_info + }); + } + } + for (i = 0; i < storage_list.length; i += 1) { + functions.finished_count += attachment_to_put.length || 1; + priv.send( + "put", + storage_list[i], + new_doc, + param.option, + functions.putAttachments(param, attachment_to_put) + ); + } + functions.finished_count += 1; + functions.finished(); + }; + functions.synchronizeConflicts = function (param) { + var rev, new_doc, new_option; + new_option = priv.clone(param.option); + new_option.synchronize_conflict = false; + for (rev in param.conflicts) { + if (param.conflicts.hasOwnProperty(rev)) { + new_doc = priv.clone(param.doc); + new_doc._rev = rev; + // no need to synchronize all the conflicts again, do it once + functions.getAllDocuments(functions.newParam( + new_doc, + new_option, + param.repair + )); + } + } + }; + functions.putAttachments = function (param, attachment_to_put) { + return function (method, index, err, response) { + var i, attachment; + if (err) { + return callback({ + "status": 40, + "statusText": "Check Failed", + "error": "check_failed", + "message": "Unable to copy attachments", + "reason": err.reason + }, undefined); + } + for (i = 0; i < attachment_to_put.length; i += 1) { + attachment = { + "_id": param.doc._id, + "_attachment": attachment_to_put[i]._id, + "_mimetype": attachment_to_put[i]._mimetype, + "_revs_info": attachment_to_put[i]._revs_info, + // "_revs_info": param.responses.list[index]._revs_info, + "_data": param.responses.attachments[attachment_to_put[i]._id] + }; + priv.send( + "putAttachment", + index, + attachment, + option, + functions.putAttachmentCallback(param) + ); + } + if (attachment_to_put.length === 0) { + functions.finished(); + } + }; + }; + functions.putAttachmentCallback = function (param) { + return function (method, index, err, response) { + if (err) { + return callback(err, undefined); + } + functions.finished(); + }; + }; + functions.finished_count = 0; + functions.finished = function () { + var response_object = {}; + functions.finished_count -= 1; + if (functions.finished_count === 0) { + response_object.ok = true; + response_object.id = doc._id; + if (doc._rev) { + response_object.rev = doc._rev; + } + callback(undefined, response_object); + } + }; + functions.begin(); + }; + + /** + * The generic method to use + * @method genericRequest + * @param {object} command The JIO command + * @param {string} method The method to use + */ + that.genericRequest = function (command, method) { + var doc = command.cloneDoc(); + doc._id = doc._id || priv.generateUuid(); + priv.sendToAllFastestResponseOnly( + method, + doc, + command.cloneOption(), + function (method, err, response) { + if (err) { + return that.error(err); + } + that.success(response); + } + ); + }; + + /** + * Post the document metadata to all sub storages + * @method post + * @param {object} command The JIO command + */ + that.post = function (command) { + that.genericRequest(command, "put"); + }; + + /** + * Put the document metadata to all sub storages + * @method put + * @param {object} command The JIO command + */ + that.put = function (command) { + that.genericRequest(command, "post"); + }; + + /** + * Put an attachment to a document to all sub storages + * @method putAttachment + * @param {object} command The JIO command + */ + that.putAttachment = function (command) { + that.genericRequest(command, "putAttachment"); + }; + + /** + * Get the document from all sub storages, get the fastest. + * @method get + * @param {object} command The JIO command + */ + that.get = function (command) { + that.genericRequest(command, "get"); + }; + + /** + * Get the attachment from all sub storages, get the fastest. + * @method getAttachment + * @param {object} command The JIO command + */ + that.getAttachment = function (command) { + that.genericRequest(command, "getAttachment"); + }; + + /** + * Remove the document from all sub storages. + * @method remove + * @param {object} command The JIO command + */ + that.remove = function (command) { + that.genericRequest(command, "remove"); + }; + + /** + * Remove the attachment from all sub storages. + * @method remove + * @param {object} command The JIO command + */ + that.removeAttachment = function (command) { + that.genericRequest(command, "removeAttachment"); + }; + + return that; +}); diff --git a/lib/jio/revisionstorage.js b/lib/jio/revisionstorage.js new file mode 100644 index 0000000..9d1c8cf --- /dev/null +++ b/lib/jio/revisionstorage.js @@ -0,0 +1,1033 @@ +/*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true */ +/*global jIO: true, hex_sha256: true, setTimeout: true */ +/** + * JIO Revision Storage. + * It manages document version and can generate conflicts. + * Description: + * { + * "type": "revision", + * "sub_storage": + * } + */ +jIO.addStorageType("revision", function (spec, my) { + "use strict"; + var that = {}, priv = {}; + spec = spec || {}; + that = my.basicStorage(spec, my); + // ATTRIBUTES // + priv.doc_tree_suffix = ".revision_tree.json"; + priv.sub_storage = spec.sub_storage; + // METHODS // + /** + * Constructor + */ + priv.RevisionStorage = function () { + // no init + }; + + /** + * Description to store in order to be restored later + * @method specToStore + * @return {object} Descriptions to store + */ + that.specToStore = function () { + return { + "sub_storage": priv.sub_storage + }; + }; + + /** + * Method to validate the storage current state + * @method validateState + * @return {string} The error message if exist + */ + that.validateState = function () { + if (typeof priv.sub_storage !== "object") { + return "Need a sub storage description: \"sub_storage\""; + } + return ""; + }; + + /** + * Clones an object in deep (without functions) + * @method clone + * @param {any} object The object to clone + * @return {any} The cloned object + */ + priv.clone = function (object) { + var tmp = JSON.stringify(object); + if (tmp === undefined) { + return undefined; + } + return JSON.parse(tmp); + }; + + /** + * Generate a new uuid + * @method generateUuid + * @return {string} The new uuid + */ + priv.generateUuid = function () { + var S4 = function () { + /* 65536 */ + var i, string = Math.floor( + Math.random() * 0x10000 + ).toString(16); + for (i = string.length; i < 4; i += 1) { + string = '0' + string; + } + return string; + }; + return S4() + S4() + "-" + S4() + "-" + S4() + "-" + S4() + "-" + S4() + + S4() + S4(); + }; + + /** + * Generates a hash code of a string + * @method hashCode + * @param {string} string The string to hash + * @return {string} The string hash code + */ + priv.hashCode = function (string) { + return hex_sha256(string); + }; + + /** + * Checks a revision format + * @method checkDocumentRevisionFormat + * @param {object} doc The document object + * @return {object} null if ok, else error object + */ + priv.checkDocumentRevisionFormat = function (doc) { + var send_error = function (message) { + return { + "status": 31, + "statusText": "Wrong Revision Format", + "error": "wrong_revision_format", + "message": message, + "reason": "Revision is wrong" + }; + }; + if (typeof doc._rev === "string") { + if (/^[0-9]+-[0-9a-zA-Z]+$/.test(doc._rev) === false) { + return send_error("The document revision does not match " + + "^[0-9]+-[0-9a-zA-Z]+$"); + } + } + if (typeof doc._revs === "object") { + if (typeof doc._revs.start !== "number" || + typeof doc._revs.ids !== "object" || + typeof doc._revs.ids.length !== "number") { + return send_error("The document revision history is not well formated"); + } + } + if (typeof doc._revs_info === "object") { + if (typeof doc._revs_info.length !== "number") { + return send_error("The document revision information " + + "is not well formated"); + } + } + }; + + /** + * Creates a new document tree + * @method newDocTree + * @return {object} The new document tree + */ + priv.newDocTree = function () { + return {"children": []}; + }; + + /** + * Convert revs_info to a simple revisions history + * @method revsInfoToHistory + * @param {array} revs_info The revs info + * @return {object} The revisions history + */ + priv.revsInfoToHistory = function (revs_info) { + var i, revisions = { + "start": 0, + "ids": [] + }; + revs_info = revs_info || []; + if (revs_info.length > 0) { + revisions.start = parseInt(revs_info[0].rev.split('-')[0], 10); + } + for (i = 0; i < revs_info.length; i += 1) { + revisions.ids.push(revs_info[i].rev.split('-')[1]); + } + return revisions; + }; + + /** + * Convert the revision history object to an array of revisions. + * @method revisionHistoryToList + * @param {object} revs The revision history + * @return {array} The revision array + */ + priv.revisionHistoryToList = function (revs) { + var i, start = revs.start, new_list = []; + for (i = 0; i < revs.ids.length; i += 1, start -= 1) { + new_list.push(start + "-" + revs.ids[i]); + } + return new_list; + }; + + /** + * Convert revision list to revs info. + * @method revisionListToRevsInfo + * @param {array} revision_list The revision list + * @param {object} doc_tree The document tree + * @return {array} The document revs info + */ + priv.revisionListToRevsInfo = function (revision_list, doc_tree) { + var revisionListToRevsInfoRec, revs_info = [], j; + for (j = 0; j < revision_list.length; j += 1) { + revs_info.push({"rev": revision_list[j], "status": "missing"}); + } + revisionListToRevsInfoRec = function (index, doc_tree) { + var child, i; + if (index < 0) { + return; + } + for (i = 0; i < doc_tree.children.length; i += 1) { + child = doc_tree.children[i]; + if (child.rev === revision_list[index]) { + revs_info[index].status = child.status; + revisionListToRevsInfoRec(index - 1, child); + } + } + }; + revisionListToRevsInfoRec(revision_list.length - 1, doc_tree); + return revs_info; + }; + + /** + * Update a document metadata revision properties + * @method fillDocumentRevisionProperties + * @param {object} doc The document object + * @param {object} doc_tree The document tree + */ + priv.fillDocumentRevisionProperties = function (doc, doc_tree) { + if (doc._revs_info) { + doc._revs = priv.revsInfoToHistory(doc._revs_info); + } else if (doc._revs) { + doc._revs_info = priv.revisionListToRevsInfo( + priv.revisionHistoryToList(doc._revs), + doc_tree + ); + } else if (doc._rev) { + doc._revs_info = priv.getRevisionInfo(doc._rev, doc_tree); + doc._revs = priv.revsInfoToHistory(doc._revs_info); + } else { + doc._revs_info = []; + doc._revs = {"start": 0, "ids": []}; + } + if (doc._revs.start > 0) { + doc._rev = doc._revs.start + "-" + doc._revs.ids[0]; + } else { + delete doc._rev; + } + }; + + /** + * Generates the next revision of a document. + * @methode generateNextRevision + * @param {object} doc The document metadata + * @param {boolean} deleted_flag The deleted flag + * @return {array} 0:The next revision number and 1:the hash code + */ + priv.generateNextRevision = function (doc, deleted_flag) { + var string, revision_history, revs_info, pseudo_revision; + doc = priv.clone(doc) || {}; + revision_history = doc._revs; + revs_info = doc._revs_info; + delete doc._rev; + delete doc._revs; + delete doc._revs_info; + string = JSON.stringify(doc) + JSON.stringify(revision_history) + + JSON.stringify(deleted_flag ? true : false); + revision_history.start += 1; + revision_history.ids.unshift(priv.hashCode(string)); + doc._revs = revision_history; + doc._rev = revision_history.start + "-" + revision_history.ids[0]; + revs_info.unshift({ + "rev": doc._rev, + "status": deleted_flag ? "deleted" : "available" + }); + doc._revs_info = revs_info; + return doc; + }; + + /** + * Gets the revs info from the document tree + * @method getRevisionInfo + * @param {string} revision The revision to search for + * @param {object} doc_tree The document tree + * @return {array} The revs info + */ + priv.getRevisionInfo = function (revision, doc_tree) { + var getRevisionInfoRec; + getRevisionInfoRec = function (doc_tree) { + var i, child, revs_info; + for (i = 0; i < doc_tree.children.length; i += 1) { + child = doc_tree.children[i]; + if (child.rev === revision) { + return [{"rev": child.rev, "status": child.status}]; + } + revs_info = getRevisionInfoRec(child); + if (revs_info.length > 0 || revision === undefined) { + revs_info.push({"rev": child.rev, "status": child.status}); + return revs_info; + } + } + return []; + }; + return getRevisionInfoRec(doc_tree); + }; + + priv.updateDocumentTree = function (doc, doc_tree) { + var revs_info, updateDocumentTreeRec, next_rev; + doc = priv.clone(doc); + revs_info = doc._revs_info; + updateDocumentTreeRec = function (doc_tree, revs_info) { + var i, child, info; + if (revs_info.length === 0) { + return; + } + info = revs_info.pop(); + for (i = 0; i < doc_tree.children.length; i += 1) { + child = doc_tree.children[i]; + if (child.rev === info.rev) { + return updateDocumentTreeRec(child, revs_info); + } + } + doc_tree.children.unshift({ + "rev": info.rev, + "status": info.status, + "children": [] + }); + updateDocumentTreeRec(doc_tree.children[0], revs_info); + }; + updateDocumentTreeRec(doc_tree, priv.clone(revs_info)); + }; + + priv.send = function (method, doc, option, callback) { + that.addJob( + method, + priv.sub_storage, + doc, + option, + function (success) { + callback(undefined, success); + }, + function (err) { + callback(err, undefined); + } + ); + }; + + priv.getWinnerRevsInfo = function (doc_tree) { + var revs_info = [], getWinnerRevsInfoRec; + getWinnerRevsInfoRec = function (doc_tree, tmp_revs_info) { + var i; + if (doc_tree.rev) { + tmp_revs_info.unshift({"rev": doc_tree.rev, "status": doc_tree.status}); + } + if (doc_tree.children.length === 0) { + if (revs_info.length === 0 || + (revs_info[0].status !== "available" && + tmp_revs_info[0].status === "available") || + (tmp_revs_info[0].status === "available" && + revs_info.length < tmp_revs_info.length)) { + revs_info = priv.clone(tmp_revs_info); + } + } + for (i = 0; i < doc_tree.children.length; i += 1) { + getWinnerRevsInfoRec(doc_tree.children[i], tmp_revs_info); + } + tmp_revs_info.shift(); + }; + getWinnerRevsInfoRec(doc_tree, []); + return revs_info; + }; + + priv.getConflicts = function (revision, doc_tree) { + var conflicts = [], getConflictsRec; + getConflictsRec = function (doc_tree) { + var i; + if (doc_tree.rev === revision) { + return; + } + if (doc_tree.children.length === 0) { + if (doc_tree.status !== "deleted") { + conflicts.push(doc_tree.rev); + } + } + for (i = 0; i < doc_tree.children.length; i += 1) { + getConflictsRec(doc_tree.children[i]); + } + }; + getConflictsRec(doc_tree); + return conflicts.length === 0 ? undefined : conflicts; + }; + + priv.get = function (doc, option, callback) { + priv.send("get", doc, option, callback); + }; + priv.put = function (doc, option, callback) { + priv.send("put", doc, option, callback); + }; + priv.remove = function (doc, option, callback) { + priv.send("remove", doc, option, callback); + }; + priv.getAttachment = function (attachment, option, callback) { + priv.send("getAttachment", attachment, option, callback); + }; + priv.putAttachment = function (attachment, option, callback) { + priv.send("putAttachment", attachment, option, callback); + }; + priv.removeAttachment = function (attachment, option, callback) { + priv.send("removeAttachment", attachment, option, callback); + }; + + priv.getDocument = function (doc, option, callback) { + doc = priv.clone(doc); + doc._id = doc._id + "." + doc._rev; + delete doc._attachment; + delete doc._rev; + delete doc._revs; + delete doc._revs_info; + priv.get(doc, option, callback); + }; + priv.putDocument = function (doc, option, callback) { + doc = priv.clone(doc); + doc._id = doc._id + "." + doc._rev; + delete doc._attachment; + delete doc._data; + delete doc._mimetype; + delete doc._rev; + delete doc._revs; + delete doc._revs_info; + priv.put(doc, option, callback); + }; + + priv.getRevisionTree = function (doc, option, callback) { + doc = priv.clone(doc); + doc._id = doc._id + priv.doc_tree_suffix; + priv.get(doc, option, callback); + }; + + priv.getAttachmentList = function (doc, option, callback) { + var attachment_id, dealResults, state = "ok", result_list = [], count = 0; + dealResults = function (attachment_id, attachment_meta) { + return function (err, attachment) { + if (state !== "ok") { + return; + } + count -= 1; + if (err) { + if (err.status === 404) { + result_list.push(undefined); + } else { + state = "error"; + return callback(err, undefined); + } + } + result_list.push({ + "_attachment": attachment_id, + "_data": attachment, + "_mimetype": attachment_meta.content_type + }); + if (count === 0) { + state = "finished"; + callback(undefined, result_list); + } + }; + }; + for (attachment_id in doc._attachments) { + if (doc._attachments.hasOwnProperty(attachment_id)) { + count += 1; + priv.getAttachment( + {"_id": doc._id, "_attachment": attachment_id}, + option, + dealResults(attachment_id, doc._attachments[attachment_id]) + ); + } + } + if (count === 0) { + callback(undefined, []); + } + }; + + priv.putAttachmentList = function (doc, option, attachment_list, callback) { + var i, dealResults, state = "ok", count = 0, attachment; + attachment_list = attachment_list || []; + dealResults = function (index) { + return function (err, response) { + if (state !== "ok") { + return; + } + count -= 1; + if (err) { + state = "error"; + return callback(err, undefined); + } + if (count === 0) { + state = "finished"; + callback(undefined, {"id": doc._id, "ok": true}); + } + }; + }; + for (i = 0; i < attachment_list.length; i += 1) { + attachment = attachment_list[i]; + if (attachment !== undefined) { + count += 1; + attachment._id = doc._id + "." + doc._rev; + priv.putAttachment(attachment, option, dealResults(i)); + } + } + if (count === 0) { + return callback(undefined, {"id": doc._id, "ok": true}); + } + }; + + priv.putDocumentTree = function (doc, option, doc_tree, callback) { + doc_tree = priv.clone(doc_tree); + doc_tree._id = doc._id + priv.doc_tree_suffix; + priv.put(doc_tree, option, callback); + }; + + priv.notFoundError = function (message, reason) { + return { + "status": 404, + "statusText": "Not Found", + "error": "not_found", + "message": message, + "reason": reason + }; + }; + + priv.conflictError = function (message, reason) { + return { + "status": 409, + "statusText": "Conflict", + "error": "conflict", + "message": message, + "reason": reason + }; + }; + + priv.revisionGenericRequest = function (doc, option, + specific_parameter, onEnd) { + var prev_doc, doc_tree, attachment_list, callback = {}; + if (specific_parameter.doc_id) { + doc._id = specific_parameter.doc_id; + } + if (specific_parameter.attachment_id) { + doc._attachment = specific_parameter.attachment_id; + } + callback.begin = function () { + var check_error; + doc._id = doc._id || priv.generateUuid(); + if (specific_parameter.revision_needed && !doc._rev) { + return onEnd(priv.conflictError( + "Document update conflict", + "No document revision was provided" + ), undefined); + } + // check revision format + check_error = priv.checkDocumentRevisionFormat(doc); + if (check_error !== undefined) { + return onEnd(check_error, undefined); + } + priv.getRevisionTree(doc, option, callback.getRevisionTree); + }; + callback.getRevisionTree = function (err, response) { + var winner_info, previous_revision = doc._rev, + generate_new_revision = doc._revs || doc._revs_info ? false : true; + if (err) { + if (err.status !== 404) { + err.message = "Cannot get document revision tree"; + return onEnd(err, undefined); + } + } + doc_tree = response || priv.newDocTree(); + if (specific_parameter.get || specific_parameter.getAttachment) { + if (!doc._rev) { + winner_info = priv.getWinnerRevsInfo(doc_tree); + if (winner_info.length === 0) { + return onEnd(priv.notFoundError( + "Document not found", + "missing" + ), undefined); + } + if (winner_info[0].status === "deleted") { + return onEnd(priv.notFoundError( + "Document not found", + "deleted" + ), undefined); + } + doc._rev = winner_info[0].rev; + } + priv.fillDocumentRevisionProperties(doc, doc_tree); + return priv.getDocument(doc, option, callback.getDocument); + } + priv.fillDocumentRevisionProperties(doc, doc_tree); + if (generate_new_revision) { + if (previous_revision && doc._revs_info.length === 0) { + // the document history has changed, it means that the document + // revision was wrong. Add a pseudo history to the document + doc._rev = previous_revision; + doc._revs = { + "start": parseInt(previous_revision.split("-")[0], 10), + "ids": [previous_revision.split("-")[1]] + }; + doc._revs_info = [{"rev": previous_revision, "status": "missing"}]; + } + doc = priv.generateNextRevision( + doc, + specific_parameter.remove + ); + } + if (doc._revs_info.length > 1) { + prev_doc = { + "_id": doc._id, + "_rev": doc._revs_info[1].rev + }; + if (!generate_new_revision && specific_parameter.putAttachment) { + prev_doc._rev = doc._revs_info[0].rev; + } + } + // force revs_info status + doc._revs_info[0].status = (specific_parameter.remove ? + "deleted" : "available"); + priv.updateDocumentTree(doc, doc_tree); + if (prev_doc) { + return priv.getDocument(prev_doc, option, callback.getDocument); + } + if (specific_parameter.remove || specific_parameter.removeAttachment) { + return onEnd(priv.notFoundError( + "Unable to remove an inexistent document", + "missing" + ), undefined); + } + priv.putDocument(doc, option, callback.putDocument); + }; + callback.getDocument = function (err, res_doc) { + var k, conflicts; + if (err) { + if (err.status === 404) { + if (specific_parameter.remove || + specific_parameter.removeAttachment) { + return onEnd(priv.conflictError( + "Document update conflict", + "Document is missing" + ), undefined); + } + if (specific_parameter.get) { + return onEnd(priv.notFoundError( + "Unable to find the document", + "missing" + ), undefined); + } + res_doc = {}; + } else { + err.message = "Cannot get document"; + return onEnd(err, undefined); + } + } + if (specific_parameter.get) { + res_doc._id = doc._id; + res_doc._rev = doc._rev; + if (option.conflicts === true) { + conflicts = priv.getConflicts(doc._rev, doc_tree); + if (conflicts) { + res_doc._conflicts = conflicts; + } + } + if (option.revs === true) { + res_doc._revisions = doc._revs; + } + if (option.revs_info === true) { + res_doc._revs_info = doc._revs_info; + } + return onEnd(undefined, res_doc); + } + if (specific_parameter.putAttachment || + specific_parameter.removeAttachment) { + // copy metadata (not beginning by "_" to document + for (k in res_doc) { + if (res_doc.hasOwnProperty(k) && !k.match("^_")) { + doc[k] = res_doc[k]; + } + } + } + if (specific_parameter.remove) { + priv.putDocumentTree(doc, option, doc_tree, callback.putDocumentTree); + } else { + priv.getAttachmentList(res_doc, option, callback.getAttachmentList); + } + }; + callback.getAttachmentList = function (err, res_list) { + var i, attachment_found = false; + if (err) { + err.message = "Cannot get attachment"; + return onEnd(err, undefined); + } + attachment_list = res_list || []; + if (specific_parameter.getAttachment) { + // getting specific attachment + for (i = 0; i < attachment_list.length; i += 1) { + if (attachment_list[i] && + doc._attachment === + attachment_list[i]._attachment) { + return onEnd(undefined, attachment_list[i]._data); + } + } + return onEnd(priv.notFoundError( + "Unable to get an inexistent attachment", + "missing" + ), undefined); + } + if (specific_parameter.remove_from_attachment_list) { + // removing specific attachment + for (i = 0; i < attachment_list.length; i += 1) { + if (attachment_list[i] && + specific_parameter.remove_from_attachment_list._attachment === + attachment_list[i]._attachment) { + attachment_found = true; + attachment_list[i] = undefined; + break; + } + } + if (!attachment_found) { + return onEnd(priv.notFoundError( + "Unable to remove an inexistent attachment", + "missing" + ), undefined); + } + } + priv.putDocument(doc, option, callback.putDocument); + }; + callback.putDocument = function (err, response) { + var i, attachment_found = false; + if (err) { + err.message = "Cannot post the document"; + return onEnd(err, undefined); + } + if (specific_parameter.add_to_attachment_list) { + // adding specific attachment + attachment_list = attachment_list || []; + for (i = 0; i < attachment_list.length; i += 1) { + if (attachment_list[i] && + specific_parameter.add_to_attachment_list._attachment === + attachment_list[i]._attachment) { + attachment_found = true; + attachment_list[i] = specific_parameter.add_to_attachment_list; + break; + } + } + if (!attachment_found) { + attachment_list.unshift(specific_parameter.add_to_attachment_list); + } + } + priv.putAttachmentList( + doc, + option, + attachment_list, + callback.putAttachmentList + ); + }; + callback.putAttachmentList = function (err, response) { + if (err) { + err.message = "Cannot copy attacments to the document"; + return onEnd(err, undefined); + } + priv.putDocumentTree(doc, option, doc_tree, callback.putDocumentTree); + }; + callback.putDocumentTree = function (err, response) { + var response_object; + if (err) { + err.message = "Cannot update the document history"; + return onEnd(err, undefined); + } + response_object = { + "ok": true, + "id": doc._id, + "rev": doc._rev + }; + if (specific_parameter.putAttachment || + specific_parameter.removeAttachment || + specific_parameter.getAttachment) { + response_object.attachment = doc._attachment; + } + onEnd(undefined, response_object); + // if (option.keep_revision_history !== true) { + // // priv.remove(prev_doc, option, function () { + // // - change "available" status to "deleted" + // // - remove attachments + // // - done, no callback + // // }); + // } + }; + callback.begin(); + }; + + /** + * Post the document metadata and create or update a document tree. + * Options: + * - {boolean} keep_revision_history To keep the previous revisions + * (false by default) (NYI). + * @method post + * @param {object} command The JIO command + */ + that.post = function (command) { + priv.revisionGenericRequest( + command.cloneDoc(), + command.cloneOption(), + {}, + function (err, response) { + if (err) { + return that.error(err); + } + that.success(response); + } + ); + }; + + /** + * Put the document metadata and create or update a document tree. + * Options: + * - {boolean} keep_revision_history To keep the previous revisions + * (false by default) (NYI). + * @method put + * @param {object} command The JIO command + */ + that.put = function (command) { + priv.revisionGenericRequest( + command.cloneDoc(), + command.cloneOption(), + {}, + function (err, response) { + if (err) { + return that.error(err); + } + that.success(response); + } + ); + }; + + + that.putAttachment = function (command) { + priv.revisionGenericRequest( + command.cloneDoc(), + command.cloneOption(), + { + "doc_id": command.getDocId(), + "attachment_id": command.getAttachmentId(), + "add_to_attachment_list": { + "_attachment": command.getAttachmentId(), + "_mimetype": command.getAttachmentMimeType(), + "_data": command.getAttachmentData() + }, + "putAttachment": true + }, + function (err, response) { + if (err) { + return that.error(err); + } + that.success(response); + } + ); + }; + + that.remove = function (command) { + if (command.getAttachmentId()) { + return that.removeAttachment(command); + } + priv.revisionGenericRequest( + command.cloneDoc(), + command.cloneOption(), + { + "revision_needed": true, + "remove": true + }, + function (err, response) { + if (err) { + return that.error(err); + } + that.success(response); + } + ); + }; + + that.removeAttachment = function (command) { + priv.revisionGenericRequest( + command.cloneDoc(), + command.cloneOption(), + { + "doc_id": command.getDocId(), + "attachment_id": command.getAttachmentId(), + "revision_needed": true, + "removeAttachment": true, + "remove_from_attachment_list": { + "_attachment": command.getAttachmentId() + } + }, + function (err, response) { + if (err) { + return that.error(err); + } + that.success(response); + } + ); + }; + + that.get = function (command) { + if (command.getAttachmentId()) { + return that.getAttachment(command); + } + priv.revisionGenericRequest( + command.cloneDoc(), + command.cloneOption(), + { + "get": true + }, + function (err, response) { + if (err) { + return that.error(err); + } + that.success(response); + } + ); + }; + + that.getAttachment = function (command) { + priv.revisionGenericRequest( + command.cloneDoc(), + command.cloneOption(), + { + "doc_id": command.getDocId(), + "attachment_id": command.getAttachmentId(), + "getAttachment": true + }, + function (err, response) { + if (err) { + return that.error(err); + } + that.success(response); + } + ); + }; + + that.allDocs = function (command) { + var rows, result = {"total_rows": 0, "rows": []}, functions = {}; + functions.finished = 0; + functions.falseResponseGenerator = function (response, callback) { + callback(undefined, response); + }; + functions.fillResultGenerator = function (doc_id) { + return function (err, doc_tree) { + var document_revision, row, revs_info; + if (err) { + return that.error(err); + } + revs_info = priv.getWinnerRevsInfo(doc_tree); + document_revision = + rows.document_revisions[doc_id + "." + revs_info[0].rev] + if (document_revision) { + row = { + "id": doc_id, + "key": doc_id, + "value": { + "rev": revs_info[0].rev + } + }; + if (document_revision.doc && command.getOption("include_docs")) { + document_revision.doc._id = doc_id; + document_revision.doc._rev = revs_info[0].rev; + row.doc = document_revision.doc; + } + result.rows.push(row); + result.total_rows += 1; + } + functions.success(); + }; + }; + functions.success = function () { + functions.finished -= 1; + if (functions.finished === 0) { + that.success(result); + } + }; + priv.send("allDocs", null, command.cloneOption(), function (err, response) { + var i, j, row, selector, selected; + if (err) { + return that.error(err); + } + selector = /^(.*)\.revision_tree\.json$/; + rows = { + "revision_trees": { + // id.revision_tree.json: { + // id: blabla + // doc: {...} + // } + }, + "document_revisions": { + // id.rev: { + // id: blabla + // rev: 1-1 + // doc: {...} + // } + } + }; + while (response.rows.length > 0) { + // filling rows + row = response.rows.shift(); + selected = selector.exec(row.id) + if (selected) { + // this is a revision tree + rows.revision_trees[row.id] = { + "id": selected[1] + }; + if (row.doc) { + rows.revision_trees[row.id].doc = row.doc; + } + } else { + // this is a simple revision + rows.document_revisions[row.id] = { + "id": row.id.split(".").slice(0,-1), + "rev": row.id.split(".").slice(-1) + }; + if (row.doc) { + rows.document_revisions[row.id].doc = row.doc; + } + } + } + functions.finished += 1; + for (i in rows.revision_trees) { + if (rows.revision_trees.hasOwnProperty(i)) { + functions.finished += 1; + if (rows.revision_trees[i].doc) { + functions.falseResponseGenerator( + rows.revision_trees[i].doc, + functions.fillResultGenerator(rows.revision_trees[i].id) + ); + } else { + priv.getRevisionTree( + {"_id": rows.revision_trees[i].id}, + command.cloneOption(), + functions.fillResultGenerator(rows.revision_trees[i].id) + ); + } + } + } + functions.success(); + }); + }; + + // END // + priv.RevisionStorage(); + return that; +}); // end RevisionStorage diff --git a/lib/md5/md5.js b/lib/md5/md5.js new file mode 100644 index 0000000..24d190e --- /dev/null +++ b/lib/md5/md5.js @@ -0,0 +1,379 @@ +/* + * A JavaScript implementation of the RSA Data Security, Inc. MD5 Message + * Digest Algorithm, as defined in RFC 1321. + * Version 2.2 Copyright (C) Paul Johnston 1999 - 2009 + * Other contributors: Greg Holt, Andrew Kepert, Ydnar, Lostinet + * Distributed under the BSD License + * See http://pajhome.org.uk/crypt/md5 for more info. + */ + +/* + * Configurable variables. You may need to tweak these to be compatible with + * the server-side, but the defaults work in most cases. + */ +var hexcase = 0; /* hex output format. 0 - lowercase; 1 - uppercase */ +var b64pad = ""; /* base-64 pad character. "=" for strict RFC compliance */ + +/* + * These are the functions you'll usually want to call + * They take string arguments and return either hex or base-64 encoded strings + */ +function hex_md5(s) { return rstr2hex(rstr_md5(str2rstr_utf8(s))); } +function b64_md5(s) { return rstr2b64(rstr_md5(str2rstr_utf8(s))); } +function any_md5(s, e) { return rstr2any(rstr_md5(str2rstr_utf8(s)), e); } +function hex_hmac_md5(k, d) + { return rstr2hex(rstr_hmac_md5(str2rstr_utf8(k), str2rstr_utf8(d))); } +function b64_hmac_md5(k, d) + { return rstr2b64(rstr_hmac_md5(str2rstr_utf8(k), str2rstr_utf8(d))); } +function any_hmac_md5(k, d, e) + { return rstr2any(rstr_hmac_md5(str2rstr_utf8(k), str2rstr_utf8(d)), e); } + +/* + * Perform a simple self-test to see if the VM is working + */ +function md5_vm_test() +{ + return hex_md5("abc").toLowerCase() == "900150983cd24fb0d6963f7d28e17f72"; +} + +/* + * Calculate the MD5 of a raw string + */ +function rstr_md5(s) +{ + return binl2rstr(binl_md5(rstr2binl(s), s.length * 8)); +} + +/* + * Calculate the HMAC-MD5, of a key and some data (raw strings) + */ +function rstr_hmac_md5(key, data) +{ + var bkey = rstr2binl(key); + if(bkey.length > 16) bkey = binl_md5(bkey, key.length * 8); + + var ipad = Array(16), opad = Array(16); + for(var i = 0; i < 16; i++) + { + ipad[i] = bkey[i] ^ 0x36363636; + opad[i] = bkey[i] ^ 0x5C5C5C5C; + } + + var hash = binl_md5(ipad.concat(rstr2binl(data)), 512 + data.length * 8); + return binl2rstr(binl_md5(opad.concat(hash), 512 + 128)); +} + +/* + * Convert a raw string to a hex string + */ +function rstr2hex(input) +{ + try { hexcase } catch(e) { hexcase=0; } + var hex_tab = hexcase ? "0123456789ABCDEF" : "0123456789abcdef"; + var output = ""; + var x; + for(var i = 0; i < input.length; i++) + { + x = input.charCodeAt(i); + output += hex_tab.charAt((x >>> 4) & 0x0F) + + hex_tab.charAt( x & 0x0F); + } + return output; +} + +/* + * Convert a raw string to a base-64 string + */ +function rstr2b64(input) +{ + try { b64pad } catch(e) { b64pad=''; } + var tab = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + var output = ""; + var len = input.length; + for(var i = 0; i < len; i += 3) + { + var triplet = (input.charCodeAt(i) << 16) + | (i + 1 < len ? input.charCodeAt(i+1) << 8 : 0) + | (i + 2 < len ? input.charCodeAt(i+2) : 0); + for(var j = 0; j < 4; j++) + { + if(i * 8 + j * 6 > input.length * 8) output += b64pad; + else output += tab.charAt((triplet >>> 6*(3-j)) & 0x3F); + } + } + return output; +} + +/* + * Convert a raw string to an arbitrary string encoding + */ +function rstr2any(input, encoding) +{ + var divisor = encoding.length; + var i, j, q, x, quotient; + + /* Convert to an array of 16-bit big-endian values, forming the dividend */ + var dividend = Array(Math.ceil(input.length / 2)); + for(i = 0; i < dividend.length; i++) + { + dividend[i] = (input.charCodeAt(i * 2) << 8) | input.charCodeAt(i * 2 + 1); + } + + /* + * Repeatedly perform a long division. The binary array forms the dividend, + * the length of the encoding is the divisor. Once computed, the quotient + * forms the dividend for the next step. All remainders are stored for later + * use. + */ + var full_length = Math.ceil(input.length * 8 / + (Math.log(encoding.length) / Math.log(2))); + var remainders = Array(full_length); + for(j = 0; j < full_length; j++) + { + quotient = Array(); + x = 0; + for(i = 0; i < dividend.length; i++) + { + x = (x << 16) + dividend[i]; + q = Math.floor(x / divisor); + x -= q * divisor; + if(quotient.length > 0 || q > 0) + quotient[quotient.length] = q; + } + remainders[j] = x; + dividend = quotient; + } + + /* Convert the remainders to the output string */ + var output = ""; + for(i = remainders.length - 1; i >= 0; i--) + output += encoding.charAt(remainders[i]); + + return output; +} + +/* + * Encode a string as utf-8. + * For efficiency, this assumes the input is valid utf-16. + */ +function str2rstr_utf8(input) +{ + var output = ""; + var i = -1; + var x, y; + + while(++i < input.length) + { + /* Decode utf-16 surrogate pairs */ + x = input.charCodeAt(i); + y = i + 1 < input.length ? input.charCodeAt(i + 1) : 0; + if(0xD800 <= x && x <= 0xDBFF && 0xDC00 <= y && y <= 0xDFFF) + { + x = 0x10000 + ((x & 0x03FF) << 10) + (y & 0x03FF); + i++; + } + + /* Encode output as utf-8 */ + if(x <= 0x7F) + output += String.fromCharCode(x); + else if(x <= 0x7FF) + output += String.fromCharCode(0xC0 | ((x >>> 6 ) & 0x1F), + 0x80 | ( x & 0x3F)); + else if(x <= 0xFFFF) + output += String.fromCharCode(0xE0 | ((x >>> 12) & 0x0F), + 0x80 | ((x >>> 6 ) & 0x3F), + 0x80 | ( x & 0x3F)); + else if(x <= 0x1FFFFF) + output += String.fromCharCode(0xF0 | ((x >>> 18) & 0x07), + 0x80 | ((x >>> 12) & 0x3F), + 0x80 | ((x >>> 6 ) & 0x3F), + 0x80 | ( x & 0x3F)); + } + return output; +} + +/* + * Encode a string as utf-16 + */ +function str2rstr_utf16le(input) +{ + var output = ""; + for(var i = 0; i < input.length; i++) + output += String.fromCharCode( input.charCodeAt(i) & 0xFF, + (input.charCodeAt(i) >>> 8) & 0xFF); + return output; +} + +function str2rstr_utf16be(input) +{ + var output = ""; + for(var i = 0; i < input.length; i++) + output += String.fromCharCode((input.charCodeAt(i) >>> 8) & 0xFF, + input.charCodeAt(i) & 0xFF); + return output; +} + +/* + * Convert a raw string to an array of little-endian words + * Characters >255 have their high-byte silently ignored. + */ +function rstr2binl(input) +{ + var output = Array(input.length >> 2); + for(var i = 0; i < output.length; i++) + output[i] = 0; + for(var i = 0; i < input.length * 8; i += 8) + output[i>>5] |= (input.charCodeAt(i / 8) & 0xFF) << (i%32); + return output; +} + +/* + * Convert an array of little-endian words to a string + */ +function binl2rstr(input) +{ + var output = ""; + for(var i = 0; i < input.length * 32; i += 8) + output += String.fromCharCode((input[i>>5] >>> (i % 32)) & 0xFF); + return output; +} + +/* + * Calculate the MD5 of an array of little-endian words, and a bit length. + */ +function binl_md5(x, len) +{ + /* append padding */ + x[len >> 5] |= 0x80 << ((len) % 32); + x[(((len + 64) >>> 9) << 4) + 14] = len; + + var a = 1732584193; + var b = -271733879; + var c = -1732584194; + var d = 271733878; + + for(var i = 0; i < x.length; i += 16) + { + var olda = a; + var oldb = b; + var oldc = c; + var oldd = d; + + a = md5_ff(a, b, c, d, x[i+ 0], 7 , -680876936); + d = md5_ff(d, a, b, c, x[i+ 1], 12, -389564586); + c = md5_ff(c, d, a, b, x[i+ 2], 17, 606105819); + b = md5_ff(b, c, d, a, x[i+ 3], 22, -1044525330); + a = md5_ff(a, b, c, d, x[i+ 4], 7 , -176418897); + d = md5_ff(d, a, b, c, x[i+ 5], 12, 1200080426); + c = md5_ff(c, d, a, b, x[i+ 6], 17, -1473231341); + b = md5_ff(b, c, d, a, x[i+ 7], 22, -45705983); + a = md5_ff(a, b, c, d, x[i+ 8], 7 , 1770035416); + d = md5_ff(d, a, b, c, x[i+ 9], 12, -1958414417); + c = md5_ff(c, d, a, b, x[i+10], 17, -42063); + b = md5_ff(b, c, d, a, x[i+11], 22, -1990404162); + a = md5_ff(a, b, c, d, x[i+12], 7 , 1804603682); + d = md5_ff(d, a, b, c, x[i+13], 12, -40341101); + c = md5_ff(c, d, a, b, x[i+14], 17, -1502002290); + b = md5_ff(b, c, d, a, x[i+15], 22, 1236535329); + + a = md5_gg(a, b, c, d, x[i+ 1], 5 , -165796510); + d = md5_gg(d, a, b, c, x[i+ 6], 9 , -1069501632); + c = md5_gg(c, d, a, b, x[i+11], 14, 643717713); + b = md5_gg(b, c, d, a, x[i+ 0], 20, -373897302); + a = md5_gg(a, b, c, d, x[i+ 5], 5 , -701558691); + d = md5_gg(d, a, b, c, x[i+10], 9 , 38016083); + c = md5_gg(c, d, a, b, x[i+15], 14, -660478335); + b = md5_gg(b, c, d, a, x[i+ 4], 20, -405537848); + a = md5_gg(a, b, c, d, x[i+ 9], 5 , 568446438); + d = md5_gg(d, a, b, c, x[i+14], 9 , -1019803690); + c = md5_gg(c, d, a, b, x[i+ 3], 14, -187363961); + b = md5_gg(b, c, d, a, x[i+ 8], 20, 1163531501); + a = md5_gg(a, b, c, d, x[i+13], 5 , -1444681467); + d = md5_gg(d, a, b, c, x[i+ 2], 9 , -51403784); + c = md5_gg(c, d, a, b, x[i+ 7], 14, 1735328473); + b = md5_gg(b, c, d, a, x[i+12], 20, -1926607734); + + a = md5_hh(a, b, c, d, x[i+ 5], 4 , -378558); + d = md5_hh(d, a, b, c, x[i+ 8], 11, -2022574463); + c = md5_hh(c, d, a, b, x[i+11], 16, 1839030562); + b = md5_hh(b, c, d, a, x[i+14], 23, -35309556); + a = md5_hh(a, b, c, d, x[i+ 1], 4 , -1530992060); + d = md5_hh(d, a, b, c, x[i+ 4], 11, 1272893353); + c = md5_hh(c, d, a, b, x[i+ 7], 16, -155497632); + b = md5_hh(b, c, d, a, x[i+10], 23, -1094730640); + a = md5_hh(a, b, c, d, x[i+13], 4 , 681279174); + d = md5_hh(d, a, b, c, x[i+ 0], 11, -358537222); + c = md5_hh(c, d, a, b, x[i+ 3], 16, -722521979); + b = md5_hh(b, c, d, a, x[i+ 6], 23, 76029189); + a = md5_hh(a, b, c, d, x[i+ 9], 4 , -640364487); + d = md5_hh(d, a, b, c, x[i+12], 11, -421815835); + c = md5_hh(c, d, a, b, x[i+15], 16, 530742520); + b = md5_hh(b, c, d, a, x[i+ 2], 23, -995338651); + + a = md5_ii(a, b, c, d, x[i+ 0], 6 , -198630844); + d = md5_ii(d, a, b, c, x[i+ 7], 10, 1126891415); + c = md5_ii(c, d, a, b, x[i+14], 15, -1416354905); + b = md5_ii(b, c, d, a, x[i+ 5], 21, -57434055); + a = md5_ii(a, b, c, d, x[i+12], 6 , 1700485571); + d = md5_ii(d, a, b, c, x[i+ 3], 10, -1894986606); + c = md5_ii(c, d, a, b, x[i+10], 15, -1051523); + b = md5_ii(b, c, d, a, x[i+ 1], 21, -2054922799); + a = md5_ii(a, b, c, d, x[i+ 8], 6 , 1873313359); + d = md5_ii(d, a, b, c, x[i+15], 10, -30611744); + c = md5_ii(c, d, a, b, x[i+ 6], 15, -1560198380); + b = md5_ii(b, c, d, a, x[i+13], 21, 1309151649); + a = md5_ii(a, b, c, d, x[i+ 4], 6 , -145523070); + d = md5_ii(d, a, b, c, x[i+11], 10, -1120210379); + c = md5_ii(c, d, a, b, x[i+ 2], 15, 718787259); + b = md5_ii(b, c, d, a, x[i+ 9], 21, -343485551); + + a = safe_add(a, olda); + b = safe_add(b, oldb); + c = safe_add(c, oldc); + d = safe_add(d, oldd); + } + return Array(a, b, c, d); +} + +/* + * These functions implement the four basic operations the algorithm uses. + */ +function md5_cmn(q, a, b, x, s, t) +{ + return safe_add(bit_rol(safe_add(safe_add(a, q), safe_add(x, t)), s),b); +} +function md5_ff(a, b, c, d, x, s, t) +{ + return md5_cmn((b & c) | ((~b) & d), a, b, x, s, t); +} +function md5_gg(a, b, c, d, x, s, t) +{ + return md5_cmn((b & d) | (c & (~d)), a, b, x, s, t); +} +function md5_hh(a, b, c, d, x, s, t) +{ + return md5_cmn(b ^ c ^ d, a, b, x, s, t); +} +function md5_ii(a, b, c, d, x, s, t) +{ + return md5_cmn(c ^ (b | (~d)), a, b, x, s, t); +} + +/* + * Add integers, wrapping at 2^32. This uses 16-bit operations internally + * to work around bugs in some JS interpreters. + */ +function safe_add(x, y) +{ + var lsw = (x & 0xFFFF) + (y & 0xFFFF); + var msw = (x >> 16) + (y >> 16) + (lsw >> 16); + return (msw << 16) | (lsw & 0xFFFF); +} + +/* + * Bitwise rotate a 32-bit number to the left. + */ +function bit_rol(num, cnt) +{ + return (num << cnt) | (num >>> (32 - cnt)); +}