Merge 607a57a into 740839c

.gitignore

@@ -25,4 +25,7 @@ config/runtime.json
 apiary.out.html
 
 config/oo.yml
+config/development.yml
+config/certs/goo.pem
+
 npm-shrinkwrap.json

.npmignore

@@ -10,4 +10,6 @@ config/runtime.json
 
 apiary.out.html
 
-config/oo.yml
+config/oo.yml
+config/development.yml
+config/certs/goo.pem

config/default.yml

@@ -1,4 +1,4 @@
-version: 0.0.95
+version: 0.0.110
 interfaces:
   webserver:
     enabled: true
@@ -58,8 +58,8 @@ store:
     file:
       level: trace
       path: /tmp/joola.io/
-  cache:
-    mongo:
+  datastore:
+    mongodb:
       dsn: mongodb://localhost:27017/cache
       expireafterseconds: 0
 dispatch:

docs/technical-documentation/Technical-Documentation.md

@@ -7,27 +7,28 @@ The technical documentation review the joola.io architecture, know-how and detai
 - [Push Data](pushing-data)
 - [Query, Analyze and Visualize](analytics-and-visualization)
 
-#### Useful Resources
-- [[Basic Concepts]]
-- [API Documentation](api-documentation)
-- [[Using the SDK]]
-- [[Examples]]
-- [[Workshops]]
-- [[Labs]]
-
 ## Architecture
 We've tried to keep things simple and divided the framework into the listed logical entities, each is aimed at serving a different aspect of the framework.
 
-- [Overview](architecture) - Internal processes
+- [Overview](architecture) - A high level overview of the framework
 - [Core](The-Core-Subsystem) - Internal processes
 - [Common](The-Common-Subsystem) - Shared modules and code
 - [Dispatch](The-Dispatch-Subsystem) - The grid messaging system
+- [Datastore](The-Datastore-Subsystem) - Where our data lives
 - [Query](The-Query-Subsystem) - Manages the aspects of querying the system
 - [Beacon](The-Beacon-Subsystem) - Handles the framework's internal cache
 - [Authentication](The-Authentication-Subsystem) - All authentication aspects of the framework
 - [Web Server](The-Webserver-Subsystem) - Serves web content to end users
 - [SDK](The-SDK-Subsystem) - Used to communicate with joola.io framework and manage it
 
+## Useful Resources
+- [[Basic Concepts]]
+- [API Documentation](api-documentation)
+- [[Using the SDK]]
+- [[Examples]]
+- [[Workshops]]
+- [[Labs]]
+
 ## The Development Process
 We aim to create the world's best mass-scale data analytics framework and for that, we need to have a solid and robust development process.
 This section describes the process and protocols we use throughout the development of the framework.

docs/technical-documentation/_Sidebar.md

@@ -1,6 +1,5 @@
 [**HOME**](Home) > **TECHNICAL DOCUMENTATION**
 
-**Using joola.io**
 - [Setup Guide](setting-up-joola.io)
 - [Configuration](Configuration)
 - [The joola.io SDK](using-the-sdk)

docs/technical-documentation/architecture/The-Datastore-Subsystem.md

@@ -0,0 +1,70 @@
+[HOME](Home) > [TECHNICAL DOCUMENTATION](technical-documentation) > [ARCHITECTURE](architecture) > **DATASTORE**
+
+joola.io uses a datastore to support its need for storing documents, querying, analyzing, etc...
+We've designed joola.io so it is agnostic to the datastore provider, it uses a logical layer that exposes API endpoints for pushing documents ([beacon](the-beacon-subsystem)) and querying analytics ([query](the-query-subsystem)).
+
+These are the current datastore providers joola.io supports:
+- [MongoDB](http://github.com/joola/joola.io.datastore-mongodb)
+
+There's a wealth of datastore providers, ranging from Google Datastore to influxdb. We invite the community to contribute plugins to extend the support of joola.io.
+
+### Datastore Plugins
+Extending joola.io with additional datastore plugins is not an overly complex task. The framework supports easy integration of additional plugins.
+
+##### Install the plugin
+When joola.io boots, it tries to load all relevant datastore plugins and initialize them. Discovery of datastores is done by inspecting configuration under the `store:datastore` key.
+  Each datastore found is initialized by requiring the plugin `require('joola.io.datastore-<datastorename>')`. If the plugin is not installed, an error is shown and the boot is terminated.
+
+For this example, we'll use the `joola.io.datastore-mongodb` plugin.
+
+```bash
+$ npm install joola.io.datastore-mongodb
+```
+
+##### Add to configuration
+```yaml
+store:
+  datastore:
+    mongodb:
+      dsn: mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]]
+```
+
+When you'll run joola.io, it will attempt to `require('joola.io.datastore-mongodb')` and then initialize it with the dsn provided in the configuration.
+
+### Writing Plugins
+In order to allow communication between joola.io and the plugin it's required for the plugin to follow strict guidelines and syntax.
+
+ We have created a template repository named [joola.io.datastore-template](http://github.com/joola/joola.io.datastore-template). Please fork this repository to use as the basis for the plugin you are about to develop.
+ The template implements all required exports for communication back and forth between joola.io and your plugin.
+
+ A working example is the [joola.io.datastore-mongodb](http://github.com/joola/joola.io.datastore-mongodb) provider plugin.
+
+ The template is rich with comments to support your development, please feel free to submit pull-request to the template repository with additional know-how, examples and comments to support the community development.
+
+#### Exports
+
+The following exports are needed from each plugin. The template includes more in-depth comments relevant to each export.
+
+###### `init`
+
+###### `destroy`
+
+###### `openConnection`
+
+###### `closeConnection`
+
+###### `checkConnection`
+
+###### `insert`
+
+###### `find`
+
+###### `delete`
+
+###### `update`
+
+###### `query`
+
+###### `drop`
+
+###### `purge`

lib/auth/index.js → lib/common/auth.js

@@ -11,7 +11,7 @@
 'use strict';
 
 var
-  joola = require('../joola.io'),
+  joola = require('../joola.io.js'),
   crypto = require('crypto'),
   path = require('path'),
   url = require('url'),

lib/common/datastore.js

@@ -0,0 +1,80 @@
+/**
+ *  @title joola.io
+ *  @overview the open-source data analytics framework
+ *  @copyright Joola Smart Solutions, Ltd. <info@joo.la>
+ *  @license GPL-3.0+ <http://spdx.org/licenses/GPL-3.0+>
+ *
+ *  Licensed under GNU General Public License 3.0 or later.
+ *  Some rights reserved. See LICENSE, AUTHORS.
+ **/
+
+'use strict';
+
+var
+  joola = require('../joola.io'),
+
+  domain = require('domain'),
+  async = require('async'),
+  _ = require('underscore');
+
+var datastore = module.exports;
+
+datastore.providers = {};
+
+datastore.init = function (callback) {
+  //find out what datastores we have in config
+  var stores = joola.config.get('store:datastore');
+
+  var calls = [];
+  Object.keys(stores).forEach(function (key) {
+    var store = stores[key];
+    if (store.enabled !== false) {
+      try {
+        var call = function (callback) {
+          var Provider = require('joola.io.datastore-' + key);
+          if (!Provider)
+            return callback(new Error('Provider [' + key + '] cannot be required. Did you forget to `npm install` it?'));
+
+          joola.logger.trace('Initializing datastore provider [' + key + '].');
+          new Provider(store, {logger: joola.logger, common: joola.common}, function (err, provider) {
+            if (err)
+              return callback(err);
+            joola.logger.debug('Datastore [' + key + '] is now ready.');
+            datastore.providers[key] = provider;
+            if (!datastore.providers.default)
+              datastore.providers.default = provider;
+            return callback(null, provider);
+          });
+        };
+        calls.push(call);
+      }
+      catch (ex) {
+        return callback(ex);
+      }
+    }
+  });
+
+  async.series(calls, function (err) {
+    if (err)
+      return callback(err);
+
+    return callback(null, stores);
+  });
+};
+
+datastore.destroy = function (callback) {
+  var calls = [];
+  Object.keys(datastore.providers).forEach(function (key) {
+    var call = function (callback) {
+      joola.logger.trace('Destroying datastore provider [' + key + '].');
+      var provider = datastore.providers[key];
+      provider.destroy(callback);
+    };
+    calls.push(call);
+  });
+  async.series(calls, function (err) {
+    if (err)
+      return callback(err);
+    return callback(null);
+  });
+};

lib/common/globals.js

@@ -29,20 +29,24 @@ joola.silentShutdown = function (code, callback) {
     joola.state.set('core', 'stop', 'received code [' + code + ']');
     joola.dispatch.emit('nodes:state:change', [joola.UID, joola.state.get()]);
 
-    var node = joola.nodeState();
-    if (node.http) {
-      //We're running an http node, let's try to start another one.
-      joola.logger.debug('Trying to start Web Services on another node');
-      joola.dispatch.request(0, 'startWebServer', [node], function () {
+
+    joola.datastore.destroy(function () {
+      var node = joola.nodeState();
+      if (node.http) {
+        //We're running an http node, let's try to start another one.
+        joola.logger.debug('Trying to start Web Services on another node');
+        joola.dispatch.request(0, 'startWebServer', [node], function () {
+          //do we care?
+        });
+      }
+      joola.redis.del('nodes:' + joola.UID, function (err) {
         //do we care?
       });
-    }
-    joola.redis.del('nodes:' + joola.UID, function (err) {
-      //do we care?
-    });
 
-    if (typeof callback === 'function')
-      return callback(null);
+      if (typeof callback === 'function')
+        return callback(null);
+      return;
+    });
   }
   catch (ex) {
     return callback(ex);

lib/common/logger.js

@@ -14,7 +14,6 @@
 var
   joola = require('../joola.io'),
   fs = require('fs'),
-  mongo_url_parser = require('mongodb/lib/mongodb/connection/url_parser'),
   bunyan = require('bunyan'),
   Bunyan2Loggly = require('bunyan-loggly').Bunyan2Loggly,
   bformat = require('bunyan-format'),