Merge a8741a3 into 3d12c72

application/bin/delete

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const config = require('../configuration');
+const solrClient = require('../solr/solrClient');
+
+
+solrClient.deleteAll().then( (res) => {
+    console.log(res);
+}).catch( (error) => {
+    console.log(error.message);
+});

application/bin/index

@@ -0,0 +1,80 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const config = require('../configuration');
+const async = require('async');
+const ProgressBar = require('progress');
+const db = require('../database/databaseConnection');
+
+const decks = require('../solr/objectCollections/decks');
+const slides = require('../solr/objectCollections/slides');
+const users = require('../solr/objectCollections/users');
+
+function index(collection, indexFunction){
+	let limit = 50;
+	let allowErrors = 20;
+
+	return new Promise( (resolve, reject) => {
+		db.getTotalCount(collection).then( (totalCount) => {
+			console.log(`Indexing ${totalCount} ${collection}`);
+
+		    let offset = 0;
+		    let progressBar = new ProgressBar(`${collection} progress [:bar] :percent`, { total: totalCount });
+
+		    async.doWhilst(
+		        (callback) => {
+		            db.getAll(collection, offset, limit).then( (docs) => {
+		            	async.eachSeries(docs, (doc, callback2) => {
+		            		progressBar.tick();
+		            		indexFunction(doc).then( () => {
+		            			callback2();
+		            		}).catch( (err) => {
+		            			console.log(err);
+
+		            			// if there are too many errors, reject promise
+		            			if(--allowErrors === 0){
+		            				reject('Too many errors, please check configuration and try again');
+		            			}
+		            			callback2();
+		            		});
+		            	}, () => {
+		            		offset += limit;
+		            		callback();
+		            	});
+		            }).catch( (err) => {
+		               reject(err.message);
+		            });
+		        },
+		        () => { return offset <= totalCount; },
+		        () => { resolve(`Initial index of ${collection} has been completed`); }
+		    );
+		}).catch( (err) => { reject(err.message); });
+	});
+
+}
+
+
+let collectionsToIndex = [
+	{ name: 'users', indexMethod: users.new },
+	{ name: 'decks', indexMethod: decks.newDeck }, 
+	{ name: 'slides', indexMethod: slides.newSlide }
+];
+
+async.eachSeries(collectionsToIndex, (col, callback) => {
+	console.log();
+	index(col.name, col.indexMethod).then( (res) => {
+		console.log(res);
+		callback();
+	}).catch( (err) => {
+		callback(err);
+	});
+}, (err) => {
+	if(err){
+		console.log(err);
+		process.exit(-1);
+	}
+
+	console.log('Initial index has been successfully completed');
+	process.exit(0);
+});

application/configuration.js

@@ -20,7 +20,7 @@ solrConfig.CORE = (!co.isEmpty(process.env.SOLR_CORE)) ? process.env.SOLR_CORE :
 solrConfig.PATH = (!co.isEmpty(process.env.SOLR_PATH)) ? process.env.SOLR_PATH : '/solr';
 solrConfig.PROTOCOL = (!co.isEmpty(process.env.SOLR_PROTOCOL)) ? process.env.SOLR_PROTOCOL : 'http';
 //local testing SOLR config:
-//solrConfig.HOST = (!co.isEmpty(process.env.SOLR_HOST)) ? process.env.SOLR_HOST : 'slidewiki.imis.athena-innovation.gr';
+// solrConfig.HOST = (!co.isEmpty(process.env.SOLR_HOST)) ? process.env.SOLR_HOST : 'slidewiki.imis.athena-innovation.gr';
 //production SOLR config:
 solrConfig.HOST = (!co.isEmpty(process.env.SOLR_HOST)) ? process.env.SOLR_HOST : 'solr';
 

application/controllers/handler.js

@@ -6,7 +6,6 @@ Handles the requests by executing stuff and replying to the client. Uses promise
 
 const boom = require('boom'), //Boom gives us some predefined http codes and proper responses
     solrClient = require('../solr/solrClient'),
-    initIndex = require('../solr/initIndex'),
     searchResults = require('../solr/searchResults'),
     suggest = require('../solr/suggestions');
 
@@ -22,26 +21,6 @@ module.exports = {
         });
     },
 
-    // index a collection from DB to SOLR
-    indexAll: function(request, reply){
-        initIndex.indexAll(request.params.collection).then( (res) => {
-            reply(res);
-        }).catch( (error) => {
-            request.log('initIndex.indexAll', error);
-            reply(boom.badImplementation());
-        });
-    },
-
-    // delete all documents from SOLR
-    deleteAll: function(request, reply){
-        solrClient.deleteAll().then( (res) => {
-            reply(res);
-        }).catch( (error) => {
-            request.log('solrClient.deleteAll', error);
-            reply(boom.badImplementation());
-        });
-    },
-
     // suggest keywords or users
     suggest: function(request, reply){
         let suggestFunction = (request.params.source === 'keywords') ? suggest.findKeywords : suggest.findUsers;

application/database/databaseConnection.js

@@ -7,10 +7,16 @@ Controller for handling mongodb and the data model slide while providing CRUD'is
 const helper = require('./helper');
 
 module.exports = {
-    getAllFromCollection: function(collectionName) {
+    getAll: function (collection, offset, limit) {
         return helper.connectToDatabase()
-            .then((db) => db.collection(collectionName))
-            .then((col) => col.find())
-            .then((stream) => stream.toArray());
+        .then((db) => db.collection(collection))
+        .then((col) => col.find({}).skip(offset).limit(limit).sort({_id:1}))
+        .then((cursor) => cursor.toArray());
+    },
+
+    getTotalCount: function(collection){
+        return helper.connectToDatabase()
+        .then((db) => db.collection(collection))
+        .then((col) => col.find({}).count());
     }
 };

application/package.json

@@ -40,7 +40,9 @@
     "solr-client": "^0.6.0",
     "mongo-trigger": "^0.1.5",
     "html-to-text": "^3.1.0",
-    "request-promise": "^4.1.1"
+    "request-promise": "^4.1.1",
+    "progress":"^1.1.8",
+    "async": "^2.2.0"
   },
   "engines": {
     "node": ">=6.9.0"

application/routes.js

@@ -47,38 +47,7 @@ module.exports = function(server) {
             description: 'Get SOLR search results'
         }
     });
-
-    // index all data from db
-    server.route({
-        method: 'GET',
-        path: '/index/{collection}',
-        handler: handlers.indexAll,
-        config: {
-            validate: {
-                params: {
-                    collection: Joi.string().valid('decks', 'slides', 'users', 'all')
-                },
-            },
-            tags: ['api'],
-            description: 'Index all data from DB in SOLR'
-        }
-    });
-
-    // delete all data from db
-    server.route({
-        method: 'GET',
-        path: '/delete',
-        handler: handlers.deleteAll,
-        config: {
-            validate: {
-                params: {
-                },
-            },
-            tags: ['api'],
-            description: 'Delete all documents from SOLR'
-        }
-    });
-
+
     // suggest keywords or users
     server.route({
         method: 'GET',

application/solr/solrClient.js

@@ -17,7 +17,10 @@ module.exports = {
         let promise = new Promise( (resolve, reject) => {
             client.add(data, (err, obj) => {
                 if(err){
-                    reject('addDocs : ' + JSON.stringify(err) + '\ndata: ' + JSON.stringify(data) + '\n');
+                    reject({
+                        message: err.message, 
+                        data: data
+                    });
                 }else{
                     resolve(obj);
                 }
@@ -45,7 +48,7 @@ module.exports = {
         let requestUri = solrUri + '/update?stream.body=<delete><query>*:*</query></delete>&commit=true';
 
         return rp.get({uri: requestUri}).then( () => {
-            return Promise.resolve('All documents are deleted from SOLR Index');
+            return Promise.resolve('All documents were deleted from SOLR Index');
         }).catch( (err) => {
             return Promise.reject(err);
         });