remove jQuery and replace by underscorejs

monbro · Dec 16, 2014 · a804758 · a804758
1 parent 4451c81
commit a804758
Show file tree

Hide file tree

Showing 6 changed files with 22 additions and 24 deletions.
diff --git a/app.js b/app.js
@@ -9,12 +9,12 @@
  * @version 0.1
  */
 
-/** 
+/**
  * laod config
  */
 var config = require('./config');
 
-/** 
+/**
  * Basic Objects
  */
 
@@ -27,7 +27,7 @@ var app = new App();
 var Model = require('./app/model');
 
 App.prototype.getModel = function(s) {
-  return new Model(s); 
+  return new Model(s);
 };
 
 // var test = app.getModel('test');
@@ -41,12 +41,12 @@ if(!config.creds.http_server) {
     scraper.wikiSearch('database');
 }
 
-/** 
+/**
  * Server
  */
 
 // Start HTTP API RESTFUL Server
 if(config.creds.http_server) {
     var Http = require("./app/http");
-    var http = new Http(); 
+    var http = new Http();
 }
diff --git a/app/analyse.js b/app/analyse.js
@@ -1,15 +1,15 @@
 var Tools = require('../lib/tools');
 var Db = require('./db');
 var config = require('../config.js');
-var $ = require("jquery");
+var _ = require("underscore");
 
 var Analyse = function() {
     tools = Tools;
     // tools = new Tools();
     db = new Db();
 };
 
-Analyse.prototype.scanTextBlock = function(snippet,counter) {
+Analyse.prototype.scanTextBlock = function(snippet,counter, callback) {
     if(config.creds.debug) {
         console.log('scanTextBlock');
     }
@@ -44,7 +44,7 @@ Analyse.prototype.scanTextBlock = function(snippet,counter) {
 
       // var base;
 
-      $.each(obj, function(index, value) {
+      _.each(obj, function(value, index) {
 
         // skip if not valid
         if(typeof index == 'undefined' || typeof index.toLowerCase == 'undefined')
@@ -54,7 +54,7 @@ Analyse.prototype.scanTextBlock = function(snippet,counter) {
         // base = new Base(index.toLowerCase());
 
         // loop all words
-        $.each(obj, function(index2, value2) {
+        _.each(obj, function(value2, index2) {
           if(index != index2) {
             // add relation, value2 is the counter of how often the word was seen in the recent textblock
             // base.pushRelation(index2.toLowerCase(),value2);
@@ -73,6 +73,7 @@ Analyse.prototype.scanTextBlock = function(snippet,counter) {
 
     // flush changes to database
     db.flush(function(err, replies) {
+        callback();
         return true;
     });
 

diff --git a/app/db.js b/app/db.js
@@ -44,7 +44,7 @@ Db.prototype.removePageFromQueue = function(s) {
 
 Db.prototype.getRandomItemFromQueue = function(callback) {
     client.srandmember('____sites2do____', function (err, result) {
-        return callback(result);
+        callback(result);
     });
 };
 
@@ -86,9 +86,10 @@ Db.prototype.addToGlobalCounter = function(owner,s) {
 };
 
 Db.prototype.getTopRelations = function(owner, callback, res) {
-    client.sort('____all____', "by", "____all____:*", 'LIMIT', 0, 500, 'DESC', "get", "#", function (err1, items1) {
+    client.sort('____all____', "by", "____all____:*", 'LIMIT', 0, 300, 'DESC', "get", "#", function (err1, items1) {
         // get most often realted keywords for the given keyword
         client.sort(owner, "by", owner+":*", 'LIMIT', 0, 120, 'DESC', "get", "#", function (err2, items2) {
+
           // remove the noise by removing the most often used keywords
           callback(tools.inAButNotInB(items2,items1),res);
         });

diff --git a/app/scraping.js b/app/scraping.js
@@ -2,7 +2,6 @@ var Db = require('./db');
 var Analyse = require('./analyse');
 var restify = require('restify');
 var config = require('../config.js');
-var $ = require("jquery");
 
 var Scraper = function() {
     db = new Db();
@@ -17,7 +16,7 @@ var Scraper = function() {
 
 Scraper.prototype.wikiSearch = function(s) {
     if(config.creds.debug) {
-        console.log('wikiSearch');
+        console.log('wikiSearch with: '+s);
     }
 
     // check if not empty string
@@ -65,13 +64,13 @@ Scraper.prototype.wikiSearch = function(s) {
           db.addPageToQueue(data[1][i]);
         }
 
-        if(config.creds.debug) {
+        if(config.creds.debug_once) {
             debug_once = true;
         }
     });
 };
 
-Scraper.prototype.wikiGrab = function(s) { 
+Scraper.prototype.wikiGrab = function(s) {
     if(config.creds.debug) {
         console.log('wikiGrab with '+s);
     }
@@ -118,7 +117,7 @@ Scraper.prototype.wikiGrab = function(s) {
     });
 };
 
-Scraper.prototype.loopWorker = function(snippets) { 
+Scraper.prototype.loopWorker = function(snippets) {
     if(config.creds.debug) {
         console.log('loopWorker');
     }
@@ -131,15 +130,12 @@ Scraper.prototype.loopWorker = function(snippets) {
         return;
     }
 
-    // analyze full text block
-    $.when(analyse.scanTextBlock(snippets.pop(),snippets.length)).done(function() {
-        // set a timeout to be gently to the memory and cpu 
-        // (can be changed in the config file)
+    analyse.scanTextBlock(snippets.pop(),snippets.length,function() {
         var t=setTimeout(function(){that.loopWorker(snippets);},config.creds.sleeptime);
     });
 };
 
-Scraper.prototype.goToNext = function() { 
+Scraper.prototype.goToNext = function() {
     if(config.creds.debug) {
         console.log('goToNext');
     }

diff --git a/config.js b/config.js
@@ -5,7 +5,8 @@ exports.creds = {
   sleeptime: 0, // sleep time in ms between the iteration of all textblocks
   mongoose_auth_local: 'mongodb://localhost/opensemanticapi', // default: mongodb://localhost/opensemanticapi
   min_text_block_length: 120, // default: 120
-  debug: false, // default: true
+  debug: true, // default: true
+  debug_once: false,
   lang: 'en' // default: english
   // lang: 'de' // german
   // lang: 'es' // spanish

diff --git a/package.json b/package.json
@@ -7,8 +7,7 @@
     "restify": "2.x",
     "mongoose": "3.x.x",
     "redis": "0.8.x",
-    "underscore": "1.x.x",
-    "jquery": "1.x.x"
+    "underscore": "1.x.x"
   },
   "repository": {
     "type": "git",