experiencing problem in phantom 2.1

bin/chopsticks.js

@@ -2,36 +2,51 @@ var _ = require('lodash');
 var Promise = require('bluebird');
 var debug = require('debug')('↻ chopsticks');
 var request = Promise.promisifyAll(require('request'));
-var plugins = require('./plugins');
 var nconf = require('nconf');
 
+var plugins = require('../plugins');
+
 var cfgFile = "config/chopsticks.json";
 
 nconf.argv()
      .env()
      .file({ file: cfgFile });
 
-var url = nconf.get('source') + '/api/v1/getTasks/' + nconf.get('vp');
+var VP = nconf.get('VP');
+if(_.isUndefined(VP)) {
+    console.log("VP, vantage point, is needed in the Environment. forced 'dummy'");
+    VP = 'dummy';
+}
+
+var directionByKind = {
+    "basic": {
+        "plugins": [ "systemState", "phantom", "saver", "reportBack" ],
+        "config": null
+    }
+};
+
+function keepPromises(N) {
+    var direction = directionByKind[N.needName];
+    debug("%j ", direction);
+    return Promise.reduce(direction.plugins, function(state, p) {
+        debug("Call %s about %s state key (%j)", p, state.href, _.keys(state));
+        return plugins[p](state, direction.config);
+    }, N);
+};
+
+var url = nconf.get('source') + '/api/v1/getTasks/' + VP + '/' + nconf.get('amount');
 debug("Looking for some needs in %s...", url);
 return request
-    .getAsync({
-        url: url
+    .getAsync(url)
+    .then(function(response) {
+        return JSON.parse(response.body);
     })
     .then(function(needs) {
-        debug("Received %d needs, can be filtered/reduced in size",
-            _.size(needs));
+        debug("Received %d needs", _.size(needs));
+        /* estimation of load might define concurrency and delay */
         return needs;
     })
-    .then(function(needs) {
-        debug("Iterating over the needs:");
-        console.log(JSON.stringify(needs, undefined, 2));
-        return Promise.reduce(needs.chain, function(memo, p) {
-            debug("Executing %s", p.plugin);
-            var state = plugins[p.plugin](memo, p.config);
-            debug("Execution completed from %s", p.plugin);
-            return state;
-        }, [ needs.promises ]);
-    })
+    .map(keepPromises, { concurrency: 1})
     .then(function(solutions) {
         debug("TODO, posts the solutions to the promises");
     })

bin/directionTool.js

@@ -24,9 +24,13 @@ function loadJSONfile(fname) {
 
 function uniqueTargets(memo, subject) {
     var alist = _.map(subject.pages, function(site) {
-        return _.pick(site, ['href', 'id', 'rank']);
+        return {
+            subjectId: site.id,
+            href: site.href,
+            rank: site.rank
+        };
     });
-    var uniqued = _.uniqBy(_.concat(memo, alist), 'id');
+    var uniqued = _.uniqBy(_.concat(memo, alist), 'subjectId');
     /* reject forcefully everything with a rank < than 100 */
     return _.reject(uniqued, function(entry) {
         return entry.rank > 100;
@@ -41,10 +45,16 @@ function insertNeeds(fname) {
             mongo.read(nconf.get('schema').subjects)
         ])
         .then(function(inputs) {
-            debug("Stripping everything with rank < 100 off");
             var targets = _.reduce(inputs[1], uniqueTargets, []);
+            debug("Remind, everything with rank < 100 has been stripped off");
             return _.map(targets, function(t) {
-                return _.extend(t, inputs[0]);
+                var p = _.extend(t, inputs[0]);
+                p.id = various.hash({
+                    'href': p.href,
+                    'start': p.start,
+                    'end': p.end
+                });
+                return p;
             });
         })
         .then(function(needs) {
@@ -78,8 +88,8 @@ function timeRanges(fname) {
             }
             return {
                 needName: content.needName,
-                start: start.toISOString(),
-                end: end.toISOString()
+                start: new Date(start.toISOString()),
+                end: new Date(end.toISOString())
             };
         });
 }

bin/storyteller.js

@@ -14,9 +14,9 @@ var nconf = require('nconf');
  * con un nome caratteristico per la funzionalità d'alto livello, che
  * prende sempre come argomento un oggetto o una collection e ci lavora
  * facendo tornare un nuovo valore */
-var various = require('./lib/various');
-var routes = require('./routes/_storyteller');
-var dispatchPromise = require('./lib/dispatchPromise');
+var various = require('../lib/various');
+var routes = require('../routes/_storyteller');
+var dispatchPromise = require('../lib/dispatchPromise');
 
 var cfgFile = "config/storyteller.json";
 var redOn = "\033[31m";

bin/vigile.js

@@ -9,9 +9,9 @@ var mongodb = Promise.promisifyAll(require('mongodb'));
 var debug = require('debug')('vigile');
 var nconf = require('nconf');
 
-var various = require('./lib/various');
-var routes = require('./routes/_vigile');
-var dispatchPromise = require('./lib/dispatchPromise');
+var various = require('../lib/various');
+var routes = require('../routes/_vigile');
+var dispatchPromise = require('../lib/dispatchPromise');
 
 var cfgFile = "config/vigile.json";
 var redOn = "\033[31m";
@@ -34,7 +34,7 @@ app.get('/api/v:version/system/info', function(req, res) {
     return dispatchPromise('systemInfo', routes, req, res);
 });
 
-app.get('/api/v:version/getTasks/:agentName', function(req, res) {
+app.get('/api/v:version/getTasks/:vantagePoint/:amount', function(req, res) {
     return dispatchPromise('getTasks', routes, req, res);
 });
 

config/chopsticks.json

@@ -2,10 +2,11 @@
   "name": "chopsticks",
   "source": "http://localhost:7200",
   "schema": {
-    "anomalies": "anomalies",
     "log": "log",
+    "thug": "thug",
     "phantom": "phantom",
-    "thug": "thug"
+    "anomalies": "anomalies"
   },
+  "amount": 30,
   "mongodb": "mongodb://localhost/chopsticks"
 }

lib/mongo.js

@@ -24,14 +24,10 @@ var writeOne = function(cName, dataObject) {
 };
 
 var upsertOne = function(cName, selector, updated) {
-    debug("upsertOne in %s selector %j ", cName, selector);
     return Promise.using(dbConnection(), function(db) {
         return db
             .collection(cName)
             .updateOne(selector, updated, { upsert: true});
-    })
-    .tap(function(upsertRet) {
-        debug("upsertOne return: %j", upsertRet);
     });
 };
 

package.json

@@ -8,10 +8,10 @@
     "watch-css": "stylus styles/index.styl -o dist/css -w",
     "clean": "rm -rf dist/css dist/js",
     "build": "./.gruntlame",
-    "storyteller": "nodemon --config config/storyteller.json storyteller",
-    "vigile": "nodemon --config config/vigile.json vigile",
-    "chopsticks": "nodemon --config config/chopsticks.json chopsticks",
-    "start": "echo 'nop, call run storyteller or the others components separately' "
+    "start": "echo 'nop, call run storyteller or the others components separately' ",
+    "storyteller": "nodemon --config config/storyteller.json bin/storyteller",
+    "vigile": "nodemon --config config/vigile.json bin/vigile",
+    "chopsticks": "DEBUG=* node bin/chopsticks"
   },
   "repository": {
     "type": "git",
@@ -47,6 +47,7 @@
     "nconf": "^0.8.4",
     "node-geoip": "^1.0.1",
     "nodemon": "^1.11.0",
+    "phantomjs": "^2.1.7",
     "pug": "^2.0.0-beta6",
     "request": "^2.79.0",
     "reveal.js": "^3.3.0",

plugins/index.js

@@ -1,4 +1,6 @@
 module.exports = {
     systemState: require('./systemState'),
-    // phantom: require('./phantom')
+    phantom: require('./phantom'),
+    saver: require('./saver'),
+    reportBack: require('./reportBack')
 };

routes/getTasks.js

@@ -7,58 +7,52 @@ var mongo = require('../lib/mongo');
 var subjectsOps = require('../lib/subjectsOps');
 var prand = require('../lib/pseudoRandom');
 
+
+function markVantagePoint(vp, siteList) {
+
+    return Promise.map(siteList, function(s) {
+        /* has to be fixed to 'toArray' returned by readLimit */
+        var start = new Date(s.start);
+        var end = new Date(s.end);
+        var update = _.set(_.extend(_.omit(s, ['start', 'end' ]), {
+            start: start,
+            end: end
+        }), vp, true);
+
+        return mongo.upsertOne(nconf.get('schema').promises, {
+            id: update.id
+        }, update);
+    });
+};
+
 /* this function is constantly called, like, every minute, 
  * through this function might be possible organize a coordinated
  * test to the same site in the same moment from N-vantage points */
 function getTasks(req) {
 
-    /* cambiare a post ? */
-    var agentName = req.params.agentName;
-    var agentInfo = req.params.agentInfo || "amen";
-    var referenceTime = moment().subtract(1, 'd');
-
-    debug("%s %s (%s) asks getTasksId since %s",
-        req.randomUnicode, agentName,
-        agentInfo, referenceTime.toISOString() );
-
-    return mongo
-        .read(nconf.get('schema').subjects)
-        .then(prand.getPseudoRandomSample)
-        .map(subjectsOps.getSites)
-        .then(prand.getPseudoRandomSample)
-        .then(function(siteList) {
-            return {
-                json: siteList
-            };
-        });
-};
+    var vantagePoint = req.params.vantagePoint;
+    var amount = _.parseInt(req.params.amount);
 
-/* quello che ho è un una richiesta da parte di un VP,
- * ho dalla mia parte una lista di promesse giornaliere che devono essere assolte, quando vigile le generale, schedulate, le scrive su DB nella tabella needs */
+    debug("%s %s asks getTasks %d",
+        req.randomUnicode, vantagePoint, amount);
 
-    /*
-     * Avere la lista di test collezionati e fare una sottrazione, per
-     * far si che chopstick possa loopare su tutto
-    return mongo
-        .read(nconf.get('schema').promises, {
-            "$lt": new Date(moment().subtract(1, 'd')),
-            "agent": agentName
-        })
-        .then(function(lists) {
-            return {
-                json: subjectsOps.serializeLists(lists)
-            };
-        });
+    var selector = {
+        "start": { "$lt": new Date() },
+        "end": { "$gt": new Date() }
+    };
+    _.set(selector, vantagePoint, { "$exists": false });
 
     return mongo
-        .read(nconf.get('schema').subjects, {
-            'public': true
+        .readLimit(nconf.get('schema').promises, selector, {}, amount, 0)
+        .map(function(site) {
+            return _.omit(site, ['_id']);
         })
-        .then(function(lists) {
-            return {
-                json: subjectsOps.serializeLists(lists)
-            };
+        .then(function(siteList) {
+            return markVantagePoint(vantagePoint, siteList)
+                .return({
+                    json: siteList
+                });
         });
-    */
+};
 
 module.exports = getTasks;

test-2.1.js

@@ -0,0 +1,14 @@
+var page = require('webpage').create();
+console.log('The default user agent is ' + page.settings.userAgent);
+page.settings.userAgent = 'SpecialAgent';
+page.open('http://www.httpuseragent.org', function(status) {
+  if (status !== 'success') {
+    console.log('Unable to access network');
+  } else {
+    var ua = page.evaluate(function() {
+      return document.getElementById('myagent').textContent;
+    });
+    console.log(ua);
+  }
+  phantom.exit();
+});