Skip to content
This repository
Browse code

ALPHA web interface

  • Loading branch information...
commit 29addaac54b70a16e72a116235cba54acd167d6e 1 parent 8eb1e61
Chris O'Hara authored December 15, 2010
4  HISTORY.md
Source Rendered
... ...
@@ -1,3 +1,7 @@
  1
+### v0.2.1-1
  2
+    * Better support for multiple jobs running in the same process
  3
+    * ALPHA Web interface
  4
+
1 5
 ### v0.2.0-4
2 6
     * Bug fixes
3 7
     * Added -d (--daemon) switch
2  README.md
Source Rendered
@@ -10,6 +10,7 @@ node.io can streamline the process of:
10 10
 - Transferring data, e.g. CSV => a database
11 11
 - Distributing work across multiple processes, and multiple servers (soon)
12 12
 - Working with continuous data & streams
  13
+- Managing jobs through a web interface
13 14
 
14 15
 ## Why node.io?
15 16
 
@@ -27,6 +28,7 @@ node.io can streamline the process of:
27 28
 - Includes a robust framework for scraping, selecting and traversing data from the web + support for a variety of proxies
28 29
 - Includes a data validation and sanitization framework
29 30
 - Provides support for retries, timeouts, dynamically adding input, etc.
  31
+- Comes with a web interface for managing jobs
30 32
 
31 33
 ## Installation
32 34
 
200  bin/node.io-web
... ...
@@ -0,0 +1,200 @@
  1
+#!/usr/bin/env node
  2
+
  3
+var nodeio = require('node.io'),
  4
+    fs = require('fs'),
  5
+    http = require('http'),
  6
+    utils = nodeio.utils,
  7
+    args = process.argv.slice(2),
  8
+    querystring = require('querystring');
  9
+
  10
+var exit = function (msg, is_error) {
  11
+    utils.status[is_error ? 'error' : 'info'](msg);
  12
+    process.exit(1);
  13
+};
  14
+
  15
+var usage = ''
  16
+  + '\x1b[1mUsage\x1b[0m: node.io-web [OPTIONS] <JOB_DIR>\n'
  17
+  + '\n'
  18
+  + 'Note: <JOB_DIR> defaults to ~/.node_modules\n'
  19
+  + '\n'
  20
+  + '\x1b[1mOptions\x1b[0m:\n'
  21
+  + '  -p, --port [PORT]      Port to listen on. Default is 8080\n'
  22
+  + '  -d, --daemon           Daemonize the process\n'
  23
+  + '  -v, --version          Display the current version\n'
  24
+  + '  -h, --help             Display help information\n\n'
  25
+  + '\x1b[1mUsage 1\x1b[0m:\n    1. Visit http://localhost:<PORT>/ for a web interface\n\n'
  26
+  + '\x1b[1mUsage 2\x1b[0m: \n    1. `/jobs` has a JSON representation of available jobs\n'
  27
+  + '    2. `/run?job=<JOB>&input=<INPUT>` where input is \\r\\n separated'
  28
+  ;
  29
+  
  30
+var port = 8080, daemonize = false, daemon_arg;
  31
+
  32
+var module_dir;
  33
+for (var i = 0, l = require.paths.length; i < l; i++) {
  34
+    if (require.paths[i].indexOf('.node_modules') >= 0) {
  35
+        module_dir = require.paths[i];
  36
+        break;
  37
+    }
  38
+}
  39
+
  40
+while (args.length) {
  41
+    arg = args.shift();
  42
+    switch (arg) {
  43
+    case '-p':
  44
+    case '--port':
  45
+        port = args.shift();
  46
+        break;
  47
+    case '-h':
  48
+    case '--help':
  49
+        exit(usage);
  50
+        break;
  51
+    case '-v':
  52
+    case '--version':
  53
+        exit('v' + require('node.io').version);
  54
+        break;
  55
+    case '-d':
  56
+    case '--daemon':
  57
+        if (args.length && args[0][0] !== '-') {
  58
+            daemon_arg = args.shift();
  59
+        }
  60
+        daemonize = true;
  61
+        break;
  62
+    default:
  63
+        module_dir = arg;
  64
+        break;
  65
+    }
  66
+}
  67
+
  68
+//Check that module_dir exists and has some jobs in it
  69
+try {
  70
+    var files = fs.readdirSync(module_dir);
  71
+    if (files.length === 0) throw new Error();
  72
+} catch (e) {
  73
+    exit('No jobs found in ' + module_dir, true);
  74
+}
  75
+
  76
+var start_job = function (params, res) {
  77
+    if (typeof params.job === 'undefined') {
  78
+        res.writeHead(400);
  79
+        res.end();
  80
+        return;
  81
+    }
  82
+    
  83
+    if (params.job.indexOf('../') >= 0) {
  84
+        res.writeHead(403);
  85
+        res.end();
  86
+        return;
  87
+    }
  88
+    
  89
+    console.log('Running job "' + params.job + '"');
  90
+    
  91
+    var methods = {
  92
+        output: function (lines) {
  93
+            lines.forEach(function (line) {
  94
+                res.write(line + '<br />');
  95
+            });
  96
+        }
  97
+    };
  98
+    
  99
+    if (params.input && params.input != '') {
  100
+        methods.input = params.input.split(params.sep || '\r\n');
  101
+    }
  102
+    
  103
+    //Remove the job from the require cache (basic hot-reloading)
  104
+    require.cache[module_dir + '/' + params.job] = null;
  105
+    
  106
+    job = require(module_dir + '/' + params.job).job.extend({}, methods);
  107
+    nodeio.start(job, {silent: true}, function (err) {
  108
+        res.end();
  109
+    });
  110
+};
  111
+
  112
+var interface_a = ''
  113
++ '<!DOCTYPE html><head><title>node.io</title></head><body>'
  114
++ '<script type="text/javascript" src="http://www.google.com/jsapi"></script><script type="text/javascript">google.load("jquery", "1.4.4");</script><style type="text/css">'
  115
++ '.radio, .checkbox{vertical-align:middle;margin:0px;padding:0px;}.radio-item,.checkbox-item{margin-top:5px;float:left;}.radio-item label,.checkbox-item label{margin-left:5px;}.radio-item br,.checkbox-item br{clear:left;}'
  116
++ '.submit-button,.submit-reset,.submit-print{margin:0px;overflow:visible;padding:1px 6px;width:auto;}.submit-button::-moz-focus-inner,.submit-reset::-moz-focus-inner{border:0px;padding:1px 6px;}.header{margin:0px;}'
  117
++ '.header-group{background:#f5f5f5;border-bottom:1px solid #ccc;padding:12px;clear:both;}.header-group-b{background:#f5f5f5;border-bottom:1px solid #ccc;padding:7px 12px;clear:both;}.label{width: 150px;margin-bottom:6px;'
  118
++ 'display:inline-block;width:137px !important;}.label-left{float:left;display:inline-block;text-align:left;padding:3px;width:137px !important;}.label-right{float:left;display:inline-block;text-align:right;margin-right:6px;'
  119
++ 'margin-bottom:6px;width:137px !important;padding:3px;}.section,.section-closed{list-style:none;list-style-position:outside;margin:0px;padding:0px;position:relative;zoom:1;}.input{display:inline-block;}.line{clear:both;'
  120
++ 'padding:10px;margin:0px;display:inline-block;width:97%;width:-moz-available;position:relative;}.single-column .clearfix{display:inline-block;}.single-column .clearfix{display:block;}.all{list-style:none;'
  121
++ 'list-style-position:outside;margin:0px;width:650px;color:#000000 !important;font-family:Arial;font-size:12px;}.advanced{display: none;}'
  122
++ '</style><form action="/run" method="post"><div class="all"><ul class="section"><li class="input-wide"><div class="header-group"><h2 class="header">Node.io</h2></div></li>'
  123
++ '<li class="line"><label class="label-left">Job</label><div class="input">'
  124
+;
  125
+
  126
+var interface_b = ''
  127
++ '<span class="clearfix"></span></div></li>'
  128
++ '<li class="line"><label class="label-left">Arguments:</label><div class="input"><input type="text" class="textbox" name="args" size="64" /></div></li>'
  129
++ '<li class="line"><label class="label-left">Input</label><div class="input"><textarea class="textarea" name="input" cols="50" rows="8"></textarea></div></li>'
  130
++ '<li class="line"><label class="label-left">Advanced Options</label><div class="input"><input type="checkbox" onclick="$(\'.advanced\').toggle()" /></div></li>'
  131
++ '<li class="line advanced"><label class="label-left">Timeout: (s)</label><div class="input"><input type="text" class="textbox" name="timeout" size="20" /></div></li>'
  132
++ '<li class="line advanced"><label class="label-left">Debug</label><div class="input"><div class="single-column"><span class="checkbox-item" style="clear:left;"><input type="checkbox" class="checkbox" name="debug" value="1" /></div></div></li>'
  133
++ '<li class="line"><div class="input-wide"><div style="margin-left:143px" class="buttons-wrapper"><button type="submit" class="submit-button">Run</button></div></div></li></ul></div></form>'
  134
+;
  135
+
  136
+var handle = function (req, res) {
  137
+    if (req.url === '/') {
  138
+        res.writeHead(200, {'Content-Type': 'text/html'});
  139
+        utils.getFiles(module_dir, function(files) {
  140
+            res.write(interface_a);
  141
+            files.sort();
  142
+            var file_name;
  143
+            files.forEach(function (file) {
  144
+                //Make the filename pretty... my_job.js => My job
  145
+                if (file.substr(file.length - 3) === '.js') {
  146
+                    file_name = file.substr(0, file.length - 3);
  147
+                } else {
  148
+                    file_name = file;
  149
+                }
  150
+                file_name = file_name.replace('_', ' ');
  151
+                file_name = file_name.substr(0, 1).toUpperCase() + file_name.substr(1);
  152
+                //Output the job radio button
  153
+                res.write('<span class="radio-item" style="clear:left;margin-left:4px;"><input type="radio" class="radio" name="job" value="' + file + '" /><label>' + file_name + '</label></span>');
  154
+            })
  155
+            res.end(interface_b);
  156
+        });
  157
+    } else if (req.url === '/jobs') {
  158
+        res.writeHead(200, {'Content-Type': 'text/html'});
  159
+        utils.getFiles(module_dir, function(files) {
  160
+            res.end(JSON.stringify(files));
  161
+        });
  162
+    } else if (req.url.indexOf('/run') === 0) {
  163
+        res.writeHead(200, {'Content-Type': 'text/html'});
  164
+        var params;
  165
+        if (req.method.toUpperCase() === 'GET') {
  166
+            params = querystring.parse(req.url.substr(5));
  167
+            start_job(params, res);
  168
+        } else {
  169
+            var data = '';
  170
+            req.setEncoding('utf8');
  171
+            req.addListener('data', function(chunk) { data += chunk; });
  172
+            req.addListener('end', function() {
  173
+                params = querystring.parse(data);
  174
+                start_job(params, res);
  175
+            });
  176
+        }
  177
+    } else {
  178
+        res.writeHead(404);
  179
+        res.end();
  180
+    }
  181
+};
  182
+
  183
+var start_server = function () {
  184
+    http.createServer(handle).listen(port);
  185
+    console.log('\x1B[33mINFO\x1B[0m: To daemonize the process, use the -d switch');
  186
+    console.log('\x1B[33mINFO\x1B[0m: Listening on port ' + port + '\n');
  187
+    
  188
+    console.log('\x1b[1mUsage 1\x1b[0m:\n    1. Visit http://localhost:' + port + '/ for a web interface\n');
  189
+    console.log('\x1b[1mUsage 2\x1b[0m: \n    1. `/jobs` has a JSON representation of available jobs');
  190
+    console.log('    2. `/run?job=<JOB>&input=<INPUT>` where input is \\r\\n separated\n');
  191
+    
  192
+};
  193
+
  194
+if (daemonize) {
  195
+    utils.daemonize(daemon_arg, function () {
  196
+        start_server();
  197
+    });
  198
+} else {
  199
+    start_server();
  200
+}
2  lib/node.io/index.js
@@ -9,7 +9,7 @@ var processor = require('./processor'),
9 9
     job = require('./job');
10 10
 
11 11
 exports = module.exports = {
12  
-    version: '0.2.0-4',
  12
+    version: '0.2.1-1',
13 13
     Processor: processor.Processor,
14 14
     JobProto: job.JobProto, //A reference to the underlying Job.prototype
15 15
     JobClass: job.JobClass, //A reference to a new prototype identical to Job.prototype (so Job.prototype isn't modified)
2  package.json
... ...
@@ -1,6 +1,6 @@
1 1
 { "name"          : "node.io",
2 2
   "description"   : "A distributed data scraping and processing framework for node.js",
3  
-  "version"       : "0.2.0-4",
  3
+  "version"       : "0.2.1-1",
4 4
   "homepage"      : "http://github.com/chriso/node.io",
5 5
   "keywords"      : ["data","mapreduce","map","reduce","scraping","html","parsing","parse","scrape","process","processing","data"],
6 6
   "author"        : "Chris O'Hara <cohara87@gmail.com>",

0 notes on commit 29addaa

Please sign in to comment.
Something went wrong with that request. Please try again.