Browse files

switch to newer dbemitter and xml2js

  • Loading branch information...
1 parent f574129 commit 9aa8d3463c7f1c553e471289986f98c878f5a7a8 @maxogden committed Apr 7, 2011
Showing with 71 additions and 67 deletions.
  1. +6 −0 .gitmodules
  2. +1 −1 changes.js
  3. +18 −26 db-watcher.js
  4. +1 −0 dbemitter
  5. +43 −39 feed-archiver.js
  6. +1 −1 feed-watcher.js
  7. +1 −0 node-xml2js
View
6 .gitmodules
@@ -0,0 +1,6 @@
+[submodule "dbemitter"]
+ path = dbemitter
+ url = https://github.com/mikeal/dbemitter.git
+[submodule "node-xml2js"]
+ path = node-xml2js
+ url = https://maxogden@github.com/maxogden/node-xml2js.git
View
2 changes.js
@@ -14,4 +14,4 @@ exports.listener = function(doc) {
archiver.processFeed(doc.feed, feedSaveFunc);
}
-}
+}
View
44 db-watcher.js
@@ -1,35 +1,27 @@
-// adapted from https://github.com/szayat/node.couch.js
-
-var stdin = process.openStdin();
-var vm = require('vm');
-var sys = require('sys');
-
+var stdin = process.openStdin(),
+ sys = require('sys'),
+ dbemitter = require('./dbemitter/main'),
+ request = require('request'),
+ child = require('child_process'),
+ path = require('path'),
+ db = process.argv[2],
+ h = {'content-type':'application/json', 'accept':'application/json'},
+ emitter = dbemitter.createCouchDBEmitter(db),
+ children = {};
+
stdin.setEncoding('utf8');
-var buffer = ''
- , docs = {}
- , listenerDoc
- ;
-
var spawnFeedProcess = function( doc ) {
sys.puts( "Starting process for " + doc._id )
- var p = child.spawn( process.execPath, [ path.join(__dirname, 'feed-watcher.js') ] );
+ var p = child.spawn( process.execPath, [ path.join(__dirname, 'feed-archiver.js') ] );
p.stderr.on( "data", function ( chunk ) { sys.error( "data error: " + chunk.toString() ) } )
- p.stdin.write(JSON.stringify(["listenerDoc", listenerDoc])+'\n');
p.stdin.write(JSON.stringify(["doc", doc])+'\n');
- docs[ doc._id ]._feed_process = function() { return p };
+ children[ doc._id ] = {'feed_process': function() { return p }};
}
-stdin.on('data', function (chunk) {
- buffer += chunk.toString();
- while (buffer.indexOf('\n') !== -1) {
- line = buffer.slice(0, buffer.indexOf('\n'));
- buffer = buffer.slice(buffer.indexOf('\n') + 1);
- var obj = JSON.parse(line);
- if ((obj[1]) === "ddoc") {
- listenerDoc = obj[1];
- } else if (obj[0] === "doc") {
- spawnFeedProcess(obj[1]);
- }
+emitter.on('change', function (change) {
+ var doc = change.doc;
+ if (doc.feed && doc.db) {
+ spawnFeedProcess(doc);
}
-});
+});
1 dbemitter
@@ -0,0 +1 @@
+Subproject commit 2d93aba7d0de7486a85984b06833a9ced73167ee
View
82 feed-archiver.js
@@ -1,46 +1,26 @@
-#!/usr/bin/env node
+var fs = require('fs'),
+ sys = require('sys'),
+ xml2js = require('./node-xml2js/lib/xml2js'),
+ url = require('url'),
+ request = require('request'),
+ http = require('http'),
+ crypto = require('crypto'),
+ couchdb = require('./lib/node-couchdb-min/couchdb'),
+ stdin = process.openStdin(),
+ buffer = '',
+ debug = false;
-var fs = require('fs')
- , sys = require('sys')
- , xml2js = require('xml2js')
- , url = require('url')
- , http = require('http')
- , crypto = require('crypto')
- , parser = new xml2js.Parser();
-
-var debug = false;
-
-exports.processFeed = function(feedUrl, feedSaveFunc) {
-
+function processFeed(feedUrl, feedSaveFunc) {
var feed = url.parse(feedUrl);
-
- var feedClient = http.createClient(80, feed.host);
-
- var request = feedClient.request("GET", feed.pathname, {'host' : feed.host});
- request.end();
-
- request.on('response', function(response) {
- response.setEncoding('utf8');
-
- var data;
-
- response.on('data', function(chunk) {
- data += chunk;
- });
-
- response.addListener('end', function() {
- parser.parseString(data);
- })
-
- });
-
- parser.addListener('end', feedSaveFunc);
+ request({uri:feed.href, headers: {'host' : feed.host}}, function (error, resp, body) {
+ if (error) throw error;
+ var parser = new xml2js.Parser(feedSaveFunc);
+ parser.parseString(body);
+ })
}
-exports.saveItem = function(item, db, uniqueKey) {
-
+function saveItem(item, db, uniqueKey) {
var _id = crypto.createHash('md5').update(item[uniqueKey]).digest("hex");
-
db.get(_id, function(err, doc) {
if (err) {
if (err.couchDbError == "not_found") {
@@ -51,4 +31,28 @@ exports.saveItem = function(item, db, uniqueKey) {
}
}
});
-}
+}
+
+function fetchFeed (doc) {
+ var starttime = new Date();
+ function feedSaveFunc(data) {
+ for (var item in data.channel.item) {
+ saveItem(data.channel.item[item], new couchdb.Db(doc.db), "description");
+ }
+ }
+ processFeed(doc.feed, feedSaveFunc);
+ var endtime = new Date();
+ setTimeout(fetchFeed, doc.interval ? doc.interval : (((endtime - starttime) * 5) + 300000));
+}
+
+stdin.on('data', function (chunk) {
+ buffer += chunk.toString();
+ while (buffer.indexOf('\n') !== -1) {
+ line = buffer.slice(0, buffer.indexOf('\n'));
+ buffer = buffer.slice(buffer.indexOf('\n') + 1);
+ var obj = JSON.parse(line);
+ if (obj[0] === "doc") {
+ fetchFeed(obj[1]);
+ }
+ }
+});
View
2 feed-watcher.js
@@ -45,4 +45,4 @@ stdin.on('data', function (chunk) {
fetchFeed(obj[1]);
}
}
-});
+});
1 node-xml2js
@@ -0,0 +1 @@
+Subproject commit 241375cde0f6da7ae57f4a2b32b3831c281e9840

0 comments on commit 9aa8d34

Please sign in to comment.