/
iconv.js
90 lines (78 loc) · 2.71 KB
/
iconv.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
/**
* Tips
* ====
* - Set `user-agent` and `accept` headers when sending requests. Some services will not respond as expected without them.
* - Set `pool` to false if you send lots of requests using "request" library.
*/
var request = require('request')
, FeedParser = require(__dirname+'/..')
, Iconv = require('iconv').Iconv;
function fetch(feed) {
// Define our streams
var req = request(feed, {timeout: 10000, pool: false});
req.setMaxListeners(50);
// Some feeds do not respond without user-agent and accept headers.
req.setHeader('user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36')
.setHeader('accept', 'text/html,application/xhtml+xml');
var feedparser = new FeedParser();
// Define our handlers
req.on('error', done);
req.on('response', function(res) {
var stream = this
, iconv
, charset;
if (res.statusCode != 200) return this.emit('error', new Error('Bad status code'));
charset = getParams(res.headers['content-type'] || '').charset;
// Use iconv if its not utf8 already.
if (!iconv && charset && !/utf-*8/i.test(charset)) {
try {
iconv = new Iconv(charset, 'utf-8');
console.log('Converting from charset %s to utf-8', charset);
iconv.on('error', done);
// If we're using iconv, stream will be the output of iconv
// otherwise it will remain the output of request
stream = this.pipe(iconv);
} catch(err) {
this.emit('error', err);
}
}
// And boom goes the dynamite
stream.pipe(feedparser);
});
feedparser.on('error', done);
feedparser.on('end', done);
feedparser.on('readable', function() {
var post;
while (post = this.read()) {
console.log(post);
}
});
}
function getParams(str) {
var params = str.split(';').reduce(function (params, param) {
var parts = param.split('=').map(function (part) { return part.trim(); });
if (parts.length === 2) {
params[parts[0]] = parts[1];
}
return params;
}, {});
return params;
}
function done(err) {
if (err) {
console.log(err, err.stack);
return process.exit(1);
}
server.close();
process.exit();
}
// Don't worry about this. It's just a localhost file server so you can be
// certain the "remote" feed is available when you run this example.
var server = require('http').createServer(function (req, res) {
var stream = require('fs').createReadStream(require('path').resolve(__dirname, '../test/feeds' + req.url));
res.setHeader('Content-Type', 'text/xml; charset=Windows-1251');
stream.pipe(res);
});
server.listen(0, function () {
fetch('http://localhost:' + this.address().port + '/iconv.xml');
});