forked from componentjs/search-api
/
fetch.js
107 lines (90 loc) · 2.49 KB
/
fetch.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
/**
* Module dependencies.
*/
var wiki = require('component-wiki')
, request = require('superagent')
, redis = require('redis')
, db = redis.createClient()
, fs = require('fs');
// NOTE: quick / horrid code lives here ;D
var pending = 0;
var packages;
fetch();
var auth = fs.readFileSync(process.env.HOME + '/.component-search-auth', 'ascii');
auth = new Buffer(auth.trim()).toString('base64');
function fetch() {
var batch = wiki(function(err, pkgs){
if (err) throw err;
packages = pkgs;
pkgs.forEach(function(pkg){
if (!pkg) return;
console.log('%s', pkg.name);
var words = [pkg.name];
if (!pkg.description) console.log('"description" missing for %s', pkg.name);
words = words.concat(parse(pkg.description));
words = words.concat(pkg.keywords || []);
pkg.dependents = dependents(pkg, pkgs)
pkg.stars = 0;
++pending;
request
.get('https://api.github.com/repos/' + pkg.repo)
.set('Authorization', 'Basic ' + auth)
.end(function(res){
done();
if (res.ok) {
pkg.stars = res.body.watchers_count;
console.log('%s stars: %d', pkg.repo, pkg.stars);
} else {
console.log('github: %s %s', res.status, res.text);
}
++pending;
db.set('component:' + pkg.repo, JSON.stringify(pkg), done);
})
++pending;
db.sadd('components', pkg.repo, done);
words.forEach(function(word){
++pending;
db.sadd('word:' + word, pkg.repo, done);
});
});
});
batch.on('error', function(err){
if (err.json) {
console.error('invalid json: %s', err.url);
} else {
console.error(err.stack);
}
});
}
/**
* List packages that depend on the supplied package, `pkg`.
*
* @param pkg
* @param pkgs {Array} All packages to analyse.
* @return {Array} Repos of packages that depend on `pkg`.
*/
function dependents(pkg, pkgs) {
var repo = pkg.repo
return pkgs.filter(blank).filter(function(pkg) {
pkg.dependencies = pkg.dependencies || []
return Object.keys(pkg.dependencies).indexOf(repo) !== -1
}).map(function(pkg) {
return pkg.repo
})
}
function blank(pkg) {
return !!pkg
}
function parse(str) {
str = String(str).trim();
if (!str) return [];
return str.match(/\w+/).map(function(word){
return word.toLowerCase();
});
}
function done() {
--pending || (function(){
fs.writeFileSync(__dirname + '/components.json', JSON.stringify(packages));
process.exit();
})();
}