-
Notifications
You must be signed in to change notification settings - Fork 0
/
scraper.js
58 lines (46 loc) · 1.27 KB
/
scraper.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
// From https://github.com/ariya/phantomjs/issues/10389
var page;
var timer;
var system = require('system');
var myurl = system.args[1];
var renderPage = function (url) {
url = url.trim();
page = require('webpage').create();
page.settings.resourceTimeout = 10000;
clearTimeout(timer)
timer = setTimeout(function() { endProcess(); }, 5000);
page.onNavigationRequested = function(url, type, willNavigate, main) {
if (main && url != myurl) {
myurl = url;
console.log(myurl)
page.close()
//setTimeout('renderPage(myurl)',1); // recurse
setTimeout(function() { renderPage(myurl); }, 1 ) // recurse
}
};
page.open(url, function(status) {
if (status === 'success') {
page.close();
phantom.exit(0);
} else {
page.close();
phantom.exit(1);
}
});
page.onError = function(msg, trace) {
page.close();
phantom.exit(1);
}
page.onResourceTimeout = function(request) {
console.log('Response (#' + request.id + '): ' + JSON.stringify(request));
//phantom.exit(1);
}
}
function endProcess() {
clearTimeout(globalTimer);
page.close();
phantom.exit(1);
}
// Set a timer before we start, just to be sure
var globalTimer = setTimeout(function() { endProcess(); }, 6000);
renderPage(myurl);