-
Notifications
You must be signed in to change notification settings - Fork 0
/
index.js
40 lines (34 loc) · 1.04 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
'use strict';
var request = require("request-promise");
var fs = require("fs");
var cheerio = require("cheerio");
console.log("runned");
var baseUrl = 'http://www.brejas.com.br';
var page = 1;
var pageMax = 10;
var itemsPerPage = 10;
var brejas = [];
console.time("crawler time");
while(true) {
if(page > pageMax) break;
page++;
request(baseUrl + "/cerveja?page=" + page).then(function(html) {
var $ = cheerio.load(html);
var items = $(".jrRow").not(".jrDataListHeader");
items.each(function() {
var breja = {};
breja.nome = $(this).find(".jrContentTitle a").text();
breja.url = baseUrl + $(this).find(".jrContentTitle a").attr("href");
request(breja.url).then(function(html) {
var $ = cheerio.load(html);
breja.img = $(".jrListingMainImage img").attr("src");
brejas.push(breja);
console.log(brejas.length);
if(brejas.length === pageMax*itemsPerPage) {
console.timeEnd("crawler time");
fs.writeFile('output.json', JSON.stringify(brejas, null, 4), function(err){});
}
});
});
});
}