-
Notifications
You must be signed in to change notification settings - Fork 0
/
index.js
115 lines (78 loc) · 2.43 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
(function() {
var pageCount2015 = 3;
var all_entries = [];
var all_titles = [];
var counter=0;
for (var mainIndex = 1; mainIndex <= pageCount2015; mainIndex++){
console.log("page: " + mainIndex);
var request = require("request"),
cheerio = require("cheerio"),
url = "http://whopays.scratchmag.net/2015/10/page/" + mainIndex;
console.log(url);
request(url, function (error, response, body) {
//console.log(i);
counter++;
if (!error) {
// load the body of the page into Cheerio so we can traverse the DOM
var $ = cheerio.load(body);
//global vars
var titles = $("[rel='bookmark']");
var entry_contents = $(".entry-content");
createEntries();
function createEntries(){
titles.each(function (i, title) {
all_titles.push($(title).html());
}); //titles.each
//console.log(entry_contents.length);
entry_contents.each(function (i, entry) {
//all_entries.push($(entry).html());
all_entries.push({
title: all_titles[i],
range: getTheRange(entry),
price: getThePrice(entry)
});
});
//console.log("title: " + $(title).html() + ", i: " + i);
if (counter==pageCount2015){
var json2csv = require('json2csv');
var fields = ['title', 'range', 'price'];
json2csv({ data: all_entries, fields: fields }, function(err, csv) {
if (err) console.log(err);
console.log(csv);
});
// all_entries.forEach(function (entry, i){
// console.log("key: " + entry.key);
// console.log("title: " + entry.title);
// console.log("price: " + entry.price);
// console.log("range: " + entry.range);
// });
}
}
function getTheRange(entry){
entry = $(entry).html();
var thisRange = entry.match(/\d,?\d*- to \d,?\d*/);
if(thisRange !== null){
return thisRange[0];
} else {
//console.log("manually enter the range");
return "0 to 0-";
}
}
function getThePrice(entry){
entry = $(entry).html();
var thisPrice = entry.match(/\$\d,?\d*( per word)?/);
if(thisPrice !== null){
return thisPrice[0];
} else {
//console.log("manually enter the price");
return "$0";
}
}
} else {
console.log("We’ve encountered an error: " + error);
console.log(i);
}
});
}
//console.log("all entries: " + all_entries);
})();