-
Notifications
You must be signed in to change notification settings - Fork 0
/
scrapper.js
119 lines (114 loc) · 5.95 KB
/
scrapper.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
module.exports = {
scrapper : (query, req, response) => {
var Scraper = require ('images-scraper'),
scraper = require('./scraper_logic'),
google = new Scraper.Google(),
fs = require('fs'),
Jimp = require("jimp");
// request = require('request'),
// rp = require('request-promise'),
// sharp = require('sharp');
// var transformer = sharp()
// .resize(300)
// .toColourspace('b-w');
// var download = function(uri, filename, callback){
// request.head(uri, function(err, res, body){
// request(uri)
// .on('response', (response) => {
// if(response.statusCode != 200) {
// return console.log('image fetch error!');
// }
// })
// .on('error', function(err) {
// return console.log(err)
// })
// // .pipe(transformer)
// .pipe(fs.createWriteStream(filename))
// .on('close', callback);
// });
// };
// var download = function(image){
// var ext = image.type.substring(6);
// console.log(`${image.url}\n`);
// var dir = `./public/images/${query}/`;
// var path = `${dir}${Math.floor((Math.random() * 10000) + 1).toString()}.${ext}`;
// if (!fs.existsSync(dir)){
// fs.mkdirSync(dir);
// }
// return new Promise((resolve, reject) => {
// request.head(image.url, function(err, res, body){
// if(err) {
// console.log('Request error!');
// reject();
// }
// var imageStream = request(image.url)
// .on('response', (response) => {
// if(response.statusCode != 200) {
// console.log('image fetch error!');
// reject();
// }
// })
// .on('error', function(err) {
// console.log(err)
// reject();
// })
// // .pipe(transformer)
// .pipe(fs.createWriteStream(path))
// .on('close', () =>{
// console.log('done: '+path);
// resolve();
// });
// });
// });
// };
var download = function(image) {
var ext = image.type.substring(6);
console.log(`${image.url}\n`);
var dir = `./public/images/${query}/`;
var path = `${dir}${Math.floor((Math.random() * 10000) + 1).toString()}.${ext}`;
if (!fs.existsSync('./public/images/')){
fs.mkdirSync('./public/images/');
}
if (!fs.existsSync(dir)){
fs.mkdirSync(dir);
}
return new Promise((resolve, reject) => {
Jimp.read(image.url).then(function (img) {
img.quality(60)
.greyscale()
.write(path);
resolve('final image saved');
}).catch(function (err) {
console.log('Image download failed');
reject('error final image not saved');
});
});
}
scraper.fetch(
{ q: query, tbm: 'isch' }
).then((res) => {
console.log('first 15 results from google', res);
var ac = res.map(download);
var final = Promise.all(ac);
final.then((msg) => {
console.log(msg);
return response.status(200).json({
message: 'Scrapping and image processing success',
obj: null
});
}, (msg) => {
console.log(msg);
});
response.status(200).json({
message: 'Scrapping and image processing success',
obj: null
});
}).catch((err) => {
console.log('err', err);
response.status(500).json({
title: 'An error occurred while scrapping',
error: err
})
})
}
}