/
index.js
104 lines (104 loc) · 2.77 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
const request = require('request')
const cheerio = require('cheerio')
const moment = require('moment')
const serverUrl = 'https://mxavtj-3000-gchfab.studio.ide.live/request'
let interval = 1
let requestTime = moment()
let poolSize = 1
let currentPoolSize = 0
let pool = []
function setOptions(options) {
if (options && options.interval) {
interval = options.interval
}
if (options && options.poolSize) {
poolSize = options.poolSize
}
}
function queue() {
return new Promise((resolve, reject) => {
const now = moment()
if (now.isBefore(requestTime)) {
const milliseconds = requestTime.valueOf() - now.valueOf()
setTimeout(() => {
resolve()
}, milliseconds)
} else {
requestTime = now
resolve()
}
requestTime.add(interval, 's')
})
}
function inPool(cb) {
pool.push(cb)
outPool()
}
function outPool() {
if (pool.length > 0 && currentPoolSize > -1 && currentPoolSize < poolSize) {
let cb = pool.shift()
currentPoolSize += 1
// console.log('outPool')
// console.log('currentPoolSize', currentPoolSize)
// console.log('time', moment().format('HH:mm:ss'))
queue().then(() => {
cb()
},() => {
cb()
})
}
}
function requestPage(options) {
let p = new Promise((resolve, reject) => {
inPool(() => {
// console.log('requet', moment().format('HH:mm:ss'))
request
.post(serverUrl, { form: { url: options.url } }, (err, response, body) => {
if (err) {
// console.log('err', err)
reject(err)
} else {
let $
// console.log('body', body)
if (body.indexOf('<title>504 Gateway Time-out</title>')>-1) {
/*
<html>
<head><title>504 Gateway Time-out</title></head>
<body bgcolor="white">
<center><h1>504 Gateway Time-out</h1></center>
<hr><center>openresty/1.11.2.2</center>
</body>
</html>
*/
reject({code: 504, message: 'time-out'})
} else if (body.indexOf('<title>502 Bad Gateway</title>')>-1) {
reject({code: 502, message: 'bad gateway'})
}
else {
body = JSON.parse(body)
// console.log('content', body.content)
$ = cheerio.load(body.content)
resolve($)
}
}
})
})
})
p.then(() => {
currentPoolSize -= 1
// console.log('done')
// console.log('currentPoolSize', currentPoolSize)
// console.log('time', moment().format('HH:mm:ss'))
outPool()
}, () => {
currentPoolSize -= 1
outPool()
})
return p
}
const obj = {
queue: queue,
request: requestPage,
setOptions: setOptions
}
module.exports = obj;