-
Notifications
You must be signed in to change notification settings - Fork 2
/
verify.go
50 lines (43 loc) · 1.17 KB
/
verify.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
package crawler
import (
"net/http"
"time"
"github.com/naiba/proxyinabox"
"github.com/naiba/proxyinabox/service"
"github.com/parnurzeal/gorequest"
)
var verifyJob chan proxyinabox.Proxy
var proxyServiceInstance proxyinabox.ProxyService
//Init crawler
func Init() {
initV()
initC()
}
func initV() {
proxyServiceInstance = &service.ProxyService{DB: proxyinabox.DB}
verifyJob = make(chan proxyinabox.Proxy, proxyinabox.Config.Sys.ProxyVerifyWorker)
for i := 0; i < proxyinabox.Config.Sys.ProxyVerifyWorker; i++ {
go getDelay(verifyJob)
}
}
//Verify verify proxies in database
func Verify() {
list, _ := proxyServiceInstance.GetUnVerified()
for _, p := range list {
verifyJob <- p
}
}
func getDelay(pc chan proxyinabox.Proxy) {
for p := range pc {
proxy := p.URI()
start := time.Now().Unix()
var resp validateJSON
_, _, errs := gorequest.New().Timeout(time.Second*5).Retry(3, time.Second*2, http.StatusInternalServerError).Proxy(proxy).Get("http://api.ip.la/cn?json").EndStruct(&resp)
delay := time.Now().Unix() - start
if len(errs) != 0 || resp.IP != p.IP {
proxyinabox.CI.DeleteProxy(p)
continue
}
proxyinabox.DB.Model(&p).Update("delay", delay)
}
}