/
proxy.go
46 lines (40 loc) · 1.13 KB
/
proxy.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
/*
* @Author: thepoy
* @Email: thepoy@163.com
* @File Name: proxy.go
* @Created: 2021-07-27 12:15:35
* @Modified: 2022-02-11 09:30:53
*/
package predator
import (
"time"
"github.com/go-predator/predator/log"
"github.com/go-predator/predator/proxy"
"github.com/valyala/fasthttp"
)
// 可以从一些代理网站的 api 中请求指定数量的代理 ip
type AcquireProxies func(n int) []string
func (c *Crawler) ProxyDialerWithTimeout(proxyAddr string, timeout time.Duration) fasthttp.DialFunc {
c.lock.Lock()
c.proxyInUse = proxyAddr
c.lock.Unlock()
if proxyAddr[:7] == "http://" || proxyAddr[:8] == "https://" {
return proxy.HttpProxyDialerWithTimeout(proxyAddr, timeout)
} else if proxyAddr[:9] == "socks5://" {
return proxy.Socks5ProxyDialer(proxyAddr)
} else {
err := proxy.ProxyErr{
Code: proxy.ErrUnknownProtocolCode,
Args: map[string]string{
"proxy_addr": proxyAddr,
},
Msg: "only support http and socks5 protocol, but the incoming proxy address uses an unknown protocol",
}
if c.log != nil {
c.Fatal(err, log.Arg{Key: "proxy", Value: proxyAddr})
} else {
panic(err)
}
return nil
}
}