/
web_fuzz_sub.go
103 lines (89 loc) · 3.38 KB
/
web_fuzz_sub.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
package getter
import (
"github.com/jth445600/proxypool/log"
"io/ioutil"
"regexp"
"sync"
"github.com/jth445600/proxypool/pkg/proxy"
"github.com/jth445600/proxypool/pkg/tool"
)
func init() {
Register("webfuzzsub", NewWebFuzzSubGetter)
}
type WebFuzzSub struct {
Url string
}
func (w *WebFuzzSub) Get() proxy.ProxyList {
resp, err := tool.GetHttpClient().Get(w.Url)
if err != nil {
return nil
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil
}
text := string(body)
subUrls := urlRe.FindAllString(text, -1)
result := make(proxy.ProxyList, 0)
for _, url := range subUrls {
newResult := (&Subscribe{Url: url}).Get()
if len(newResult) == 0 {
newResult = (&Clash{Url: url}).Get()
}
result = result.UniqAppendProxyList(newResult)
}
return result
}
func (w *WebFuzzSub) Get2ChanWG(pc chan proxy.Proxy, wg *sync.WaitGroup) {
defer wg.Done()
nodes := w.Get()
log.Infoln("STATISTIC: WebFuzzSub\tcount=%d\turl=%s\n", len(nodes), w.Url)
for _, node := range nodes {
pc <- node
}
}
func (w *WebFuzzSub) Get2Chan(pc chan proxy.Proxy) {
nodes := w.Get()
log.Infoln("STATISTIC: WebFuzzSub\tcount=%d\turl=%s\n", len(nodes), w.Url)
for _, node := range nodes {
pc <- node
}
}
func NewWebFuzzSubGetter(options tool.Options) (getter Getter, err error) {
urlInterface, found := options["url"]
if found {
url, err := AssertTypeStringNotNull(urlInterface)
if err != nil {
return nil, err
}
return &WebFuzzSub{Url: url}, nil
}
return nil, ErrorUrlNotFound
}
var urlRe = regexp.MustCompile(urlPattern)
const (
// 匹配 IP4
ip4Pattern = `((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)`
// 匹配 IP6，参考以下网页内容：
// http://blog.csdn.net/jiangfeng08/article/details/7642018
ip6Pattern = `(([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|` +
`(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|` +
`(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|` +
`(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|` +
`(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|` +
`(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|` +
`(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|` +
`(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))`
// 同时匹配 IP4 和 IP6
ipPattern = "(" + ip4Pattern + ")|(" + ip6Pattern + ")"
// 匹配域名
domainPattern = `[a-zA-Z0-9][a-zA-Z0-9_-]{0,62}(\.[a-zA-Z0-9][a-zA-Z0-9_-]{0,62})*(\.[a-zA-Z][a-zA-Z0-9]{0,10}){1}`
// 匹配 URL
urlPattern = `((https|http)?://)?` + // 协议
`(([0-9a-zA-Z]+:)?[0-9a-zA-Z_-]+@)?` + // pwd:user@
"(" + ipPattern + "|(" + domainPattern + "))" + // IP 或域名
`(:\d{1,5})?` + // 端口
`(/+[a-zA-Z0-9][a-zA-Z0-9_.-]*)*/*` + // path
`(\?([a-zA-Z0-9_-]+(=.*&?)*)*)*` // query
)