forked from YorkMoon/proxypool
/
tgchannel.go
123 lines (111 loc) · 2.94 KB
/
tgchannel.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
package getter
import (
"fmt"
"io/ioutil"
"strings"
"sync"
"github.com/selastmomo/proxypool/log"
"github.com/selastmomo/proxypool/pkg/proxy"
"github.com/selastmomo/proxypool/pkg/tool"
"github.com/gocolly/colly"
)
func init() {
Register("tgchannel", NewTGChannelGetter)
}
type TGChannelGetter struct {
c *colly.Collector
NumNeeded int
results []string
Url string
apiUrl string
}
func NewTGChannelGetter(options tool.Options) (getter Getter, err error) {
num, found := options["num"]
t := 200
switch num := num.(type) {
case int:
t = num
case float64:
t = int(num)
}
if !found || t <= 0 {
t = 200
}
urlInterface, found := options["channel"]
if found {
url, err := AssertTypeStringNotNull(urlInterface)
if err != nil {
return nil, err
}
return &TGChannelGetter{
c: tool.GetColly(),
NumNeeded: t,
Url: "https://t.me/s/" + url,
apiUrl: "https://tg.i-c-a.su/rss/" + url,
}, nil
}
return nil, ErrorUrlNotFound
}
func (g *TGChannelGetter) Get() proxy.ProxyList {
result := make(proxy.ProxyList, 0)
g.results = make([]string, 0)
// 找到所有的文字消息
g.c.OnHTML("div.tgme_widget_message_text", func(e *colly.HTMLElement) {
g.results = append(g.results, GrepLinksFromString(e.Text)...)
// 抓取到http链接,有可能是订阅链接或其他链接,无论如何试一下
subUrls := urlRe.FindAllString(e.Text, -1)
for _, url := range subUrls {
result = append(result, (&Subscribe{Url: url}).Get()...)
}
})
// 找到之前消息页面的链接,加入访问队列
g.c.OnHTML("link[rel=prev]", func(e *colly.HTMLElement) {
if len(g.results) < g.NumNeeded {
_ = e.Request.Visit(e.Attr("href"))
}
})
g.results = make([]string, 0)
err := g.c.Visit(g.Url)
if err != nil {
_ = fmt.Errorf("%s", err.Error())
}
result = append(result, StringArray2ProxyArray(g.results)...)
// 获取文件(api需要维护)
resp, err := tool.GetHttpClient().Get(g.apiUrl)
if err != nil {
return result
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil
}
items := strings.Split(string(body), "\n")
for _, s := range items {
if strings.Contains(s, "enclosure url") { // get to xml node
elements := strings.Split(s, "\"")
for _, e := range elements {
if strings.Contains(e, "https://") {
// Webfuzz的可能性比较大,也有可能是订阅链接,为了不拖慢运行速度不写了
result = append(result, (&WebFuzz{Url: e}).Get()...)
}
}
}
}
return result
}
func (g *TGChannelGetter) Get2ChanWG(pc chan proxy.Proxy, wg *sync.WaitGroup) {
defer wg.Done()
nodes := g.Get()
log.Infoln("STATISTIC: TGChannel\tcount=%d\turl=%s\n", len(nodes), g.Url)
for _, node := range nodes {
pc <- node
}
}
func (g *TGChannelGetter) Get2Chan(pc chan proxy.Proxy) {
nodes := g.Get()
log.Infoln("STATISTIC: TGChannel\tcount=%d\turl=%s\n", len(nodes), g.Url)
for _, node := range nodes {
pc <- node
}
}