-
-
Notifications
You must be signed in to change notification settings - Fork 327
/
website.go
250 lines (208 loc) · 6.47 KB
/
website.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
package updater
import (
"context"
"errors"
"fmt"
"net/http"
"strings"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/provider/common"
htmlutils "github.com/qdm12/gluetun/internal/updater/html"
"golang.org/x/net/html"
)
func fetchServers(ctx context.Context, client *http.Client,
warner common.Warner) (servers []models.Server, err error) {
const url = "https://www.vpnsecure.me/vpn-locations/"
rootNode, err := htmlutils.Fetch(ctx, client, url)
if err != nil {
return nil, fmt.Errorf("fetching HTML code: %w", err)
}
servers, warnings, err := parseHTML(rootNode)
for _, warning := range warnings {
warner.Warn(warning)
}
if err != nil {
return nil, fmt.Errorf("parsing HTML code: %w", err)
}
return servers, nil
}
var (
ErrHTMLServersDivNotFound = errors.New("HTML servers container div not found")
)
const divString = "div"
func parseHTML(rootNode *html.Node) (servers []models.Server,
warnings []string, err error) {
// Find div container for all servers, searching with BFS.
serversDiv := findServersDiv(rootNode)
if serversDiv == nil {
return nil, nil, htmlutils.WrapError(ErrHTMLServersDivNotFound, rootNode)
}
for countryNode := serversDiv.FirstChild; countryNode != nil; countryNode = countryNode.NextSibling {
if countryNode.Data != divString {
// empty line(s) and tab(s)
continue
}
country := findCountry(countryNode)
if country == "" {
warnings = append(warnings, htmlutils.WrapWarning("country not found", countryNode))
continue
}
grid := htmlutils.BFS(countryNode, matchGridDiv)
if grid == nil {
warnings = append(warnings, htmlutils.WrapWarning("grid div not found", countryNode))
continue
}
gridItems := htmlutils.DirectChildren(grid, matchGridItem)
if len(gridItems) == 0 {
warnings = append(warnings, htmlutils.WrapWarning("no grid item found", grid))
continue
}
for _, gridItem := range gridItems {
server, warning := parseHTMLGridItem(gridItem)
if warning != "" {
warnings = append(warnings, warning)
continue
}
server.Country = country
servers = append(servers, server)
}
}
return servers, warnings, nil
}
func parseHTMLGridItem(gridItem *html.Node) (
server models.Server, warning string) {
gridItemDT := htmlutils.DirectChild(gridItem, matchDT)
if gridItemDT == nil {
return server, htmlutils.WrapWarning("grid item <dt> not found", gridItem)
}
host := findHost(gridItemDT)
host = naToEmpty(host)
if host == "" {
return server, htmlutils.WrapWarning("host not found", gridItemDT)
}
status := findStatus(gridItemDT)
if !strings.EqualFold(status, "up") {
warning := fmt.Sprintf("skipping server with host %s which has status %q", host, status)
warning = htmlutils.WrapWarning(warning, gridItemDT)
return server, warning
}
gridItemDD := htmlutils.DirectChild(gridItem, matchDD)
if gridItemDD == nil {
return server, htmlutils.WrapWarning("grid item dd not found", gridItem)
}
region := findSpanStrong(gridItemDD, "Region:")
region = naToEmpty(region)
if region == "" {
warning := fmt.Sprintf("region for host %s not found", host)
return server, htmlutils.WrapWarning(warning, gridItemDD)
}
city := findSpanStrong(gridItemDD, "City:")
city = naToEmpty(city)
if city == "" {
warning := fmt.Sprintf("region for host %s not found", host)
return server, htmlutils.WrapWarning(warning, gridItemDD)
}
premiumString := findSpanStrong(gridItemDD, "Premium:")
premiumString = naToEmpty(premiumString)
if premiumString == "" {
warning := fmt.Sprintf("premium for host %s not found", host)
return server, htmlutils.WrapWarning(warning, gridItemDD)
}
return models.Server{
Region: region,
City: city,
Hostname: host + ".isponeder.com",
Premium: strings.EqualFold(premiumString, "yes"),
}, ""
}
func naToEmpty(current string) (output string) {
if current == "N / A" {
return ""
}
return current
}
func findCountry(countryNode *html.Node) (country string) {
for node := countryNode.FirstChild; node != nil; node = node.NextSibling {
if node.Data != "a" {
continue
}
for subNode := node.FirstChild; subNode != nil; subNode = subNode.NextSibling {
if subNode.Data != "h4" {
continue
}
return subNode.FirstChild.Data
}
}
return ""
}
func findServersDiv(rootNode *html.Node) (serversDiv *html.Node) {
locationsDiv := htmlutils.BFS(rootNode, matchLocationsListDiv)
if locationsDiv == nil {
return nil
}
return htmlutils.BFS(locationsDiv, matchServersDiv)
}
func findHost(gridItemDT *html.Node) (host string) {
hostNode := htmlutils.DirectChild(gridItemDT, matchText)
return strings.TrimSpace(hostNode.Data)
}
func matchText(node *html.Node) (match bool) {
if node.Type != html.TextNode {
return false
}
data := strings.TrimSpace(node.Data)
return data != ""
}
func findStatus(gridItemDT *html.Node) (status string) {
statusNode := htmlutils.DirectChild(gridItemDT, matchStatusSpan)
return strings.TrimSpace(statusNode.FirstChild.Data)
}
func matchServersDiv(node *html.Node) (match bool) {
return node != nil && node.Data == divString &&
htmlutils.HasClassStrings(node, "blk__i")
}
func matchLocationsListDiv(node *html.Node) (match bool) {
return node != nil && node.Data == divString &&
htmlutils.HasClassStrings(node, "locations-list")
}
func matchGridDiv(node *html.Node) (match bool) {
return node != nil && node.Data == divString &&
htmlutils.HasClassStrings(node, "grid--locations")
}
func matchGridItem(node *html.Node) (match bool) {
return node != nil && node.Data == "dl" &&
htmlutils.HasClassStrings(node, "grid__i")
}
func matchDT(node *html.Node) (match bool) {
return node != nil && node.Data == "dt"
}
func matchDD(node *html.Node) (match bool) {
return node != nil && node.Data == "dd"
}
func matchStatusSpan(node *html.Node) (match bool) {
return node.Data == "span" && htmlutils.HasClassStrings(node, "status")
}
func findSpanStrong(gridItemDD *html.Node, spanData string) (
strongValue string) {
spanFound := false
for child := gridItemDD.FirstChild; child != nil; child = child.NextSibling {
if !htmlutils.MatchData("div")(child) {
continue
}
for subchild := child.FirstChild; subchild != nil; subchild = subchild.NextSibling {
if htmlutils.MatchData("span")(subchild) && subchild.FirstChild.Data == spanData {
spanFound = true
break
}
}
if !spanFound {
continue
}
for subchild := child.FirstChild; subchild != nil; subchild = subchild.NextSibling {
if htmlutils.MatchData("strong")(subchild) {
return subchild.FirstChild.Data
}
}
}
return ""
}