-
Notifications
You must be signed in to change notification settings - Fork 35
/
info_urls.go
117 lines (104 loc) · 3.12 KB
/
info_urls.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
package build
import (
"io"
"io/ioutil"
"net/http"
"strings"
"time"
)
func UpdateInfoURLs(zones map[string]*Zone) {
Trace("@{.}Updating info URLs for %d zones...\n", len(zones))
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.TLSHandshakeTimeout = 5 * time.Second
transport.MaxIdleConnsPerHost = 10
client := &http.Client{
Transport: transport,
Timeout: 10 * time.Second,
}
mapZones(zones, func(z *Zone) {
var urls []string
if strings.HasPrefix(z.InfoURL, "http:") {
urls = []string{
strings.Replace(z.InfoURL, "http:", "https:", 1),
z.InfoURL,
}
} else if !strings.HasPrefix(z.InfoURL, "https://newgtlds.icann.org") {
urls = []string{
z.InfoURL,
}
}
if z.IsTLD() {
urls = append(urls,
// Try NIC websites
"https://nic."+z.Domain,
"https://www.nic."+z.Domain,
"http://nic."+z.Domain,
"http://www.nic."+z.Domain,
// Try ICANN first
"https://www.icann.org/en/registry-agreements/details/"+z.ASCII(),
// Then fall back to IANA
"https://www.iana.org/domains/root/db/"+z.ASCII()+".html",
)
}
var infoURL string
for _, u := range urls {
if u == "" {
continue
}
u = NormalizeURL(u)
req, err := http.NewRequest(http.MethodGet, u, nil)
if err != nil {
Trace("@{y!}Warning:@{y} error fetching info URL for @{y!}%s@{y}: (%s): %v\n", z.Domain, u, err)
continue
}
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36")
res, err := client.Do(req)
if err != nil {
if u == z.InfoURL {
Trace("@{y!}Warning:@{y} error fetching info URL for @{y!}%s@{y}: (%s): %v\n", z.Domain, u, err)
}
continue
} else if res.StatusCode != http.StatusOK {
if u == z.InfoURL {
Trace("@{y!}Warning:@{y} non-200 status for info URL for @{y!}%s@{y}: (%s): %s\n", z.Domain, u, res.Status)
}
continue
}
CloseN(res.Body, 10_000_000)
// Don’t use redirected URL, use the URL we crafted
// infoURL = NormalizeURL(res.Request.URL.String())
infoURL = u
break
}
// Do not rewrite URLs that just add tracking or query string info
if infoURL != z.InfoURL && !strings.HasPrefix(infoURL, z.InfoURL) {
if infoURL == "" {
Trace("@{.!}Removed@{.} info URL for @{c}%s@{c}: @{y}%s@{c}\n", z.Domain, z.InfoURL)
} else {
Trace("@{.}Updated info URL for @{c}%s@{c}: @{y}%s@{c} → @{g}%s\n", z.Domain, z.InfoURL, infoURL)
}
z.InfoURL = infoURL
}
})
}
// CloseN drains rc up to a maximum of n bytes and closes rc.
// It returns the number of bytes drained and the first error encountered.
// Regardless of any errors, rc.Close() is guaranteed to be called.
func CloseN(rc io.ReadCloser, n int64) (int64, error) {
n, cerr := io.Copy(ioutil.Discard, io.LimitReader(rc, n))
err := rc.Close()
if cerr != nil {
err = cerr
}
return n, err
}
// trimURL trims query strings and /index.htm(l)? from a URL.
func trimURL(u string) string {
u, _, _ = strings.Cut(u, "?")
if strings.HasSuffix(u, "/index.html") {
u = strings.TrimSuffix(u, "/index.html")
} else {
u = strings.TrimSuffix(u, "/index.htm")
}
return u
}