-
Notifications
You must be signed in to change notification settings - Fork 0
/
fasthttp_html_loader.go
48 lines (40 loc) · 1.19 KB
/
fasthttp_html_loader.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
package visitor
import (
"fmt"
"time"
"github.com/valyala/fasthttp"
)
func CreateLoader() {
// return
}
// FastHTTPPageLoader represents a way to load HTML by URL via fasthttp package
type FastHTTPPageLoader struct {
Timeout time.Duration
MaxRedirects int
UserAgent string
Buf []byte
}
func (l FastHTTPPageLoader) LoadPage(url string) (statusCode int, finalUrl string, body []byte, error error) {
req := fasthttp.AcquireRequest()
defer fasthttp.ReleaseRequest(req)
res := fasthttp.AcquireResponse()
defer fasthttp.ReleaseResponse(res)
req.Header.Add("User-Agent", l.UserAgent)
finalURL := url
for i := 0; i < l.MaxRedirects; i++ {
req.SetRequestURI(finalURL)
err := fasthttp.DoTimeout(req, res, l.Timeout)
if err != nil {
return 0, finalURL, nil, fmt.Errorf("loadPage: %w", err)
}
statusCode := res.StatusCode()
if statusCode >= 300 && statusCode < 400 && res.Header.Peek("location") != nil {
finalURL = string(res.Header.Peek("location"))
continue
}
copy := fasthttp.AcquireResponse()
res.CopyTo(copy)
return statusCode, finalURL, copy.Body(), nil
}
return 0, finalURL, nil, fmt.Errorf("loadPage: maxRedirects (%v) exceeded", l.MaxRedirects)
}