/
provider.go
90 lines (77 loc) · 2.28 KB
/
provider.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
package provider
import (
"fmt"
"io"
"net/http"
"net/url"
"strings"
"github.com/apex/log"
"go.uber.org/ratelimit"
)
// ErrMediaNotSupported is an error for unsupported media
type ErrMediaNotSupported struct {
error
}
func mediaNotSupported(api string) ErrMediaNotSupported {
return ErrMediaNotSupported{
fmt.Errorf("media not supported %v", api),
}
}
// IsErrMediaNotSupported return true if the error dictates taht the media
// type was not supported by the scraper
func IsErrMediaNotSupported(err error) bool {
if err == nil {
return false
}
if _, ok := err.(ErrMediaNotSupported); ok {
return true
}
return false
}
// APIClient is a http client with rate limiting
type APIClient struct {
*http.Client
ratelimit.Limiter
name string
}
func simpleURL(url string) string {
return strings.Split(url, "?")[0]
}
// NewAPIClient return a new APIClient
func NewAPIClient(name string, limit int) *APIClient {
return &APIClient{
Client: http.DefaultClient,
Limiter: ratelimit.New(limit),
name: name,
}
}
// Do performs a http request with rate limiting
func (a *APIClient) Do(req *http.Request) (*http.Response, error) {
a.Take()
log.WithField("url", simpleURL(req.URL.String())).WithField("method", req.Method).Debug(a.name)
return a.Client.Do(req)
}
// Get performs a http get request with rate limiting
func (a *APIClient) Get(url string) (*http.Response, error) {
a.Take()
log.WithField("url", simpleURL(url)).WithField("method", "GET").Debug(a.name)
return a.Client.Get(url)
}
// Head performs a http head request with rate limiting
func (a *APIClient) Head(url string) (*http.Response, error) {
a.Take()
log.WithField("url", simpleURL(url)).WithField("method", "HEAD").Debug(a.name)
return a.Client.Head(url)
}
// Post performs a http post request with rate limiting
func (a *APIClient) Post(url string, contentType string, body io.Reader) (*http.Response, error) {
a.Take()
log.WithField("url", simpleURL(url)).WithField("method", "POST").Debug(a.name)
return a.Client.Post(url, contentType, body)
}
// PostForm performs a http post form request with rate limiting
func (a *APIClient) PostForm(url string, data url.Values) (*http.Response, error) {
a.Take()
log.WithField("url", simpleURL(url)).WithField("method", "POST").Debug(a.name)
return a.Client.PostForm(url, data)
}