/
requests.go
168 lines (149 loc) · 4.58 KB
/
requests.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
package matrix
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"os"
"time"
"github.com/t2bot/matrix-media-repo/common/rcontext"
)
// Based in part on https://github.com/matrix-org/gomatrix/blob/072b39f7fa6b40257b4eead8c958d71985c28bdd/client.go#L180-L243
func doRequest(ctx rcontext.RequestContext, method string, urlStr string, body interface{}, result interface{}, accessToken string, ipAddr string) error {
ctx.Log.Debugf("Calling %s %s", method, urlStr)
var bodyBytes []byte
if body != nil {
jsonStr, err := json.Marshal(body)
if err != nil {
return err
}
bodyBytes = jsonStr
}
req, err := http.NewRequest(method, urlStr, bytes.NewBuffer(bodyBytes))
if err != nil {
return err
}
req.Header.Set("User-Agent", "matrix-media-repo")
req.Header.Set("Content-Type", "application/json; charset=UTF-8")
if accessToken != "" {
req.Header.Set("Authorization", "Bearer "+accessToken)
}
if ipAddr != "" {
req.Header.Set("X-Forwarded-For", ipAddr)
req.Header.Set("X-Real-IP", ipAddr)
}
client := &http.Client{
Timeout: time.Duration(ctx.Config.TimeoutSeconds.ClientServer) * time.Second,
}
res, err := client.Do(req)
if res != nil {
defer res.Body.Close()
}
if err != nil {
return err
}
contents, err := io.ReadAll(res.Body)
if err != nil {
return err
}
if res.StatusCode != http.StatusOK {
mtxErr := &errorResponse{}
err = json.Unmarshal(contents, mtxErr)
if err == nil && mtxErr.ErrorCode != "" {
return mtxErr
}
return errors.New("failed to perform request: " + string(contents))
}
if result != nil {
err = json.Unmarshal(contents, &result)
if err != nil {
return err
}
}
return nil
}
func FederatedGet(url string, realHost string, ctx rcontext.RequestContext) (*http.Response, error) {
ctx.Log.Debug("Doing federated GET to " + url + " with host " + realHost)
cb := getFederationBreaker(realHost)
var resp *http.Response
replyError := cb.CallContext(ctx, func() error {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return err
}
// Override the host to be compliant with the spec
req.Header.Set("Host", realHost)
req.Header.Set("User-Agent", "matrix-media-repo")
req.Host = realHost
var client *http.Client
if os.Getenv("MEDIA_REPO_UNSAFE_FEDERATION") != "true" {
// This is how we verify the certificate is valid for the host we expect.
// Previously using `req.URL.Host` we'd end up changing which server we were
// connecting to (ie: matrix.org instead of matrix.org.cdn.cloudflare.net),
// which obviously doesn't help us. We needed to do that though because the
// HTTP client doesn't verify against the req.Host certificate, but it does
// handle it off the req.URL.Host. So, we need to tell it which certificate
// to verify.
h, _, err := net.SplitHostPort(realHost)
if err == nil {
// Strip the port first, certs are port-insensitive
realHost = h
}
client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
ServerName: realHost,
},
},
}
} else {
ctx.Log.Warn("Ignoring any certificate errors while making request")
tr := &http.Transport{
DisableKeepAlives: true,
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
// Based on https://github.com/matrix-org/gomatrixserverlib/blob/51152a681e69a832efcd934b60080b92bc98b286/client.go#L74-L90
DialTLSContext: func(ctx2 context.Context, network, addr string) (net.Conn, error) {
rawconn, err := net.Dial(network, addr)
if err != nil {
return nil, err
}
// Wrap a raw connection ourselves since tls.Dial defaults the SNI
conn := tls.Client(rawconn, &tls.Config{
ServerName: "",
InsecureSkipVerify: true,
})
if err := conn.Handshake(); err != nil {
return nil, err
}
return conn, nil
},
}
client = &http.Client{
Transport: tr,
}
}
client.Timeout = time.Duration(ctx.Config.TimeoutSeconds.UrlPreviews) * time.Second
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
if len(via) > 5 { // arbitrary
return errors.New("too many redirects")
}
ctx.Log.Debugf("Redirected to %s", req.URL.String())
client.Transport = nil // Clear our TLS handler as we're out of the Matrix certificate verification steps
return nil
}
resp, err = client.Do(req)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNotFound {
return fmt.Errorf("response not ok: %d", resp.StatusCode)
}
return nil
}, 1*time.Minute)
return resp, replyError
}