/
goproxy.go
656 lines (594 loc) · 17.4 KB
/
goproxy.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
/*
Package goproxy implements a minimalist Go module proxy handler.
*/
package goproxy
import (
"context"
"errors"
"fmt"
"io"
"io/fs"
"log"
"math"
"math/rand"
"net/http"
"net/url"
"os"
"path"
"strconv"
"strings"
"sync"
"time"
"golang.org/x/mod/sumdb"
)
// Goproxy is the top-level struct of this project.
//
// Note that Goproxy will still adhere to your environment variables. This means
// you can set GOPROXY to serve Goproxy itself under other proxies. By setting
// GONOPROXY and GOPRIVATE, you can instruct Goproxy on which modules to fetch
// directly, rather than using those proxies. Additionally, you can set GOSUMDB,
// GONOSUMDB, and GOPRIVATE to specify how Goproxy should verify the modules it
// has just fetched. Importantly, all of these mentioned environment variables
// are built-in supported, resulting in fewer external command calls and a
// significant performance boost.
//
// For requests involving the download of a large number of modules (e.g., for
// bulk static analysis), Goproxy supports a non-standard header,
// "Disable-Module-Fetch: true", which instructs it to return only cached
// content.
//
// Make sure that all fields of Goproxy have been finalized before calling any
// of its methods.
type Goproxy struct {
// GoBinName is the name of the Go binary.
//
// If GoBinName is empty, "go" is used.
//
// Note that the version of the Go binary targeted by GoBinName must be
// at least version 1.11.
GoBinName string
// GoBinEnv is the environment of the Go binary. Each entry is in the
// form "key=value".
//
// If GoBinEnv is nil, [os.Environ] is used.
//
// If GoBinEnv contains duplicate environment keys, only the last value
// in the slice for each duplicate key is used.
//
// Note that GOPROXY, GONOPROXY, GOSUMDB, GONOSUMDB, and GOPRIVATE are
// built-in supported. This means they can be set, even if the version
// of the Go binary targeted by [Goproxy.GoBinName] is before version
// 1.13.
GoBinEnv []string
// GoBinMaxWorkers is the maximum number of concurrently executing
// commands for the Go binary.
//
// If GoBinMaxWorkers is zero, there is no limit.
GoBinMaxWorkers int
// PathPrefix is the prefix for all request paths. It is used to trim
// the request paths using [strings.TrimPrefix].
//
// If PathPrefix is not empty, it must start with "/" and typically end
// with "/".
PathPrefix string
// Cacher is used to cache module files.
//
// If Cacher is nil, module files will be temporarily stored on the
// local disk and discarded when the request ends.
Cacher Cacher
// CacherMaxCacheBytes is the maximum number of bytes allowed for
// storing a new module file in [Goproxy.Cacher].
//
// If CacherMaxCacheBytes is zero, there is no limit.
CacherMaxCacheBytes int
// ProxiedSUMDBs is a list of proxied checksum databases (see
// https://go.dev/design/25530-sumdb#proxying-a-checksum-database). Each
// entry is in the form "<sumdb-name>" or "<sumdb-name> <sumdb-URL>".
// The first form is a shorthand for the second, where the corresponding
// <sumdb-URL> will be the <sumdb-name> itself as a host with an "https"
// scheme.
//
// If ProxiedSUMDBs contains duplicate checksum database names, only the
// last value in the slice for each duplicate checksum database name is
// used.
ProxiedSUMDBs []string
// Transport is used to perform all requests except those initiated by
// calling the Go binary targeted by [Goproxy.GoBinName].
//
// If Transport is nil, [http.DefaultTransport] is used.
Transport http.RoundTripper
// TempDir is the directory for storing temporary files.
//
// If TempDir is empty, [os.TempDir] is used.
TempDir string
// ErrorLogger is used to log errors that occur during proxying.
//
// If ErrorLogger is nil, [log.Default] is used.
ErrorLogger *log.Logger
initOnce sync.Once
goBinName string
goBinEnv []string
goBinEnvGOPROXY string
goBinEnvGONOPROXY string
goBinEnvGOSUMDB string
goBinEnvGONOSUMDB string
goBinWorkerChan chan struct{}
proxiedSUMDBs map[string]*url.URL
httpClient *http.Client
sumdbClient *sumdb.Client
}
// init initializes the g.
func (g *Goproxy) init() {
g.goBinName = g.GoBinName
if g.goBinName == "" {
g.goBinName = "go"
}
goBinEnv := g.GoBinEnv
if goBinEnv == nil {
goBinEnv = os.Environ()
}
var goBinEnvGOPRIVATE string
for _, env := range goBinEnv {
if k, v, ok := strings.Cut(env, "="); ok {
switch strings.TrimSpace(k) {
case "GO111MODULE":
case "GOPROXY":
g.goBinEnvGOPROXY = v
case "GONOPROXY":
g.goBinEnvGONOPROXY = v
case "GOSUMDB":
g.goBinEnvGOSUMDB = v
case "GONOSUMDB":
g.goBinEnvGONOSUMDB = v
case "GOPRIVATE":
goBinEnvGOPRIVATE = v
default:
g.goBinEnv = append(g.goBinEnv, k+"="+v)
}
}
}
g.goBinEnv = append(
g.goBinEnv,
"GO111MODULE=on",
"GOPROXY=direct",
"GONOPROXY=",
"GOSUMDB=off",
"GONOSUMDB=",
"GOPRIVATE=",
)
var goBinEnvGOPROXY string
for goproxy := g.goBinEnvGOPROXY; goproxy != ""; {
var proxy, sep string
if i := strings.IndexAny(goproxy, ",|"); i >= 0 {
proxy = goproxy[:i]
sep = string(goproxy[i])
goproxy = goproxy[i+1:]
if goproxy == "" {
sep = ""
}
} else {
proxy = goproxy
goproxy = ""
}
proxy = strings.TrimSpace(proxy)
switch proxy {
case "":
continue
case "direct", "off":
sep = ""
goproxy = ""
}
goBinEnvGOPROXY += proxy + sep
}
if goBinEnvGOPROXY != "" {
g.goBinEnvGOPROXY = goBinEnvGOPROXY
} else if g.goBinEnvGOPROXY == "" {
g.goBinEnvGOPROXY = "https://proxy.golang.org,direct"
} else {
g.goBinEnvGOPROXY = "off"
}
if g.goBinEnvGONOPROXY == "" {
g.goBinEnvGONOPROXY = goBinEnvGOPRIVATE
}
var goBinEnvGONOPROXYParts []string
for _, noproxy := range strings.Split(g.goBinEnvGONOPROXY, ",") {
if noproxy = strings.TrimSpace(noproxy); noproxy != "" {
goBinEnvGONOPROXYParts = append(goBinEnvGONOPROXYParts, noproxy)
}
}
if len(goBinEnvGONOPROXYParts) > 0 {
g.goBinEnvGONOPROXY = strings.Join(goBinEnvGONOPROXYParts, ",")
}
g.goBinEnvGOSUMDB = strings.TrimSpace(g.goBinEnvGOSUMDB)
if g.goBinEnvGOSUMDB == "" {
g.goBinEnvGOSUMDB = "sum.golang.org"
}
if g.goBinEnvGONOSUMDB == "" {
g.goBinEnvGONOSUMDB = goBinEnvGOPRIVATE
}
var goBinEnvGONOSUMDBParts []string
for _, nosumdb := range strings.Split(g.goBinEnvGONOSUMDB, ",") {
if nosumdb = strings.TrimSpace(nosumdb); nosumdb != "" {
goBinEnvGONOSUMDBParts = append(goBinEnvGONOSUMDBParts, nosumdb)
}
}
if len(goBinEnvGONOSUMDBParts) > 0 {
g.goBinEnvGONOSUMDB = strings.Join(goBinEnvGONOSUMDBParts, ",")
}
if g.GoBinMaxWorkers != 0 {
g.goBinWorkerChan = make(chan struct{}, g.GoBinMaxWorkers)
}
g.proxiedSUMDBs = map[string]*url.URL{}
for _, proxiedSUMDB := range g.ProxiedSUMDBs {
sumdbParts := strings.Fields(proxiedSUMDB)
if len(sumdbParts) == 0 {
continue
}
sumdbName := sumdbParts[0]
rawSUMDBURL := sumdbName
if len(sumdbParts) > 1 {
rawSUMDBURL = sumdbParts[1]
}
sumdbURL, err := parseRawURL(rawSUMDBURL)
if err != nil {
continue
}
g.proxiedSUMDBs[sumdbName] = sumdbURL
}
g.httpClient = &http.Client{Transport: g.Transport}
g.sumdbClient = sumdb.NewClient(&sumdbClientOps{
envGOPROXY: g.goBinEnvGOPROXY,
envGOSUMDB: g.goBinEnvGOSUMDB,
httpClient: g.httpClient,
})
}
// ServeHTTP implements [http.Handler].
func (g *Goproxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
g.initOnce.Do(g.init)
switch req.Method {
case http.MethodGet, http.MethodHead:
default:
responseMethodNotAllowed(rw, req, 86400)
return
}
name, _ := url.PathUnescape(req.URL.Path)
if name == "" || name[0] != '/' || name[len(name)-1] == '/' || strings.Contains(name, "..") {
responseNotFound(rw, req, 86400)
return
}
name = path.Clean(name)
if g.PathPrefix != "" {
name = strings.TrimPrefix(name, g.PathPrefix)
} else {
name = strings.TrimPrefix(name, "/")
}
tempDir, err := os.MkdirTemp(g.TempDir, "goproxy")
if err != nil {
g.logErrorf("failed to create temporary directory: %v", err)
responseInternalServerError(rw, req)
return
}
defer os.RemoveAll(tempDir)
if strings.HasPrefix(name, "sumdb/") {
g.serveSUMDB(rw, req, name, tempDir)
return
}
g.serveFetch(rw, req, name, tempDir)
}
// serveFetch serves fetch requests.
func (g *Goproxy) serveFetch(rw http.ResponseWriter, req *http.Request, name, tempDir string) {
f, err := newFetch(g, name, tempDir)
if err != nil {
responseNotFound(rw, req, 86400, err)
return
}
var isDownload bool
switch f.ops {
case fetchOpsDownloadInfo, fetchOpsDownloadMod, fetchOpsDownloadZip:
isDownload = true
}
noFetch, _ := strconv.ParseBool(req.Header.Get("Disable-Module-Fetch"))
if noFetch {
var cacheControlMaxAge int
if isDownload {
cacheControlMaxAge = 604800
} else {
cacheControlMaxAge = 60
}
g.serveCache(rw, req, f.name, f.contentType, cacheControlMaxAge, func() {
responseNotFound(rw, req, 60, "temporarily unavailable")
})
return
}
if isDownload {
g.serveCache(rw, req, f.name, f.contentType, 604800, func() {
g.serveFetchDownload(rw, req, f)
})
return
}
fr, err := f.do(req.Context())
if err != nil {
g.serveCache(rw, req, f.name, f.contentType, 60, func() {
g.logErrorf("failed to %s module version: %s: %v", f.ops, f.name, err)
responseError(rw, req, err, true)
})
return
}
content, err := fr.Open()
if err != nil {
g.logErrorf("failed to open fetch result: %s: %v", f.name, err)
responseInternalServerError(rw, req)
return
}
defer content.Close()
if err := g.putCache(req.Context(), f.name, content); err != nil {
g.logErrorf("failed to cache module file: %s: %v", f.name, err)
responseInternalServerError(rw, req)
return
} else if _, err := content.Seek(0, io.SeekStart); err != nil {
g.logErrorf("failed to seek fetch result content: %s: %v", f.name, err)
responseInternalServerError(rw, req)
return
}
responseSuccess(rw, req, content, f.contentType, 60)
}
// serveFetchDownload serves fetch download requests.
func (g *Goproxy) serveFetchDownload(rw http.ResponseWriter, req *http.Request, f *fetch) {
fr, err := f.do(req.Context())
if err != nil {
g.logErrorf("failed to download module version: %s: %v", f.name, err)
responseError(rw, req, err, false)
return
}
nameWithoutExt := strings.TrimSuffix(f.name, path.Ext(f.name))
for _, cache := range []struct{ nameExt, localFile string }{
{".info", fr.Info},
{".mod", fr.GoMod},
{".zip", fr.Zip},
} {
if cache.localFile == "" {
continue
}
if err := g.putCacheFile(req.Context(), nameWithoutExt+cache.nameExt, cache.localFile); err != nil {
g.logErrorf("failed to cache module file: %s: %v", f.name, err)
responseInternalServerError(rw, req)
return
}
}
content, err := fr.Open()
if err != nil {
g.logErrorf("failed to open fetch result: %s: %v", f.name, err)
responseInternalServerError(rw, req)
return
}
defer content.Close()
responseSuccess(rw, req, content, f.contentType, 604800)
}
// serveSUMDB serves checksum database proxy requests.
func (g *Goproxy) serveSUMDB(rw http.ResponseWriter, req *http.Request, name, tempDir string) {
sumdbURL, err := parseRawURL(strings.TrimPrefix(name, "sumdb/"))
if err != nil {
responseNotFound(rw, req, 86400)
return
}
proxiedSUMDBURL, ok := g.proxiedSUMDBs[sumdbURL.Host]
if !ok {
responseNotFound(rw, req, 86400)
return
}
var (
contentType string
cacheControlMaxAge int
)
if sumdbURL.Path == "/supported" {
setResponseCacheControlHeader(rw, 86400)
rw.WriteHeader(http.StatusOK)
return
} else if sumdbURL.Path == "/latest" {
contentType = "text/plain; charset=utf-8"
cacheControlMaxAge = 3600
} else if strings.HasPrefix(sumdbURL.Path, "/lookup/") {
contentType = "text/plain; charset=utf-8"
cacheControlMaxAge = 86400
} else if strings.HasPrefix(sumdbURL.Path, "/tile/") {
contentType = "application/octet-stream"
cacheControlMaxAge = 86400
} else {
responseNotFound(rw, req, 86400)
return
}
tempFile, err := os.CreateTemp(tempDir, "")
if err != nil {
g.logErrorf("failed to create temporary file: %v", err)
responseInternalServerError(rw, req)
return
}
if err := httpGet(req.Context(), g.httpClient, appendURL(proxiedSUMDBURL, sumdbURL.Path).String(), tempFile); err != nil {
g.serveCache(rw, req, name, contentType, cacheControlMaxAge, func() {
g.logErrorf("failed to proxy checksum database: %s: %v", name, err)
responseError(rw, req, err, true)
})
return
}
if err := tempFile.Close(); err != nil {
g.logErrorf("failed to close temporary file: %v", err)
responseInternalServerError(rw, req)
return
}
if err := g.putCacheFile(req.Context(), name, tempFile.Name()); err != nil {
g.logErrorf("failed to cache module file: %s: %v", name, err)
responseInternalServerError(rw, req)
return
}
content, err := os.Open(tempFile.Name())
if err != nil {
g.logErrorf("failed to open temporary file: %s: %v", name, err)
responseInternalServerError(rw, req)
return
}
defer content.Close()
responseSuccess(rw, req, content, contentType, cacheControlMaxAge)
}
// serveCache serves requests with cached module files.
func (g *Goproxy) serveCache(rw http.ResponseWriter, req *http.Request, name, contentType string, cacheControlMaxAge int, onNotFound func()) {
content, err := g.cache(req.Context(), name)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
onNotFound()
return
}
g.logErrorf("failed to get cached module file: %s: %v", name, err)
responseInternalServerError(rw, req)
return
}
defer content.Close()
responseSuccess(rw, req, content, contentType, cacheControlMaxAge)
}
// cache returns the matched cache for the name from the g.Cacher.
func (g *Goproxy) cache(ctx context.Context, name string) (io.ReadCloser, error) {
if g.Cacher == nil {
return nil, fs.ErrNotExist
}
return g.Cacher.Get(ctx, name)
}
// putCache puts a cache to the g.Cacher for the name with the content.
func (g *Goproxy) putCache(ctx context.Context, name string, content io.ReadSeeker) error {
if g.Cacher == nil {
return nil
}
if g.CacherMaxCacheBytes != 0 {
if size, err := content.Seek(0, io.SeekEnd); err != nil {
return err
} else if size > int64(g.CacherMaxCacheBytes) {
return nil
} else if _, err := content.Seek(0, io.SeekStart); err != nil {
return err
}
}
return g.Cacher.Put(ctx, name, content)
}
// putCacheFile puts a cache to the g.Cacher for the name with the targeted local file.
func (g *Goproxy) putCacheFile(ctx context.Context, name, file string) error {
f, err := os.Open(file)
if err != nil {
return err
}
defer f.Close()
return g.putCache(ctx, name, f)
}
// logErrorf formats according to a format specifier and writes to the g.ErrorLogger.
func (g *Goproxy) logErrorf(format string, v ...any) {
msg := "goproxy: " + fmt.Sprintf(format, v...)
if g.ErrorLogger != nil {
g.ErrorLogger.Output(2, msg)
} else {
log.Output(2, msg)
}
}
// walkGOPROXY walks through the proxy list parsed from the goproxy.
func walkGOPROXY(goproxy string, onProxy func(proxy string) error, onDirect, onOff func() error) error {
if goproxy == "" {
return errors.New("missing GOPROXY")
}
var proxyError error
for goproxy != "" {
var (
proxy string
fallBackOnError bool
)
if i := strings.IndexAny(goproxy, ",|"); i >= 0 {
proxy = goproxy[:i]
fallBackOnError = goproxy[i] == '|'
goproxy = goproxy[i+1:]
} else {
proxy = goproxy
goproxy = ""
}
switch proxy {
case "direct":
return onDirect()
case "off":
return onOff()
}
if err := onProxy(proxy); err != nil {
if fallBackOnError || errors.Is(err, errNotFound) {
proxyError = err
continue
}
return err
}
return nil
}
return proxyError
}
var (
backoffRand = rand.New(rand.NewSource(time.Now().UnixNano()))
backoffRandMutex sync.Mutex
)
// backoffSleep computes the exponential backoff sleep duration based on the
// algorithm described in https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/.
func backoffSleep(base, cap time.Duration, attempt int) time.Duration {
var pow time.Duration
if attempt < 63 {
pow = 1 << attempt
} else {
pow = math.MaxInt64
}
sleep := base * pow
if sleep > cap || sleep/pow != base {
sleep = cap
}
backoffRandMutex.Lock()
sleep = time.Duration(backoffRand.Int63n(int64(sleep)))
backoffRandMutex.Unlock()
return sleep
}
// stringSliceContains reports whether the ss contains the s.
func stringSliceContains(ss []string, s string) bool {
for _, v := range ss {
if v == s {
return true
}
}
return false
}
// globsMatchPath reports whether any path prefix of target matches one of the
// glob patterns (as defined by [path.Match]) in the comma-separated globs list.
// It ignores any empty or malformed patterns in the list.
func globsMatchPath(globs, target string) bool {
for globs != "" {
// Extract next non-empty glob in comma-separated list.
var glob string
if i := strings.Index(globs, ","); i >= 0 {
glob, globs = globs[:i], globs[i+1:]
} else {
glob, globs = globs, ""
}
if glob == "" {
continue
}
// A glob with N+1 path elements (N slashes) needs to be matched
// against the first N+1 path elements of target, which end just
// before the N+1'th slash.
n := strings.Count(glob, "/")
prefix := target
// Walk target, counting slashes, truncating at the N+1'th slash.
for i := 0; i < len(target); i++ {
if target[i] == '/' {
if n == 0 {
prefix = target[:i]
break
}
n--
}
}
if n > 0 {
// Not enough prefix elements.
continue
}
if matched, _ := path.Match(glob, prefix); matched {
return true
}
}
return false
}