all: use RateLimiter where it seems appropriate

This also sets up a cookie jar correctly so that any rate limiting
schemes based on Cookie headers will actually work.

Signed-off-by: Hank Donnay <hdonnay@redhat.com>

cmd/clairctl/export.go

@@ -3,6 +3,7 @@ package main
 import (
 	"errors"
 	"io"
+	"net/http"
 	"os"
 	"regexp"
 
@@ -11,6 +12,8 @@ import (
 	"github.com/quay/claircore/updater"
 	_ "github.com/quay/claircore/updater/defaults"
 	"github.com/urfave/cli/v2"
+
+	"github.com/quay/clair/v4/internal/httputil"
 )
 
 // ExportCmd is the "export-updaters" subcommand.
@@ -66,7 +69,8 @@ func exportAction(c *cli.Context) error {
 		cfgs[name] = node.Decode
 	}
 
-	cl, _, err := cfg.Client(nil, commonClaim)
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	cl, _, err := cfg.Client(httputil.RateLimiter(tr), commonClaim)
 	if err != nil {
 		return err
 	}

config/httpclient.go

@@ -2,8 +2,10 @@ package config
 
 import (
 	"net/http"
+	"net/http/cookiejar"
 	"time"
 
+	"golang.org/x/net/publicsuffix"
 	"gopkg.in/square/go-jose.v2"
 	"gopkg.in/square/go-jose.v2/jwt"
 )
@@ -13,7 +15,7 @@ import (
 //
 // It returns an *http.Client and a boolean indicating whether the client is
 // configured for authentication, or an error that occurred during construction.
-func (cfg *Config) Client(next *http.Transport, cl jwt.Claims) (c *http.Client, authed bool, err error) {
+func (cfg *Config) Client(next http.RoundTripper, cl jwt.Claims) (c *http.Client, authed bool, err error) {
 	if next == nil {
 		next = http.DefaultTransport.(*http.Transport).Clone()
 	}
@@ -29,11 +31,20 @@ func (cfg *Config) Client(next *http.Transport, cl jwt.Claims) (c *http.Client,
 		sk.Key = cfg.Auth.PSK.Key
 	default:
 	}
+	jar, err := cookiejar.New(&cookiejar.Options{
+		PublicSuffixList: publicsuffix.List,
+	})
+	if err != nil {
+		return nil, false, err
+	}
 	rt := &transport{
 		next: next,
 		base: cl,
 	}
-	c = &http.Client{Transport: rt}
+	c = &http.Client{
+		Jar:       jar,
+		Transport: rt,
+	}
 
 	// Both of the JWT-based methods set the signing key.
 	if sk.Key != nil {

go.mod

@@ -31,8 +31,8 @@ require (
 	go.opentelemetry.io/otel/exporters/trace/jaeger v0.16.0
 	go.opentelemetry.io/otel/sdk v0.16.0
 	golang.org/x/mod v0.4.0 // indirect
+	golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4
 	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
-	golang.org/x/sys v0.0.0-20210122093101-04d7465088b8 // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	golang.org/x/tools v0.0.0-20210112235408-75fd75db8797 // indirect
 	gopkg.in/square/go-jose.v2 v2.4.1

go.sum

@@ -809,6 +809,8 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 h1:b0LrWgu8+q7z4J+0Y3Umo5q1dL7NXBkKBWkaVkAq17E=
+golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -880,10 +882,12 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210122093101-04d7465088b8 h1:de2yTH1xuxjmGB7i6Z5o2z3RCHVa0XlpSZzjd8Fe6bE=
-golang.org/x/sys v0.0.0-20210122093101-04d7465088b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005 h1:pDMpM2zh2MT0kHy037cKlSby2nEhD50SYqwQk76Nm40=
+golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

initialize/services.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/http"
+	"net/http/cookiejar"
 	"time"
 
 	"github.com/quay/claircore/libindex"
@@ -12,13 +14,15 @@ import (
 	"github.com/quay/zlog"
 	"go.opentelemetry.io/otel/baggage"
 	"go.opentelemetry.io/otel/label"
+	"golang.org/x/net/publicsuffix"
 	"gopkg.in/square/go-jose.v2/jwt"
 
 	clairerror "github.com/quay/clair/v4/clair-error"
 	"github.com/quay/clair/v4/config"
 	"github.com/quay/clair/v4/httptransport"
 	"github.com/quay/clair/v4/httptransport/client"
 	"github.com/quay/clair/v4/indexer"
+	"github.com/quay/clair/v4/internal/httputil"
 	"github.com/quay/clair/v4/matcher"
 	notifier "github.com/quay/clair/v4/notifier/service"
 )
@@ -153,7 +157,8 @@ func remoteIndexer(ctx context.Context, cfg *config.Config, addr string) (indexe
 }
 
 func remoteClient(ctx context.Context, cfg *config.Config, claim jwt.Claims, addr string) (*client.HTTP, error) {
-	c, auth, err := cfg.Client(nil, claim)
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	c, auth, err := cfg.Client(tr, claim)
 	switch {
 	case err != nil:
 		return nil, err
@@ -172,6 +177,17 @@ func localMatcher(ctx context.Context, cfg *config.Config) (matcher.Service, err
 		}
 	}
 
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	jar, err := cookiejar.New(&cookiejar.Options{
+		PublicSuffixList: publicsuffix.List,
+	})
+	if err != nil {
+		return nil, err
+	}
+	cl := &http.Client{
+		Jar:       jar,
+		Transport: httputil.RateLimiter(tr),
+	}
 	updaterConfigs := make(map[string]driver.ConfigUnmarshaler)
 	for name, node := range cfg.Updaters.Config {
 		updaterConfigs[name] = node.Decode
@@ -184,6 +200,7 @@ func localMatcher(ctx context.Context, cfg *config.Config) (matcher.Service, err
 		UpdateInterval:  cfg.Matcher.Period,
 		UpdaterConfigs:  updaterConfigs,
 		UpdateRetention: cfg.Matcher.UpdateRetention,
+		Client:          cl,
 	})
 	if err != nil {
 		return nil, mkErr(err)
@@ -211,7 +228,8 @@ func localNotifier(ctx context.Context, cfg *config.Config, i indexer.Service, m
 		}
 	}
 
-	c, _, err := cfg.Client(nil, notifierClaim)
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	c, _, err := cfg.Client(tr, notifierClaim)
 	if err != nil {
 		return nil, mkErr(err)
 	}