From 028c4b3c9e71ef132278200eb71251b1888a610f Mon Sep 17 00:00:00 2001 From: Arve Knudsen Date: Mon, 6 Sep 2021 17:25:52 +0200 Subject: [PATCH] Chore: Upgrade weaveworks/common (#4462) * Chore: Upgrade weaveworks/common Signed-off-by: Arve Knudsen * Add changelog entries Signed-off-by: Arve Knudsen * Use instrument.NewHistogramCollector Signed-off-by: Arve Knudsen * Use tracing.ExtractSampledTraceID Signed-off-by: Arve Knudsen --- CHANGELOG.md | 2 ++ go.mod | 2 +- go.sum | 3 +- pkg/chunk/cache/memcached.go | 18 +++------- pkg/chunk/cache/redis_cache.go | 8 ++--- pkg/querier/queryrange/instrumentation.go | 4 +-- pkg/util/log/wrappers.go | 4 +-- .../common/instrument/instrument.go | 33 ++++++++++++++----- .../common/middleware/grpc_instrumentation.go | 9 ++--- .../common/middleware/http_tracing.go | 31 ----------------- .../common/middleware/instrument.go | 14 ++------ .../weaveworks/common/middleware/logging.go | 3 +- .../weaveworks/common/server/server.go | 14 ++++++-- .../weaveworks/common/tracing/tracing.go | 32 ++++++++++++++++++ vendor/modules.txt | 2 +- 15 files changed, 95 insertions(+), 84 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd3a341aa3..db88ee0084 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,8 @@ * [ENHANCEMENT] Memberlist: expose configuration of memberlist packet compression via `-memberlist.compression=enabled`. #4346 * [ENHANCEMENT] Update Go version to 1.16.6. #4362 * [ENHANCEMENT] Updated Prometheus to include changes from prometheus/prometheus#9083. Now whenever `/labels` API calls include matchers, blocks store is queried for `LabelNames` with matchers instead of `Series` calls which was inefficient. #4380 +* [ENHANCEMENT] Exemplars are now emitted for all gRPC calls and many operations tracked by histograms. #4462 +* [ENHANCEMENT] New options `-server.http-listen-network` and `-server.grpc-listen-network` allow binding as 'tcp4' or 'tcp6'. #4462 * [BUGFIX] Compactor: compactor will no longer try to compact blocks that are already marked for deletion. Previously compactor would consider blocks marked for deletion within `-compactor.deletion-delay / 2` period as eligible for compaction. #4328 * [BUGFIX] HA Tracker: when cleaning up obsolete elected replicas from KV store, tracker didn't update number of cluster per user correctly. #4336 * [BUGFIX] Ruler: fixed counting of PromQL evaluation errors as user-errors when updating `cortex_ruler_queries_failed_total`. #4335 diff --git a/go.mod b/go.mod index 24d328ff0d..83f8b93ed5 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/thanos-io/thanos v0.22.0 github.com/uber/jaeger-client-go v2.29.1+incompatible - github.com/weaveworks/common v0.0.0-20210722103813-e649eff5ab4a + github.com/weaveworks/common v0.0.0-20210901124008-1fa3f9fa874c go.etcd.io/bbolt v1.3.6 go.uber.org/atomic v1.9.0 golang.org/x/net v0.0.0-20210610132358-84b48f89b13b diff --git a/go.sum b/go.sum index 2ee420f83e..b67d785b9a 100644 --- a/go.sum +++ b/go.sum @@ -1765,8 +1765,9 @@ github.com/weaveworks/common v0.0.0-20200914083218-61ffdd448099/go.mod h1:hz10LO github.com/weaveworks/common v0.0.0-20201119133501-0619918236ec/go.mod h1:ykzWac1LtVfOxdCK+jD754at1Ws9dKCwFeUzkFBffPs= github.com/weaveworks/common v0.0.0-20210112142934-23c8d7fa6120/go.mod h1:ykzWac1LtVfOxdCK+jD754at1Ws9dKCwFeUzkFBffPs= github.com/weaveworks/common v0.0.0-20210419092856-009d1eebd624/go.mod h1:ykzWac1LtVfOxdCK+jD754at1Ws9dKCwFeUzkFBffPs= -github.com/weaveworks/common v0.0.0-20210722103813-e649eff5ab4a h1:ALomSnvy/NPeVoc4a1o7keaHHgLS76r9ZYIlwWWF+KA= github.com/weaveworks/common v0.0.0-20210722103813-e649eff5ab4a/go.mod h1:YU9FvnS7kUnRt6HY10G+2qHkwzP3n3Vb1XsXDsJTSp8= +github.com/weaveworks/common v0.0.0-20210901124008-1fa3f9fa874c h1:+yzwVr4/12cUgsdjbEHq6MsKB7jWBZpZccAP6xvqTzQ= +github.com/weaveworks/common v0.0.0-20210901124008-1fa3f9fa874c/go.mod h1:YU9FvnS7kUnRt6HY10G+2qHkwzP3n3Vb1XsXDsJTSp8= github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M= github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= diff --git a/pkg/chunk/cache/memcached.go b/pkg/chunk/cache/memcached.go index 6b5a2ad36c..98c1dd3e8d 100644 --- a/pkg/chunk/cache/memcached.go +++ b/pkg/chunk/cache/memcached.go @@ -20,16 +20,6 @@ import ( "github.com/cortexproject/cortex/pkg/util/spanlogger" ) -type observableVecCollector struct { - v prometheus.ObserverVec -} - -func (observableVecCollector) Register() {} -func (observableVecCollector) Before(method string, start time.Time) {} -func (o observableVecCollector) After(method, statusCode string, start time.Time) { - o.v.WithLabelValues(method, statusCode).Observe(time.Since(start).Seconds()) -} - // MemcachedConfig is config to make a Memcached type MemcachedConfig struct { Expiration time.Duration `yaml:"expiration"` @@ -51,7 +41,7 @@ type Memcached struct { memcache MemcachedClient name string - requestDuration observableVecCollector + requestDuration *instr.HistogramCollector wg sync.WaitGroup inputCh chan *work @@ -67,8 +57,8 @@ func NewMemcached(cfg MemcachedConfig, client MemcachedClient, name string, reg memcache: client, name: name, logger: logger, - requestDuration: observableVecCollector{ - v: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ + requestDuration: instr.NewHistogramCollector( + promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ Namespace: "cortex", Name: "memcache_request_duration_seconds", Help: "Total time spent in seconds doing memcache requests.", @@ -76,7 +66,7 @@ func NewMemcached(cfg MemcachedConfig, client MemcachedClient, name string, reg Buckets: prometheus.ExponentialBuckets(0.000016, 4, 8), ConstLabels: prometheus.Labels{"name": name}, }, []string{"method", "status_code"}), - }, + ), } if cfg.BatchSize == 0 || cfg.Parallelism == 0 { diff --git a/pkg/chunk/cache/redis_cache.go b/pkg/chunk/cache/redis_cache.go index 3a88d0ab47..5f504067ab 100644 --- a/pkg/chunk/cache/redis_cache.go +++ b/pkg/chunk/cache/redis_cache.go @@ -19,7 +19,7 @@ type RedisCache struct { name string redis *RedisClient logger log.Logger - requestDuration observableVecCollector + requestDuration *instr.HistogramCollector } // NewRedisCache creates a new RedisCache @@ -29,15 +29,15 @@ func NewRedisCache(name string, redisClient *RedisClient, reg prometheus.Registe name: name, redis: redisClient, logger: logger, - requestDuration: observableVecCollector{ - v: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ + requestDuration: instr.NewHistogramCollector( + promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ Namespace: "cortex", Name: "rediscache_request_duration_seconds", Help: "Total time spent in seconds doing Redis requests.", Buckets: prometheus.ExponentialBuckets(0.000016, 4, 8), ConstLabels: prometheus.Labels{"name": name}, }, []string{"method", "status_code"}), - }, + ), } if err := cache.redis.Ping(context.Background()); err != nil { level.Error(logger).Log("msg", "error connecting to redis", "name", name, "err", err) diff --git a/pkg/querier/queryrange/instrumentation.go b/pkg/querier/queryrange/instrumentation.go index f06e4785f6..39f6ac379e 100644 --- a/pkg/querier/queryrange/instrumentation.go +++ b/pkg/querier/queryrange/instrumentation.go @@ -58,7 +58,7 @@ type NoopCollector struct{} func (c *NoopCollector) Register() {} // Before implements instrument.Collector. -func (c *NoopCollector) Before(method string, start time.Time) {} +func (c *NoopCollector) Before(ctx context.Context, method string, start time.Time) {} // After implements instrument.Collector. -func (c *NoopCollector) After(method, statusCode string, start time.Time) {} +func (c *NoopCollector) After(ctx context.Context, method, statusCode string, start time.Time) {} diff --git a/pkg/util/log/wrappers.go b/pkg/util/log/wrappers.go index 9c37472728..a76d87e326 100644 --- a/pkg/util/log/wrappers.go +++ b/pkg/util/log/wrappers.go @@ -5,7 +5,7 @@ import ( "github.com/go-kit/kit/log" kitlog "github.com/go-kit/kit/log" - "github.com/weaveworks/common/middleware" + "github.com/weaveworks/common/tracing" "github.com/cortexproject/cortex/pkg/tenant" ) @@ -38,7 +38,7 @@ func WithContext(ctx context.Context, l kitlog.Logger) kitlog.Logger { l = WithUserID(userID, l) } - traceID, ok := middleware.ExtractTraceID(ctx) + traceID, ok := tracing.ExtractSampledTraceID(ctx) if !ok { return l } diff --git a/vendor/github.com/weaveworks/common/instrument/instrument.go b/vendor/github.com/weaveworks/common/instrument/instrument.go index 1256910063..07aa033c0c 100644 --- a/vendor/github.com/weaveworks/common/instrument/instrument.go +++ b/vendor/github.com/weaveworks/common/instrument/instrument.go @@ -11,6 +11,7 @@ import ( oldcontext "golang.org/x/net/context" "github.com/weaveworks/common/grpc" + "github.com/weaveworks/common/tracing" "github.com/weaveworks/common/user" ) @@ -21,8 +22,8 @@ var DefBuckets = []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25, // Collector describes something that collects data before and/or after a task. type Collector interface { Register() - Before(method string, start time.Time) - After(method, statusCode string, start time.Time) + Before(ctx context.Context, method string, start time.Time) + After(ctx context.Context, method, statusCode string, start time.Time) } // HistogramCollector collects the duration of a request @@ -52,16 +53,30 @@ func (c *HistogramCollector) Register() { } // Before collects for the upcoming request. -func (c *HistogramCollector) Before(method string, start time.Time) { +func (c *HistogramCollector) Before(ctx context.Context, method string, start time.Time) { } // After collects when the request is done. -func (c *HistogramCollector) After(method, statusCode string, start time.Time) { +func (c *HistogramCollector) After(ctx context.Context, method, statusCode string, start time.Time) { if c.metric != nil { - c.metric.WithLabelValues(method, statusCode).Observe(time.Now().Sub(start).Seconds()) + ObserveWithExemplar(ctx, c.metric.WithLabelValues(method, statusCode), time.Since(start).Seconds()) } } +// ObserveWithExemplar adds a sample to a histogram, and adds an exemplar if the context has a sampled trace. +// 'histogram' parameter must be castable to prometheus.ExemplarObserver or function will panic +// (this will always work for a HistogramVec). +func ObserveWithExemplar(ctx context.Context, histogram prometheus.Observer, seconds float64) { + if traceID, ok := tracing.ExtractSampledTraceID(ctx); ok { + histogram.(prometheus.ExemplarObserver).ObserveWithExemplar( + seconds, + prometheus.Labels{"traceID": traceID}, + ) + return + } + histogram.Observe(seconds) +} + // JobCollector collects metrics for jobs. Designed for batch jobs which run on a regular, // not-too-frequent, non-overlapping interval. We can afford to measure duration directly // with gauges, and compute quantile with quantile_over_time. @@ -116,13 +131,13 @@ func (c *JobCollector) Register() { } // Before collects for the upcoming request. -func (c *JobCollector) Before(method string, start time.Time) { +func (c *JobCollector) Before(ctx context.Context, method string, start time.Time) { c.start.WithLabelValues(method).Set(float64(start.UTC().Unix())) c.started.WithLabelValues(method).Inc() } // After collects when the request is done. -func (c *JobCollector) After(method, statusCode string, start time.Time) { +func (c *JobCollector) After(ctx context.Context, method, statusCode string, start time.Time) { end := time.Now() c.end.WithLabelValues(method, statusCode).Set(float64(end.UTC().Unix())) c.duration.WithLabelValues(method, statusCode).Set(end.Sub(start).Seconds()) @@ -148,9 +163,9 @@ func CollectedRequest(ctx context.Context, method string, col Collector, toStatu } start := time.Now() - col.Before(method, start) + col.Before(newCtx, method, start) err := f(newCtx) - col.After(method, toStatusCode(err), start) + col.After(newCtx, method, toStatusCode(err), start) if err != nil { if !grpc.IsCanceled(err) { diff --git a/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go b/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go index 58748d5fd4..5ced3989e8 100644 --- a/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go +++ b/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go @@ -7,11 +7,12 @@ import ( "github.com/prometheus/client_golang/prometheus" grpcUtils "github.com/weaveworks/common/grpc" "github.com/weaveworks/common/httpgrpc" + "github.com/weaveworks/common/instrument" "golang.org/x/net/context" "google.golang.org/grpc" ) -func observe(hist *prometheus.HistogramVec, method string, err error, duration time.Duration) { +func observe(ctx context.Context, hist *prometheus.HistogramVec, method string, err error, duration time.Duration) { respStatus := "success" if err != nil { if errResp, ok := httpgrpc.HTTPResponseFromError(err); ok { @@ -22,7 +23,7 @@ func observe(hist *prometheus.HistogramVec, method string, err error, duration t respStatus = "error" } } - hist.WithLabelValues(gRPC, method, respStatus, "false").Observe(duration.Seconds()) + instrument.ObserveWithExemplar(ctx, hist.WithLabelValues(gRPC, method, respStatus, "false"), duration.Seconds()) } // UnaryServerInstrumentInterceptor instruments gRPC requests for errors and latency. @@ -30,7 +31,7 @@ func UnaryServerInstrumentInterceptor(hist *prometheus.HistogramVec) grpc.UnaryS return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { begin := time.Now() resp, err := handler(ctx, req) - observe(hist, info.FullMethod, err, time.Since(begin)) + observe(ctx, hist, info.FullMethod, err, time.Since(begin)) return resp, err } } @@ -40,7 +41,7 @@ func StreamServerInstrumentInterceptor(hist *prometheus.HistogramVec) grpc.Strea return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { begin := time.Now() err := handler(srv, ss) - observe(hist, info.FullMethod, err, time.Since(begin)) + observe(ss.Context(), hist, info.FullMethod, err, time.Since(begin)) return err } } diff --git a/vendor/github.com/weaveworks/common/middleware/http_tracing.go b/vendor/github.com/weaveworks/common/middleware/http_tracing.go index dfcafa8eba..ded22aaa51 100644 --- a/vendor/github.com/weaveworks/common/middleware/http_tracing.go +++ b/vendor/github.com/weaveworks/common/middleware/http_tracing.go @@ -6,8 +6,6 @@ import ( "github.com/opentracing-contrib/go-stdlib/nethttp" "github.com/opentracing/opentracing-go" - jaeger "github.com/uber/jaeger-client-go" - "golang.org/x/net/context" ) // Dummy dependency to enforce that we have a nethttp version newer @@ -40,32 +38,3 @@ func (t Tracer) Wrap(next http.Handler) http.Handler { return nethttp.Middleware(opentracing.GlobalTracer(), next, options...) } - -// ExtractTraceID extracts the trace id, if any from the context. -func ExtractTraceID(ctx context.Context) (string, bool) { - sp := opentracing.SpanFromContext(ctx) - if sp == nil { - return "", false - } - sctx, ok := sp.Context().(jaeger.SpanContext) - if !ok { - return "", false - } - - return sctx.TraceID().String(), true -} - -// ExtractSampledTraceID works like ExtractTraceID but the returned bool is only -// true if the returned trace id is sampled. -func ExtractSampledTraceID(ctx context.Context) (string, bool) { - sp := opentracing.SpanFromContext(ctx) - if sp == nil { - return "", false - } - sctx, ok := sp.Context().(jaeger.SpanContext) - if !ok { - return "", false - } - - return sctx.TraceID().String(), sctx.IsSampled() -} diff --git a/vendor/github.com/weaveworks/common/middleware/instrument.go b/vendor/github.com/weaveworks/common/middleware/instrument.go index 9b9ea2299b..ac1f76f007 100644 --- a/vendor/github.com/weaveworks/common/middleware/instrument.go +++ b/vendor/github.com/weaveworks/common/middleware/instrument.go @@ -10,6 +10,8 @@ import ( "github.com/felixge/httpsnoop" "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus" + + "github.com/weaveworks/common/instrument" ) const mb = 1024 * 1024 @@ -71,17 +73,7 @@ func (i Instrument) Wrap(next http.Handler) http.Handler { i.RequestBodySize.WithLabelValues(r.Method, route).Observe(float64(rBody.read)) i.ResponseBodySize.WithLabelValues(r.Method, route).Observe(float64(respMetrics.Written)) - histogram := i.Duration.WithLabelValues(r.Method, route, strconv.Itoa(respMetrics.Code), isWS) - if traceID, ok := ExtractSampledTraceID(r.Context()); ok { - // Need to type-convert the Observer to an - // ExemplarObserver. This will always work for a - // HistogramVec. - histogram.(prometheus.ExemplarObserver).ObserveWithExemplar( - respMetrics.Duration.Seconds(), prometheus.Labels{"traceID": traceID}, - ) - return - } - histogram.Observe(respMetrics.Duration.Seconds()) + instrument.ObserveWithExemplar(r.Context(), i.Duration.WithLabelValues(r.Method, route, strconv.Itoa(respMetrics.Code), isWS), respMetrics.Duration.Seconds()) }) } diff --git a/vendor/github.com/weaveworks/common/middleware/logging.go b/vendor/github.com/weaveworks/common/middleware/logging.go index 00270df95f..81e1e48a27 100644 --- a/vendor/github.com/weaveworks/common/middleware/logging.go +++ b/vendor/github.com/weaveworks/common/middleware/logging.go @@ -8,6 +8,7 @@ import ( "time" "github.com/weaveworks/common/logging" + "github.com/weaveworks/common/tracing" "github.com/weaveworks/common/user" ) @@ -21,7 +22,7 @@ type Log struct { // logWithRequest information from the request and context as fields. func (l Log) logWithRequest(r *http.Request) logging.Interface { localLog := l.Log - traceID, ok := ExtractTraceID(r.Context()) + traceID, ok := tracing.ExtractTraceID(r.Context()) if ok { localLog = localLog.WithField("traceID", traceID) } diff --git a/vendor/github.com/weaveworks/common/server/server.go b/vendor/github.com/weaveworks/common/server/server.go index 0cf936dbbc..ee6cafee16 100644 --- a/vendor/github.com/weaveworks/common/server/server.go +++ b/vendor/github.com/weaveworks/common/server/server.go @@ -171,8 +171,12 @@ func New(cfg Config) (*Server, error) { }, []string{"protocol"}) prometheus.MustRegister(tcpConnections) + network := cfg.HTTPListenNetwork + if network == "" { + network = DefaultNetwork + } // Setup listeners first, so we can fail early if the port is in use. - httpListener, err := net.Listen(cfg.HTTPListenNetwork, fmt.Sprintf("%s:%d", cfg.HTTPListenAddress, cfg.HTTPListenPort)) + httpListener, err := net.Listen(network, fmt.Sprintf("%s:%d", cfg.HTTPListenAddress, cfg.HTTPListenPort)) if err != nil { return nil, err } @@ -182,7 +186,11 @@ func New(cfg Config) (*Server, error) { httpListener = netutil.LimitListener(httpListener, cfg.HTTPConnLimit) } - grpcListener, err := net.Listen(cfg.HTTPListenNetwork, fmt.Sprintf("%s:%d", cfg.GRPCListenAddress, cfg.GRPCListenPort)) + network = cfg.GRPCListenNetwork + if network == "" { + network = DefaultNetwork + } + grpcListener, err := net.Listen(network, fmt.Sprintf("%s:%d", cfg.GRPCListenAddress, cfg.GRPCListenPort)) if err != nil { return nil, err } @@ -258,8 +266,8 @@ func New(cfg Config) (*Server, error) { } grpcMiddleware := []grpc.UnaryServerInterceptor{ serverLog.UnaryServerInterceptor, - middleware.UnaryServerInstrumentInterceptor(requestDuration), otgrpc.OpenTracingServerInterceptor(opentracing.GlobalTracer()), + middleware.UnaryServerInstrumentInterceptor(requestDuration), } grpcMiddleware = append(grpcMiddleware, cfg.GRPCMiddleware...) diff --git a/vendor/github.com/weaveworks/common/tracing/tracing.go b/vendor/github.com/weaveworks/common/tracing/tracing.go index 7eb1c9b1f6..55fdfde972 100644 --- a/vendor/github.com/weaveworks/common/tracing/tracing.go +++ b/vendor/github.com/weaveworks/common/tracing/tracing.go @@ -1,9 +1,12 @@ package tracing import ( + "context" "io" + "github.com/opentracing/opentracing-go" "github.com/pkg/errors" + jaeger "github.com/uber/jaeger-client-go" jaegercfg "github.com/uber/jaeger-client-go/config" jaegerprom "github.com/uber/jaeger-lib/metrics/prometheus" ) @@ -45,3 +48,32 @@ func NewFromEnv(serviceName string, options ...jaegercfg.Option) (io.Closer, err return installJaeger(serviceName, cfg, options...) } + +// ExtractTraceID extracts the trace id, if any from the context. +func ExtractTraceID(ctx context.Context) (string, bool) { + sp := opentracing.SpanFromContext(ctx) + if sp == nil { + return "", false + } + sctx, ok := sp.Context().(jaeger.SpanContext) + if !ok { + return "", false + } + + return sctx.TraceID().String(), true +} + +// ExtractSampledTraceID works like ExtractTraceID but the returned bool is only +// true if the returned trace id is sampled. +func ExtractSampledTraceID(ctx context.Context) (string, bool) { + sp := opentracing.SpanFromContext(ctx) + if sp == nil { + return "", false + } + sctx, ok := sp.Context().(jaeger.SpanContext) + if !ok { + return "", false + } + + return sctx.TraceID().String(), sctx.IsSampled() +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 8c76fc7b35..667479ce40 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -672,7 +672,7 @@ github.com/uber/jaeger-client-go/utils # github.com/uber/jaeger-lib v2.4.1+incompatible github.com/uber/jaeger-lib/metrics github.com/uber/jaeger-lib/metrics/prometheus -# github.com/weaveworks/common v0.0.0-20210722103813-e649eff5ab4a +# github.com/weaveworks/common v0.0.0-20210901124008-1fa3f9fa874c ## explicit github.com/weaveworks/common/aws github.com/weaveworks/common/errors