Merge pull request #2466 from tifayuki/prometheus_go_metrics

add prometheus metrics

cmd/registry/config-dev.yml

@@ -31,7 +31,10 @@ storage:
 http:
     addr: :5000
     debug:
-        addr: localhost:5001
+        addr: :5001
+        prometheus:
+            enabled: true
+            path: /metrics
     headers:
         X-Content-Type-Options: [nosniff]
 redis:

configuration/configuration.go

@@ -133,6 +133,11 @@ type Configuration struct {
 		Debug struct {
 			// Addr specifies the bind address for the debug server.
 			Addr string `yaml:"addr,omitempty"`
+			// Prometheus configures the Prometheus telemetry endpoint.
+			Prometheus struct {
+				Enabled bool   `yaml:"enabled,omitempty"`
+				Path    string `yaml:"path,omitempty"`
+			} `yaml:"prometheus,omitempty"`
 		} `yaml:"debug,omitempty"`
 
 		// HTTP2 configuration options

configuration/configuration_test.go

@@ -85,7 +85,11 @@ var configStruct = Configuration{
 		} `yaml:"tls,omitempty"`
 		Headers http.Header `yaml:"headers,omitempty"`
 		Debug   struct {
-			Addr string `yaml:"addr,omitempty"`
+			Addr       string `yaml:"addr,omitempty"`
+			Prometheus struct {
+				Enabled bool   `yaml:"enabled,omitempty"`
+				Path    string `yaml:"path,omitempty"`
+			} `yaml:"prometheus,omitempty"`
 		} `yaml:"debug,omitempty"`
 		HTTP2 struct {
 			Disabled bool `yaml:"disabled,omitempty"`

docs/configuration.md

@@ -218,6 +218,9 @@ http:
       hosts: [myregistryaddress.org]
   debug:
     addr: localhost:5001
+    prometheus:
+      enabled: true
+      path: /metrics
   headers:
     X-Content-Type-Options: [nosniff]
   http2:
@@ -805,6 +808,19 @@ access to the debug endpoint is locked down in a production environment.
 The `debug` section takes a single required `addr` parameter, which specifies
 the `HOST:PORT` on which the debug server should accept connections.
 
+## `prometheus`
+
+The `prometheus` option defines whether the prometheus metrics is enable, as well
+as the path to access the metrics.
+
+| Parameter | Required | Description                                           |
+|-----------|----------|-------------------------------------------------------|
+| `enabled` | no       | Set `true` to enable the prometheus server            |
+| `path`    | no       | The path to access the metrics, `/metrics` by default |
+
+The url to access the metrics is `HOST:PORT/path`, where `HOST:PORT` is defined
+in `addr` under `debug`.
+
 ### `headers`
 
 The `headers` option is **optional** . Use it to specify headers that the HTTP

metrics/prometheus.go

@@ -0,0 +1,13 @@
+package metrics
+
+import "github.com/docker/go-metrics"
+
+const (
+	// NamespacePrefix is the namespace of prometheus metrics
+	NamespacePrefix = "registry"
+)
+
+var (
+	// StorageNamespace is the prometheus namespace of blob/cache related operations
+	StorageNamespace = metrics.NewNamespace(NamespacePrefix, "storage", nil)
+)

registry/handlers/app.go

@@ -20,6 +20,7 @@ import (
 	dcontext "github.com/docker/distribution/context"
 	"github.com/docker/distribution/health"
 	"github.com/docker/distribution/health/checks"
+	prometheus "github.com/docker/distribution/metrics"
 	"github.com/docker/distribution/notifications"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/distribution/registry/api/errcode"
@@ -35,6 +36,7 @@ import (
 	"github.com/docker/distribution/registry/storage/driver/factory"
 	storagemiddleware "github.com/docker/distribution/registry/storage/driver/middleware"
 	"github.com/docker/distribution/version"
+	"github.com/docker/go-metrics"
 	"github.com/docker/libtrust"
 	"github.com/garyburd/redigo/redis"
 	"github.com/gorilla/mux"
@@ -412,14 +414,23 @@ func (app *App) RegisterHealthChecks(healthRegistries ...*health.Registry) {
 // passed through the application filters and context will be constructed at
 // request time.
 func (app *App) register(routeName string, dispatch dispatchFunc) {
+	handler := app.dispatcher(dispatch)
+
+	// Chain the handler with prometheus instrumented handler
+	if app.Config.HTTP.Debug.Prometheus.Enabled {
+		namespace := metrics.NewNamespace(prometheus.NamespacePrefix, "http", nil)
+		httpMetrics := namespace.NewDefaultHttpMetrics(strings.Replace(routeName, "-", "_", -1))
+		metrics.Register(namespace)
+		handler = metrics.InstrumentHandler(httpMetrics, handler)
+	}
 
 	// TODO(stevvooe): This odd dispatcher/route registration is by-product of
 	// some limitations in the gorilla/mux router. We are using it to keep
 	// routing consistent between the client and server, but we may want to
 	// replace it with manual routing and structure-based dispatch for better
 	// control over the request execution.
 
-	app.router.GetRoute(routeName).Handler(app.dispatcher(dispatch))
+	app.router.GetRoute(routeName).Handler(handler)
 }
 
 // configureEvents prepares the event sink for action.

registry/registry.go

@@ -21,6 +21,7 @@ import (
 	"github.com/docker/distribution/registry/listener"
 	"github.com/docker/distribution/uuid"
 	"github.com/docker/distribution/version"
+	"github.com/docker/go-metrics"
 	gorhandlers "github.com/gorilla/handlers"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -58,6 +59,15 @@ var ServeCmd = &cobra.Command{
 			log.Fatalln(err)
 		}
 
+		if config.HTTP.Debug.Prometheus.Enabled {
+			path := config.HTTP.Debug.Prometheus.Path
+			if path == "" {
+				path = "/metrics"
+			}
+			log.Info("providing prometheus metrics on ", path)
+			http.Handle(path, metrics.Handler())
+		}
+
 		if err = registry.ListenAndServe(); err != nil {
 			log.Fatalln(err)
 		}

registry/storage/cache/cachedblobdescriptorstore.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/docker/distribution"
+	prometheus "github.com/docker/distribution/metrics"
 	"github.com/opencontainers/go-digest"
 )
 
@@ -38,6 +39,11 @@ type cachedBlobStatter struct {
 	tracker MetricsTracker
 }
 
+var (
+	// cacheCount is the number of total cache request received/hits/misses
+	cacheCount = prometheus.StorageNamespace.NewLabeledCounter("cache", "The number of cache request received", "type")
+)
+
 // NewCachedBlobStatter creates a new statter which prefers a cache and
 // falls back to a backend.
 func NewCachedBlobStatter(cache distribution.BlobDescriptorService, backend distribution.BlobDescriptorService) distribution.BlobDescriptorService {
@@ -58,6 +64,7 @@ func NewCachedBlobStatterWithMetrics(cache distribution.BlobDescriptorService, b
 }
 
 func (cbds *cachedBlobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
+	cacheCount.WithValues("Request").Inc(1)
 	desc, err := cbds.cache.Stat(ctx, dgst)
 	if err != nil {
 		if err != distribution.ErrBlobUnknown {
@@ -66,12 +73,13 @@ func (cbds *cachedBlobStatter) Stat(ctx context.Context, dgst digest.Digest) (di
 
 		goto fallback
 	}
-
+	cacheCount.WithValues("Hit").Inc(1)
 	if cbds.tracker != nil {
 		cbds.tracker.Hit()
 	}
 	return desc, nil
 fallback:
+	cacheCount.WithValues("Miss").Inc(1)
 	if cbds.tracker != nil {
 		cbds.tracker.Miss()
 	}

registry/storage/driver/base/base.go

@@ -42,9 +42,21 @@ import (
 	"io"
 
 	dcontext "github.com/docker/distribution/context"
+	prometheus "github.com/docker/distribution/metrics"
 	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/go-metrics"
+	"time"
 )
 
+var (
+	// storageAction is the metrics of blob related operations
+	storageAction = prometheus.StorageNamespace.NewLabeledTimer("action", "The number of seconds that the storage action takes", "driver", "action")
+)
+
+func init() {
+	metrics.Register(prometheus.StorageNamespace)
+}
+
 // Base provides a wrapper around a storagedriver implementation that provides
 // common path and bounds checking.
 type Base struct {
@@ -87,7 +99,9 @@ func (base *Base) GetContent(ctx context.Context, path string) ([]byte, error) {
 		return nil, storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()}
 	}
 
+	start := time.Now()
 	b, e := base.StorageDriver.GetContent(ctx, path)
+	storageAction.WithValues(base.Name(), "GetContent").UpdateSince(start)
 	return b, base.setDriverName(e)
 }
 
@@ -100,7 +114,10 @@ func (base *Base) PutContent(ctx context.Context, path string, content []byte) e
 		return storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()}
 	}
 
-	return base.setDriverName(base.StorageDriver.PutContent(ctx, path, content))
+	start := time.Now()
+	err := base.setDriverName(base.StorageDriver.PutContent(ctx, path, content))
+	storageAction.WithValues(base.Name(), "PutContent").UpdateSince(start)
+	return err
 }
 
 // Reader wraps Reader of underlying storage driver.
@@ -142,7 +159,9 @@ func (base *Base) Stat(ctx context.Context, path string) (storagedriver.FileInfo
 		return nil, storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()}
 	}
 
+	start := time.Now()
 	fi, e := base.StorageDriver.Stat(ctx, path)
+	storageAction.WithValues(base.Name(), "Stat").UpdateSince(start)
 	return fi, base.setDriverName(e)
 }
 
@@ -155,7 +174,9 @@ func (base *Base) List(ctx context.Context, path string) ([]string, error) {
 		return nil, storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()}
 	}
 
+	start := time.Now()
 	str, e := base.StorageDriver.List(ctx, path)
+	storageAction.WithValues(base.Name(), "List").UpdateSince(start)
 	return str, base.setDriverName(e)
 }
 
@@ -170,7 +191,10 @@ func (base *Base) Move(ctx context.Context, sourcePath string, destPath string)
 		return storagedriver.InvalidPathError{Path: destPath, DriverName: base.StorageDriver.Name()}
 	}
 
-	return base.setDriverName(base.StorageDriver.Move(ctx, sourcePath, destPath))
+	start := time.Now()
+	err := base.setDriverName(base.StorageDriver.Move(ctx, sourcePath, destPath))
+	storageAction.WithValues(base.Name(), "Move").UpdateSince(start)
+	return err
 }
 
 // Delete wraps Delete of underlying storage driver.
@@ -182,7 +206,10 @@ func (base *Base) Delete(ctx context.Context, path string) error {
 		return storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()}
 	}
 
-	return base.setDriverName(base.StorageDriver.Delete(ctx, path))
+	start := time.Now()
+	err := base.setDriverName(base.StorageDriver.Delete(ctx, path))
+	storageAction.WithValues(base.Name(), "Delete").UpdateSince(start)
+	return err
 }
 
 // URLFor wraps URLFor of underlying storage driver.
@@ -194,7 +221,9 @@ func (base *Base) URLFor(ctx context.Context, path string, options map[string]in
 		return "", storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()}
 	}
 
+	start := time.Now()
 	str, e := base.StorageDriver.URLFor(ctx, path, options)
+	storageAction.WithValues(base.Name(), "URLFor").UpdateSince(start)
 	return str, base.setDriverName(e)
 }
 

vendor.conf

@@ -3,11 +3,13 @@ github.com/Azure/go-autorest ec5f4903f77ed9927ac95b19ab8e44ada64c1356
 github.com/sirupsen/logrus 3d4380f53a34dcdc95f0c1db702615992b38d9a4
 github.com/aws/aws-sdk-go 5bcc0a238d880469f949fc7cd24e35f32ab80cbd
 github.com/bshuster-repo/logrus-logstash-hook d2c0ecc1836d91814e15e23bb5dc309c3ef51f4a
+github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9
 github.com/bugsnag/bugsnag-go b1d153021fcd90ca3f080db36bec96dc690fb274
 github.com/bugsnag/osext 0dd3f918b21bec95ace9dc86c7e70266cfc5c702
 github.com/bugsnag/panicwrap e2c28503fcd0675329da73bf48b33404db873782
 github.com/denverdino/aliyungo afedced274aa9a7fcdd47ac97018f0f8db4e5de2
 github.com/dgrijalva/jwt-go a601269ab70c205d26370c16f7c81e9017c14e04
+github.com/docker/go-metrics 399ea8c73916000c64c2c76e8da00ca82f8387ab
 github.com/docker/goamz f0a21f5b2e12f83a505ecf79b633bb2035cf6f85
 github.com/docker/libtrust fa567046d9b14f6aa788882a950d69651d230b21
 github.com/garyburd/redigo 535138d7bcd717d6531c701ef5933d98b1866257
@@ -18,9 +20,14 @@ github.com/gorilla/handlers 60c7bfde3e33c201519a200a4507a158cc03a17b
 github.com/gorilla/mux 599cba5e7b6137d46ddf58fb1765f5d928e69604
 github.com/inconshreveable/mousetrap 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
 github.com/jmespath/go-jmespath bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d
+github.com/matttproud/golang_protobuf_extensions c12348ce28de40eed0136aa2b644d0ee0650e56c
 github.com/miekg/dns 271c58e0c14f552178ea321a545ff9af38930f39
 github.com/mitchellh/mapstructure 482a9fd5fa83e8c4e7817413b80f3eb8feec03ef
 github.com/ncw/swift b964f2ca856aac39885e258ad25aec08d5f64ee6
+github.com/prometheus/client_golang c332b6f63c0658a65eca15c0e5247ded801cf564
+github.com/prometheus/client_model 99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c
+github.com/prometheus/common 89604d197083d4781071d3c65855d24ecfb0a563
+github.com/prometheus/procfs cb4147076ac75738c9a7d279075a253c0cc5acbd
 github.com/spf13/cobra 312092086bed4968099259622145a0c9ae280064
 github.com/spf13/pflag 5644820622454e71517561946e3d94b9f9db6842
 github.com/stevvooe/resumable 2aaf90b2ceea5072cb503ef2a620b08ff3119870
@@ -40,4 +47,4 @@ gopkg.in/check.v1 64131543e7896d5bcc6bd5a76287eb75ea96c673
 gopkg.in/square/go-jose.v1 40d457b439244b546f023d056628e5184136899b
 gopkg.in/yaml.v2 bef53efd0c76e49e6de55ead051f886bea7e9420
 rsc.io/letsencrypt e770c10b0f1a64775ae91d240407ce00d1a5bdeb https://github.com/dmcgowan/letsencrypt.git
-github.com/opencontainers/go-digest a6d0ee40d4207ea02364bd3b9e8e77b9159ba1eb
+github.com/opencontainers/go-digest a6d0ee40d4207ea02364bd3b9e8e77b9159ba1eb