Metrics Server and Config support

Signed-off-by: hfuss <haydenfuss@gmail.com>

internal/apiserver/metrics_server.go

@@ -0,0 +1,15 @@
+package apiserver
+
+import (
+	"github.com/hyperledger/firefly/internal/config"
+)
+
+const (
+	MetricsEnabled = "enabled"
+	MetricsPath    = "path"
+)
+
+func initMetricsConfPrefix(prefix config.Prefix) {
+	prefix.AddKnownKey(MetricsEnabled, true)
+	prefix.AddKnownKey(MetricsPath, "/metrics")
+}

internal/apiserver/server.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/hyperledger/firefly/internal/metrics"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"io"
 	"io/ioutil"
 	"mime/multipart"
@@ -42,15 +44,15 @@ import (
 	"github.com/hyperledger/firefly/pkg/database"
 	"github.com/hyperledger/firefly/pkg/fftypes"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	muxprom "gitlab.com/msvechla/mux-prometheus/pkg/middleware"
 )
 
 var ffcodeExtractor = regexp.MustCompile(`^(FF\d+):`)
 
 var (
-	adminConfigPrefix = config.NewPluginConfig("admin")
-	apiConfigPrefix   = config.NewPluginConfig("http")
+	adminConfigPrefix   = config.NewPluginConfig("admin")
+	apiConfigPrefix     = config.NewPluginConfig("http")
+	metricsConfigPrefix = config.NewPluginConfig("metrics")
 )
 
 // Server is the external interface for the API Server
@@ -65,12 +67,14 @@ type apiServer struct {
 	maxFilterSkip      uint64
 	apiTimeout         time.Duration
 	apiMaxTimeout      time.Duration
-	metricsRegistery   *prometheus.Registry
+	metricsEnabled     bool
 }
 
 func InitConfig() {
 	initHTTPConfPrefx(apiConfigPrefix, 5000)
 	initHTTPConfPrefx(adminConfigPrefix, 5001)
+	initHTTPConfPrefx(metricsConfigPrefix, 6000)
+	initMetricsConfPrefix(metricsConfigPrefix)
 }
 
 func NewAPIServer() Server {
@@ -80,13 +84,15 @@ func NewAPIServer() Server {
 		maxFilterSkip:      uint64(config.GetUint(config.APIMaxFilterSkip)),
 		apiTimeout:         config.GetDuration(config.APIRequestTimeout),
 		apiMaxTimeout:      config.GetDuration(config.APIRequestMaxTimeout),
+		metricsEnabled:     config.GetBool(config.MetricsEnabled),
 	}
 }
 
 // Serve is the main entry point for the API Server
 func (as *apiServer) Serve(ctx context.Context, o orchestrator.Orchestrator) (err error) {
 	httpErrChan := make(chan error)
 	adminErrChan := make(chan error)
+	metricsErrChan := make(chan error)
 
 	if !o.IsPreInit() {
 		apiHTTPServer, err := newHTTPServer(ctx, "api", as.createMuxRouter(ctx, o), httpErrChan, apiConfigPrefix)
@@ -104,15 +110,25 @@ func (as *apiServer) Serve(ctx context.Context, o orchestrator.Orchestrator) (er
 		go adminHTTPServer.serveHTTP(ctx)
 	}
 
-	return as.waitForServerStop(httpErrChan, adminErrChan)
+	if as.metricsEnabled {
+		metricsHTTPServer, err := newHTTPServer(ctx, "metrics", as.createMetricsMuxRouter(ctx), metricsErrChan, metricsConfigPrefix)
+		if err != nil {
+			return err
+		}
+		go metricsHTTPServer.serveHTTP(ctx)
+	}
+
+	return as.waitForServerStop(httpErrChan, adminErrChan, metricsErrChan)
 }
 
-func (as *apiServer) waitForServerStop(httpErrChan, adminErrChan chan error) error {
+func (as *apiServer) waitForServerStop(httpErrChan, adminErrChan, metricsErrChan chan error) error {
 	select {
 	case err := <-httpErrChan:
 		return err
 	case err := <-adminErrChan:
 		return err
+	case err := <-metricsErrChan:
+		return err
 	}
 }
 
@@ -419,22 +435,17 @@ func (as *apiServer) swaggerHandler(routes []*oapispec.Route, url string) func(r
 }
 
 func (as *apiServer) configurePrometheusInstrumentation(namespace, subsystem string, r *mux.Router) {
-	if as.metricsRegistery == nil {
-		as.metricsRegistery = prometheus.NewRegistry()
-		as.metricsRegistery.MustRegister(prometheus.NewGoCollector())
-		as.metricsRegistery.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
+	if as.metricsEnabled {
+		instrumentation := muxprom.NewCustomInstrumentation(
+			true,
+			namespace,
+			subsystem,
+			prometheus.DefBuckets,
+			map[string]string{},
+			metrics.Registry(),
+		)
+		r.Use(instrumentation.Middleware)
 	}
-	instrumentation := muxprom.NewCustomInstrumentation(
-		true,
-		namespace,
-		subsystem,
-		prometheus.DefBuckets,
-		map[string]string{},
-		as.metricsRegistery,
-	)
-	r.Use(instrumentation.Middleware)
-	r.Path("/metrics").Handler(promhttp.InstrumentMetricHandler(as.metricsRegistery,
-		promhttp.HandlerFor(as.metricsRegistery, promhttp.HandlerOpts{})))
 }
 
 func (as *apiServer) createMuxRouter(ctx context.Context, o orchestrator.Orchestrator) *mux.Router {
@@ -481,3 +492,12 @@ func (as *apiServer) createAdminMuxRouter(o orchestrator.Orchestrator) *mux.Rout
 
 	return r
 }
+
+func (as *apiServer) createMetricsMuxRouter(ctx context.Context) *mux.Router {
+	r := mux.NewRouter()
+
+	r.Path(config.GetString(config.MetricsPath)).Handler(promhttp.InstrumentMetricHandler(metrics.Registry(),
+		promhttp.HandlerFor(metrics.Registry(), promhttp.HandlerOpts{})))
+
+	return r
+}

internal/config/config.go

@@ -40,7 +40,7 @@ import (
 )
 
 // The following keys can be access from the root configuration.
-// Plugins are resonsible for defining their own keys using the Config interface
+// Plugins are responsible for defining their own keys using the Config interface
 var (
 	// APIDefaultFilterLimit is the default limit that will be applied to filtered queries on the API
 	APIDefaultFilterLimit = rootKey("api.defaultFilterLimit")
@@ -148,7 +148,7 @@ var (
 	GroupCacheSize = rootKey("group.cache.size")
 	// GroupCacheTTL cache time-to-live for private group addresses
 	GroupCacheTTL = rootKey("group.cache.ttl")
-	// AdminHTTPEnabled determines whether the admin interface will be enabled or not
+	// AdminEnabled determines whether the admin interface will be enabled or not
 	AdminEnabled = rootKey("admin.enabled")
 	// AdminPreinit waits for at least one ConfigREcord to be posted to the server before it starts (the database must be available on startup)
 	AdminPreinit = rootKey("admin.preinit")
@@ -180,6 +180,10 @@ var (
 	LogMaxAge = rootKey("log.maxAge")
 	// LogCompress sets whether to compress backups
 	LogCompress = rootKey("log.compress")
+	// MetricsEnabled determines whether metrics will be instrumented and if the metrics server will be enabled or not
+	MetricsEnabled = rootKey("metrics.enabled")
+	// MetricsPath determines what path to serve the Prometheus metrics from
+	MetricsPath = rootKey("metrics.path")
 	// NamespacesDefault is the default namespace - must be in the predefines list
 	NamespacesDefault = rootKey("namespaces.default")
 	// NamespacesPredefined is a list of namespaces to ensure exists, without requiring a broadcast from the network

internal/metrics/metrics.go

@@ -0,0 +1,21 @@
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var registry *prometheus.Registry
+
+// Registry returns FireFly's customized Prometheus registry
+func Registry() *prometheus.Registry {
+	if registry == nil {
+		registry = prometheus.NewRegistry()
+		registry.MustRegister(prometheus.NewGoCollector())
+		registry.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
+	}
+
+	return registry
+}
+
+// Clear will reset the Prometheus metrics registry, useful for testing
+func Clear() {
+	registry = nil
+}