Prometheus Metrics for Mux Router

go.mod

@@ -5,21 +5,15 @@ go 1.16
 require (
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/Masterminds/squirrel v1.5.1
-	github.com/Microsoft/go-winio v0.5.0 // indirect
 	github.com/aidarkhanov/nanoid v1.0.8
 	github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 // indirect
-	github.com/cockroachdb/cockroach-go v0.0.0-20190925194419-606b3d062051 // indirect
-	github.com/containerd/containerd v1.5.7 // indirect
-	github.com/coreos/etcd v3.3.13+incompatible // indirect
 	github.com/docker/go-units v0.4.0
 	github.com/getkin/kin-openapi v0.80.0
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-openapi/swag v0.19.15 // indirect
 	github.com/go-resty/resty/v2 v2.6.0
 	github.com/golang-migrate/migrate/v4 v4.15.1
-	github.com/google/go-github v17.0.0+incompatible // indirect
 	github.com/google/uuid v1.3.0
-	github.com/gorilla/context v1.1.1 // indirect
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -37,6 +31,7 @@ require (
 	github.com/onsi/ginkgo v1.16.1 // indirect
 	github.com/onsi/gomega v1.11.0 // indirect
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.8.0
 	github.com/qeesung/image2ascii v1.0.1
 	github.com/rs/cors v1.8.0
 	github.com/sirupsen/logrus v1.8.1
@@ -45,15 +40,14 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/wayneashleyberry/terminal-dimensions v1.0.0 // indirect
 	github.com/x-cray/logrus-prefixed-formatter v0.5.2
-	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
-	github.com/xdg/stringprep v1.0.0 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0
+	gitlab.com/msvechla/mux-prometheus v0.0.2
 	go.uber.org/atomic v1.9.0 // indirect
 	golang.org/x/net v0.0.0-20211020060615-d418f374d309 // indirect
 	golang.org/x/sys v0.0.0-20211025112917-711f33c9992c // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7
-	google.golang.org/grpc v1.41.0 // indirect
+	golang.org/x/tools v0.1.7 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )

internal/apiserver/metrics_server.go

@@ -0,0 +1,31 @@
+// Copyright © 2021 Kaleido, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package apiserver
+
+import (
+	"github.com/hyperledger/firefly/internal/config"
+)
+
+const (
+	MetricsEnabled = "enabled"
+	MetricsPath    = "path"
+)
+
+func initMetricsConfPrefix(prefix config.Prefix) {
+	prefix.AddKnownKey(MetricsEnabled, true)
+	prefix.AddKnownKey(MetricsPath, "/metrics")
+}

internal/apiserver/server.go

@@ -29,8 +29,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/hyperledger/firefly/internal/metrics"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
 	"github.com/ghodss/yaml"
 	"github.com/gorilla/mux"
+
 	"github.com/hyperledger/firefly/internal/config"
 	"github.com/hyperledger/firefly/internal/events/eifactory"
 	"github.com/hyperledger/firefly/internal/events/websockets"
@@ -40,13 +44,16 @@ import (
 	"github.com/hyperledger/firefly/internal/orchestrator"
 	"github.com/hyperledger/firefly/pkg/database"
 	"github.com/hyperledger/firefly/pkg/fftypes"
+	"github.com/prometheus/client_golang/prometheus"
+	muxprom "gitlab.com/msvechla/mux-prometheus/pkg/middleware"
 )
 
 var ffcodeExtractor = regexp.MustCompile(`^(FF\d+):`)
 
 var (
-	adminConfigPrefix = config.NewPluginConfig("admin")
-	apiConfigPrefix   = config.NewPluginConfig("http")
+	adminConfigPrefix   = config.NewPluginConfig("admin")
+	apiConfigPrefix     = config.NewPluginConfig("http")
+	metricsConfigPrefix = config.NewPluginConfig("metrics")
 )
 
 // Server is the external interface for the API Server
@@ -61,11 +68,14 @@ type apiServer struct {
 	maxFilterSkip      uint64
 	apiTimeout         time.Duration
 	apiMaxTimeout      time.Duration
+	metricsEnabled     bool
 }
 
 func InitConfig() {
 	initHTTPConfPrefx(apiConfigPrefix, 5000)
 	initHTTPConfPrefx(adminConfigPrefix, 5001)
+	initHTTPConfPrefx(metricsConfigPrefix, 6000)
+	initMetricsConfPrefix(metricsConfigPrefix)
 }
 
 func NewAPIServer() Server {
@@ -75,13 +85,15 @@ func NewAPIServer() Server {
 		maxFilterSkip:      uint64(config.GetUint(config.APIMaxFilterSkip)),
 		apiTimeout:         config.GetDuration(config.APIRequestTimeout),
 		apiMaxTimeout:      config.GetDuration(config.APIRequestMaxTimeout),
+		metricsEnabled:     config.GetBool(config.MetricsEnabled),
 	}
 }
 
 // Serve is the main entry point for the API Server
 func (as *apiServer) Serve(ctx context.Context, o orchestrator.Orchestrator) (err error) {
 	httpErrChan := make(chan error)
 	adminErrChan := make(chan error)
+	metricsErrChan := make(chan error)
 
 	if !o.IsPreInit() {
 		apiHTTPServer, err := newHTTPServer(ctx, "api", as.createMuxRouter(ctx, o), httpErrChan, apiConfigPrefix)
@@ -99,15 +111,25 @@ func (as *apiServer) Serve(ctx context.Context, o orchestrator.Orchestrator) (er
 		go adminHTTPServer.serveHTTP(ctx)
 	}
 
-	return as.waitForServerStop(httpErrChan, adminErrChan)
+	if as.metricsEnabled {
+		metricsHTTPServer, err := newHTTPServer(ctx, "metrics", as.createMetricsMuxRouter(ctx), metricsErrChan, metricsConfigPrefix)
+		if err != nil {
+			return err
+		}
+		go metricsHTTPServer.serveHTTP(ctx)
+	}
+
+	return as.waitForServerStop(httpErrChan, adminErrChan, metricsErrChan)
 }
 
-func (as *apiServer) waitForServerStop(httpErrChan, adminErrChan chan error) error {
+func (as *apiServer) waitForServerStop(httpErrChan, adminErrChan, metricsErrChan chan error) error {
 	select {
 	case err := <-httpErrChan:
 		return err
 	case err := <-adminErrChan:
 		return err
+	case err := <-metricsErrChan:
+		return err
 	}
 }
 
@@ -413,8 +435,24 @@ func (as *apiServer) swaggerHandler(routes []*oapispec.Route, url string) func(r
 	}
 }
 
+func (as *apiServer) configurePrometheusInstrumentation(namespace, subsystem string, r *mux.Router) {
+	if as.metricsEnabled {
+		instrumentation := muxprom.NewCustomInstrumentation(
+			true,
+			namespace,
+			subsystem,
+			prometheus.DefBuckets,
+			map[string]string{},
+			metrics.Registry(),
+		)
+		r.Use(instrumentation.Middleware)
+	}
+}
+
 func (as *apiServer) createMuxRouter(ctx context.Context, o orchestrator.Orchestrator) *mux.Router {
 	r := mux.NewRouter()
+	as.configurePrometheusInstrumentation("apiserver", "rest", r)
+
 	for _, route := range routes {
 		if route.JSONHandler != nil {
 			r.HandleFunc(fmt.Sprintf("/api/v1/%s", route.Path), as.routeHandler(o, route)).
@@ -440,6 +478,8 @@ func (as *apiServer) createMuxRouter(ctx context.Context, o orchestrator.Orchest
 
 func (as *apiServer) createAdminMuxRouter(o orchestrator.Orchestrator) *mux.Router {
 	r := mux.NewRouter()
+	as.configurePrometheusInstrumentation("apiserver", "admin", r)
+
 	for _, route := range adminRoutes {
 		if route.JSONHandler != nil {
 			r.HandleFunc(fmt.Sprintf("/admin/api/v1/%s", route.Path), as.routeHandler(o, route)).
@@ -453,3 +493,12 @@ func (as *apiServer) createAdminMuxRouter(o orchestrator.Orchestrator) *mux.Rout
 
 	return r
 }
+
+func (as *apiServer) createMetricsMuxRouter(_ context.Context) *mux.Router {
+	r := mux.NewRouter()
+
+	r.Path(config.GetString(config.MetricsPath)).Handler(promhttp.InstrumentMetricHandler(metrics.Registry(),
+		promhttp.HandlerFor(metrics.Registry(), promhttp.HandlerOpts{})))
+
+	return r
+}

internal/apiserver/server_test.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/hyperledger/firefly/internal/metrics"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -61,6 +62,7 @@ func newTestAdminServer() (*orchestratormocks.Orchestrator, *mux.Router) {
 
 func TestStartStopServer(t *testing.T) {
 	config.Reset()
+	metrics.Clear()
 	InitConfig()
 	apiConfigPrefix.Set(HTTPConfPort, 0)
 	adminConfigPrefix.Set(HTTPConfPort, 0)
@@ -77,6 +79,7 @@ func TestStartStopServer(t *testing.T) {
 
 func TestStartAPIFail(t *testing.T) {
 	config.Reset()
+	metrics.Clear()
 	InitConfig()
 	apiConfigPrefix.Set(HTTPConfAddress, "...://")
 	ctx, cancel := context.WithCancel(context.Background())
@@ -90,6 +93,7 @@ func TestStartAPIFail(t *testing.T) {
 
 func TestStartAdminFail(t *testing.T) {
 	config.Reset()
+	metrics.Clear()
 	InitConfig()
 	adminConfigPrefix.Set(HTTPConfAddress, "...://")
 	config.Set(config.AdminEnabled, true)
@@ -375,16 +379,20 @@ func TestWaitForServerStop(t *testing.T) {
 
 	chl1 := make(chan error, 1)
 	chl2 := make(chan error, 1)
+	chl3 := make(chan error, 1)
 	chl1 <- fmt.Errorf("pop1")
 
 	as := &apiServer{}
-	err := as.waitForServerStop(chl1, chl2)
+	err := as.waitForServerStop(chl1, chl2, chl3)
 	assert.EqualError(t, err, "pop1")
 
 	chl2 <- fmt.Errorf("pop2")
-	err = as.waitForServerStop(chl1, chl2)
+	err = as.waitForServerStop(chl1, chl2, chl3)
 	assert.EqualError(t, err, "pop2")
 
+	chl3 <- fmt.Errorf("pop3")
+	err = as.waitForServerStop(chl1, chl2, chl3)
+	assert.EqualError(t, err, "pop3")
 }
 
 func TestGetTimeoutMax(t *testing.T) {

internal/config/config.go

@@ -40,7 +40,7 @@ import (
 )
 
 // The following keys can be access from the root configuration.
-// Plugins are resonsible for defining their own keys using the Config interface
+// Plugins are responsible for defining their own keys using the Config interface
 var (
 	// APIDefaultFilterLimit is the default limit that will be applied to filtered queries on the API
 	APIDefaultFilterLimit = rootKey("api.defaultFilterLimit")
@@ -148,7 +148,7 @@ var (
 	GroupCacheSize = rootKey("group.cache.size")
 	// GroupCacheTTL cache time-to-live for private group addresses
 	GroupCacheTTL = rootKey("group.cache.ttl")
-	// AdminHTTPEnabled determines whether the admin interface will be enabled or not
+	// AdminEnabled determines whether the admin interface will be enabled or not
 	AdminEnabled = rootKey("admin.enabled")
 	// AdminPreinit waits for at least one ConfigREcord to be posted to the server before it starts (the database must be available on startup)
 	AdminPreinit = rootKey("admin.preinit")
@@ -180,6 +180,10 @@ var (
 	LogMaxAge = rootKey("log.maxAge")
 	// LogCompress sets whether to compress backups
 	LogCompress = rootKey("log.compress")
+	// MetricsEnabled determines whether metrics will be instrumented and if the metrics server will be enabled or not
+	MetricsEnabled = rootKey("metrics.enabled")
+	// MetricsPath determines what path to serve the Prometheus metrics from
+	MetricsPath = rootKey("metrics.path")
 	// NamespacesDefault is the default namespace - must be in the predefines list
 	NamespacesDefault = rootKey("namespaces.default")
 	// NamespacesPredefined is a list of namespaces to ensure exists, without requiring a broadcast from the network

internal/metrics/metrics.go

@@ -0,0 +1,37 @@
+// Copyright © 2021 Kaleido, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var registry *prometheus.Registry
+
+// Registry returns FireFly's customized Prometheus registry
+func Registry() *prometheus.Registry {
+	if registry == nil {
+		registry = prometheus.NewRegistry()
+		registry.MustRegister(prometheus.NewGoCollector())
+		registry.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
+	}
+
+	return registry
+}
+
+// Clear will reset the Prometheus metrics registry, useful for testing
+func Clear() {
+	registry = nil
+}