Skip to content

Commit

Permalink
Merge pull request #321 from kaleido-io/prometheus-poc
Browse files Browse the repository at this point in the history
Prometheus Metrics for Mux Router
  • Loading branch information
peterbroadhurst committed Nov 16, 2021
2 parents feac44a + cd70f40 commit 7e4fa6a
Show file tree
Hide file tree
Showing 7 changed files with 270 additions and 131 deletions.
12 changes: 3 additions & 9 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,15 @@ go 1.16
require (
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/Masterminds/squirrel v1.5.1
github.com/Microsoft/go-winio v0.5.0 // indirect
github.com/aidarkhanov/nanoid v1.0.8
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 // indirect
github.com/cockroachdb/cockroach-go v0.0.0-20190925194419-606b3d062051 // indirect
github.com/containerd/containerd v1.5.7 // indirect
github.com/coreos/etcd v3.3.13+incompatible // indirect
github.com/docker/go-units v0.4.0
github.com/getkin/kin-openapi v0.80.0
github.com/ghodss/yaml v1.0.0
github.com/go-openapi/swag v0.19.15 // indirect
github.com/go-resty/resty/v2 v2.6.0
github.com/golang-migrate/migrate/v4 v4.15.1
github.com/google/go-github v17.0.0+incompatible // indirect
github.com/google/uuid v1.3.0
github.com/gorilla/context v1.1.1 // indirect
github.com/gorilla/mux v1.8.0
github.com/gorilla/websocket v1.4.2
github.com/hashicorp/errwrap v1.1.0 // indirect
Expand All @@ -37,6 +31,7 @@ require (
github.com/onsi/ginkgo v1.16.1 // indirect
github.com/onsi/gomega v1.11.0 // indirect
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.8.0
github.com/qeesung/image2ascii v1.0.1
github.com/rs/cors v1.8.0
github.com/sirupsen/logrus v1.8.1
Expand All @@ -45,15 +40,14 @@ require (
github.com/stretchr/testify v1.7.0
github.com/wayneashleyberry/terminal-dimensions v1.0.0 // indirect
github.com/x-cray/logrus-prefixed-formatter v0.5.2
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
github.com/xdg/stringprep v1.0.0 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonschema v1.2.0
gitlab.com/msvechla/mux-prometheus v0.0.2
go.uber.org/atomic v1.9.0 // indirect
golang.org/x/net v0.0.0-20211020060615-d418f374d309 // indirect
golang.org/x/sys v0.0.0-20211025112917-711f33c9992c // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7
google.golang.org/grpc v1.41.0 // indirect
golang.org/x/tools v0.1.7 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0
)
244 changes: 130 additions & 114 deletions go.sum

Large diffs are not rendered by default.

31 changes: 31 additions & 0 deletions internal/apiserver/metrics_server.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Copyright © 2021 Kaleido, Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package apiserver

import (
"github.com/hyperledger/firefly/internal/config"
)

const (
MetricsEnabled = "enabled"
MetricsPath = "path"
)

func initMetricsConfPrefix(prefix config.Prefix) {
prefix.AddKnownKey(MetricsEnabled, true)
prefix.AddKnownKey(MetricsPath, "/metrics")
}
57 changes: 53 additions & 4 deletions internal/apiserver/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,12 @@ import (
"strings"
"time"

"github.com/hyperledger/firefly/internal/metrics"
"github.com/prometheus/client_golang/prometheus/promhttp"

"github.com/ghodss/yaml"
"github.com/gorilla/mux"

"github.com/hyperledger/firefly/internal/config"
"github.com/hyperledger/firefly/internal/events/eifactory"
"github.com/hyperledger/firefly/internal/events/websockets"
Expand All @@ -40,13 +44,16 @@ import (
"github.com/hyperledger/firefly/internal/orchestrator"
"github.com/hyperledger/firefly/pkg/database"
"github.com/hyperledger/firefly/pkg/fftypes"
"github.com/prometheus/client_golang/prometheus"
muxprom "gitlab.com/msvechla/mux-prometheus/pkg/middleware"
)

var ffcodeExtractor = regexp.MustCompile(`^(FF\d+):`)

var (
adminConfigPrefix = config.NewPluginConfig("admin")
apiConfigPrefix = config.NewPluginConfig("http")
adminConfigPrefix = config.NewPluginConfig("admin")
apiConfigPrefix = config.NewPluginConfig("http")
metricsConfigPrefix = config.NewPluginConfig("metrics")
)

// Server is the external interface for the API Server
Expand All @@ -61,11 +68,14 @@ type apiServer struct {
maxFilterSkip uint64
apiTimeout time.Duration
apiMaxTimeout time.Duration
metricsEnabled bool
}

func InitConfig() {
initHTTPConfPrefx(apiConfigPrefix, 5000)
initHTTPConfPrefx(adminConfigPrefix, 5001)
initHTTPConfPrefx(metricsConfigPrefix, 6000)
initMetricsConfPrefix(metricsConfigPrefix)
}

func NewAPIServer() Server {
Expand All @@ -75,13 +85,15 @@ func NewAPIServer() Server {
maxFilterSkip: uint64(config.GetUint(config.APIMaxFilterSkip)),
apiTimeout: config.GetDuration(config.APIRequestTimeout),
apiMaxTimeout: config.GetDuration(config.APIRequestMaxTimeout),
metricsEnabled: config.GetBool(config.MetricsEnabled),
}
}

// Serve is the main entry point for the API Server
func (as *apiServer) Serve(ctx context.Context, o orchestrator.Orchestrator) (err error) {
httpErrChan := make(chan error)
adminErrChan := make(chan error)
metricsErrChan := make(chan error)

if !o.IsPreInit() {
apiHTTPServer, err := newHTTPServer(ctx, "api", as.createMuxRouter(ctx, o), httpErrChan, apiConfigPrefix)
Expand All @@ -99,15 +111,25 @@ func (as *apiServer) Serve(ctx context.Context, o orchestrator.Orchestrator) (er
go adminHTTPServer.serveHTTP(ctx)
}

return as.waitForServerStop(httpErrChan, adminErrChan)
if as.metricsEnabled {
metricsHTTPServer, err := newHTTPServer(ctx, "metrics", as.createMetricsMuxRouter(ctx), metricsErrChan, metricsConfigPrefix)
if err != nil {
return err
}
go metricsHTTPServer.serveHTTP(ctx)
}

return as.waitForServerStop(httpErrChan, adminErrChan, metricsErrChan)
}

func (as *apiServer) waitForServerStop(httpErrChan, adminErrChan chan error) error {
func (as *apiServer) waitForServerStop(httpErrChan, adminErrChan, metricsErrChan chan error) error {
select {
case err := <-httpErrChan:
return err
case err := <-adminErrChan:
return err
case err := <-metricsErrChan:
return err
}
}

Expand Down Expand Up @@ -413,8 +435,24 @@ func (as *apiServer) swaggerHandler(routes []*oapispec.Route, url string) func(r
}
}

func (as *apiServer) configurePrometheusInstrumentation(namespace, subsystem string, r *mux.Router) {
if as.metricsEnabled {
instrumentation := muxprom.NewCustomInstrumentation(
true,
namespace,
subsystem,
prometheus.DefBuckets,
map[string]string{},
metrics.Registry(),
)
r.Use(instrumentation.Middleware)
}
}

func (as *apiServer) createMuxRouter(ctx context.Context, o orchestrator.Orchestrator) *mux.Router {
r := mux.NewRouter()
as.configurePrometheusInstrumentation("apiserver", "rest", r)

for _, route := range routes {
if route.JSONHandler != nil {
r.HandleFunc(fmt.Sprintf("/api/v1/%s", route.Path), as.routeHandler(o, route)).
Expand All @@ -440,6 +478,8 @@ func (as *apiServer) createMuxRouter(ctx context.Context, o orchestrator.Orchest

func (as *apiServer) createAdminMuxRouter(o orchestrator.Orchestrator) *mux.Router {
r := mux.NewRouter()
as.configurePrometheusInstrumentation("apiserver", "admin", r)

for _, route := range adminRoutes {
if route.JSONHandler != nil {
r.HandleFunc(fmt.Sprintf("/admin/api/v1/%s", route.Path), as.routeHandler(o, route)).
Expand All @@ -453,3 +493,12 @@ func (as *apiServer) createAdminMuxRouter(o orchestrator.Orchestrator) *mux.Rout

return r
}

func (as *apiServer) createMetricsMuxRouter(_ context.Context) *mux.Router {
r := mux.NewRouter()

r.Path(config.GetString(config.MetricsPath)).Handler(promhttp.InstrumentMetricHandler(metrics.Registry(),
promhttp.HandlerFor(metrics.Registry(), promhttp.HandlerOpts{})))

return r
}
12 changes: 10 additions & 2 deletions internal/apiserver/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/hyperledger/firefly/internal/metrics"
"io/ioutil"
"net/http"
"net/http/httptest"
Expand Down Expand Up @@ -61,6 +62,7 @@ func newTestAdminServer() (*orchestratormocks.Orchestrator, *mux.Router) {

func TestStartStopServer(t *testing.T) {
config.Reset()
metrics.Clear()
InitConfig()
apiConfigPrefix.Set(HTTPConfPort, 0)
adminConfigPrefix.Set(HTTPConfPort, 0)
Expand All @@ -77,6 +79,7 @@ func TestStartStopServer(t *testing.T) {

func TestStartAPIFail(t *testing.T) {
config.Reset()
metrics.Clear()
InitConfig()
apiConfigPrefix.Set(HTTPConfAddress, "...://")
ctx, cancel := context.WithCancel(context.Background())
Expand All @@ -90,6 +93,7 @@ func TestStartAPIFail(t *testing.T) {

func TestStartAdminFail(t *testing.T) {
config.Reset()
metrics.Clear()
InitConfig()
adminConfigPrefix.Set(HTTPConfAddress, "...://")
config.Set(config.AdminEnabled, true)
Expand Down Expand Up @@ -375,16 +379,20 @@ func TestWaitForServerStop(t *testing.T) {

chl1 := make(chan error, 1)
chl2 := make(chan error, 1)
chl3 := make(chan error, 1)
chl1 <- fmt.Errorf("pop1")

as := &apiServer{}
err := as.waitForServerStop(chl1, chl2)
err := as.waitForServerStop(chl1, chl2, chl3)
assert.EqualError(t, err, "pop1")

chl2 <- fmt.Errorf("pop2")
err = as.waitForServerStop(chl1, chl2)
err = as.waitForServerStop(chl1, chl2, chl3)
assert.EqualError(t, err, "pop2")

chl3 <- fmt.Errorf("pop3")
err = as.waitForServerStop(chl1, chl2, chl3)
assert.EqualError(t, err, "pop3")
}

func TestGetTimeoutMax(t *testing.T) {
Expand Down
8 changes: 6 additions & 2 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ import (
)

// The following keys can be access from the root configuration.
// Plugins are resonsible for defining their own keys using the Config interface
// Plugins are responsible for defining their own keys using the Config interface
var (
// APIDefaultFilterLimit is the default limit that will be applied to filtered queries on the API
APIDefaultFilterLimit = rootKey("api.defaultFilterLimit")
Expand Down Expand Up @@ -148,7 +148,7 @@ var (
GroupCacheSize = rootKey("group.cache.size")
// GroupCacheTTL cache time-to-live for private group addresses
GroupCacheTTL = rootKey("group.cache.ttl")
// AdminHTTPEnabled determines whether the admin interface will be enabled or not
// AdminEnabled determines whether the admin interface will be enabled or not
AdminEnabled = rootKey("admin.enabled")
// AdminPreinit waits for at least one ConfigREcord to be posted to the server before it starts (the database must be available on startup)
AdminPreinit = rootKey("admin.preinit")
Expand Down Expand Up @@ -180,6 +180,10 @@ var (
LogMaxAge = rootKey("log.maxAge")
// LogCompress sets whether to compress backups
LogCompress = rootKey("log.compress")
// MetricsEnabled determines whether metrics will be instrumented and if the metrics server will be enabled or not
MetricsEnabled = rootKey("metrics.enabled")
// MetricsPath determines what path to serve the Prometheus metrics from
MetricsPath = rootKey("metrics.path")
// NamespacesDefault is the default namespace - must be in the predefines list
NamespacesDefault = rootKey("namespaces.default")
// NamespacesPredefined is a list of namespaces to ensure exists, without requiring a broadcast from the network
Expand Down
37 changes: 37 additions & 0 deletions internal/metrics/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Copyright © 2021 Kaleido, Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metrics

import "github.com/prometheus/client_golang/prometheus"

var registry *prometheus.Registry

// Registry returns FireFly's customized Prometheus registry
func Registry() *prometheus.Registry {
if registry == nil {
registry = prometheus.NewRegistry()
registry.MustRegister(prometheus.NewGoCollector())
registry.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
}

return registry
}

// Clear will reset the Prometheus metrics registry, useful for testing
func Clear() {
registry = nil
}

0 comments on commit 7e4fa6a

Please sign in to comment.