feat(api): low level metrics (#3207)

engine/api/api.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/go-gorp/gorp"
 	"github.com/gorilla/mux"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
 
 	"github.com/ovh/cds/engine/api/action"
 	"github.com/ovh/cds/engine/api/auth"
@@ -25,6 +27,7 @@ import (
 	"github.com/ovh/cds/engine/api/migrate"
 	"github.com/ovh/cds/engine/api/notification"
 	"github.com/ovh/cds/engine/api/objectstore"
+	"github.com/ovh/cds/engine/api/observability"
 	"github.com/ovh/cds/engine/api/pipeline"
 	"github.com/ovh/cds/engine/api/platform"
 	"github.com/ovh/cds/engine/api/poller"
@@ -200,6 +203,10 @@ type API struct {
 	eventsBroker        *eventsBroker
 	warnChan            chan sdk.Event
 	Cache               cache.Store
+	Stats               struct {
+		WorkflowRuns *stats.Int64Measure
+		Sessions     *stats.Int64Measure
+	}
 }
 
 // ApplyConfiguration apply an object of type api.Configuration after checking it
@@ -532,6 +539,13 @@ func (a *API) Serve(ctx context.Context) error {
 		Background: ctx,
 	}
 	a.InitRouter()
+	if err := a.Router.InitStats("cds-api", a.Name); err != nil {
+		log.Error("unable to init router stats: %v", err)
+	}
+
+	if err := a.initStats(); err != nil {
+		log.Error("unable to init api stats: %v", err)
+	}
 
 	//Initiliaze hook package
 	hook.Init(a.Config.URL.API)
@@ -701,3 +715,19 @@ func (a *API) Serve(ctx context.Context) error {
 
 	return nil
 }
+
+func (a *API) initStats() error {
+	label := fmt.Sprintf("cds/cds-api/%s/workflow_runs", a.Name)
+	a.Stats.WorkflowRuns = stats.Int64(label, "number of workflow runs", stats.UnitDimensionless)
+
+	log.Info("api> Stats initialized")
+
+	return observability.RegisterView(
+		&view.View{
+			Name:        "workflow_runs",
+			Description: a.Stats.WorkflowRuns.Description(),
+			Measure:     a.Stats.WorkflowRuns,
+			Aggregation: view.Count(),
+		},
+	)
+}

engine/api/api_routes.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"sync"
 
+	"github.com/ovh/cds/engine/api/observability"
 	"github.com/ovh/cds/sdk"
 )
 
@@ -102,6 +103,7 @@ func (api *API) InitRouter() {
 	r.Handle("/mon/building", r.GET(api.getBuildingPipelinesHandler))
 	r.Handle("/mon/building/{hash}", r.GET(api.getPipelineBuildingCommitHandler))
 	r.Handle("/mon/metrics", r.GET(api.getMetricsHandler, Auth(false)))
+	r.Handle("/mon/stats", r.GET(observability.StatsHandler, Auth(false)))
 
 	r.Handle("/navbar", r.GET(api.getNavbarHandler))
 

engine/api/observability/stats.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"net/http"
 
+	"go.opencensus.io/stats"
+
 	"go.opencensus.io/stats/view"
 
 	"github.com/ovh/cds/engine/service"
@@ -26,3 +28,33 @@ func StatsHandler() service.Handler {
 		return nil
 	}
 }
+
+// Record an in64 measure
+func Record(ctx context.Context, m stats.Measure, v int64) {
+	if m == nil {
+		return
+	}
+	mInt64, ok := m.(*stats.Int64Measure)
+	if !ok {
+		return
+	}
+	if mInt64 == nil {
+		return
+	}
+	stats.Record(ctx, mInt64.M(v))
+}
+
+// RecordFloat64 a float64 measure
+func RecordFloat64(ctx context.Context, m stats.Measure, v float64) {
+	if m == nil {
+		return
+	}
+	mFloat64, ok := m.(*stats.Float64Measure)
+	if !ok {
+		return
+	}
+	if mFloat64 == nil {
+		return
+	}
+	stats.Record(ctx, mFloat64.M(v))
+}

engine/api/router.go

@@ -17,8 +17,11 @@ import (
 	muxcontext "github.com/gorilla/context"
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
 
 	"github.com/ovh/cds/engine/api/auth"
+	"github.com/ovh/cds/engine/api/observability"
 	"github.com/ovh/cds/engine/service"
 	"github.com/ovh/cds/sdk"
 	"github.com/ovh/cds/sdk/cdsclient"
@@ -43,6 +46,10 @@ type Router struct {
 	panicked               bool
 	nbPanic                int
 	lastPanic              *time.Time
+	Stats                  struct {
+		Errors *stats.Int64Measure
+		Hits   *stats.Int64Measure
+	}
 }
 
 // NewHandlerConfig returns a new HandlerConfig pointer
@@ -53,7 +60,7 @@ func NewHandlerConfig() *service.HandlerConfig {
 }
 
 func newRouter(a auth.Driver, m *mux.Router, p string) *Router {
-	return &Router{
+	r := &Router{
 		AuthDriver:             a,
 		Mux:                    m,
 		Prefix:                 p,
@@ -62,6 +69,7 @@ func newRouter(a auth.Driver, m *mux.Router, p string) *Router {
 		mapAsynchronousHandler: map[string]service.HandlerFunc{},
 		Background:             context.Background(),
 	}
+	return r
 }
 
 // HandlerConfigParam is a type used in handler configuration, to set specific config on a route given a method
@@ -188,10 +196,12 @@ func (r *Router) Handle(uri string, handlers ...*service.HandlerConfig) {
 			w.WriteHeader(http.StatusOK)
 			return
 		}
+		observability.Record(ctx, r.Stats.Hits, 1)
 
 		//Get route configuration
 		rc := cfg.Config[req.Method]
 		if rc == nil || rc.Handler == nil {
+			observability.Record(ctx, r.Stats.Errors, 1)
 			service.WriteError(w, req, sdk.ErrNotFound)
 			return
 		}
@@ -212,12 +222,14 @@ func (r *Router) Handle(uri string, handlers ...*service.HandlerConfig) {
 			var err error
 			ctx, err = m(ctx, w, req, rc)
 			if err != nil {
+				observability.Record(ctx, r.Stats.Errors, 1)
 				service.WriteError(w, req, err)
 				return
 			}
 		}
 
 		if err := rc.Handler(ctx, w, req); err != nil {
+			observability.Record(ctx, r.Stats.Errors, 1)
 			service.WriteError(w, req, err)
 			return
 		}
@@ -496,3 +508,28 @@ func (r *Router) StatusPanic() sdk.MonitoringStatusLine {
 	}
 	return sdk.MonitoringStatusLine{Component: "Nb of Panics", Value: fmt.Sprintf("%d", r.nbPanic), Status: statusPanic}
 }
+
+// InitStats initialize prometheus metrics
+func (r *Router) InitStats(service, name string) error {
+	label := fmt.Sprintf("cds/%s/%s/router_errors", service, name)
+	r.Stats.Errors = stats.Int64(label, "number of errors", stats.UnitDimensionless)
+	label = fmt.Sprintf("cds/%s/%s/router_hits", service, name)
+	r.Stats.Hits = stats.Int64(label, "number of hits", stats.UnitDimensionless)
+
+	log.Info("api> Stats initialized")
+
+	return observability.RegisterView(
+		&view.View{
+			Name:        "router_errors",
+			Description: r.Stats.Errors.Description(),
+			Measure:     r.Stats.Errors,
+			Aggregation: view.Count(),
+		},
+		&view.View{
+			Name:        "router_hits",
+			Description: r.Stats.Hits.Description(),
+			Measure:     r.Stats.Hits,
+			Aggregation: view.Count(),
+		},
+	)
+}

engine/api/workflow_run.go

@@ -594,6 +594,7 @@ func (api *API) postWorkflowRunHandler() service.Handler {
 		u := getUser(ctx)
 
 		observability.Current(ctx, observability.Tag(observability.TagWorkflow, name))
+		observability.Record(ctx, api.Stats.WorkflowRuns, 1)
 
 		_, next := observability.Span(ctx, "project.Load")
 		p, errP := project.Load(api.mustDB(), api.Cache, key, u,