[observability] Add gRPC client call and additional functional metrics

chart/templates/ws-manager-bridge-deployment.yaml

@@ -43,6 +43,7 @@ spec:
 {{ include "gitpod.databaseWaiter.container" $this | indent 6 }}
 {{ include "gitpod.msgbusWaiter.container" $this | indent 6 }}
       containers:
+{{ include "gitpod.kube-rbac-proxy" $this | indent 6 }}
       - name: ws-manager-bridge
         image: {{ template "gitpod.comp.imageFull" $this }}
 {{ include "gitpod.container.resources" $this | indent 8 }}

chart/templates/ws-proxy-configmap.yaml

@@ -65,6 +65,6 @@ data:
         },
         "pprofAddr": ":6060",
         "readinessProbeAddr": ":60088",
-        "prometheusAddr": ":60095"
+        "prometheusAddr": "localhost:9500"
     }
 {{- end -}}

chart/templates/ws-proxy-deployment.yaml

@@ -54,6 +54,7 @@ spec:
 {{- end }}
       enableServiceLinks: false
       containers:
+{{ include "gitpod.kube-rbac-proxy" $this | indent 6 }}
       - name: ws-proxy
         image: {{ template "gitpod.comp.imageFull" $this }}
         args: ["run", "-v", "/config/config.json"]

components/common-go/go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/go-test/deep v1.0.5
 	github.com/google/go-cmp v0.5.6
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/prometheus/client_golang v1.11.0
 	github.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3 // indirect

components/common-go/go.sum

@@ -103,6 +103,8 @@ github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97Dwqy
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=
 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=

components/common-go/grpc/grpc.go

@@ -10,7 +10,9 @@ import (
 	"github.com/gitpod-io/gitpod/common-go/log"
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_opentracing "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing"
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/opentracing/opentracing-go"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/backoff"
@@ -22,14 +24,38 @@ import (
 // grpc library default is 4MB
 const maxMsgSize = 1024 * 1024 * 16
 
+var defaultClientOptionsConfig struct {
+	Metrics *grpc_prometheus.ClientMetrics
+}
+
+// ClientMetrics produces client-side gRPC metrics
+func ClientMetrics() prometheus.Collector {
+	res := grpc_prometheus.NewClientMetrics()
+	defaultClientOptionsConfig.Metrics = res
+	return res
+}
+
 // DefaultClientOptions returns the default grpc client connection options
 func DefaultClientOptions() []grpc.DialOption {
 	bfConf := backoff.DefaultConfig
 	bfConf.MaxDelay = 5 * time.Second
 
-	return []grpc.DialOption{
-		grpc.WithUnaryInterceptor(grpc_opentracing.UnaryClientInterceptor(grpc_opentracing.WithTracer(opentracing.GlobalTracer()))),
-		grpc.WithStreamInterceptor(grpc_opentracing.StreamClientInterceptor(grpc_opentracing.WithTracer(opentracing.GlobalTracer()))),
+	var (
+		unaryInterceptor = []grpc.UnaryClientInterceptor{
+			grpc_opentracing.UnaryClientInterceptor(grpc_opentracing.WithTracer(opentracing.GlobalTracer())),
+		}
+		streamInterceptor = []grpc.StreamClientInterceptor{
+			grpc_opentracing.StreamClientInterceptor(grpc_opentracing.WithTracer(opentracing.GlobalTracer())),
+		}
+	)
+	if defaultClientOptionsConfig.Metrics != nil {
+		unaryInterceptor = append(unaryInterceptor, defaultClientOptionsConfig.Metrics.UnaryClientInterceptor())
+		streamInterceptor = append(streamInterceptor, defaultClientOptionsConfig.Metrics.StreamClientInterceptor())
+	}
+
+	res := []grpc.DialOption{
+		grpc.WithUnaryInterceptor(grpc_middleware.ChainUnaryClient(unaryInterceptor...)),
+		grpc.WithStreamInterceptor(grpc_middleware.ChainStreamClient(streamInterceptor...)),
 		grpc.WithBlock(),
 		grpc.WithConnectParams(grpc.ConnectParams{
 			Backoff: bfConf,
@@ -41,6 +67,8 @@ func DefaultClientOptions() []grpc.DialOption {
 		}),
 		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)),
 	}
+
+	return res
 }
 
 // DefaultServerOptions returns the default ServerOption sets options for internal components

components/content-service-api/typescript/src/client-call-metrics.ts

@@ -0,0 +1,88 @@
+/**
+ * Copyright (c) 2021 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License-AGPL.txt in the project root for license information.
+ */
+
+import * as grpc from "@grpc/grpc-js";
+import { Status } from "@grpc/grpc-js/build/src/constants";
+
+type GrpcMethodType = 'unary' | 'client_stream' | 'server_stream' | 'bidi_stream';
+export interface IGrpcCallMetricsLabels {
+	service: string,
+	method: string,
+	type: GrpcMethodType,
+}
+
+export interface IGrpcCallMetricsLabelsWithCode extends IGrpcCallMetricsLabels {
+	code: string
+}
+
+export const IClientCallMetrics = Symbol("IClientCallMetrics");
+
+export interface IClientCallMetrics {
+    handled(labels: IGrpcCallMetricsLabelsWithCode) : void;
+    received(labels: IGrpcCallMetricsLabels) : void;
+    sent(labels: IGrpcCallMetricsLabels) : void;
+    started(labels: IGrpcCallMetricsLabels) : void;
+}
+
+export function getGrpcMethodType(requestStream: boolean, responseStream: boolean): GrpcMethodType {
+    if (requestStream) {
+        if (responseStream) {
+            return 'bidi_stream';
+        } else {
+            return 'client_stream';
+        }
+    } else {
+        if (responseStream) {
+            return 'server_stream';
+        } else {
+            return 'unary';
+        }
+    }
+}
+
+export function createClientCallMetricsInterceptor(metrics: IClientCallMetrics): grpc.Interceptor {
+    return (options, nextCall): grpc.InterceptingCall => {
+        const methodDef = options.method_definition;
+        const method = methodDef.path.substring(methodDef.path.lastIndexOf('/') + 1);
+        const service = methodDef.path.substring(1, methodDef.path.length - method.length - 1);
+        const labels = {
+            service,
+            method,
+            type: getGrpcMethodType(options.method_definition.requestStream, options.method_definition.responseStream)
+        };
+        const requester = new grpc.RequesterBuilder()
+            .withStart((metadata, listener, next) => {
+                const newListener = new grpc.ListenerBuilder().withOnReceiveStatus((status, next) => {
+                    try {
+                        metrics.handled({
+                            ...labels,
+                            code: Status[status.code]
+                        });
+                    } finally {
+                        next(status);
+                    }
+                }).withOnReceiveMessage((message, next) => {
+                    try {
+                        metrics.received(labels);
+                    } finally {
+                        next(message);
+                    }
+                }).build()
+                try {
+                    metrics.started(labels);
+                } finally {
+                    next(metadata, newListener);
+                }
+            }).withSendMessage((message, next) => {
+                try {
+                    metrics.sent(labels);
+                } finally {
+                    next(message);
+                }
+            }).build();
+        return new grpc.InterceptingCall(nextCall(options), requester);
+    };
+}

components/ee/ws-scheduler/cmd/run.go

@@ -15,6 +15,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
 
+	common_grpc "github.com/gitpod-io/gitpod/common-go/grpc"
 	"github.com/gitpod-io/gitpod/common-go/log"
 	"github.com/gitpod-io/gitpod/common-go/pprof"
 	"github.com/gitpod-io/gitpod/ws-scheduler/pkg/scaler"
@@ -40,6 +41,27 @@ var runCmd = &cobra.Command{
 		}
 		log.Info("connected to Kubernetes")
 
+		reg := prometheus.NewRegistry()
+		if config.Prometheus.Addr != "" {
+			prometheus.WrapRegistererWithPrefix("gitpod_ws_scheduler_", reg).MustRegister(schedMetrics.AllMetrics...)
+			reg.MustRegister(common_grpc.ClientMetrics())
+
+			handler := http.NewServeMux()
+			handler.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
+
+			go func() {
+				err := http.ListenAndServe(config.Prometheus.Addr, handler)
+				if err != nil {
+					log.WithError(err).Error("Prometheus metrics server failed")
+				}
+			}()
+			log.WithField("addr", config.Prometheus.Addr).Info("started Prometheus metrics server")
+		}
+
+		if config.PProf.Addr != "" {
+			go pprof.Serve(config.PProf.Addr)
+		}
+
 		scheduler, err := sched.NewScheduler(config.Scheduler, clientSet)
 		if err != nil {
 			log.WithError(err).Fatal("cannot create scheduler")
@@ -59,8 +81,6 @@ var runCmd = &cobra.Command{
 			log.Info("ws-scheduler shut down")
 		}()
 
-		reg := prometheus.NewRegistry()
-
 		if config.Scaler.Enabled {
 			controller, err := scaler.NewController(config.Scaler.Controller)
 			if err != nil {
@@ -80,25 +100,6 @@ var runCmd = &cobra.Command{
 			log.WithField("controller", config.Scaler.Controller.Kind).Info("started scaler")
 		}
 
-		if config.Prometheus.Addr != "" {
-			prometheus.WrapRegistererWithPrefix("gitpod_ws_scheduler_", reg).MustRegister(schedMetrics.AllMetrics...)
-
-			handler := http.NewServeMux()
-			handler.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
-
-			go func() {
-				err := http.ListenAndServe(config.Prometheus.Addr, handler)
-				if err != nil {
-					log.WithError(err).Error("Prometheus metrics server failed")
-				}
-			}()
-			log.WithField("addr", config.Prometheus.Addr).Info("started Prometheus metrics server")
-		}
-
-		if config.PProf.Addr != "" {
-			go pprof.Serve(config.PProf.Addr)
-		}
-
 		log.Info("🗓️ ws-scheduler is up and running. Stop with SIGINT or CTRL+C")
 
 		// Run until we're told to stop

components/ee/ws-scheduler/go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000
 	github.com/gitpod-io/gitpod/content-service/api v0.0.0-00010101000000-000000000000
 	github.com/gitpod-io/gitpod/ws-manager/api v0.0.0-00010101000000-000000000000
-	github.com/go-ozzo/ozzo-validation v3.5.0+incompatible
+	github.com/go-ozzo/ozzo-validation v3.6.0+incompatible
 	github.com/golang/mock v1.6.0
 	github.com/google/go-cmp v0.5.6
 	github.com/google/uuid v1.1.2
@@ -19,8 +19,8 @@ require (
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
 	google.golang.org/grpc v1.39.1
 	google.golang.org/protobuf v1.27.1
-	k8s.io/api v0.22.0
-	k8s.io/apimachinery v0.22.0
+	k8s.io/api v0.22.1
+	k8s.io/apimachinery v0.22.1
 	k8s.io/client-go v0.22.0
 	k8s.io/component-helpers v0.22.0
 )
@@ -37,6 +37,7 @@ require (
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/imdario/mergo v0.3.5 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.11 // indirect

components/ee/ws-scheduler/go.sum

@@ -112,8 +112,8 @@ github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-ozzo/ozzo-validation v3.5.0+incompatible h1:sUy/in/P6askYr16XJgTKq/0SZhiWsdg4WZGaLsGQkM=
-github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
+github.com/go-ozzo/ozzo-validation v3.6.0+incompatible h1:msy24VGS42fKO9K1vLz82/GeYW1cILu7Nuuj1N3BBkE=
+github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -184,6 +184,7 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=
 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=