improve stackdriver metric type (googleforgames#1132)

* improve stackdriver metric type

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>

cmd/allocator/main.go

@@ -65,6 +65,7 @@ const (
 	enableStackdriverMetricsFlag = "stackdriver-exporter"
 	enablePrometheusMetricsFlag  = "prometheus-exporter"
 	projectIDFlag                = "gcp-project-id"
+	stackdriverLabels            = "stackdriver-labels"
 )
 
 func init() {
@@ -333,29 +334,34 @@ type config struct {
 	PrometheusMetrics bool
 	Stackdriver       bool
 	GCPProjectID      string
+	StackdriverLabels string
 }
 
 func parseEnvFlags() config {
 
 	viper.SetDefault(enablePrometheusMetricsFlag, true)
 	viper.SetDefault(enableStackdriverMetricsFlag, false)
 	viper.SetDefault(projectIDFlag, "")
+	viper.SetDefault(stackdriverLabels, "")
 
 	pflag.Bool(enablePrometheusMetricsFlag, viper.GetBool(enablePrometheusMetricsFlag), "Flag to activate metrics of Agones. Can also use PROMETHEUS_EXPORTER env variable.")
 	pflag.Bool(enableStackdriverMetricsFlag, viper.GetBool(enableStackdriverMetricsFlag), "Flag to activate stackdriver monitoring metrics for Agones. Can also use STACKDRIVER_EXPORTER env variable.")
 	pflag.String(projectIDFlag, viper.GetString(projectIDFlag), "GCP ProjectID used for Stackdriver, if not specified ProjectID from Application Default Credentials would be used. Can also use GCP_PROJECT_ID env variable.")
+	pflag.String(stackdriverLabels, viper.GetString(stackdriverLabels), "A set of default labels to add to all stackdriver metrics generated. By default metadata are automatically added using Kubernetes API and GCP metadata enpoint.")
 	pflag.Parse()
 
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
 	runtime.Must(viper.BindEnv(enablePrometheusMetricsFlag))
 	runtime.Must(viper.BindEnv(enableStackdriverMetricsFlag))
 	runtime.Must(viper.BindEnv(projectIDFlag))
+	runtime.Must(viper.BindEnv(stackdriverLabels))
 	runtime.Must(viper.BindPFlags(pflag.CommandLine))
 
 	return config{
 		PrometheusMetrics: viper.GetBool(enablePrometheusMetricsFlag),
 		Stackdriver:       viper.GetBool(enableStackdriverMetricsFlag),
 		GCPProjectID:      viper.GetString(projectIDFlag),
+		StackdriverLabels: viper.GetString(stackdriverLabels),
 	}
 }
 
@@ -371,7 +377,7 @@ func setupMetricsRecorder(conf config) (health healthcheck.Handler, closer func(
 
 	// Stackdriver metrics
 	if conf.Stackdriver {
-		sd, err := metrics.RegisterStackdriverExporter(conf.GCPProjectID)
+		sd, err := metrics.RegisterStackdriverExporter(conf.GCPProjectID, conf.StackdriverLabels)
 		if err != nil {
 			logger.WithError(err).Fatal("Could not register stackdriver exporter")
 		}

cmd/controller/main.go

@@ -53,6 +53,7 @@ import (
 
 const (
 	enableStackdriverMetricsFlag = "stackdriver-exporter"
+	stackdriverLabels            = "stackdriver-labels"
 	enablePrometheusMetricsFlag  = "prometheus-exporter"
 	projectIDFlag                = "gcp-project-id"
 	sidecarImageFlag             = "sidecar-image"
@@ -158,7 +159,7 @@ func main() {
 
 	// Stackdriver metrics
 	if ctlConf.Stackdriver {
-		sd, err := metrics.RegisterStackdriverExporter(ctlConf.GCPProjectID)
+		sd, err := metrics.RegisterStackdriverExporter(ctlConf.GCPProjectID, ctlConf.StackdriverLabels)
 		if err != nil {
 			logger.WithError(err).Fatal("Could not register stackdriver exporter")
 		}
@@ -240,6 +241,8 @@ func parseEnvFlags() config {
 	viper.SetDefault(keyFileFlag, filepath.Join(base, "certs/server.key"))
 	viper.SetDefault(enablePrometheusMetricsFlag, true)
 	viper.SetDefault(enableStackdriverMetricsFlag, false)
+	viper.SetDefault(stackdriverLabels, "")
+
 	viper.SetDefault(projectIDFlag, "")
 	viper.SetDefault(numWorkersFlag, 64)
 	viper.SetDefault(apiServerSustainedQPSFlag, 100)
@@ -260,6 +263,7 @@ func parseEnvFlags() config {
 	pflag.String(kubeconfigFlag, viper.GetString(kubeconfigFlag), "Optional. kubeconfig to run the controller out of the cluster. Only use it for debugging as webhook won't works.")
 	pflag.Bool(enablePrometheusMetricsFlag, viper.GetBool(enablePrometheusMetricsFlag), "Flag to activate metrics of Agones. Can also use PROMETHEUS_EXPORTER env variable.")
 	pflag.Bool(enableStackdriverMetricsFlag, viper.GetBool(enableStackdriverMetricsFlag), "Flag to activate stackdriver monitoring metrics for Agones. Can also use STACKDRIVER_EXPORTER env variable.")
+	pflag.String(stackdriverLabels, viper.GetString(stackdriverLabels), "A set of default labels to add to all stackdriver metrics generated. By default metadata are automatically added using Kubernetes API and GCP metadata enpoint.")
 	pflag.String(projectIDFlag, viper.GetString(projectIDFlag), "GCP ProjectID used for Stackdriver, if not specified ProjectID from Application Default Credentials would be used. Can also use GCP_PROJECT_ID env variable.")
 	pflag.Int32(numWorkersFlag, 64, "Number of controller workers per resource type")
 	pflag.Int32(apiServerSustainedQPSFlag, 100, "Maximum sustained queries per second to send to the API server")
@@ -282,6 +286,7 @@ func parseEnvFlags() config {
 	runtime.Must(viper.BindEnv(kubeconfigFlag))
 	runtime.Must(viper.BindEnv(enablePrometheusMetricsFlag))
 	runtime.Must(viper.BindEnv(enableStackdriverMetricsFlag))
+	runtime.Must(viper.BindEnv(stackdriverLabels))
 	runtime.Must(viper.BindEnv(projectIDFlag))
 	runtime.Must(viper.BindPFlags(pflag.CommandLine))
 	runtime.Must(viper.BindEnv(numWorkersFlag))
@@ -321,6 +326,7 @@ func parseEnvFlags() config {
 		LogDir:                viper.GetString(logDirFlag),
 		LogLevel:              viper.GetString(logLevelFlag),
 		LogSizeLimitMB:        int(viper.GetInt32(logSizeLimitMBFlag)),
+		StackdriverLabels:     viper.GetString(stackdriverLabels),
 	}
 }
 
@@ -335,6 +341,7 @@ type config struct {
 	AlwaysPullSidecar     bool
 	PrometheusMetrics     bool
 	Stackdriver           bool
+	StackdriverLabels     string
 	KeyFile               string
 	CertFile              string
 	KubeConfig            string

go.mod

@@ -3,7 +3,7 @@ module agones.dev/agones
 go 1.12
 
 require (
-	cloud.google.com/go v0.34.0 // indirect
+	cloud.google.com/go v0.34.0
 	contrib.go.opencensus.io/exporter/stackdriver v0.8.0
 	fortio.org/fortio v1.3.1
 	github.com/ahmetb/gen-crd-api-reference-docs v0.1.1

go.sum

@@ -139,6 +139,8 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV
 github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
+github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 h1:zNBQb37RGLmJybyMcs983HfUfpkw9OTFD9tbBfAViHE=
+github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34=
 github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=

install/helm/agones/templates/controller.yaml

@@ -90,6 +90,8 @@ spec:
           value: {{ .Values.agones.metrics.prometheusEnabled | quote }}
         - name: STACKDRIVER_EXPORTER
           value: {{ .Values.agones.metrics.stackdriverEnabled | quote }}
+        - name: STACKDRIVER_LABELS
+          value: {{ .Values.agones.metrics.stackdriverLabels | quote }}
         - name: GCP_PROJECT_ID
           value: {{ .Values.agones.metrics.stackdriverProjectID | quote }}
         - name: SIDECAR_CPU_LIMIT
@@ -108,6 +110,16 @@ spec:
         - name: LOG_SIZE_LIMIT_MB
           value: {{ .Values.agones.controller.persistentLogsSizeLimitMB | quote }}
 {{- end }}
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: CONTAINER_NAME
+          value: "agones-controller"
         livenessProbe:
           httpGet:
             path: /live

install/helm/agones/templates/service/allocation.yaml

@@ -112,6 +112,18 @@ spec:
           value: {{ .Values.agones.metrics.stackdriverEnabled | quote }}
         - name: GCP_PROJECT_ID
           value: {{ .Values.agones.metrics.stackdriverProjectID | quote }}
+        - name: STACKDRIVER_LABELS
+          value: {{ .Values.agones.metrics.stackdriverLabels | quote }}
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: CONTAINER_NAME
+          value: "agones-allocator"
         ports:
         - name: https
           containerPort: 8443
@@ -192,7 +204,7 @@ roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
   name: agones-allocator
- 
+
 {{- end }}
 
 ---

install/helm/agones/values.yaml

@@ -20,6 +20,7 @@ agones:
     prometheusServiceDiscovery: true
     stackdriverEnabled: false
     stackdriverProjectID: ""
+    stackdriverLabels: ""
   rbacEnabled: true
   registerServiceAccounts: true
   registerWebhooks: true

install/yaml/install.yaml

@@ -1137,6 +1137,18 @@ spec:
           value: "false"
         - name: GCP_PROJECT_ID
           value: ""
+        - name: STACKDRIVER_LABELS
+          value: ""
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: CONTAINER_NAME
+          value: "agones-allocator"
         ports:
         - name: https
           containerPort: 8443
@@ -1352,6 +1364,8 @@ spec:
           value: "true"
         - name: STACKDRIVER_EXPORTER
           value: "false"
+        - name: STACKDRIVER_LABELS
+          value: ""
         - name: GCP_PROJECT_ID
           value: ""
         - name: SIDECAR_CPU_LIMIT
@@ -1368,6 +1382,16 @@ spec:
           value: "/home/agones/logs"
         - name: LOG_SIZE_LIMIT_MB
           value: "10000"
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: CONTAINER_NAME
+          value: "agones-controller"
         livenessProbe:
           httpGet:
             path: /live

pkg/metrics/exporter.go

@@ -16,12 +16,16 @@ package metrics
 
 import (
 	"net/http"
+	"os"
 	"time"
 
+	"cloud.google.com/go/compute/metadata"
 	"contrib.go.opencensus.io/exporter/stackdriver"
+	"github.com/pkg/errors"
 	prom "github.com/prometheus/client_golang/prometheus"
 	"go.opencensus.io/exporter/prometheus"
 	"go.opencensus.io/stats/view"
+	"google.golang.org/genproto/googleapis/api/monitoredres"
 )
 
 // RegisterPrometheusExporter register a prometheus exporter to OpenCensus with a given prometheus metric registry.
@@ -48,20 +52,30 @@ func RegisterPrometheusExporter(registry *prom.Registry) (http.Handler, error) {
 
 // RegisterStackdriverExporter register a Stackdriver exporter to OpenCensus.
 // It will add Agones metrics into Stackdriver on Google Cloud.
-func RegisterStackdriverExporter(projectID string) (sd *stackdriver.Exporter, err error) {
-	// Default project will be used
-	sd, err = stackdriver.NewExporter(stackdriver.Options{
+func RegisterStackdriverExporter(projectID string, defaultLabels string) (*stackdriver.Exporter, error) {
+	monitoredRes, err := getMonitoredResource(projectID)
+	if err != nil {
+		logger.WithError(err).Warn("error discovering monitored resource")
+	}
+	labels, err := parseLabels(defaultLabels)
+	if err != nil {
+		return nil, err
+	}
+
+	sd, err := stackdriver.NewExporter(stackdriver.Options{
 		ProjectID: projectID,
 		// MetricPrefix helps uniquely identify your metrics.
-		MetricPrefix: "agones",
+		MetricPrefix:            "agones",
+		Resource:                monitoredRes,
+		DefaultMonitoringLabels: labels,
 	})
 	if err != nil {
-		return
+		return nil, err
 	}
 
 	// Register it as a metrics exporter
 	view.RegisterExporter(sd)
-	return
+	return sd, nil
 }
 
 // SetReportingPeriod set appropriate reporting period which depends on exporters
@@ -79,3 +93,33 @@ func SetReportingPeriod(prometheus, stackdriver bool) {
 		view.SetReportingPeriod(reportingPeriod)
 	}
 }
+
+func getMonitoredResource(projectID string) (*monitoredres.MonitoredResource, error) {
+	instanceID, err := metadata.InstanceID()
+	if err != nil {
+		return nil, errors.Wrap(err, "error getting instance ID")
+	}
+	zone, err := metadata.Zone()
+	if err != nil {
+		return nil, errors.Wrap(err, "error getting zone")
+	}
+	clusterName, err := metadata.InstanceAttributeValue("cluster-name")
+	if err != nil {
+		return nil, errors.Wrap(err, "error getting cluster-name")
+	}
+
+	return &monitoredres.MonitoredResource{
+		Type: "k8s_container",
+		Labels: map[string]string{
+			"project_id":   projectID,
+			"instance_id":  instanceID,
+			"zone":         zone,
+			"cluster_name": clusterName,
+
+			// See: https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/
+			"namespace_id":   os.Getenv("POD_NAMESPACE"),
+			"pod_id":         os.Getenv("POD_NAME"),
+			"container_name": os.Getenv("CONTAINER_NAME"),
+		},
+	}, nil
+}

pkg/metrics/util.go

@@ -16,8 +16,13 @@ package metrics
 
 import (
 	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"unicode/utf8"
 
 	"agones.dev/agones/pkg/util/runtime"
+	"contrib.go.opencensus.io/exporter/stackdriver"
 	"go.opencensus.io/stats"
 	"go.opencensus.io/tag"
 )
@@ -48,3 +53,27 @@ func MustTagKey(key string) tag.Key {
 	}
 	return t
 }
+
+func parseLabels(s string) (*stackdriver.Labels, error) {
+	res := &stackdriver.Labels{}
+	if s == "" {
+		return res, nil
+	}
+	pairs := strings.Split(s, ",")
+	for _, p := range pairs {
+		keyValue := strings.Split(p, "=")
+		if len(keyValue) != 2 {
+			return nil, fmt.Errorf("invalid labels: %s, expect key=value,key2=value2", s)
+		}
+		key := strings.TrimSpace(keyValue[0])
+		value := strings.TrimSpace(keyValue[1])
+		if !utf8.ValidString(key) || !utf8.ValidString(value) {
+			return nil, errors.New("invalid labels: must be a valid utf-8 string")
+		}
+		if key == "" || value == "" {
+			return nil, errors.New("invalid labels: must not be empty string")
+		}
+		res.Set(key, value, "")
+	}
+	return res, nil
+}