server: add go gc metrics (#2977) (#3000)

pingcap · Oct 11, 2021 · 8e95c90 · 8e95c90
1 parent d0be116
commit 8e95c90
Show file tree

Hide file tree

Showing 3 changed files with 64 additions and 12 deletions.
diff --git a/cdc/metrics_server.go b/cdc/metrics_server.go
@@ -14,19 +14,56 @@
 package cdc
 
 import (
+	"os"
+	"runtime"
+	"strconv"
+
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-var etcdHealthCheckDuration = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: "ticdc",
-		Subsystem: "server",
-		Name:      "etcd_health_check_duration",
-		Help:      "Bucketed histogram of processing time (s) of flushing events in processor",
-		Buckets:   prometheus.ExponentialBuckets(0.0001 /* 0.1ms */, 2, 18),
-	}, []string{"capture", "pd"})
+var (
+	etcdHealthCheckDuration = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: "ticdc",
+			Subsystem: "server",
+			Name:      "etcd_health_check_duration",
+			Help:      "Bucketed histogram of processing time (s) of flushing events in processor",
+			Buckets:   prometheus.ExponentialBuckets(0.0001 /* 0.1ms */, 2, 18),
+		}, []string{"capture", "pd"})
+
+	goGC = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: "ticdc",
+			Subsystem: "server",
+			Name:      "go_gc",
+			Help:      "The value of GOGC",
+		})
+
+	goMaxProcs = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: "ticdc",
+			Subsystem: "server",
+			Name:      "go_max_procs",
+			Help:      "The value of GOMAXPROCS",
+		})
+)
+
+// RecordGoRuntimeSettings records GOGC settings.
+func RecordGoRuntimeSettings() {
+	// The default GOGC value is 100. See debug.SetGCPercent.
+	gogcValue := 100
+	if val, err := strconv.Atoi(os.Getenv("GOGC")); err == nil {
+		gogcValue = val
+	}
+	goGC.Set(float64(gogcValue))
+
+	maxProcs := runtime.GOMAXPROCS(0)
+	goMaxProcs.Set(float64(maxProcs))
+}
 
 // initServerMetrics registers all metrics used in processor
 func initServerMetrics(registry *prometheus.Registry) {
 	registry.MustRegister(etcdHealthCheckDuration)
+	registry.MustRegister(goGC)
+	registry.MustRegister(goMaxProcs)
 }
diff --git a/metrics/grafana/ticdc.json b/metrics/grafana/ticdc.json
@@ -125,7 +125,7 @@
   "gnetId": null,
   "graphTooltip": 1,
   "id": null,
-  "iteration": 1631019842531,
+  "iteration": 1633763775536,
   "links": [],
   "panels": [
     {
@@ -558,7 +558,14 @@
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
-          "seriesOverrides": [],
+          "seriesOverrides": [
+            {
+              "alias": "/.*MaxProcs/",
+              "fill": 0,
+              "linewidth": 2,
+              "yaxis": 2
+            }
+          ],
           "spaceLength": 10,
           "stack": false,
           "steppedLine": false,
@@ -569,6 +576,13 @@
               "intervalFactor": 1,
               "legendFormat": "{{instance}}",
               "refId": "A"
+            },
+            {
+              "expr": "ticdc_server_go_max_procs{tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}-MaxProcs",
+              "refId": "B"
             }
           ],
           "thresholds": [],
@@ -8528,15 +8542,15 @@
               "refId": "A"
             },
             {
-              "expr": "go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + tidb_server_gogc{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)",
+              "expr": "go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 1,
               "legendFormat": "estimate-inuse",
               "refId": "H"
             },
             {
-              "expr": "go_memstats_heap_alloc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + tidb_server_gogc{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)",
+              "expr": "go_memstats_heap_alloc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 1,

diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go
@@ -120,6 +120,7 @@ func (o *options) run(cmd *cobra.Command) error {
 	}
 
 	util.LogHTTPProxies()
+	cdc.RecordGoRuntimeSettings()
 	server, err := cdc.NewServer(strings.Split(o.serverPdAddr, ","))
 	if err != nil {
 		return errors.Annotate(err, "new server")