Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

server: add go gc metrics (#2977) #3000

Merged
merged 4 commits into from Oct 11, 2021
Merged

server: add go gc metrics (#2977) #3000

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
566d033
server: add go gc metrics
overvenus Sep 21, 2021
277dfa3
address lints
overvenus Oct 6, 2021
1282b01
address comments
overvenus Oct 8, 2021
66a2fd7
address comments
overvenus Oct 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
53 changes: 45 additions & 8 deletions cdc/metrics_server.go
View file
Open in desktop
Expand Up @@ -14,19 +14,56 @@
package cdc

import (
"os"
"runtime"
"strconv"

"github.com/prometheus/client_golang/prometheus"
)

var etcdHealthCheckDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: "ticdc",
Subsystem: "server",
Name: "etcd_health_check_duration",
Help: "Bucketed histogram of processing time (s) of flushing events in processor",
Buckets: prometheus.ExponentialBuckets(0.0001 /* 0.1ms */, 2, 18),
}, []string{"capture", "pd"})
var (
etcdHealthCheckDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: "ticdc",
Subsystem: "server",
Name: "etcd_health_check_duration",
Help: "Bucketed histogram of processing time (s) of flushing events in processor",
Buckets: prometheus.ExponentialBuckets(0.0001 /* 0.1ms */, 2, 18),
}, []string{"capture", "pd"})

goGC = prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: "ticdc",
Subsystem: "server",
Name: "go_gc",
Help: "The value of GOGC",
})

goMaxProcs = prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: "ticdc",
Subsystem: "server",
Name: "go_max_procs",
Help: "The value of GOMAXPROCS",
})
)

// RecordGoRuntimeSettings records GOGC settings.
func RecordGoRuntimeSettings() {
// The default GOGC value is 100. See debug.SetGCPercent.
gogcValue := 100
if val, err := strconv.Atoi(os.Getenv("GOGC")); err == nil {
gogcValue = val
}
goGC.Set(float64(gogcValue))

maxProcs := runtime.GOMAXPROCS(0)
goMaxProcs.Set(float64(maxProcs))
}

// initServerMetrics registers all metrics used in processor
func initServerMetrics(registry *prometheus.Registry) {
registry.MustRegister(etcdHealthCheckDuration)
registry.MustRegister(goGC)
registry.MustRegister(goMaxProcs)
}
22 changes: 18 additions & 4 deletions metrics/grafana/ticdc.json
View file
Open in desktop
Expand Up @@ -125,7 +125,7 @@
"gnetId": null,
"graphTooltip": 1,
"id": null,
"iteration": 1631019842531,
"iteration": 1633763775536,
"links": [],
"panels": [
{
Expand Down Expand Up @@ -558,7 +558,14 @@
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"seriesOverrides": [
{
"alias": "/.*MaxProcs/",
"fill": 0,
"linewidth": 2,
"yaxis": 2
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
Expand All @@ -569,6 +576,13 @@
"intervalFactor": 1,
"legendFormat": "{{instance}}",
"refId": "A"
},
{
"expr": "ticdc_server_go_max_procs{tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{instance}}-MaxProcs",
"refId": "B"
}
],
"thresholds": [],
Expand Down Expand Up @@ -8528,15 +8542,15 @@
"refId": "A"
},
{
"expr": "go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + tidb_server_gogc{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)",
"expr": "go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)",
"format": "time_series",
"hide": false,
"intervalFactor": 1,
"legendFormat": "estimate-inuse",
"refId": "H"
},
{
"expr": "go_memstats_heap_alloc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + tidb_server_gogc{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)",
"expr": "go_memstats_heap_alloc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)",
"format": "time_series",
"hide": false,
"intervalFactor": 1,
Expand Down
1 change: 1 addition & 0 deletions pkg/cmd/server/server.go
View file
Open in desktop
Expand Up @@ -120,6 +120,7 @@ func (o *options) run(cmd *cobra.Command) error {
}

util.LogHTTPProxies()
cdc.RecordGoRuntimeSettings()
server, err := cdc.NewServer(strings.Split(o.serverPdAddr, ","))
if err != nil {
return errors.Annotate(err, "new server")
Expand Down