From e6789549443223d687abfe53ed706119f2cf059c Mon Sep 17 00:00:00 2001 From: husharp Date: Mon, 3 Jun 2024 17:44:57 +0800 Subject: [PATCH] refine code Signed-off-by: husharp --- tools/pd-heartbeat-bench/main.go | 13 ++---- tools/pd-heartbeat-bench/metrics/util.go | 52 ++++++++++++++++-------- 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/tools/pd-heartbeat-bench/main.go b/tools/pd-heartbeat-bench/main.go index 46dd234bdfe..b93d6c2701e 100644 --- a/tools/pd-heartbeat-bench/main.go +++ b/tools/pd-heartbeat-bench/main.go @@ -484,7 +484,6 @@ func main() { log.Fatal("initialize logger error", zap.Error(err)) } - withMetric := metrics.InitMetric2Collect(cfg.MetricsAddr) maxVersion = cfg.InitEpochVer options := config.NewOptions(cfg) // let PD have enough time to start @@ -531,6 +530,7 @@ func main() { defer heartbeatTicker.Stop() var resolvedTSTicker = time.NewTicker(time.Second) defer resolvedTSTicker.Stop() + withMetric := metrics.InitMetric2Collect(cfg.MetricsAddr) for { select { case <-heartbeatTicker.C: @@ -557,14 +557,8 @@ func main() { close(rep.Results()) regions.result(cfg.RegionCount, since) stats := <-r - log.Info("region heartbeat stats", zap.String("total", fmt.Sprintf("%.4fs", stats.Total.Seconds())), - zap.String("slowest", fmt.Sprintf("%.4fs", stats.Slowest)), - zap.String("fastest", fmt.Sprintf("%.4fs", stats.Fastest)), - zap.String("average", fmt.Sprintf("%.4fs", stats.Average)), - zap.String("stddev", fmt.Sprintf("%.4fs", stats.Stddev)), - zap.String("rps", fmt.Sprintf("%.4f", stats.RPS)), - zap.Uint64("max-epoch-version", maxVersion), - ) + log.Info("region heartbeat stats", + metrics.RegionFields(stats, zap.Uint64("max-epoch-version", maxVersion))...) log.Info("store heartbeat stats", zap.String("max", fmt.Sprintf("%.4fs", since))) metrics.CollectRegionAndStoreStats(&stats, &since) regions.update(cfg, options) @@ -602,6 +596,7 @@ func main() { } func exit(code int) { + metrics.OutputConclusion() os.Exit(code) } diff --git a/tools/pd-heartbeat-bench/metrics/util.go b/tools/pd-heartbeat-bench/metrics/util.go index 96ecf0da15c..b25a28fd26a 100644 --- a/tools/pd-heartbeat-bench/metrics/util.go +++ b/tools/pd-heartbeat-bench/metrics/util.go @@ -31,16 +31,16 @@ import ( ) var ( - prometheusCli api.Client - avgMetrics2Collect []Metric - avgRegionStats report.Stats - avgStoreTime float64 - collectRound = 1.0 + prometheusCli api.Client + finalMetrics2Collect []Metric + avgRegionStats report.Stats + avgStoreTime float64 + collectRound = 1.0 metrics2Collect = []Metric{ - {promSQL: cpuMetric, name: "max cpu usage(%)"}, - {promSQL: memoryMetric, name: "max memory usage(G)"}, - {promSQL: goRoutineMetric, name: "max go routines"}, + {promSQL: cpuMetric, name: "max cpu usage(%)", max: true}, + {promSQL: memoryMetric, name: "max memory usage(G)", max: true}, + {promSQL: goRoutineMetric, name: "max go routines", max: true}, {promSQL: hbLatency99Metric, name: "99% Heartbeat Latency(ms)"}, {promSQL: hbLatencyAvgMetric, name: "Avg Heartbeat Latency(ms)"}, } @@ -53,8 +53,7 @@ var ( hbLatencyAvgMetric = `sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_sum{}[1m])) / sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_count{}[1m]))` // Heartbeat Performance Duration BreakDown - hbBreakdownName = "Heartbeat Performance Duration BreakDown (Accumulation)(ms)" - breakdownNames = []string{ + breakdownNames = []string{ "AsyncHotStatsDuration", "CollectRegionStats", "Other", @@ -74,16 +73,18 @@ type Metric struct { promSQL string name string value float64 + // max indicates whether the metric is a max value + max bool } func InitMetric2Collect(endpoint string) (withMetric bool) { for _, name := range breakdownNames { metrics2Collect = append(metrics2Collect, Metric{ promSQL: hbBreakdownMetricByName(name), - name: fmt.Sprintf("%s with %s", hbBreakdownName, name), + name: name, }) } - avgMetrics2Collect = metrics2Collect + finalMetrics2Collect = metrics2Collect if j := strings.Index(endpoint, "//"); j == -1 { endpoint = "http://" + endpoint @@ -150,11 +151,15 @@ func CollectMetrics(curRound int, wait time.Duration) { } for i := 0; i < len(metrics2Collect); i++ { metrics2Collect[i].value = getRes(i) - avgMetrics2Collect[i].value = (avgMetrics2Collect[i].value*collectRound + metrics2Collect[i].value) / (collectRound + 1) + if metrics2Collect[i].max { + finalMetrics2Collect[i].value = max(finalMetrics2Collect[i].value, metrics2Collect[i].value) + } else { + finalMetrics2Collect[i].value = (finalMetrics2Collect[i].value*collectRound + metrics2Collect[i].value) / (collectRound + 1) + } } collectRound += 1 - log.Info("metrics collected", zap.String("metrics", formatMetrics(metrics2Collect))) + log.Info("metrics collected", zap.Float64("round", collectRound), zap.String("metrics", formatMetrics(metrics2Collect))) } func getMetric(cli api.Client, query string, ts time.Time) ([]float64, error) { @@ -207,7 +212,20 @@ func formatMetrics(ms []Metric) string { } func OutputConclusion() { - log.Info("average metrics", zap.Float64("avg store time", avgStoreTime), - zap.Any("avg region stats", avgRegionStats), - zap.String("metrics", formatMetrics(avgMetrics2Collect))) + logFields := RegionFields(avgRegionStats, + zap.Float64("avg store time", avgStoreTime), + zap.Float64("current round", collectRound), + zap.String("metrics", formatMetrics(finalMetrics2Collect))) + log.Info("final metrics collected", logFields...) +} + +func RegionFields(stats report.Stats, fields ...zap.Field) []zap.Field { + return append([]zap.Field{ + zap.String("total", fmt.Sprintf("%.4fs", stats.Total.Seconds())), + zap.String("slowest", fmt.Sprintf("%.4fs", stats.Slowest)), + zap.String("fastest", fmt.Sprintf("%.4fs", stats.Fastest)), + zap.String("average", fmt.Sprintf("%.4fs", stats.Average)), + zap.String("stddev", fmt.Sprintf("%.4fs", stats.Stddev)), + zap.String("rps", fmt.Sprintf("%.4f", stats.RPS)), + }, fields...) }