go_metrics: adds new runtime metrics (#58)

* go_metrics: adds new runtime metrics
go_sched_latency_seconds - Distribution of the time goroutines have spent in the scheduler in a runnable state before actually running
go_mutex_wait_total_seconds - Approximate cumulative time goroutines have spent blocked on a sync.Mutex or sync.RWMutex
#54

* fixes data race

* add tests

* wip

* wip

---------

Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>

go_metrics.go

@@ -3,12 +3,26 @@ package metrics
 import (
 	"fmt"
 	"io"
+	"math"
 	"runtime"
+	runtimemetrics "runtime/metrics"
 
 	"github.com/valyala/histogram"
 )
 
+// See https://pkg.go.dev/runtime/metrics#hdr-Supported_metrics
+var runtimeMetrics = [][2]string{
+	{"/sched/latencies:seconds", "go_sched_latencies_seconds"},
+	{"/sync/mutex/wait/total:seconds", "go_mutex_wait_seconds_total"},
+	{"/cpu/classes/gc/mark/assist:cpu-seconds", "go_gc_mark_assist_cpu_seconds_total"},
+	{"/cpu/classes/gc/total:cpu-seconds", "go_gc_cpu_seconds_total"},
+	{"/cpu/classes/scavenge/total:cpu-seconds", "go_scavenge_cpu_seconds_total"},
+	{"/gc/gomemlimit:bytes", "go_memlimit_bytes"},
+}
+
 func writeGoMetrics(w io.Writer) {
+	writeRuntimeMetrics(w)
+
 	var ms runtime.MemStats
 	runtime.ReadMemStats(&ms)
 	fmt.Fprintf(w, "go_memstats_alloc_bytes %d\n", ms.Alloc)
@@ -17,6 +31,7 @@ func writeGoMetrics(w io.Writer) {
 	fmt.Fprintf(w, "go_memstats_frees_total %d\n", ms.Frees)
 	fmt.Fprintf(w, "go_memstats_gc_cpu_fraction %g\n", ms.GCCPUFraction)
 	fmt.Fprintf(w, "go_memstats_gc_sys_bytes %d\n", ms.GCSys)
+
 	fmt.Fprintf(w, "go_memstats_heap_alloc_bytes %d\n", ms.HeapAlloc)
 	fmt.Fprintf(w, "go_memstats_heap_idle_bytes %d\n", ms.HeapIdle)
 	fmt.Fprintf(w, "go_memstats_heap_inuse_bytes %d\n", ms.HeapInuse)
@@ -62,3 +77,40 @@ func writeGoMetrics(w io.Writer) {
 	fmt.Fprintf(w, "go_info_ext{compiler=%q, GOARCH=%q, GOOS=%q, GOROOT=%q} 1\n",
 		runtime.Compiler, runtime.GOARCH, runtime.GOOS, runtime.GOROOT())
 }
+
+func writeRuntimeMetrics(w io.Writer) {
+	samples := make([]runtimemetrics.Sample, len(runtimeMetrics))
+	for i, rm := range runtimeMetrics {
+		samples[i].Name = rm[0]
+	}
+	runtimemetrics.Read(samples)
+	for i, rm := range runtimeMetrics {
+		writeRuntimeMetric(w, rm[1], &samples[i])
+	}
+}
+
+func writeRuntimeMetric(w io.Writer, name string, sample *runtimemetrics.Sample) {
+	switch sample.Value.Kind() {
+	case runtimemetrics.KindBad:
+		panic(fmt.Errorf("BUG: unexpected runtimemetrics.KindBad for sample.Name=%q", sample.Name))
+	case runtimemetrics.KindUint64:
+		fmt.Fprintf(w, "%s %d\n", name, sample.Value.Uint64())
+	case runtimemetrics.KindFloat64:
+		fmt.Fprintf(w, "%s %g\n", name, sample.Value.Float64())
+	case runtimemetrics.KindFloat64Histogram:
+		writeRuntimeHistogramMetric(w, name, sample.Value.Float64Histogram())
+	}
+}
+
+func writeRuntimeHistogramMetric(w io.Writer, name string, h *runtimemetrics.Float64Histogram) {
+	runningCount := uint64(0)
+	buckets := h.Buckets
+	for i, count := range h.Counts {
+		fmt.Fprintf(w, `%s_bucket{le="%g"} %d`+"\n", name, buckets[i], runningCount)
+		runningCount += count
+	}
+	fmt.Fprintf(w, `%s_bucket{le="%g"} %d`+"\n", name, buckets[len(buckets)-1], runningCount)
+	if !math.IsInf(buckets[len(buckets)-1], 1) {
+		fmt.Fprintf(w, `%s_bucket{le="+Inf"} %d`+"\n", name, runningCount)
+	}
+}

go_metrics_test.go

@@ -0,0 +1,65 @@
+package metrics
+
+import (
+	"math"
+	runtimemetrics "runtime/metrics"
+	"strings"
+	"testing"
+)
+
+func TestWriteRuntimeHistogramMetricOk(t *testing.T) {
+	f := func(h *runtimemetrics.Float64Histogram, resultExpected string) {
+		t.Helper()
+		var wOut strings.Builder
+		writeRuntimeHistogramMetric(&wOut, "foo", h)
+		result := wOut.String()
+		if result != resultExpected {
+			t.Fatalf("unexpected result; got\n%s\nwant\n%s", result, resultExpected)
+		}
+
+	}
+
+	f(&runtimemetrics.Float64Histogram{
+		Counts:  []uint64{1, 2, 3},
+		Buckets: []float64{1, 2, 3, 4},
+	}, `foo_bucket{le="1"} 0
+foo_bucket{le="2"} 1
+foo_bucket{le="3"} 3
+foo_bucket{le="4"} 6
+foo_bucket{le="+Inf"} 6
+`)
+
+	f(&runtimemetrics.Float64Histogram{
+		Counts:  []uint64{0, 25, 1, 3},
+		Buckets: []float64{1, 2, 3, 4, math.Inf(1)},
+	}, `foo_bucket{le="1"} 0
+foo_bucket{le="2"} 0
+foo_bucket{le="3"} 25
+foo_bucket{le="4"} 26
+foo_bucket{le="+Inf"} 29
+`)
+
+	f(&runtimemetrics.Float64Histogram{
+		Counts:  []uint64{0, 25, 1, 3, 0, 44, 15, 132, 10, 11},
+		Buckets: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, math.Inf(1)},
+	}, `foo_bucket{le="1"} 0
+foo_bucket{le="2"} 0
+foo_bucket{le="3"} 25
+foo_bucket{le="4"} 26
+foo_bucket{le="5"} 29
+foo_bucket{le="6"} 29
+foo_bucket{le="7"} 73
+foo_bucket{le="8"} 88
+foo_bucket{le="9"} 220
+foo_bucket{le="10"} 230
+foo_bucket{le="+Inf"} 241
+`)
+
+	f(&runtimemetrics.Float64Histogram{
+		Counts:  []uint64{1, 5},
+		Buckets: []float64{math.Inf(-1), 4, math.Inf(1)},
+	}, `foo_bucket{le="-Inf"} 0
+foo_bucket{le="4"} 1
+foo_bucket{le="+Inf"} 6
+`)
+}