Skip to content

Commit

Permalink
Add runtime metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
Brian Hoffmann authored and panzerfahrer committed Aug 14, 2019
1 parent f6e96bf commit a39a0cd
Show file tree
Hide file tree
Showing 4 changed files with 280 additions and 13 deletions.
20 changes: 20 additions & 0 deletions framework/opencensus/Readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,26 @@ Opencensus allows you to collect data for your application and process them via
The core package already collects metrics automatically for routers and prefixrouters rendering times out of the box.
Also traces for request handling are done by default.

## Default runtime metrics

By default runtime metrics will be recorded. The following configuration is available:

```yaml
opencensus:
runtimeMetrics:
enable: true # default: true
interval: 15 # interval in seconds. default: 15
prefix: myapp # prefix metric names. default: "process_"
```
The following metrics will be recorded:
- process heap allocation
- number of objects allocated on the heap
- number of objects released from the heap
- memory used by stack spans and OS thread stacks
- number of pointer lookups
- number of current goroutines
## Adding your own metrics
The most likely usecase is in a controller, but to have your own metric, please import the following packages:
Expand Down
52 changes: 39 additions & 13 deletions framework/opencensus/module.go
Original file line number Diff line number Diff line change
@@ -1,19 +1,23 @@
package opencensus

import (
"context"
"fmt"
"log"
"math/rand"
"net"
"net/http"
"sync"
"time"

"contrib.go.opencensus.io/exporter/jaeger"
"contrib.go.opencensus.io/exporter/prometheus"
"contrib.go.opencensus.io/exporter/zipkin"
"flamingo.me/dingo"
openzipkin "github.com/openzipkin/zipkin-go"
reporterHttp "github.com/openzipkin/zipkin-go/reporter/http"
"go.opencensus.io/metric"
"go.opencensus.io/metric/metricproducer"
"go.opencensus.io/plugin/ochttp"
"go.opencensus.io/stats"
"go.opencensus.io/stats/view"
Expand Down Expand Up @@ -56,12 +60,15 @@ func (rt *correlationIDInjector) RoundTrip(req *http.Request) (*http.Response, e

// Module registers the opencensus module which in turn enables jaeger & co
type Module struct {
Endpoint string `inject:"config:opencensus.jaeger.endpoint"`
ServiceName string `inject:"config:opencensus.serviceName"`
ServiceAddr string `inject:"config:opencensus.serviceAddr"`
JaegerEnable bool `inject:"config:opencensus.jaeger.enable"`
ZipkinEnable bool `inject:"config:opencensus.zipkin.enable"`
ZipkinEndpoint string `inject:"config:opencensus.zipkin.endpoint"`
RuntimeMetricsEnable bool `inject:"config:opencensus.runtimeMetrics.enable"`
RuntimeMetricsIntervalSec int `inject:"config:opencensus.runtimeMetrics.interval"`
RuntimeMetricsPrefix string `inject:"config:opencensus.runtimeMetrics.prefix"`
ServiceName string `inject:"config:opencensus.serviceName"`
ServiceAddr string `inject:"config:opencensus.serviceAddr"`
JaegerEndpoint string `inject:"config:opencensus.jaeger.endpoint"`
JaegerEnable bool `inject:"config:opencensus.jaeger.enable"`
ZipkinEnable bool `inject:"config:opencensus.zipkin.enable"`
ZipkinEndpoint string `inject:"config:opencensus.zipkin.endpoint"`
}

// find first not-loopback ipv4 address
Expand Down Expand Up @@ -96,11 +103,26 @@ func (m *Module) Configure(injector *dingo.Injector) {
trace.ApplyConfig(trace.Config{DefaultSampler: trace.NeverSample()})
http.DefaultTransport = &correlationIDInjector{next: &ochttp.Transport{Base: http.DefaultTransport}}

reg := metric.NewRegistry()
metricproducer.GlobalManager().AddProducer(reg)

if m.RuntimeMetricsEnable {
opt := GaugeOptions{
Prefix: m.RuntimeMetricsPrefix,
}

runtimeGauges, err := NewRuntimeGauges(reg, opt)
if err != nil {
log.Fatal(err)
}
go runtimeGauges.StartRecording(context.Background(), time.Duration(m.RuntimeMetricsIntervalSec)*time.Second)
}

if m.JaegerEnable {
// Register the Jaeger exporter to be able to retrieve
// the collected spans.
exporter, err := jaeger.NewExporter(jaeger.Options{
CollectorEndpoint: m.Endpoint,
CollectorEndpoint: m.JaegerEndpoint,
Process: jaeger.Process{
ServiceName: m.ServiceName,
Tags: []jaeger.Tag{
Expand Down Expand Up @@ -130,6 +152,7 @@ func (m *Module) Configure(injector *dingo.Injector) {
trace.RegisterExporter(exporter)
}
})

exporter, err := prometheus.NewExporter(prometheus.Options{})
if err != nil {
log.Fatal(err)
Expand All @@ -142,12 +165,15 @@ func (m *Module) Configure(injector *dingo.Injector) {
func (m *Module) DefaultConfig() config.Map {
return config.Map{
"opencensus": config.Map{
"jaeger.enable": false,
"jaeger.endpoint": "http://localhost:14268/api/traces",
"zipkin.enable": false,
"zipkin.endpoint": "http://localhost:9411/api/v2/spans",
"serviceName": "flamingo",
"serviceAddr": ":13210",
"runtimeMetrics.enable": true,
"runtimeMetrics.interval": 15,
"runtimeMetrics.prefix": "process_",
"jaeger.enable": false,
"jaeger.endpoint": "http://localhost:14268/api/traces",
"zipkin.enable": false,
"zipkin.endpoint": "http://localhost:9411/api/v2/spans",
"serviceName": "flamingo",
"serviceAddr": ":13210",
"tracing": config.Map{
"sampler": config.Map{
"whitelist": config.Slice{},
Expand Down
107 changes: 107 additions & 0 deletions framework/opencensus/runtime_gauges.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package opencensus

import (
"context"
"runtime"
"time"

"github.com/pkg/errors"
"go.opencensus.io/metric"
"go.opencensus.io/metric/metricdata"
)

type (
// runtimeGauges collects runtime metrics in gauge entries.
runtimeGauges struct {
heapAlloc *metric.Int64GaugeEntry
heapObjects *metric.Int64GaugeEntry
heapReleased *metric.Int64GaugeEntry
stackSys *metric.Int64GaugeEntry
ptrLookups *metric.Int64GaugeEntry
numGoroutines *metric.Int64GaugeEntry
}

GaugeOptions struct {
Prefix string
}
)

func NewRuntimeGauges(reg *metric.Registry, gaugeOptions GaugeOptions) (*runtimeGauges, error) {
opt := gaugeOptions

rg := new(runtimeGauges)
var err error

rg.heapAlloc, err = createInt64GaugeEntry(reg, opt.Prefix+"heap_alloc", "Process heap allocation", metricdata.UnitBytes)
if err != nil {
return nil, err
}

rg.heapObjects, err = createInt64GaugeEntry(reg, opt.Prefix+"heap_objects", "The number of objects allocated on the heap", metricdata.UnitDimensionless)
if err != nil {
return nil, err
}

rg.heapReleased, err = createInt64GaugeEntry(reg, opt.Prefix+"heap_release", "The number of objects released from the heap", metricdata.UnitBytes)
if err != nil {
return nil, err
}

rg.stackSys, err = createInt64GaugeEntry(reg, opt.Prefix+"stack_sys", "The memory used by stack spans and OS thread stacks", metricdata.UnitBytes)
if err != nil {
return nil, err
}

rg.ptrLookups, err = createInt64GaugeEntry(reg, opt.Prefix+"ptr_lookups", "The number of pointer lookups", metricdata.UnitDimensionless)
if err != nil {
return nil, err
}

rg.numGoroutines, err = createInt64GaugeEntry(reg, opt.Prefix+"num_goroutines", "Number of current goroutines", metricdata.UnitDimensionless)
if err != nil {
return nil, err
}

return rg, nil
}

func (r *runtimeGauges) StartRecording(ctx context.Context, delay time.Duration) {
mem := &runtime.MemStats{}

tick := time.NewTicker(delay)
defer tick.Stop()

for {
select {
case <-ctx.Done():
return

case <-tick.C:
runtime.ReadMemStats(mem)
r.heapAlloc.Set(int64(mem.HeapAlloc))
r.heapObjects.Set(int64(mem.HeapObjects))
r.heapReleased.Set(int64(mem.HeapReleased))
r.stackSys.Set(int64(mem.StackSys))
r.ptrLookups.Set(int64(mem.Lookups))

r.numGoroutines.Set(int64(runtime.NumGoroutine()))
}
}
}

func createInt64GaugeEntry(reg *metric.Registry, name string, description string, unit metricdata.Unit) (*metric.Int64GaugeEntry, error) {
gauge, err := reg.AddInt64Gauge(
name,
metric.WithDescription(description),
metric.WithUnit(unit))
if err != nil {
return nil, errors.WithMessage(err, "error creating gauge for "+name)
}

entry, err := gauge.GetEntry()
if err != nil {
return nil, errors.WithMessage(err, "error getting gauge entry for "+name)
}

return entry, nil
}
114 changes: 114 additions & 0 deletions framework/opencensus/runtime_gauges_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
package opencensus

import (
"context"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"time"

"contrib.go.opencensus.io/exporter/prometheus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opencensus.io/metric"
"go.opencensus.io/metric/metricdata"
"go.opencensus.io/metric/metricexport"
"go.opencensus.io/metric/metricproducer"
)

type testExporter struct {
data []*metricdata.Metric
}

func (t *testExporter) ExportMetrics(ctx context.Context, data []*metricdata.Metric) error {
t.data = append(t.data, data...)
return nil
}

func TestRuntimeGauges(t *testing.T) {
tests := []struct {
name string
options GaugeOptions
expectedNames []string
}{
{
"custom prefix",
GaugeOptions{Prefix: "test_"},
[]string{"test_heap_alloc", "test_heap_objects", "test_heap_release", "test_stack_sys", "test_ptr_lookups", "test_num_goroutines"},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
reg := metric.NewRegistry()
metricproducer.GlobalManager().AddProducer(reg)
defer metricproducer.GlobalManager().DeleteProducer(reg)

gauges, err := NewRuntimeGauges(reg, test.options)
require.NoError(t, err)

ctx, cancel := context.WithCancel(context.Background())
time.AfterFunc(2*time.Second, func() {
cancel()
})
go gauges.StartRecording(ctx, 1*time.Second)

exporter := &testExporter{}
reader := metricexport.NewReader()
reader.ReadAndExport(exporter)

assertNames(t, exporter, test.expectedNames)
})
}
}

func assertNames(t *testing.T, exporter *testExporter, expectedNames []string) {
metricNames := make([]string, 0)
for _, v := range exporter.data {
metricNames = append(metricNames, v.Descriptor.Name)
}
assert.ElementsMatchf(t, expectedNames, metricNames, "actual: %v", metricNames)
}

func TestRuntimeGauges_WithPrometheus(t *testing.T) {
reg := metric.NewRegistry()
metricproducer.GlobalManager().AddProducer(reg)
defer metricproducer.GlobalManager().DeleteProducer(reg)

gauges, err := NewRuntimeGauges(reg, GaugeOptions{Prefix: "test_"})
require.NoError(t, err)

ctx, cancel := context.WithCancel(context.Background())
time.AfterFunc(2*time.Second, func() {
cancel()
})
go gauges.StartRecording(ctx, 1*time.Second)

exporter, err := prometheus.NewExporter(prometheus.Options{})
require.NoError(t, err)

server := httptest.NewServer(exporter)
defer server.Close()

// wait for at least one metric to be written
<-time.After(1 * time.Second)

resp, err := http.Get(server.URL)
require.NoError(t, err)

if resp.Body != nil {
defer resp.Body.Close()
}

bytes, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)

strBody := string(bytes)
assert.Regexp(t, "test_heap_alloc \\d+", strBody)
assert.Regexp(t, "test_heap_objects \\d+", strBody)
assert.Regexp(t, "test_heap_release \\d+", strBody)
assert.Regexp(t, "test_stack_sys \\d+", strBody)
assert.Regexp(t, "test_ptr_lookups \\d+", strBody)
assert.Regexp(t, "test_num_goroutines \\d+", strBody)
}

0 comments on commit a39a0cd

Please sign in to comment.