From 2d13a7a78cfdfcf78380a0c16d85b870dbfb0394 Mon Sep 17 00:00:00 2001 From: Ondrej Fabry Date: Tue, 19 Mar 2019 14:18:50 +0100 Subject: [PATCH] Add metrics package and refactor stats collection Signed-off-by: Ondrej Fabry --- pkg/metrics/metrics.go | 83 +++++++++++++++++++++ plugins/govppmux/stats.go | 65 +++++------------ plugins/kvscheduler/stats.go | 137 +++++++++++++++-------------------- 3 files changed, 160 insertions(+), 125 deletions(-) create mode 100644 pkg/metrics/metrics.go diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 0000000000..e5e41e1d42 --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,83 @@ +// Copyright (c) 2019 Cisco and/or its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "encoding/json" + "sort" + "time" +) + +// RoundDuration is the default value used for rounding durations. +var RoundDuration = time.Microsecond * 10 + +type Calls map[string]*CallStats + +// MarshalJSON implements json.Marshaler interface +func (m Calls) MarshalJSON() ([]byte, error) { + calls := make([]*CallStats, 0, len(m)) + for _, s := range m { + calls = append(calls, s) + } + sort.Slice(calls, func(i, j int) bool { + return calls[i].Total > calls[j].Total + }) + return json.Marshal(calls) +} + +// CallStats represents generic stats for call metrics. +type CallStats struct { + Name string `json:",omitempty"` + Count uint64 + Total Duration + Avg Duration + Min Duration + Max Duration +} + +// Increment increments call count and recalculates durations +func (m *CallStats) Increment(d time.Duration) { + took := Duration(d) + m.Count++ + m.Total += took + m.Avg = m.Total / Duration(m.Count) + if took > m.Max { + m.Max = took + } + if m.Min == 0 || took < m.Min { + m.Min = took + } +} + +/* +// MarshalJSON implements json.Marshaler interface +func (m *CallStats) MarshalJSON() ([]byte, error) { + var d string + d = fmt.Sprintf( + "count: %d, total: %s (avg/min/max: %s/%s/%s)", + m.Count, durStr(m.TotalDur), + durStr(m.AvgDur), durStr(m.MinDur), durStr(m.MaxDur), + ) + return json.Marshal(d) +} +*/ + +type Duration time.Duration + +// MarshalJSON implements json.Marshaler interface +func (m *Duration) MarshalJSON() ([]byte, error) { + s := time.Duration(*m).Round(RoundDuration).String() + return json.Marshal(s) +} diff --git a/plugins/govppmux/stats.go b/plugins/govppmux/stats.go index 662f845767..ac1b016b66 100644 --- a/plugins/govppmux/stats.go +++ b/plugins/govppmux/stats.go @@ -15,25 +15,27 @@ package govppmux import ( - "encoding/json" "expvar" - "fmt" "sync" "time" + + "github.com/ligato/vpp-agent/pkg/metrics" ) var ( - stats Stats - messageMu sync.RWMutex + stats Stats + statsMu sync.RWMutex ) func init() { - stats.Messages = map[string]*MessageStats{} + stats.Messages = make(metrics.Calls) } func GetStats() *Stats { s := new(Stats) + statsMu.RLock() *s = stats + statsMu.RUnlock() return s } @@ -42,58 +44,29 @@ type Stats struct { ChannelsOpen uint64 RequestsSent uint64 RequestsFailed uint64 - Messages map[string]*MessageStats -} - -type MessageStats struct { - Message string - Calls uint64 - TotalNs time.Duration - AvgNs time.Duration - MaxNs time.Duration -} - -func dur(d time.Duration) string { - return d.Round(time.Microsecond * 100).String() + AllMessages metrics.CallStats + Messages metrics.Calls } -func (m *MessageStats) MarshalJSON() ([]byte, error) { - d := fmt.Sprintf( - "calls: %d, total: %s, avg: %s, max: %s", - m.Calls, dur(m.TotalNs), dur(m.AvgNs), dur(m.MaxNs), - ) - return json.Marshal(d) -} - -func (s *Stats) getOrCreateMessage(msg string) *MessageStats { - messageMu.RLock() +func (s *Stats) getOrCreateMessage(msg string) *metrics.CallStats { + statsMu.RLock() ms, ok := s.Messages[msg] - messageMu.RUnlock() + statsMu.RUnlock() if !ok { - ms = &MessageStats{Message: msg} - messageMu.Lock() + ms = &metrics.CallStats{Name: msg} + statsMu.Lock() s.Messages[msg] = ms - messageMu.Unlock() + statsMu.Unlock() } return ms } -func (m *MessageStats) increment(took time.Duration) { - m.Calls++ - m.TotalNs += took - m.AvgNs = m.TotalNs / time.Duration(m.Calls) - if took > m.MaxNs { - m.MaxNs = took - } -} - func trackMsgRequestDur(m string, d time.Duration) { ms := stats.getOrCreateMessage(m) - mall := stats.getOrCreateMessage("ALL") - messageMu.Lock() - ms.increment(d) - mall.increment(d) - messageMu.Unlock() + statsMu.Lock() + ms.Increment(d) + stats.AllMessages.Increment(d) + statsMu.Unlock() } func init() { diff --git a/plugins/kvscheduler/stats.go b/plugins/kvscheduler/stats.go index ff2c662c19..6fe4f7f1eb 100644 --- a/plugins/kvscheduler/stats.go +++ b/plugins/kvscheduler/stats.go @@ -17,9 +17,10 @@ package kvscheduler import ( "encoding/json" "expvar" - "fmt" "sync" "time" + + "github.com/ligato/vpp-agent/pkg/metrics" ) var ( @@ -28,12 +29,34 @@ var ( ) func init() { - stats.Descriptors = map[string]*StructStats{ - "ALL": {}, - } - stats.Graph = &StructStats{} + stats.GraphMethods.Methods = make(metrics.Calls) + stats.AllDescriptors.Methods = make(metrics.Calls) + stats.Descriptors = make(map[string]*StructStats) } +/*func GetDescriptorStats() map[string]metrics.Calls { + ss := make(map[string]metrics.Calls, len(stats.Descriptors)) + statsMu.RLock() + for d, ds := range stats.Descriptors { + cc := make(metrics.Calls, len(ds)) + for c, cs := range ds { + css := *cs + cc[c] = &css + } + ss[d] = cc + } + statsMu.RUnlock() + return ss +}*/ + +/*func GetGraphStats() *metrics.CallStats { + s := make(metrics.Calls, len(stats.Descriptors)) + statsMu.RLock() + *s = stats.Graph + statsMu.RUnlock() + return s +}*/ + func GetStats() *Stats { s := new(Stats) statsMu.RLock() @@ -45,113 +68,69 @@ func GetStats() *Stats { type Stats struct { TransactionsProcessed uint64 - Graph *StructStats - Descriptors map[string]*StructStats + GraphMethods StructStats + AllDescriptors StructStats + Descriptors map[string]*StructStats } func (s *Stats) addDescriptor(name string) { - s.Descriptors[name] = &StructStats{} -} - -func GetDescriptorStats() map[string]*StructStats { - s := map[string]*StructStats{} - statsMu.RLock() - for d, ds := range stats.Descriptors { - dss := *ds - s[d] = &dss + s.Descriptors[name] = &StructStats{ + Methods: make(metrics.Calls), } - statsMu.RUnlock() - return s -} - -func GetGraphStats() *StructStats { - s := new(StructStats) - statsMu.RLock() - *s = *stats.Graph - statsMu.RUnlock() - return s } type StructStats struct { - Methods []*MethodStats -} - -type MethodStats struct { - Method string - Calls uint64 - TotalNs time.Duration - AvgNs time.Duration - MaxNs time.Duration -} - -func dur(d time.Duration) string { - return d.Round(time.Microsecond * 100).String() + Methods metrics.Calls `json:"-,omitempty"` } func (s *StructStats) MarshalJSON() ([]byte, error) { - d := map[string]string{} + /*d := make(map[string]*metrics.CallStats, len(s.Methods)) for _, ms := range s.Methods { - m := fmt.Sprintf("%s()", ms.Method) - d[m] = fmt.Sprintf( - "calls: %d, total: %s, avg: %s, max: %s", - ms.Calls, dur(ms.TotalNs), dur(ms.AvgNs), dur(ms.MaxNs), - ) - } - return json.Marshal(d) + m := fmt.Sprintf("%s()", ms.Name) + d[m] = ms + }*/ + return json.Marshal(s.Methods) } -func (s *StructStats) getOrCreateMethod(method string) *MethodStats { +func (s *StructStats) getOrCreateMethod(method string) *metrics.CallStats { statsMu.RLock() - for _, m := range s.Methods { - if m.Method == method { - statsMu.RUnlock() - return m - } - } + ms, ok := s.Methods[method] statsMu.RUnlock() - ms := &MethodStats{Method: method} - statsMu.Lock() - s.Methods = append(s.Methods, ms) - statsMu.Unlock() - return ms -} - -func (m *MethodStats) increment(took time.Duration) { - statsMu.Lock() - m.Calls++ - m.TotalNs += took - m.AvgNs = m.TotalNs / time.Duration(m.Calls) - if took > m.MaxNs { - m.MaxNs = took + if !ok { + ms = &metrics.CallStats{Name: method} + statsMu.Lock() + s.Methods[method] = ms + statsMu.Unlock() } - statsMu.Unlock() + return ms } func trackDescMethod(d, m string) func() { t := time.Now() method := stats.Descriptors[d].getOrCreateMethod(m) - methodall := stats.Descriptors["ALL"].getOrCreateMethod(m) + methodall := stats.AllDescriptors.getOrCreateMethod(m) return func() { took := time.Since(t) - method.increment(took) - methodall.increment(took) + statsMu.Lock() + method.Increment(took) + methodall.Increment(took) + statsMu.Unlock() } } func trackGraphMethod(m string) func() { t := time.Now() - method := stats.Graph.getOrCreateMethod(m) + method := stats.GraphMethods.getOrCreateMethod(m) return func() { took := time.Since(t) - method.increment(took) + statsMu.Lock() + method.Increment(took) + statsMu.Unlock() } } func init() { - expvar.Publish("kvdescriptors", expvar.Func(func() interface{} { - return GetDescriptorStats() - })) - expvar.Publish("kvgraph", expvar.Func(func() interface{} { - return GetGraphStats() + expvar.Publish("kvscheduler", expvar.Func(func() interface{} { + return GetStats() })) }