Add metrics package and refactor stats collection

Signed-off-by: Ondrej Fabry <ofabry@cisco.com>

pkg/metrics/metrics.go

@@ -0,0 +1,83 @@
+//  Copyright (c) 2019 Cisco and/or its affiliates.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at:
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package metrics
+
+import (
+	"encoding/json"
+	"sort"
+	"time"
+)
+
+// RoundDuration is the default value used for rounding durations.
+var RoundDuration = time.Microsecond * 10
+
+type Calls map[string]*CallStats
+
+// MarshalJSON implements json.Marshaler interface
+func (m Calls) MarshalJSON() ([]byte, error) {
+	calls := make([]*CallStats, 0, len(m))
+	for _, s := range m {
+		calls = append(calls, s)
+	}
+	sort.Slice(calls, func(i, j int) bool {
+		return calls[i].Total > calls[j].Total
+	})
+	return json.Marshal(calls)
+}
+
+// CallStats represents generic stats for call metrics.
+type CallStats struct {
+	Name  string `json:",omitempty"`
+	Count uint64
+	Total Duration
+	Avg   Duration
+	Min   Duration
+	Max   Duration
+}
+
+// Increment increments call count and recalculates durations
+func (m *CallStats) Increment(d time.Duration) {
+	took := Duration(d)
+	m.Count++
+	m.Total += took
+	m.Avg = m.Total / Duration(m.Count)
+	if took > m.Max {
+		m.Max = took
+	}
+	if m.Min == 0 || took < m.Min {
+		m.Min = took
+	}
+}
+
+/*
+// MarshalJSON implements json.Marshaler interface
+func (m *CallStats) MarshalJSON() ([]byte, error) {
+	var d string
+	d = fmt.Sprintf(
+		"count: %d, total: %s (avg/min/max: %s/%s/%s)",
+		m.Count, durStr(m.TotalDur),
+		durStr(m.AvgDur), durStr(m.MinDur), durStr(m.MaxDur),
+	)
+	return json.Marshal(d)
+}
+*/
+
+type Duration time.Duration
+
+// MarshalJSON implements json.Marshaler interface
+func (m *Duration) MarshalJSON() ([]byte, error) {
+	s := time.Duration(*m).Round(RoundDuration).String()
+	return json.Marshal(s)
+}

plugins/govppmux/stats.go

@@ -15,25 +15,27 @@
 package govppmux
 
 import (
-	"encoding/json"
 	"expvar"
-	"fmt"
 	"sync"
 	"time"
+
+	"github.com/ligato/vpp-agent/pkg/metrics"
 )
 
 var (
-	stats     Stats
-	messageMu sync.RWMutex
+	stats   Stats
+	statsMu sync.RWMutex
 )
 
 func init() {
-	stats.Messages = map[string]*MessageStats{}
+	stats.Messages = make(metrics.Calls)
 }
 
 func GetStats() *Stats {
 	s := new(Stats)
+	statsMu.RLock()
 	*s = stats
+	statsMu.RUnlock()
 	return s
 }
 
@@ -42,58 +44,29 @@ type Stats struct {
 	ChannelsOpen    uint64
 	RequestsSent    uint64
 	RequestsFailed  uint64
-	Messages        map[string]*MessageStats
-}
-
-type MessageStats struct {
-	Message string
-	Calls   uint64
-	TotalNs time.Duration
-	AvgNs   time.Duration
-	MaxNs   time.Duration
-}
-
-func dur(d time.Duration) string {
-	return d.Round(time.Microsecond * 100).String()
+	AllMessages     metrics.CallStats
+	Messages        metrics.Calls
 }
 
-func (m *MessageStats) MarshalJSON() ([]byte, error) {
-	d := fmt.Sprintf(
-		"calls: %d, total: %s, avg: %s, max: %s",
-		m.Calls, dur(m.TotalNs), dur(m.AvgNs), dur(m.MaxNs),
-	)
-	return json.Marshal(d)
-}
-
-func (s *Stats) getOrCreateMessage(msg string) *MessageStats {
-	messageMu.RLock()
+func (s *Stats) getOrCreateMessage(msg string) *metrics.CallStats {
+	statsMu.RLock()
 	ms, ok := s.Messages[msg]
-	messageMu.RUnlock()
+	statsMu.RUnlock()
 	if !ok {
-		ms = &MessageStats{Message: msg}
-		messageMu.Lock()
+		ms = &metrics.CallStats{Name: msg}
+		statsMu.Lock()
 		s.Messages[msg] = ms
-		messageMu.Unlock()
+		statsMu.Unlock()
 	}
 	return ms
 }
 
-func (m *MessageStats) increment(took time.Duration) {
-	m.Calls++
-	m.TotalNs += took
-	m.AvgNs = m.TotalNs / time.Duration(m.Calls)
-	if took > m.MaxNs {
-		m.MaxNs = took
-	}
-}
-
 func trackMsgRequestDur(m string, d time.Duration) {
 	ms := stats.getOrCreateMessage(m)
-	mall := stats.getOrCreateMessage("ALL")
-	messageMu.Lock()
-	ms.increment(d)
-	mall.increment(d)
-	messageMu.Unlock()
+	statsMu.Lock()
+	ms.Increment(d)
+	stats.AllMessages.Increment(d)
+	statsMu.Unlock()
 }
 
 func init() {

plugins/kvscheduler/stats.go

@@ -17,9 +17,10 @@ package kvscheduler
 import (
 	"encoding/json"
 	"expvar"
-	"fmt"
 	"sync"
 	"time"
+
+	"github.com/ligato/vpp-agent/pkg/metrics"
 )
 
 var (
@@ -28,12 +29,34 @@ var (
 )
 
 func init() {
-	stats.Descriptors = map[string]*StructStats{
-		"ALL": {},
-	}
-	stats.Graph = &StructStats{}
+	stats.GraphMethods.Methods = make(metrics.Calls)
+	stats.AllDescriptors.Methods = make(metrics.Calls)
+	stats.Descriptors = make(map[string]*StructStats)
 }
 
+/*func GetDescriptorStats() map[string]metrics.Calls {
+	ss := make(map[string]metrics.Calls, len(stats.Descriptors))
+	statsMu.RLock()
+	for d, ds := range stats.Descriptors {
+		cc := make(metrics.Calls, len(ds))
+		for c, cs := range ds {
+			css := *cs
+			cc[c] = &css
+		}
+		ss[d] = cc
+	}
+	statsMu.RUnlock()
+	return ss
+}*/
+
+/*func GetGraphStats() *metrics.CallStats {
+	s := make(metrics.Calls, len(stats.Descriptors))
+	statsMu.RLock()
+	*s = stats.Graph
+	statsMu.RUnlock()
+	return s
+}*/
+
 func GetStats() *Stats {
 	s := new(Stats)
 	statsMu.RLock()
@@ -45,113 +68,69 @@ func GetStats() *Stats {
 type Stats struct {
 	TransactionsProcessed uint64
 
-	Graph       *StructStats
-	Descriptors map[string]*StructStats
+	GraphMethods   StructStats
+	AllDescriptors StructStats
+	Descriptors    map[string]*StructStats
 }
 
 func (s *Stats) addDescriptor(name string) {
-	s.Descriptors[name] = &StructStats{}
-}
-
-func GetDescriptorStats() map[string]*StructStats {
-	s := map[string]*StructStats{}
-	statsMu.RLock()
-	for d, ds := range stats.Descriptors {
-		dss := *ds
-		s[d] = &dss
+	s.Descriptors[name] = &StructStats{
+		Methods: make(metrics.Calls),
 	}
-	statsMu.RUnlock()
-	return s
-}
-
-func GetGraphStats() *StructStats {
-	s := new(StructStats)
-	statsMu.RLock()
-	*s = *stats.Graph
-	statsMu.RUnlock()
-	return s
 }
 
 type StructStats struct {
-	Methods []*MethodStats
-}
-
-type MethodStats struct {
-	Method  string
-	Calls   uint64
-	TotalNs time.Duration
-	AvgNs   time.Duration
-	MaxNs   time.Duration
-}
-
-func dur(d time.Duration) string {
-	return d.Round(time.Microsecond * 100).String()
+	Methods metrics.Calls `json:"-,omitempty"`
 }
 
 func (s *StructStats) MarshalJSON() ([]byte, error) {
-	d := map[string]string{}
+	/*d := make(map[string]*metrics.CallStats, len(s.Methods))
 	for _, ms := range s.Methods {
-		m := fmt.Sprintf("%s()", ms.Method)
-		d[m] = fmt.Sprintf(
-			"calls: %d, total: %s, avg: %s, max: %s",
-			ms.Calls, dur(ms.TotalNs), dur(ms.AvgNs), dur(ms.MaxNs),
-		)
-	}
-	return json.Marshal(d)
+		m := fmt.Sprintf("%s()", ms.Name)
+		d[m] = ms
+	}*/
+	return json.Marshal(s.Methods)
 }
 
-func (s *StructStats) getOrCreateMethod(method string) *MethodStats {
+func (s *StructStats) getOrCreateMethod(method string) *metrics.CallStats {
 	statsMu.RLock()
-	for _, m := range s.Methods {
-		if m.Method == method {
-			statsMu.RUnlock()
-			return m
-		}
-	}
+	ms, ok := s.Methods[method]
 	statsMu.RUnlock()
-	ms := &MethodStats{Method: method}
-	statsMu.Lock()
-	s.Methods = append(s.Methods, ms)
-	statsMu.Unlock()
-	return ms
-}
-
-func (m *MethodStats) increment(took time.Duration) {
-	statsMu.Lock()
-	m.Calls++
-	m.TotalNs += took
-	m.AvgNs = m.TotalNs / time.Duration(m.Calls)
-	if took > m.MaxNs {
-		m.MaxNs = took
+	if !ok {
+		ms = &metrics.CallStats{Name: method}
+		statsMu.Lock()
+		s.Methods[method] = ms
+		statsMu.Unlock()
 	}
-	statsMu.Unlock()
+	return ms
 }
 
 func trackDescMethod(d, m string) func() {
 	t := time.Now()
 	method := stats.Descriptors[d].getOrCreateMethod(m)
-	methodall := stats.Descriptors["ALL"].getOrCreateMethod(m)
+	methodall := stats.AllDescriptors.getOrCreateMethod(m)
 	return func() {
 		took := time.Since(t)
-		method.increment(took)
-		methodall.increment(took)
+		statsMu.Lock()
+		method.Increment(took)
+		methodall.Increment(took)
+		statsMu.Unlock()
 	}
 }
 
 func trackGraphMethod(m string) func() {
 	t := time.Now()
-	method := stats.Graph.getOrCreateMethod(m)
+	method := stats.GraphMethods.getOrCreateMethod(m)
 	return func() {
 		took := time.Since(t)
-		method.increment(took)
+		statsMu.Lock()
+		method.Increment(took)
+		statsMu.Unlock()
 	}
 }
 
 func init() {
-	expvar.Publish("kvdescriptors", expvar.Func(func() interface{} {
-		return GetDescriptorStats()
-	}))
-	expvar.Publish("kvgraph", expvar.Func(func() interface{} {
-		return GetGraphStats()
+	expvar.Publish("kvscheduler", expvar.Func(func() interface{} {
+		return GetStats()
 	}))
 }