Skip to content

Added more metrics and small bugfix + more docs #291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jan 18, 2018
Merged

Added more metrics and small bugfix + more docs #291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
19e098c
fixed pps out using the wrong metrics
roffe Jan 18, 2018
0542386
adding debug
roffe Jan 18, 2018
51e3dc4
name fixes
roffe Jan 18, 2018
f777b44
missing mustregister
roffe Jan 18, 2018
afe1fbc
missing bracer
roffe Jan 18, 2018
3138c3a
added more metrics docs
roffe Jan 18, 2018
d91e171
fixed faulty names
roffe Jan 18, 2018
8e712fd
fixed more faulty names
roffe Jan 18, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 29 additions & 1 deletion Documentation/metrics.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,32 @@ For example:
k8s-app: kube-router
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "8080"
prometheus.io/port: "8080"

## Avail metrics

The following metrics is exposed by kube-router prefixed by `kube_router_`

* service_total_connections
Total connections made to the service since creation
* service_packets_in
Total n/o packets received by service
* service_packets_out
Total n/o packets sent by service
* service_bytes_in
Total bytes received by the service
* service_bytes_out
Total bytes sent by the service
* service_pps_in
Incoming packets per second
* service_pps_out
Outgoing packets per second
* service_cps
Connections per second
* service_bps_in
Incoming bytes per second
* service_bps_out
Outgoing bytes per second

To get a grouped list of CPS for each service a Prometheus query could look like this e.g:
`sum(kube_router_service_cps) by (namespace, service_name)`
80 changes: 69 additions & 11 deletions app/controllers/network_services_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,31 @@ const (
)

var (
h *ipvs.Handle
serviceActiveConn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
h *ipvs.Handle
serviceTotalConn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_active_connections",
Help: "Active conntection to service",
Name: "service_total_connections",
Help: "Total conntection to service",
}, []string{"namespace", "service_name", "service_vip"})
servicePacketsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_packets_in",
Help: "Total incoming packets",
}, []string{"namespace", "service_name", "service_vip"})
servicePacketsOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_packets_out",
Help: "Total outoging packets",
}, []string{"namespace", "service_name", "service_vip"})
serviceBytesIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_bytes_in",
Help: "Total incoming bytes",
}, []string{"namespace", "service_name", "service_vip"})
serviceBytesOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_bytes_out",
Help: "Total outoging bytes",
}, []string{"namespace", "service_name", "service_vip"})
servicePpsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Expand All @@ -61,6 +81,21 @@ var (
Name: "service_pps_out",
Help: "Outoging packets per second",
}, []string{"namespace", "service_name", "service_vip"})
serviceCPS = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_cps",
Help: "Service connections per second",
}, []string{"namespace", "service_name", "service_vip"})
serviceBpsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_bps_in",
Help: "Incoming bytes per second",
}, []string{"namespace", "service_name", "service_vip"})
serviceBpsOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "service_bps_out",
Help: "Outoging bytes per second",
}, []string{"namespace", "service_name", "service_vip"})
)

// NetworkServicesController enables local node as network service proxy through IPVS/LVS.
Expand Down Expand Up @@ -132,9 +167,17 @@ func (nsc *NetworkServicesController) Run(stopCh <-chan struct{}, wg *sync.WaitG
}

// register metrics
prometheus.MustRegister(serviceActiveConn)
prometheus.MustRegister(serviceBpsIn)
prometheus.MustRegister(serviceBpsOut)
prometheus.MustRegister(serviceBytesIn)
prometheus.MustRegister(serviceBytesOut)
prometheus.MustRegister(serviceCPS)
prometheus.MustRegister(servicePacketsIn)
prometheus.MustRegister(servicePacketsOut)
prometheus.MustRegister(servicePpsIn)
prometheus.MustRegister(servicePpsOut)
prometheus.MustRegister(serviceTotalConn)

http.Handle(nsc.MetricsPath, promhttp.Handler())
go http.ListenAndServe(":"+strconv.Itoa(nsc.MetricsPort), nil)

Expand Down Expand Up @@ -748,16 +791,31 @@ func (nsc *NetworkServicesController) publishMetrics(serviceInfoMap serviceInfoM
if strings.Compare(svc.clusterIP.String(), ipvsSvc.Address.String()) == 0 &&
protocol == ipvsSvc.Protocol && uint16(svc.port) == ipvsSvc.Port {
glog.Infof("Publishing prometheus metrics " + svc.clusterIP.String() + ":" + strconv.Itoa(svc.port))
serviceActiveConn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.Connections))
servicePpsIn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.PacketsIn))
servicePpsOut.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.PacketsIn))
serviceBpsIn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.BPSIn))
serviceBpsOut.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.BPSOut))
serviceBytesIn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.BytesIn))
serviceBytesOut.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.BytesOut))
serviceCPS.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.CPS))
servicePacketsIn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.PacketsIn))
servicePacketsOut.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.PacketsOut))
servicePpsIn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.PPSIn))
servicePpsOut.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.PPSOut))
serviceTotalConn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.Connections))

}
if strings.Compare(nsc.nodeIP.String(), ipvsSvc.Address.String()) == 0 &&
protocol == ipvsSvc.Protocol && uint16(svc.port) == ipvsSvc.Port {
glog.Infof("Publishing prometheus metrics " + nsc.nodeIP.String() + ":" + strconv.Itoa(svc.port))
serviceActiveConn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.Connections))
servicePpsIn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.PacketsIn))
servicePpsOut.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.PacketsIn))
serviceBpsIn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.BPSIn))
serviceBpsOut.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.BPSOut))
serviceBytesIn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.BytesIn))
serviceBytesOut.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.BytesOut))
serviceCPS.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.CPS))
servicePacketsIn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.PacketsIn))
servicePacketsOut.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.PacketsOut))
servicePpsIn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.PPSIn))
servicePpsOut.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.PPSOut))
serviceTotalConn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.Connections))
}
}
}
Expand Down