Skip to content

Commit

Permalink
add new origin inspector stats (#132)
Browse files Browse the repository at this point in the history 
Rather than adding new metrics for WafResponsesTotal and
ComputeReponsesTotal this adds a new "source" label to
the existing origin inspector metrics.
  • Loading branch information
@leklund
leklund committed Jun 26, 2023
1 parent 262be75 commit b45aca3
Show file tree
Hide file tree
Showing 3 changed files with 208 additions and 31 deletions.
12 changes: 6 additions & 6 deletions pkg/origin/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ type Metrics struct {
// Only metrics whose names pass the name filter are registered.
func NewMetrics(namespace, subsystem string, nameFilter filter.Filter, r prometheus.Registerer) *Metrics {
m := Metrics{
RespBodyBytesTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "resp_body_bytes_total", Help: `Number of body bytes from origin.`}, []string{"service_id", "service_name", "datacenter", "origin"}),
RespHeaderBytesTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "resp_header_bytes_total", Help: `Number of header bytes from origin.`}, []string{"service_id", "service_name", "datacenter", "origin"}),
ResponsesTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "responses_total", Help: `Number of responses from origin.`}, []string{"service_id", "service_name", "datacenter", "origin"}),
StatusCodeTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "status_code_total", Help: `Number of responses from origin, by status code e.g. 200, 419.`}, []string{"service_id", "service_name", "datacenter", "origin", "status_code"}),
StatusGroupTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "status_group_total", Help: `Number of responses from origin, by status group e.g. 1xx, 2xx.`}, []string{"service_id", "service_name", "datacenter", "origin", "status_group"}),
LatencySeconds: prometheus.NewHistogramVec(prometheus.HistogramOpts{Namespace: namespace, Subsystem: subsystem, Name: "latency_seconds", Help: `Response time from origin in seconds.`, Buckets: []float64{0.001, 0.005, 0.010, 0.050, 0.100, 0.250, 0.500, 1.000, 5.000, 10.000, 60.000}}, []string{"service_id", "service_name", "datacenter", "origin"}),
RespBodyBytesTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "resp_body_bytes_total", Help: `Number of body bytes from origin.`}, []string{"service_id", "service_name", "datacenter", "origin", "source"}),
RespHeaderBytesTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "resp_header_bytes_total", Help: `Number of header bytes from origin.`}, []string{"service_id", "service_name", "datacenter", "origin", "source"}),
ResponsesTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "responses_total", Help: `Number of responses from origin.`}, []string{"service_id", "service_name", "datacenter", "origin", "source"}),
StatusCodeTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "status_code_total", Help: `Number of responses from origin, by status code e.g. 200, 419.`}, []string{"service_id", "service_name", "datacenter", "origin", "source", "status_code"}),
StatusGroupTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "status_group_total", Help: `Number of responses from origin, by status group e.g. 1xx, 2xx.`}, []string{"service_id", "service_name", "datacenter", "origin", "source", "status_group"}),
LatencySeconds: prometheus.NewHistogramVec(prometheus.HistogramOpts{Namespace: namespace, Subsystem: subsystem, Name: "latency_seconds", Help: `Response time from origin in seconds.`, Buckets: []float64{0.001, 0.005, 0.010, 0.050, 0.100, 0.250, 0.500, 1.000, 5.000, 10.000, 60.000}}, []string{"service_id", "service_name", "datacenter", "origin", "source"}),
}

for i, v := 0, reflect.ValueOf(m); i < v.NumField(); i++ {
Expand Down
146 changes: 122 additions & 24 deletions pkg/origin/process.go
View file
Original file line number Diff line number Diff line change
@@ -1,34 +1,94 @@
package origin

const (
srcDelivery = "delivery"
srcCompute = "compute"
srcWaf = "waf"
)

// Process updates the metrics with data from the API response.
func Process(response *Response, serviceID, serviceName, serviceVersion string, m *Metrics) {

Check failure on line 10 in pkg/origin/process.go

View workflow job for this annotation

GitHub Actions / test (ubuntu-latest) 

parameter 'serviceVersion' seems to be unused, consider removing or renaming it as _

Check failure on line 10 in pkg/origin/process.go

View workflow job for this annotation

GitHub Actions / test (macos-latest) 

parameter 'serviceVersion' seems to be unused, consider removing or renaming it as _

Check failure on line 10 in pkg/origin/process.go

View workflow job for this annotation

GitHub Actions / test (windows-latest) 

parameter 'serviceVersion' seems to be unused, consider removing or renaming it as _
for _, d := range response.Data {
for datacenter, byOrigin := range d.Datacenter {
for origin, stats := range byOrigin {
m.RespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter, origin).Add(float64(stats.RespBodyBytes))
m.RespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter, origin).Add(float64(stats.RespHeaderBytes))
m.ResponsesTotal.WithLabelValues(serviceID, serviceName, datacenter, origin).Add(float64(stats.Responses))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "1xx").Add(float64(stats.Status1xx))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "200").Add(float64(stats.Status200))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "204").Add(float64(stats.Status204))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "2xx").Add(float64(stats.Status2xx))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "301").Add(float64(stats.Status301))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "302").Add(float64(stats.Status302))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "304").Add(float64(stats.Status304))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "3xx").Add(float64(stats.Status3xx))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "400").Add(float64(stats.Status400))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "401").Add(float64(stats.Status401))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "403").Add(float64(stats.Status403))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "404").Add(float64(stats.Status404))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "416").Add(float64(stats.Status416))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "4xx").Add(float64(stats.Status4xx))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "500").Add(float64(stats.Status500))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "501").Add(float64(stats.Status501))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "502").Add(float64(stats.Status502))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "503").Add(float64(stats.Status503))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "504").Add(float64(stats.Status504))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "505").Add(float64(stats.Status505))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "5xx").Add(float64(stats.Status5xx))
m.RespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery).Add(float64(stats.RespBodyBytes))
m.RespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute).Add(float64(stats.ComputeRespBodyBytes))
m.RespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf).Add(float64(stats.WafRespBodyBytes))
m.RespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery).Add(float64(stats.RespHeaderBytes))
m.RespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute).Add(float64(stats.ComputeRespHeaderBytes))
m.RespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf).Add(float64(stats.WafRespHeaderBytes))
m.ResponsesTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery).Add(float64(stats.Responses))
m.ResponsesTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute).Add(float64(stats.ComputeResponses))
m.ResponsesTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf).Add(float64(stats.WafResponses))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "1xx").Add(float64(stats.Status1xx))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "200").Add(float64(stats.Status200))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "204").Add(float64(stats.Status204))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "206").Add(float64(stats.Status206))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "2xx").Add(float64(stats.Status2xx))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "301").Add(float64(stats.Status301))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "302").Add(float64(stats.Status302))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "304").Add(float64(stats.Status304))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "3xx").Add(float64(stats.Status3xx))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "400").Add(float64(stats.Status400))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "401").Add(float64(stats.Status401))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "403").Add(float64(stats.Status403))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "404").Add(float64(stats.Status404))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "416").Add(float64(stats.Status416))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "429").Add(float64(stats.Status429))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "4xx").Add(float64(stats.Status4xx))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "500").Add(float64(stats.Status500))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "501").Add(float64(stats.Status501))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "502").Add(float64(stats.Status502))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "503").Add(float64(stats.Status503))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "504").Add(float64(stats.Status504))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "505").Add(float64(stats.Status505))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcDelivery, "5xx").Add(float64(stats.Status5xx))

m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "1xx").Add(float64(stats.ComputeStatus1xx))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "200").Add(float64(stats.ComputeStatus200))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "204").Add(float64(stats.ComputeStatus204))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "2xx").Add(float64(stats.ComputeStatus2xx))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "301").Add(float64(stats.ComputeStatus301))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "302").Add(float64(stats.ComputeStatus302))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "304").Add(float64(stats.ComputeStatus304))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "3xx").Add(float64(stats.ComputeStatus3xx))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "400").Add(float64(stats.ComputeStatus400))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "401").Add(float64(stats.ComputeStatus401))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "403").Add(float64(stats.ComputeStatus403))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "404").Add(float64(stats.ComputeStatus404))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "416").Add(float64(stats.ComputeStatus416))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "4xx").Add(float64(stats.ComputeStatus4xx))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "500").Add(float64(stats.ComputeStatus500))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "501").Add(float64(stats.ComputeStatus501))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "502").Add(float64(stats.ComputeStatus502))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "503").Add(float64(stats.ComputeStatus503))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "504").Add(float64(stats.ComputeStatus504))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "505").Add(float64(stats.ComputeStatus505))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcCompute, "5xx").Add(float64(stats.ComputeStatus5xx))

m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "1xx").Add(float64(stats.WafStatus1xx))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "200").Add(float64(stats.WafStatus200))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "204").Add(float64(stats.WafStatus204))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "206").Add(float64(stats.WafStatus206))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "2xx").Add(float64(stats.WafStatus2xx))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "301").Add(float64(stats.WafStatus301))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "302").Add(float64(stats.WafStatus302))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "304").Add(float64(stats.WafStatus304))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "3xx").Add(float64(stats.WafStatus3xx))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "400").Add(float64(stats.WafStatus400))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "401").Add(float64(stats.WafStatus401))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "403").Add(float64(stats.WafStatus403))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "404").Add(float64(stats.WafStatus404))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "416").Add(float64(stats.WafStatus416))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "429").Add(float64(stats.WafStatus429))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "4xx").Add(float64(stats.WafStatus4xx))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "500").Add(float64(stats.WafStatus500))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "501").Add(float64(stats.WafStatus501))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "502").Add(float64(stats.WafStatus502))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "503").Add(float64(stats.WafStatus503))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "504").Add(float64(stats.WafStatus504))
m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "505").Add(float64(stats.WafStatus505))
m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, srcWaf, "5xx").Add(float64(stats.WafStatus5xx))

// Latency stats are clearly from xxx_bucket{le="v"} metrics,
// but I don't see a good way to re-populate a histogram from
Expand Down Expand Up @@ -56,6 +116,44 @@ func Process(response *Response, serviceID, serviceName, serviceVersion string,
m.LatencySeconds.WithLabelValues(serviceID, serviceName, datacenter, origin).Observe(v)
}
}

for v, n := range map[float64]uint64{
60.00: stats.WafLatency60000plus,
10.00: stats.WafLatency10000to60000,
5.000: stats.WafLatency5000to10000,
1.000: stats.WafLatency1000to5000,
0.500: stats.WafLatency500to1000,
0.250: stats.WafLatency250to500,
0.100: stats.WafLatency100to250,
0.050: stats.WafLatency50to100,
0.010: stats.WafLatency10to50,
0.005: stats.WafLatency5to10,
0.001: stats.WafLatency1to5,
0.0005: stats.WafLatency0to1, // yolo
} {
for i := uint64(0); i < n; i++ {
m.LatencySeconds.WithLabelValues(serviceID, serviceName, datacenter, origin).Observe(v)
}
}

for v, n := range map[float64]uint64{
60.00: stats.ComputeLatency60000plus,
10.00: stats.ComputeLatency10000to60000,
5.000: stats.ComputeLatency5000to10000,
1.000: stats.ComputeLatency1000to5000,
0.500: stats.ComputeLatency500to1000,
0.250: stats.ComputeLatency250to500,
0.100: stats.ComputeLatency100to250,
0.050: stats.ComputeLatency50to100,
0.010: stats.ComputeLatency10to50,
0.005: stats.ComputeLatency5to10,
0.001: stats.ComputeLatency1to5,
0.0005: stats.ComputeLatency0to1, // yolo
} {
for i := uint64(0); i < n; i++ {
m.LatencySeconds.WithLabelValues(serviceID, serviceName, datacenter, origin).Observe(v)
}
}
}
}
}
Expand Down
Loading

0 comments on commit b45aca3

Please sign in to comment.