Skip to content

Commit

Permalink
feat(endpoint/metrics): add extra metrics to /metrics page
Browse files Browse the repository at this point in the history
  • Loading branch information
@macrat
macrat committed May 26, 2022
1 parent c6241be commit cffa3db
Show file tree
Hide file tree
Showing 3 changed files with 257 additions and 2 deletions.
93 changes: 93 additions & 0 deletions internal/endpoint/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,20 @@ package endpoint
import (
"fmt"
"net/http"
"regexp"
"strconv"
"strings"

api "github.com/macrat/ayd/lib-ayd"
)

type extraMetric struct {
Name string
Type string
Value float64
Help string
}

// metricInfo is a metric point for /metrics endpoint.
type metricInfo struct {
Timestamp int64
Expand All @@ -18,6 +27,7 @@ type metricInfo struct {
Failure int
Aborted int
Latency float64
Extra []extraMetric
}

// MetricsEndpoint implements Prometheus metrics endpoint.
Expand All @@ -34,6 +44,7 @@ func MetricsEndpoint(s Store) http.HandlerFunc {
Timestamp: last.CheckedAt.UnixMilli(),
Latency: last.Latency.Seconds(),
Target: strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(hs.Target.String(), "\\", "\\\\"), "\n", "\\\n"), "\"", "\\\""),
Extra: extractExtraMetrics(last),
}

switch last.Status {
Expand Down Expand Up @@ -88,5 +99,87 @@ func MetricsEndpoint(s Store) http.HandlerFunc {
fmt.Fprintln(w, "# HELP ayd_incident_total The number of incident happened since server started.")
fmt.Fprintln(w, "# TYPE ayd_incident_total counter")
fmt.Fprintf(w, "ayd_incident_total %d\n", s.IncidentCount())

for _, m := range metrics {
fmt.Fprintln(w)
for _, e := range m.Extra {
fmt.Fprintf(w, "# HELP ayd_%s %s\n", e.Name, e.Help)
fmt.Fprintf(w, "# TYPE ayd_%s %s\n", e.Name, e.Type)
fmt.Fprintf(w, "ayd_%s{target=\"%s\"} %v %d\n", e.Name, m.Target, e.Value, m.Timestamp)
}
}
}
}

var (
httpMessageRe = regexp.MustCompile(`^proto=HTTP/([0-9]+\.[0-9]+) length=(-?[0-9]+) status=([0-9]+)_`)
ftpMessageRe = regexp.MustCompile(`^type=(file|directory) (?:size|files)=([0-9]+)$`)
pingMessageRe = regexp.MustCompile(`^ip=[^ ]+ rtt\(min/avg/max\)=([0-9]+\.[0-9]{2})/([0-9]+\.[0-9]{2})/([0-9]+\.[0-9]{2}) recv/sent=([0-9]+)/([0-9]+)$`)
sourceMessageRe = regexp.MustCompile(`^targets=([0-9]+)$`)
)

func parseFloats(ss []string) ([]float64, error) {
buf := make([]float64, len(ss))
for i := range ss {
var err error
buf[i], err = strconv.ParseFloat(ss[i], 64)
if err != nil {
return nil, err
}
}
return buf, nil
}

func extractExtraMetrics(r api.Record) []extraMetric {
switch r.Target.Scheme {
case "http", "https":
m := httpMessageRe.FindStringSubmatch(r.Message)
if len(m) > 0 {
if f, err := parseFloats(m[1:]); err == nil {
return []extraMetric{
{"http_proto", "gauge", f[0], "HTTP protocol version."},
{"http_content_length_bytes", "gauge", f[1], "HTTP Content-Length in the response header."},
{"http_status_code", "gauge", f[2], "The response status code."},
}
}
}
case "ftp", "ftps":
m := ftpMessageRe.FindStringSubmatch(r.Message)
if len(m) > 0 {
if f, err := strconv.ParseFloat(m[2], 64); err == nil {
if m[1] == "directory" {
return []extraMetric{
{"ftp_files", "gauge", f, "The number of files in the target directory."},
}
} else {
return []extraMetric{
{"ftp_file_size_bytes", "gauge", f, "The size of the target file."},
}
}
}
}
case "ping", "ping4", "ping6":
m := pingMessageRe.FindStringSubmatch(r.Message)
if len(m) > 0 {
if f, err := parseFloats(m[1:]); err == nil {
return []extraMetric{
{"ping_min_latency_seconds", "gauge", f[0] / 1000.0, "The minimal latency in seconds."},
{"ping_average_latency_seconds", "gauge", f[1] / 1000.0, "The average latency in seconds."},
{"ping_max_latency_seconds", "gauge", f[2] / 1000.0, "The maximum latency in seconds."},
{"ping_received_packets", "gauge", f[3], "Number of packets that received in the latest probe."},
{"ping_sent_packets", "gauge", f[4], "Number of packets that sent in the latest probe."},
}
}
}
case "source", "source+http", "source+https", "source+ftp", "source+ftps", "source+exec":
m := sourceMessageRe.FindStringSubmatch(r.Message)
if len(m) > 0 {
if f, err := strconv.ParseFloat(m[1], 64); err == nil {
return []extraMetric{
{"source_targets", "gauge", f, "The number of loaded targets."},
}
}
}
}
return nil
}
162 changes: 162 additions & 0 deletions internal/endpoint/metrics_internal_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
package endpoint

import (
"fmt"
"testing"

api "github.com/macrat/ayd/lib-ayd"
)

func TestExtractExtraMetrics(t *testing.T) {
tests := []struct {
Scheme string
Message string
Want []extraMetric
}{
{
"http",
"proto=HTTP/1.1 length=123 status=200_OK",
[]extraMetric{
{"http_proto", "gauge", 1.1, ""},
{"http_content_length_bytes", "gauge", 123, ""},
{"http_status_code", "gauge", 200, ""},
},
},
{
"http",
"proto=HTTP/2.0 length=-1 status=404_Not_Found",
[]extraMetric{
{"http_proto", "gauge", 2.0, ""},
{"http_content_length_bytes", "gauge", -1, ""},
{"http_status_code", "gauge", 404, ""},
},
},
{
"https",
"proto=HTTP/1.0 length=0 status=201_No_Content",
[]extraMetric{
{"http_proto", "gauge", 1.0, ""},
{"http_content_length_bytes", "gauge", 0, ""},
{"http_status_code", "gauge", 201, ""},
},
},
{
"https",
"error: something wrong",
nil,
},
{
"ftp",
"type=file size=1234",
[]extraMetric{
{"ftp_file_size_bytes", "gauge", 1234, ""},
},
},
{
"ftp",
"type=directory files=42",
[]extraMetric{
{"ftp_files", "gauge", 42, ""},
},
},
{
"ftp",
"type=file size=12",
[]extraMetric{
{"ftp_file_size_bytes", "gauge", 12, ""},
},
},
{
"ftps",
"oh no",
nil,
},
{
"ping",
"ip=127.0.0.1 rtt(min/avg/max)=0.10/0.25/0.50 recv/sent=2/3",
[]extraMetric{
{"ping_min_latency_seconds", "gauge", 0.10 / 1000, ""},
{"ping_average_latency_seconds", "gauge", 0.25 / 1000, ""},
{"ping_max_latency_seconds", "gauge", 0.50 / 1000, ""},
{"ping_received_packets", "gauge", 2, ""},
{"ping_sent_packets", "gauge", 3, ""},
},
},
{
"ping4",
"ip=127.0.0.1 rtt(min/avg/max)=1.00/2.00/3.00 recv/sent=9/10",
[]extraMetric{
{"ping_min_latency_seconds", "gauge", 1.0 / 1000, ""},
{"ping_average_latency_seconds", "gauge", 2.0 / 1000, ""},
{"ping_max_latency_seconds", "gauge", 3.0 / 1000, ""},
{"ping_received_packets", "gauge", 9, ""},
{"ping_sent_packets", "gauge", 10, ""},
},
},
{
"ping6",
"ip=[::] rtt(min/avg/max)=100.00/200.00/300.00 recv/sent=0/1",
[]extraMetric{
{"ping_min_latency_seconds", "gauge", 100.0 / 1000, ""},
{"ping_average_latency_seconds", "gauge", 200.0 / 1000, ""},
{"ping_max_latency_seconds", "gauge", 300.0 / 1000, ""},
{"ping_received_packets", "gauge", 0, ""},
{"ping_sent_packets", "gauge", 1, ""},
},
},
{
"ping",
"failed to send!",
nil,
},
{
"source",
"targets=123",
[]extraMetric{
{"source_targets", "gauge", 123, ""},
},
},
{
"source+http",
"targets=0",
[]extraMetric{
{"source_targets", "gauge", 0, ""},
},
},
{
"source",
"no such file",
nil,
},
}

for i, tt := range tests {
t.Run(fmt.Sprintf("%d_%s", i, tt.Scheme), func(t *testing.T) {
actual := extractExtraMetrics(api.Record{
Target: &api.URL{Scheme: tt.Scheme},
Message: tt.Message,
})

if len(actual) != len(tt.Want) {
t.Fatalf("expected %d metrics but got %d metrics", len(tt.Want), len(actual))
}

for i := range actual {
a := actual[i]
w := tt.Want[i]

if a.Name != w.Name {
t.Errorf("%d: expected name is %s but got %s", i, w.Name, a.Name)
}

if a.Type != w.Type {
t.Errorf("%d: expected type is %s but got %s", i, w.Type, a.Type)
}

if a.Value != w.Value {
t.Errorf("%d: expected value is %v but got %v", i, w.Value, a.Value)
}
}
})
}
}
4 changes: 2 additions & 2 deletions internal/scheme/source.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,7 @@ func (p SourceScheme) Probe(ctx context.Context, r Reporter) {
CheckedAt: stime,
Target: p.target,
Status: api.StatusHealthy,
Message: fmt.Sprintf("target_count=%d", len(probes)),
Message: fmt.Sprintf("targets=%d", len(probes)),
Latency: d,
})

Expand Down Expand Up @@ -425,7 +425,7 @@ func (p SourceScheme) Alert(ctx context.Context, r Reporter, lastRecord api.Reco
CheckedAt: stime,
Target: p.target,
Status: api.StatusHealthy,
Message: fmt.Sprintf("target_count=%d", len(alerters)),
Message: fmt.Sprintf("targets=%d", len(alerters)),
Latency: d,
})

Expand Down

0 comments on commit cffa3db

Please sign in to comment.