SLO - http request external duration and error source

experimental/slo/slo_middleware.go

@@ -0,0 +1,95 @@
+package slo
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var duration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Namespace: "grafana",
+	Name:      "plugin_external_requests_duration_seconds",
+	Help:      "Duration of requests to external services",
+}, []string{"datasource_name", "datasource_type", "error_source"})
+
+const DataSourceSLOMiddlewareName = "slo"
+
+// Middleware captures duration of requests to external services and the source of errors
+func Middleware(plugin string) httpclient.Middleware {
+	return httpclient.NamedMiddlewareFunc(DataSourceSLOMiddlewareName, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper {
+		return RoundTripper(plugin, opts, next)
+	})
+}
+
+// RoundTripper captures duration of requests to external services and the source of errors
+func RoundTripper(plugin string, opts httpclient.Options, next http.RoundTripper) http.RoundTripper {
+	return httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+		start := time.Now()
+		var errorSource = "none"
+
+		defer func() {
+			if opts.Labels == nil {
+				return
+			}
+
+			datasourceName, exists := opts.Labels["datasource_name"]
+			if !exists {
+				return
+			}
+
+			datasourceLabelName, err := SanitizeLabelName(datasourceName)
+			// if the datasource named cannot be turned into a prometheus
+			// label we will skip instrumenting these metrics.
+			if err != nil {
+				return
+			}
+
+			datasourceType, exists := opts.Labels["datasource_type"]
+			if !exists {
+				return
+			}
+			duration.WithLabelValues(datasourceLabelName, datasourceType, errorSource).Observe(time.Since(start).Seconds())
+		}()
+
+		res, err := next.RoundTrip(req)
+		if res != nil && res.StatusCode >= 400 {
+			errorSource = string(backend.ErrorSourceFromHTTPStatus(res.StatusCode))
+		}
+		if errors.Is(err, syscall.ECONNREFUSED) {
+			errorSource = string(backend.ErrorSourceDownstream)
+		}
+		return res, err
+	})
+}
+
+// SanitizeLabelName removes all invalid chars from the label name.
+// If the label name is empty or contains only invalid chars, it
+// will return an error.
+func SanitizeLabelName(name string) (string, error) {
+	if len(name) == 0 {
+		return "", errors.New("label name cannot be empty")
+	}
+
+	out := strings.Builder{}
+	for i, b := range name {
+		if (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' || (b >= '0' && b <= '9' && i > 0) {
+			out.WriteRune(b)
+		} else if b == ' ' {
+			out.WriteRune('_')
+		}
+	}
+
+	if out.Len() == 0 {
+		return "", fmt.Errorf("label name only contains invalid chars: %q", name)
+	}
+
+	return out.String(), nil
+}

experimental/testdata/folder.golden.txt

@@ -9,7 +9,7 @@ Frame[0] {
     "pathSeparator": "/"
 }
 Name: 
-Dimensions: 2 Fields by 20 Rows
+Dimensions: 2 Fields by 21 Rows
 +----------------+------------------+
 | Name: name     | Name: media-type |
 | Labels:        | Labels:          |
@@ -29,4 +29,4 @@ Dimensions: 2 Fields by 20 Rows
 
 
 ====== TEST DATA RESPONSE (arrow base64) ======
-FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP/////YAAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAGAIAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAeAAAABQAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUAAAAAAAAAFgAAAAAAAAA9wAAAAAAAABQAQAAAAAAAAAAAAAAAAAAUAEAAAAAAABUAAAAAAAAAKgBAAAAAAAAbAAAAAAAAAAAAAAAAgAAABQAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAkAAAAQAAAAFAAAAB4AAAAsAAAALwAAADoAAABIAAAAUwAAAGMAAAByAAAAhgAAAKAAAAC/AAAAygAAANAAAADUAAAA4gAAAO8AAAD3AAAAAAAAAFJFQURNRS5tZGFjdGlvbnNhcGlzYXV0aGNsaWVudGRhdGFzb3VyY2V0ZXN0ZTJlZXJyb3Jzb3VyY2VmZWF0dXJldG9nZ2xlc2ZpbGVpbmZvLmdvZmlsZWluZm9fdGVzdC5nb2ZyYW1lX3NvcnRlci5nb2ZyYW1lX3NvcnRlcl90ZXN0LmdvZ29sZGVuX3Jlc3BvbnNlX2NoZWNrZXIuZ29nb2xkZW5fcmVzcG9uc2VfY2hlY2tlcl90ZXN0LmdvaHR0cF9sb2dnZXJtYWNyb3Ntb2NrcmVzdF9jbGllbnQuZ29zY2hlbWFidWlsZGVydGVzdGRhdGEAAAAAAAAAAAAJAAAAEgAAABsAAAAkAAAALQAAADYAAAA/AAAAPwAAAD8AAAA/AAAAPwAAAD8AAAA/AAAASAAAAFEAAABaAAAAWgAAAGMAAABsAAAAAAAAAGRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeWRpcmVjdG9yeQAAAAAQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAABAABAAAA2AEAAAAAAADgAAAAAAAAABgCAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAAC4AAAAAwAAAEwAAAAoAAAABAAAAMD+//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAA4P7//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAAA////CAAAAFAAAABEAAAAeyJ0eXBlIjoiZGlyZWN0b3J5LWxpc3RpbmciLCJ0eXBlVmVyc2lvbiI6WzAsMF0sInBhdGhTZXBhcmF0b3IiOiIvIn0AAAAABAAAAG1ldGEAAAAAAgAAAHgAAAAEAAAAov///xQAAAA8AAAAPAAAAAAAAAU4AAAAAQAAAAQAAACQ////CAAAABAAAAAGAAAAc3RyaW5nAAAGAAAAdHN0eXBlAAAAAAAAiP///woAAABtZWRpYS10eXBlAAAAABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEgAAAAAAAAFRAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAGAAAAc3RyaW5nAAAGAAAAdHN0eXBlAAAAAAAABAAEAAQAAAAEAAAAbmFtZQAAAAD4AQAAQVJST1cx
+FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP/////YAAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAKAIAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAeAAAABUAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABYAAAAAAAAAFgAAAAAAAAA+gAAAAAAAABYAQAAAAAAAAAAAAAAAAAAWAEAAAAAAABYAAAAAAAAALABAAAAAAAAdQAAAAAAAAAAAAAAAgAAABUAAAAAAAAAAAAAAAAAAAAVAAAAAAAAAAAAAAAAAAAAAAAAAAkAAAAQAAAAFAAAAB4AAAAsAAAALwAAADoAAABIAAAAUwAAAGMAAAByAAAAhgAAAKAAAAC/AAAAygAAANAAAADUAAAA4gAAAO8AAADyAAAA+gAAAFJFQURNRS5tZGFjdGlvbnNhcGlzYXV0aGNsaWVudGRhdGFzb3VyY2V0ZXN0ZTJlZXJyb3Jzb3VyY2VmZWF0dXJldG9nZ2xlc2ZpbGVpbmZvLmdvZmlsZWluZm9fdGVzdC5nb2ZyYW1lX3NvcnRlci5nb2ZyYW1lX3NvcnRlcl90ZXN0LmdvZ29sZGVuX3Jlc3BvbnNlX2NoZWNrZXIuZ29nb2xkZW5fcmVzcG9uc2VfY2hlY2tlcl90ZXN0LmdvaHR0cF9sb2dnZXJtYWNyb3Ntb2NrcmVzdF9jbGllbnQuZ29zY2hlbWFidWlsZGVyc2xvdGVzdGRhdGEAAAAAAAAAAAAAAAAAAAkAAAASAAAAGwAAACQAAAAtAAAANgAAAD8AAAA/AAAAPwAAAD8AAAA/AAAAPwAAAD8AAABIAAAAUQAAAFoAAABaAAAAYwAAAGwAAAB1AAAAZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5AAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAQAAQAAANgBAAAAAAAA4AAAAAAAAAAoAgAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAuAAAAAMAAABMAAAAKAAAAAQAAADA/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAOD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAAP///wgAAABQAAAARAAAAHsidHlwZSI6ImRpcmVjdG9yeS1saXN0aW5nIiwidHlwZVZlcnNpb24iOlswLDBdLCJwYXRoU2VwYXJhdG9yIjoiLyJ9AAAAAAQAAABtZXRhAAAAAAIAAAB4AAAABAAAAKL///8UAAAAPAAAADwAAAAAAAAFOAAAAAEAAAAEAAAAkP///wgAAAAQAAAABgAAAHN0cmluZwAABgAAAHRzdHlwZQAAAAAAAIj///8KAAAAbWVkaWEtdHlwZQAAAAASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABIAAAAAAAABUQAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABgAAAHN0cmluZwAABgAAAHRzdHlwZQAAAAAAAAQABAAEAAAABAAAAG5hbWUAAAAA+AEAAEFSUk9XMQ==