feat: add test case execution related promethus metrics (#488)

cmd/server.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/linuxsuren/api-testing/pkg/runner"
 	"github.com/linuxsuren/api-testing/pkg/util/home"
 	"net"
 	"net/http"
@@ -102,8 +103,8 @@ func createServerCmd(execer fakeruntime.Execer, httpServer server.HTTPServer) (c
 	flags.IntVarP(&opt.gcPercent, "gc-percent", "", 100, "The GC percent of Go")
 	//grpc_tls
 	flags.BoolVarP(&opt.tls, "tls-grpc", "", false, "Enable TLS mode. Set to true to enable TLS. Alow SAN certificates")
-	flags.StringVarP(&opt.tlsCert, "cert-file", "", "","The path to the certificate file, Alow SAN certificates")
-	flags.StringVarP(&opt.tlsKey, "key-file", "", "", "The path to the key file, Alow SAN certificates")  
+	flags.StringVarP(&opt.tlsCert, "cert-file", "", "", "The path to the certificate file, Alow SAN certificates")
+	flags.StringVarP(&opt.tlsKey, "key-file", "", "", "The path to the key file, Alow SAN certificates")
 
 	c.Flags().MarkHidden("dry-run")
 	c.Flags().MarkHidden("gc-percent")
@@ -146,9 +147,9 @@ type serverOption struct {
 
 	// inner fields, not as command flags
 	provider oauth.OAuthProvider
-	tls bool
-	tlsCert string
-	tlsKey string
+	tls      bool
+	tlsCert  string
+	tlsKey   string
 }
 
 func (o *serverOption) preRunE(cmd *cobra.Command, args []string) (err error) {
@@ -187,7 +188,7 @@ func (o *serverOption) preRunE(cmd *cobra.Command, args []string) (err error) {
 				return fmt.Errorf("failed to load credentials: %v", err)
 			}
 			grpcOpts = append(grpcOpts, grpc.Creds(creds))
-		}		
+		}
 	}
 	if o.dryRun {
 		o.gRPCServer = &fakeGRPCServer{}
@@ -289,15 +290,15 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) {
 	gRPCServerAddr := fmt.Sprintf("127.0.0.1:%s", gRPCServerPort)
 
 	mux := runtime.NewServeMux(runtime.WithMetadata(server.MetadataStoreFunc))
-	if o.tls { 
-		creds,err:=credentials.NewClientTLSFromFile(o.tlsCert,"localhost")
-		if err!=nil{
-				return fmt.Errorf("failed to load credentials: %v", err)
+	if o.tls {
+		creds, err := credentials.NewClientTLSFromFile(o.tlsCert, "localhost")
+		if err != nil {
+			return fmt.Errorf("failed to load credentials: %v", err)
 		}
 		err = errors.Join(
 			server.RegisterRunnerHandlerFromEndpoint(ctx, mux, gRPCServerAddr, []grpc.DialOption{grpc.WithTransportCredentials(creds)}),
 			server.RegisterMockHandlerFromEndpoint(ctx, mux, gRPCServerAddr, []grpc.DialOption{grpc.WithTransportCredentials(creds)}))
-	}else{
+	} else {
 		err = errors.Join(
 			server.RegisterRunnerHandlerFromEndpoint(ctx, mux, gRPCServerAddr, []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}),
 			server.RegisterMockHandlerFromEndpoint(ctx, mux, gRPCServerAddr, []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}))
@@ -333,6 +334,9 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) {
 		reg.MustRegister(
 			collectors.NewGoCollector(),
 			collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
+			collectors.NewBuildInfoCollector(),
+			server.ExecutionCountNum, server.ExecutionSuccessNum, server.ExecutionFailNum,
+			runner.RunnersNum,
 		)
 		mux.HandlePath(http.MethodGet, "/metrics", func(w http.ResponseWriter, r *http.Request, pathParams map[string]string) {
 			promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}).ServeHTTP(w, r)

e2e/test-suite-common.yaml

@@ -364,3 +364,15 @@ items:
   expect:
     header:
       Content-Type: image/x-icon
+
+## metrics
+- name: metrics
+  request:
+    api: |
+      {{.param.server}}/metrics
+  expect:
+    verify:
+      - indexOf(data, "atest_execution_count") != -1
+      - indexOf(data, "atest_execution_fail") != -1
+      - indexOf(data, "atest_execution_success") != -1
+      - indexOf(data, "atest_runners_count") != -1

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/linuxsuren/go-service v0.0.0-20231225060426-efabcd3a5161
 	github.com/linuxsuren/unstructured v0.0.1
 	github.com/prometheus/client_golang v1.19.0
+	github.com/prometheus/common v0.50.0
 	github.com/signintech/gopdf v0.18.0
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
@@ -69,7 +70,6 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.6.0 // indirect
-	github.com/prometheus/common v0.50.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/sergi/go-diff v1.3.1 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect

pkg/runner/reporter_prometheus.go

@@ -17,11 +17,14 @@ limitations under the License.
 package runner
 
 import (
+	"fmt"
+	"os"
 	"sync"
 
 	"github.com/linuxsuren/api-testing/pkg/logging"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/push"
+	"github.com/prometheus/common/expfmt"
 )
 
 var (
@@ -96,6 +99,23 @@ func (w *prometheusReporter) PutRecord(record *ReportRecord) {
 }
 
 func (r *prometheusReporter) GetResourceUsage() []ResourceUsage {
+	reader, err := os.Open("path")
+	if err != nil {
+		return nil
+	}
+
+	var parser expfmt.TextParser
+	mf, err := parser.TextToMetricFamilies(reader)
+	if err != nil {
+		return nil
+	}
+	for k, v := range mf {
+		fmt.Println("KEY: ", k)
+		fmt.Println("VAL: ", v)
+	}
+
+	// mf["jvm_memory_used_bytes"].Metric[0].Value
+
 	return nil
 }
 

pkg/runner/runner_factory.go

@@ -18,6 +18,8 @@ package runner
 
 import (
 	"fmt"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"strings"
 
 	"github.com/linuxsuren/api-testing/pkg/testing"
@@ -47,13 +49,19 @@ func GetTestSuiteRunner(suite *testing.TestSuite) TestCaseRunner {
 
 type RunnerCreator func(suite *testing.TestSuite) TestCaseRunner
 
-var runners map[string]RunnerCreator = make(map[string]RunnerCreator, 4)
+var runners = make(map[string]RunnerCreator, 4)
+
+var RunnersNum = promauto.NewGauge(prometheus.GaugeOpts{
+	Name: "atest_runners_count",
+	Help: "The total number of runners",
+})
 
 func RegisterRunner(kind string, runner RunnerCreator) error {
 	if _, ok := runners[kind]; ok {
 		return fmt.Errorf("duplicated kind %q", kind)
 	}
 
 	runners[kind] = runner
+	RunnersNum.Inc()
 	return nil
 }

pkg/runner/verify.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2023 API Testing Authors.
+Copyright 2023-2024 API Testing Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -94,6 +94,8 @@ func NewBodyVerify(contentType string, body BodyGetter) BodyVerifier {
 		return &jsonBodyVerifier{body: body}
 	case util.YAML:
 		return &yamlBodyVerifier{body: body}
+	case util.Plain:
+		return &plainTextBodyVerify{body: body}
 	default:
 		return nil
 	}
@@ -154,6 +156,20 @@ func (v *yamlBodyVerifier) Verify(data []byte) (err error) {
 	return
 }
 
+type plainTextBodyVerify struct {
+	body BodyGetter
+}
+
+func (v *plainTextBodyVerify) Parse(data []byte) (obj interface{}, err error) {
+	obj = string(data)
+	return
+}
+
+func (v *plainTextBodyVerify) Verify(data []byte) (err error) {
+	// no need to do anything
+	return
+}
+
 func valueCompare(expect interface{}, acutalResult gjson.Result, key string) (err error) {
 	var actual interface{}
 	actual = acutalResult.Value()

pkg/server/remote_server.go

@@ -21,6 +21,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"io"
 	"net/http"
 	"os"
@@ -494,8 +496,29 @@ func (s *server) GetTestCase(ctx context.Context, in *TestCaseIdentity) (reply *
 	return
 }
 
+var ExecutionCountNum = promauto.NewCounter(prometheus.CounterOpts{
+	Name: "atest_execution_count",
+	Help: "The total number of request execution",
+})
+var ExecutionSuccessNum = promauto.NewCounter(prometheus.CounterOpts{
+	Name: "atest_execution_success",
+	Help: "The total number of request execution success",
+})
+var ExecutionFailNum = promauto.NewCounter(prometheus.CounterOpts{
+	Name: "atest_execution_fail",
+	Help: "The total number of request execution fail",
+})
+
 func (s *server) RunTestCase(ctx context.Context, in *TestCaseIdentity) (result *TestCaseResult, err error) {
 	var targetTestSuite testing.TestSuite
+	ExecutionCountNum.Inc()
+	defer func() {
+		if result.Error == "" {
+			ExecutionSuccessNum.Inc()
+		} else {
+            ExecutionFailNum.Inc()
+		}
+	}()
 
 	result = &TestCaseResult{}
 	loader := s.getLoader(ctx)

pkg/testing/case.go

@@ -36,6 +36,7 @@ type APISpec struct {
 	URL    string   `yaml:"url,omitempty" json:"url,omitempty"`
 	RPC    *RPCDesc `yaml:"rpc,omitempty" json:"rpc,omitempty"`
 	Secure *Secure  `yaml:"secure,omitempty" json:"secure,omitempty"`
+	Metric *Metric  `yaml:"metric,omitempty" json:"metric,omitempty"`
 }
 
 type RPCDesc struct {
@@ -54,6 +55,11 @@ type Secure struct {
 	ServerName string `yaml:"serverName,omitempty" json:"serverName,omitempty"`
 }
 
+type Metric struct {
+	Type string `yaml:"type,omitempty" json:"type,omitempty"`
+	URL  string `yaml:"url,omitempty" json:"url,omitempty"`
+}
+
 // TestCase represents a test case
 type TestCase struct {
 	ID      string   `yaml:"id,omitempty" json:"id,omitempty"`