Merge pull request #84 from topolvm/csa-apps-640-add-metrics

add metrics

README.md

@@ -47,6 +47,11 @@ The number of attempts that the creation of the Pod object and the creation of t
 
 TYPE: counter
 
+### `pie_performance_probe_total`
+The number of attempts that the creation of the Pod object and the creation of the container.
+
+TYPE: counter
+
 ## Contributing
 
 ### Test It Out

e2e/suite_test.go

@@ -136,6 +136,9 @@ var _ = Describe("pie", func() {
 		onTimeTrueLabelPair := io_prometheus_client.LabelPair{Name: &onTimeLabelKey, Value: &trueValue}
 		onTimeFalseLabelPair := io_prometheus_client.LabelPair{Name: &onTimeLabelKey, Value: &falseValue}
 
+		succeedLabelKey := "succeed"
+		succeedTrueLabelPair := io_prometheus_client.LabelPair{Name: &succeedLabelKey, Value: &trueValue}
+
 		Eventually(func(g Gomega) {
 			resp, err := http.Get("http://localhost:8080/metrics")
 			g.Expect(err).NotTo(HaveOccurred())
@@ -172,6 +175,19 @@ var _ = Describe("pie", func() {
 					}
 				}
 			}
+
+			By("checking pie_performance_probe_total with succeed=true is more than zero for standard SC")
+			g.Expect("pie_performance_probe_total").Should(BeKeyOf(metricFamilies))
+			metrics = metricFamilies["pie_performance_probe_total"].Metric
+			g.Expect(metrics).Should(ContainElement(HaveField("Label", ContainElement(&standardSCLabelPair))))
+			for _, metric := range metrics {
+				for _, label := range metric.Label {
+					if reflect.DeepEqual(label, &standardSCLabelPair) {
+						g.Expect(metric.Label).Should(ContainElement(&succeedTrueLabelPair))
+						g.Expect(metric.Counter).ShouldNot(BeZero())
+					}
+				}
+			}
 		}).Should(Succeed())
 
 	})

metrics/metrics.go

@@ -13,12 +13,14 @@ const (
 type MetricsExporter interface {
 	SetLatency(node, storageClass string, readLatency, writeLatency float64)
 	IncrementCreateProbeCount(node string, storageClass string, onTime bool)
+	IncrementPerformanceProbeCount(node string, storageClass string, succeed bool)
 }
 
 type metricExporterImpl struct {
-	writeLatencyGauge *prometheus.GaugeVec
-	readLatencyGauge  *prometheus.GaugeVec
-	createProbeCount  *prometheus.CounterVec
+	writeLatencyGauge     *prometheus.GaugeVec
+	readLatencyGauge      *prometheus.GaugeVec
+	createProbeCount      *prometheus.CounterVec
+	performanceProbeCount *prometheus.CounterVec
 }
 
 func NewMetrics() MetricsExporter {
@@ -57,6 +59,16 @@ func (m *metricExporterImpl) registerMetrics() {
 		[]string{"node", "storage_class", "on_time"})
 
 	metrics.Registry.MustRegister(m.createProbeCount)
+
+	m.performanceProbeCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "pie",
+			Name:      "performance_probe_total",
+			Help:      "The number of performance tests on a probe container.",
+		},
+		[]string{"node", "storage_class", "succeed"})
+
+	metrics.Registry.MustRegister(m.performanceProbeCount)
 }
 
 func (m *metricExporterImpl) SetLatency(node string, storageClass string, readLatency, writeLatency float64) {
@@ -71,3 +83,11 @@ func (m *metricExporterImpl) IncrementCreateProbeCount(node string, storageClass
 	}
 	m.createProbeCount.WithLabelValues(node, storageClass, onTimeStr).Inc()
 }
+
+func (m *metricExporterImpl) IncrementPerformanceProbeCount(node string, storageClass string, succeed bool) {
+	succeedStr := "false"
+	if succeed {
+		succeedStr = "true"
+	}
+	m.performanceProbeCount.WithLabelValues(node, storageClass, succeedStr).Inc()
+}

metrics/receiver.go

@@ -28,6 +28,7 @@ func (rh *receiver) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 
 	rh.metrics.SetLatency(receivedData.Node, receivedData.StorageClass, receivedData.ReadLatency, receivedData.WriteLatency)
+	rh.metrics.IncrementPerformanceProbeCount(receivedData.Node, receivedData.StorageClass, receivedData.PerformanceProbeSucceed)
 
 	fmt.Fprintf(w, "OK")
 }

probe/diskmetrics.go

@@ -32,7 +32,7 @@ func execWrap(stdin []byte, command string, args ...string) ([]byte, error) {
 	return stdoutBuf.Bytes(), nil
 }
 
-func parseFioResult(fioOutput []byte, property string) (float64, error) {
+func parseFioLatency(fioOutput []byte, property string) (float64, error) {
 	jqOut, err := execWrap(
 		fioOutput,
 		"jq",
@@ -53,6 +53,26 @@ func parseFioResult(fioOutput []byte, property string) (float64, error) {
 	return actualNumber / 1_000_000_000, nil
 }
 
+func parseFioError(fioOutput []byte) (int, error) {
+	jqOut, err := execWrap(
+		fioOutput,
+		"jq",
+		".jobs[0].error",
+	)
+	if err != nil {
+		return 0, err
+	}
+
+	stringJqOut := string(jqOut)
+
+	actualNumber, err := strconv.ParseInt(stringJqOut[0:len(stringJqOut)-1], 0, 0)
+	if err != nil {
+		return 0, err
+	}
+
+	return int(actualNumber), nil
+}
+
 func (mtr *diskMetricsImpl) GetMetrics(ctx context.Context) (*DiskMetrics, error) {
 	fioStdout, err := execWrap(
 		nil,
@@ -73,12 +93,17 @@ func (mtr *diskMetricsImpl) GetMetrics(ctx context.Context) (*DiskMetrics, error
 	}
 
 	var metrics DiskMetrics
-	metrics.ReadLatency, err = parseFioResult(fioStdout, "read")
+	metrics.ReadLatency, err = parseFioLatency(fioStdout, "read")
+	if err != nil {
+		return nil, err
+	}
+
+	metrics.WriteLatency, err = parseFioLatency(fioStdout, "write")
 	if err != nil {
 		return nil, err
 	}
 
-	metrics.WriteLatency, err = parseFioResult(fioStdout, "write")
+	metrics.ErrorNumber, err = parseFioError(fioStdout)
 	if err != nil {
 		return nil, err
 	}

probe/exporter.go

@@ -28,12 +28,13 @@ func NewDiskInfoExporter(url string, node string, storageClass string) DiskInfoE
 	}
 }
 
-func (di *diskInfoImpl) Export(metrics DiskMetrics) error {
+func (di *diskInfoImpl) Export(metrics *DiskMetrics) error {
 	m := types.MetricsExchangeFormat{
-		Node:         di.node,
-		StorageClass: di.storageClass,
-		WriteLatency: metrics.WriteLatency,
-		ReadLatency:  metrics.ReadLatency,
+		Node:                    di.node,
+		StorageClass:            di.storageClass,
+		WriteLatency:            metrics.WriteLatency,
+		ReadLatency:             metrics.ReadLatency,
+		PerformanceProbeSucceed: metrics.ErrorNumber == 0,
 	}
 
 	s, err := json.Marshal(m)

probe/probe.go

@@ -14,7 +14,7 @@ func SubMain(node string, measurePath string, storageClass string, serverURI str
 		return err
 	}
 
-	err = infoExporter.Export(*metrics)
+	err = infoExporter.Export(metrics)
 	if err != nil {
 		return err
 	}

probe/types.go

@@ -5,12 +5,13 @@ import "context"
 type DiskMetrics struct {
 	ReadLatency  float64
 	WriteLatency float64
+	ErrorNumber  int
 }
 
 type DiskMetricsInterface interface {
 	GetMetrics(ctx context.Context) (*DiskMetrics, error)
 }
 
 type DiskInfoExporter interface {
-	Export(metrics DiskMetrics) error
+	Export(metrics *DiskMetrics) error
 }

types/types.go

@@ -1,8 +1,9 @@
 package types
 
 type MetricsExchangeFormat struct {
-	Node         string  `json:"node"`
-	StorageClass string  `json:"storage_class"`
-	WriteLatency float64 `json:"write_latency"`
-	ReadLatency  float64 `json:"read_latency"`
+	Node                    string  `json:"node"`
+	StorageClass            string  `json:"storage_class"`
+	WriteLatency            float64 `json:"write_latency"`
+	ReadLatency             float64 `json:"read_latency"`
+	PerformanceProbeSucceed bool    `json:"performance_probe_succeed"`
 }