feat: enable ChangeLog plugin to monitor metric value change

cmd/poller/plugin/changelog/change_log.go → cmd/poller/plugin/changelog/changelog.go

@@ -6,6 +6,7 @@ import (
 	"github.com/netapp/harvest/v2/pkg/set"
 	"github.com/netapp/harvest/v2/pkg/tree/yaml"
 	"github.com/netapp/harvest/v2/pkg/util"
+	maps2 "golang.org/x/exp/maps"
 	"maps"
 	"strconv"
 	"time"
@@ -110,7 +111,6 @@ func (c *ChangeLog) initMatrix() (map[string]*matrix.Matrix, error) {
 
 // Run processes the data and generates ChangeLog instances
 func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
-
 	data := dataMap[c.Object]
 	changeLogMap, err := c.initMatrix()
 	if err != nil {
@@ -128,7 +128,6 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u
 	}
 
 	changeMat := changeLogMap[c.matrixName]
-
 	changeMat.SetGlobalLabels(data.GetGlobalLabels())
 	object := data.Object
 	if c.changeLogConfig.Object == "" {
@@ -179,7 +178,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u
 			c.updateChangeLogLabels(object, instance, change)
 			c.createChangeLogInstance(changeMat, change)
 		} else {
-			// check for any modification
+			// check for any label modification
 			cur, old := instance.CompareDiffs(prevInstance, c.changeLogConfig.Track)
 			if len(cur) > 0 {
 				for currentLabel, nVal := range cur {
@@ -201,8 +200,33 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u
 					c.createChangeLogInstance(changeMat, change)
 				}
 			}
+
+			// check for any metric modification
+			metricChanges := c.CompareMetrics(data)
+			if changes, ok := metricChanges[key]; ok {
+				for metricName := range changes {
+					change := &Change{
+						key:    uuid + "_" + object + "_" + metricName,
+						object: object,
+						op:     update,
+						labels: make(map[string]string),
+						track:  metricName,
+						// Enabling tracking of both old and new values results in the creation of a new time series each time the pair of values changes. For metrics tracking, it is not suitable.
+						//oldValue: strconv.FormatFloat(values[0], 'f', -1, 64),
+						//newValue: strconv.FormatFloat(values[1], 'f', -1, 64),
+						time: currentTime,
+					}
+					c.updateChangeLogLabels(object, instance, change)
+					// add changed track and its old, new value
+					change.labels[track] = metricName
+					change.labels[oldValue] = change.oldValue
+					change.labels[newValue] = change.newValue
+					c.createChangeLogInstance(changeMat, change)
+				}
+			}
 		}
 	}
+
 	// create deleted instances change_log
 	for key := range oldInstances.Iter() {
 		prevInstance := prevMat.GetInstance(key)
@@ -243,12 +267,53 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u
 	return matricesArray, nil, nil
 }
 
+// CompareMetrics compares the metrics of the current and previous instances
+func (c *ChangeLog) CompareMetrics(curMat *matrix.Matrix) map[string]map[string][2]float64 {
+	metricChanges := make(map[string]map[string][2]float64)
+	prevMat := c.previousData
+
+	met := maps2.Keys(c.previousData.GetMetrics())
+
+	for _, metricKey := range met {
+		prevMetric := prevMat.GetMetric(metricKey)
+		curMetric := curMat.GetMetric(metricKey)
+		for key, currInstance := range curMat.GetInstances() {
+			prevInstance := prevMat.GetInstance(key)
+			prevIndex := prevInstance.GetIndex()
+			currIndex := currInstance.GetIndex()
+			curVal := curMetric.GetValues()[currIndex]
+			prevVal := prevMetric.GetValues()[prevIndex]
+			if prevInstance != nil {
+				if curVal != prevVal {
+					if _, ok := metricChanges[key]; !ok {
+						metricChanges[key] = make(map[string][2]float64)
+					}
+					metricChanges[key][metricKey] = [2]float64{prevVal, curVal}
+				}
+			}
+		}
+	}
+	return metricChanges
+}
+
 // copyPreviousData creates a copy of the previous data for comparison
 func (c *ChangeLog) copyPreviousData(cur *matrix.Matrix) {
 	labels := c.changeLogConfig.PublishLabels
-	labels = append(labels, c.changeLogConfig.Track...)
+	var met []string
+	for _, t := range c.changeLogConfig.Track {
+		mKey := cur.DisplayMetricKey(t)
+		if mKey != "" {
+			met = append(met, mKey)
+		} else {
+			labels = append(labels, t)
+		}
+	}
 	labels = append(labels, "uuid")
-	c.previousData = cur.Clone(matrix.With{Data: true, Metrics: false, Instances: true, ExportInstances: false, Labels: labels})
+	withMetrics := false
+	if len(met) > 0 {
+		withMetrics = true
+	}
+	c.previousData = cur.Clone(matrix.With{Data: true, Metrics: withMetrics, Instances: true, ExportInstances: false, Labels: labels, MetricsNames: met})
 }
 
 // createChangeLogInstance creates a new ChangeLog instance with the given change data

cmd/poller/plugin/changelog/change_log_test.go → cmd/poller/plugin/changelog/changelog_test.go

@@ -39,14 +39,28 @@ func createChangeLog(params, parentParams *node.Node) *ChangeLog {
 
 func newChangeLogUnsupportedTrack(object string) *ChangeLog {
 	params := node.NewS("ChangeLog")
-	t := params.NewChildS("Track", "")
+	t := params.NewChildS("track", "")
 	t.NewChildS("", "abcd")
 	parentParams := node.NewS("parent")
 	parentParams.NewChildS("object", object)
 
 	return createChangeLog(params, parentParams)
 }
 
+func newChangeLogMetricTrack(object string) *ChangeLog {
+	params := node.NewS("ChangeLog")
+	t := params.NewChildS("track", "")
+	t.NewChildS("", "svm")
+	t.NewChildS("", "type")
+	t.NewChildS("", "cpu_usage")
+	t1 := params.NewChildS("publish_labels", "")
+	t1.NewChildS("", "svm")
+	parentParams := node.NewS("parent")
+	parentParams.NewChildS("object", object)
+
+	return createChangeLog(params, parentParams)
+}
+
 func checkChangeLogInstances(t *testing.T, o []*matrix.Matrix, expectedInstances, expectedLabels int, expectedOpLabel, opLabel string) {
 	if len(o) == 1 {
 		cl := o[0]
@@ -94,6 +108,41 @@ func TestChangeLogModified(t *testing.T) {
 	checkChangeLogInstances(t, o, 2, 9, update, opLabel)
 }
 
+func TestChangeLogModifiedWithMetrics(t *testing.T) {
+	p := newChangeLogMetricTrack("svm")
+	m := matrix.New("TestChangeLog", "svm", "svm")
+	data := map[string]*matrix.Matrix{
+		"svm": m,
+	}
+	instance, _ := m.NewInstance("0")
+	instance.SetLabel("uuid", "u1")
+	instance.SetLabel("svm", "s1")
+	instance.SetLabel("type", "t1")
+
+	// Add a metric to the instance
+	metric, _ := m.NewMetricFloat64("cpu_usage")
+	_ = metric.SetValueFloat64(instance, 10.0)
+
+	_, _, _ = p.Run(data)
+
+	m1 := matrix.New("TestChangeLog", "svm", "svm")
+	data1 := map[string]*matrix.Matrix{
+		"svm": m1,
+	}
+	instance1, _ := m1.NewInstance("0")
+	instance1.SetLabel("uuid", "u1")
+	instance1.SetLabel("svm", "s2")
+	instance1.SetLabel("type", "t2")
+
+	// Modify the metric value
+	metric1, _ := m1.NewMetricFloat64("cpu_usage")
+	_ = metric1.SetValueFloat64(instance1, 20.0)
+
+	o, _, _ := p.Run(data1)
+
+	checkChangeLogInstances(t, o, 3, 7, update, opLabel)
+}
+
 func TestChangeLogCreated(t *testing.T) {
 	p := newChangeLog("svm", false)
 	m := matrix.New("TestChangeLog", "svm", "svm")

docs/plugins.md

@@ -637,7 +637,7 @@ The ChangeLog plugin is a feature of Harvest, designed to detect and track chang
 
 Please note that the ChangeLog plugin requires the `uuid` label, which is unique, to be collected by the template. Without the `uuid` label, the plugin will not function.
 
-The ChangeLog feature only detects changes when Harvest is up and running. It does not detect changes that occur when Harvest is down. Additionally, the plugin does not detect changes in metric values.
+The ChangeLog feature only detects changes when Harvest is up and running. It does not detect changes that occur when Harvest is down. Additionally, the plugin does not detect changes in metric values by default, but it can be configured to do so.
 
 ## Enabling the Plugin
 
@@ -735,7 +735,6 @@ change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter="
 
 When an object is deleted, the ChangeLog plugin will publish a metric with the following labels:
 
-
 | Label        | Description                                                                 |
 |--------------|-----------------------------------------------------------------------------|
 | object       | name of the ONTAP object that was changed                                   |
@@ -748,6 +747,23 @@ Example of metric shape for object deletion:
 change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter="u2", index="2", instance="localhost:12993", job="prometheus", node="umeng-aff300-01", object="volume", op="delete", style="flexvol", svm="harvest", volume="harvest_demo"} 1698735708
 ```
 
+### Metric Value Changes
+
+When a metric value changes, the ChangeLog plugin will publish a metric with the following labels. Note that old and new values are not tracked to avoid impacting cardinality in Prometheus:
+
+| Label        | Description                                                                 |
+|--------------|-----------------------------------------------------------------------------|
+| object       | name of the ONTAP object that was changed                                   |
+| op           | type of change that was made                                                |
+| track        | property of the object which was modified                                   |
+| metric value | timestamp when Harvest captured the change. 1698735800 in the example below |
+
+Example of metric shape for metric value change:
+
+```
+change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter="u2", index="3", instance="localhost:12993", job="prometheus", node="umeng-aff300-01", object="volume", op="metric_change", track="read_latency", svm="harvest", volume="harvest_demo"} 1698735800
+```
+
 ## Viewing the Metrics
 
 You can view the metrics published by the ChangeLog plugin in the `ChangeLog Monitor` dashboard in `Grafana`. This dashboard provides a visual representation of the changes tracked by the plugin for volume, svm, and node objects.

grafana/dashboards/cmode/changelogMonitor.json

@@ -304,7 +304,7 @@
         },
         {
           "datasource": "${DS_PROMETHEUS}",
-          "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node)  \n\n`OP:` The type of change that was made (e.g., create, update, delete)  \n\n`Track:` Property of the object which was modified  \n\n`New Value:` The new value of the object after the change was made  \n\n`Old Value:` The previous value of the object before the change was made.",
+          "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node)  \n\n`OP:` The type of change that was made (e.g., create, update, delete)  \n\n`Track:` Property of the object which was modified  \n\n`New Value:` The new value of the property after the change was made  \n\n`Old Value:` The previous value of the property before the change was made \n\n*Note:* `Old Value` and `New Value` are only available for label changes and not for metric changes.",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -718,7 +718,7 @@
         },
         {
           "datasource": "${DS_PROMETHEUS}",
-          "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node)  \n\n`OP:` The type of change that was made (e.g., create, update, delete)  \n\n`Track:` Property of the object which was modified  \n\n`New Value:` The new value of the object after the change was made  \n\n`Old Value:` The previous value of the object before the change was made.",
+          "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node)  \n\n`OP:` The type of change that was made (e.g., create, update, delete)  \n\n`Track:` Property of the object which was modified  \n\n`New Value:` The new value of the property after the change was made  \n\n`Old Value:` The previous value of the property before the change was made \n\n*Note:* `Old Value` and `New Value` are only available for label changes and not for metric changes.",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -1158,7 +1158,7 @@
         },
         {
           "datasource": "${DS_PROMETHEUS}",
-          "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node)  \n\n`OP:` The type of change that was made (e.g., create, update, delete)  \n\n`Track:` Property of the object which was modified  \n\n`New Value:` The new value of the object after the change was made  \n\n`Old Value:` The previous value of the object before the change was made.",
+          "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node)  \n\n`OP:` The type of change that was made (e.g., create, update, delete)  \n\n`Track:` Property of the object which was modified  \n\n`New Value:` The new value of the property after the change was made  \n\n`Old Value:` The previous value of the property before the change was made \n\n*Note:* `Old Value` and `New Value` are only available for label changes and not for metric changes.",
           "fieldConfig": {
             "defaults": {
               "color": {

pkg/matrix/matrix.go

@@ -38,6 +38,7 @@ type With struct {
 	ExportInstances  bool
 	PartialInstances bool
 	Labels           []string
+	MetricsNames     []string
 }
 
 func New(uuid, object string, identifier string) *Matrix {
@@ -99,11 +100,23 @@ func (m *Matrix) Clone(with With) *Matrix {
 	}
 
 	if with.Metrics {
-		clone.metrics = make(map[string]*Metric, len(m.GetMetrics()))
-		for key, metric := range m.GetMetrics() {
-			c := metric.Clone(with.Data)
-			clone.metrics[key] = c
-			clone.displayMetrics[c.GetName()] = key
+		if len(with.MetricsNames) > 0 {
+			clone.metrics = make(map[string]*Metric, len(with.MetricsNames))
+			for _, metricName := range with.MetricsNames {
+				metric, ok := m.GetMetrics()[metricName]
+				if ok {
+					c := metric.Clone(with.Data)
+					clone.metrics[metricName] = c
+					clone.displayMetrics[c.GetName()] = metricName
+				}
+			}
+		} else {
+			clone.metrics = make(map[string]*Metric, len(m.GetMetrics()))
+			for key, metric := range m.GetMetrics() {
+				c := metric.Clone(with.Data)
+				clone.metrics[key] = c
+				clone.displayMetrics[c.GetName()] = key
+			}
 		}
 	} else {
 		clone.metrics = make(map[string]*Metric)
@@ -127,6 +140,13 @@ func (m *Matrix) DisplayMetric(name string) *Metric {
 	return nil
 }
 
+func (m *Matrix) DisplayMetricKey(name string) string {
+	if metricKey, has := m.displayMetrics[name]; has {
+		return metricKey
+	}
+	return ""
+}
+
 func (m *Matrix) GetMetric(key string) *Metric {
 	if metric, has := m.metrics[key]; has {
 		return metric

pkg/matrix/metric.go

@@ -146,6 +146,10 @@ func (m *Metric) SetValueNAN(i *Instance) {
 	m.record[i.index] = false
 }
 
+func (m *Metric) GetValues() []float64 {
+	return m.values
+}
+
 // Storage resizing methods
 
 func (m *Metric) Reset(size int) {