feat: enable ChangeLog plugin to monitor metric value change

cmd/exporters/prometheus/prometheus.go

@@ -24,6 +24,7 @@ package prometheus
 import (
 	"fmt"
 	"github.com/netapp/harvest/v2/cmd/poller/exporter"
+	"github.com/netapp/harvest/v2/cmd/poller/plugin/changelog"
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/set"
@@ -337,6 +338,24 @@ func (p *Prometheus) render(data *matrix.Matrix) ([][]byte, exporter.Stats) {
 		instanceLabels := make([]string, 0)
 		instanceLabelsSet := make(map[string]struct{})
 
+		// The ChangeLog plugin tracks metric values and publishes the names of metrics that have changed.
+		// For example, it might indicate that 'volume_size_total' has been updated.
+		// If a global prefix for the exporter is defined, we need to amend the metric name with this prefix.
+		if p.globalPrefix != "" && data.Object == changelog.ObjectChangeLog {
+			categoryIsMetric := false
+			for label, value := range instance.GetLabels() {
+				if label == changelog.Category && value == changelog.Metric {
+					categoryIsMetric = true
+					break
+				}
+			}
+			if categoryIsMetric {
+				if value, ok := instance.GetLabels()[changelog.Track]; ok {
+					instance.GetLabels()[changelog.Track] = p.globalPrefix + value
+				}
+			}
+		}
+
 		if includeAllLabels {
 			for label, value := range instance.GetLabels() {
 				// temporary fix for the rarely happening duplicate labels

cmd/poller/plugin/changelog/change_log.go → cmd/poller/plugin/changelog/changelog.go

@@ -6,6 +6,7 @@ import (
 	"github.com/netapp/harvest/v2/pkg/set"
 	"github.com/netapp/harvest/v2/pkg/tree/yaml"
 	"github.com/netapp/harvest/v2/pkg/util"
+	maps2 "golang.org/x/exp/maps"
 	"maps"
 	"strconv"
 	"time"
@@ -18,16 +19,19 @@ The shape of the change_log is specific to each label change and is only applica
 
 // Constants for ChangeLog metrics and labels
 const (
-	changeLog   = "change"
-	objectLabel = "object"
-	opLabel     = "op"
-	create      = "create"
-	update      = "update"
-	del         = "delete"
-	track       = "track"
-	oldValue    = "old_value"
-	newValue    = "new_value"
-	indexLabel  = "index"
+	ObjectChangeLog = "change"
+	objectLabel     = "object"
+	opLabel         = "op"
+	create          = "create"
+	update          = "update"
+	del             = "delete"
+	Track           = "track"
+	oldValue        = "old_value"
+	newValue        = "new_value"
+	indexLabel      = "index"
+	Metric          = "metric"
+	Label           = "label"
+	Category        = "category"
 )
 
 // Metrics to be used in ChangeLog
@@ -55,6 +59,7 @@ type Change struct {
 	oldValue string
 	newValue string
 	time     int64
+	category string
 }
 
 // New initializes a new instance of the ChangeLog plugin
@@ -71,7 +76,7 @@ func (c *ChangeLog) Init() error {
 	}
 
 	object := c.ParentParams.GetChildS("object")
-	c.matrixName = object.GetContentS() + "_" + changeLog
+	c.matrixName = object.GetContentS() + "_" + ObjectChangeLog
 
 	return c.populateChangeLogConfig()
 }
@@ -94,7 +99,7 @@ func (c *ChangeLog) populateChangeLogConfig() error {
 // initMatrix initializes a new matrix with the given name
 func (c *ChangeLog) initMatrix() (map[string]*matrix.Matrix, error) {
 	changeLogMap := make(map[string]*matrix.Matrix)
-	changeLogMap[c.matrixName] = matrix.New(c.Parent+c.matrixName, changeLog, c.matrixName)
+	changeLogMap[c.matrixName] = matrix.New(c.Parent+c.matrixName, ObjectChangeLog, c.matrixName)
 	for _, changeLogMatrix := range changeLogMap {
 		changeLogMatrix.SetExportOptions(matrix.DefaultExportOptions())
 	}
@@ -110,7 +115,6 @@ func (c *ChangeLog) initMatrix() (map[string]*matrix.Matrix, error) {
 
 // Run processes the data and generates ChangeLog instances
 func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
-
 	data := dataMap[c.Object]
 	changeLogMap, err := c.initMatrix()
 	if err != nil {
@@ -128,7 +132,6 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u
 	}
 
 	changeMat := changeLogMap[c.matrixName]
-
 	changeMat.SetGlobalLabels(data.GetGlobalLabels())
 	object := data.Object
 	if c.changeLogConfig.Object == "" {
@@ -151,6 +154,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u
 
 	currentTime := time.Now().Unix()
 
+	metricChanges := c.CompareMetrics(data)
 	// loop over current instances
 	for key, instance := range data.GetInstances() {
 		uuid := instance.GetLabel("uuid")
@@ -179,7 +183,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u
 			c.updateChangeLogLabels(object, instance, change)
 			c.createChangeLogInstance(changeMat, change)
 		} else {
-			// check for any modification
+			// check for any label modification
 			cur, old := instance.CompareDiffs(prevInstance, c.changeLogConfig.Track)
 			if len(cur) > 0 {
 				for currentLabel, nVal := range cur {
@@ -189,20 +193,46 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u
 						op:       update,
 						labels:   make(map[string]string),
 						track:    currentLabel,
+						category: Label,
 						oldValue: old[currentLabel],
 						newValue: nVal,
 						time:     currentTime,
 					}
 					c.updateChangeLogLabels(object, instance, change)
-					// add changed track and its old, new value
-					change.labels[track] = currentLabel
+					// add changed Track and its old, new value
+					change.labels[Category] = change.category
+					change.labels[Track] = currentLabel
 					change.labels[oldValue] = change.oldValue
 					change.labels[newValue] = nVal
 					c.createChangeLogInstance(changeMat, change)
 				}
 			}
+
+			// check for any metric modification
+			if changes, ok := metricChanges[key]; ok {
+				for metricName := range changes {
+					change := &Change{
+						key:      uuid + "_" + object + "_" + metricName,
+						object:   object,
+						op:       update,
+						labels:   make(map[string]string),
+						track:    metricName,
+						category: Metric,
+						// Enabling tracking of both old and new values results in the creation of a new time series each time the pair of values changes. For metrics tracking, it is not suitable.
+						time: currentTime,
+					}
+					c.updateChangeLogLabels(object, instance, change)
+					// add changed Track and its old, new value
+					change.labels[Category] = change.category
+					change.labels[Track] = metricName
+					change.labels[oldValue] = change.oldValue
+					change.labels[newValue] = change.newValue
+					c.createChangeLogInstance(changeMat, change)
+				}
+			}
 		}
 	}
+
 	// create deleted instances change_log
 	for key := range oldInstances.Iter() {
 		prevInstance := prevMat.GetInstance(key)
@@ -243,12 +273,51 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u
 	return matricesArray, nil, nil
 }
 
+// CompareMetrics compares the metrics of the current and previous instances
+func (c *ChangeLog) CompareMetrics(curMat *matrix.Matrix) map[string]map[string]struct{} {
+	metricChanges := make(map[string]map[string]struct{})
+	prevMat := c.previousData
+	met := maps2.Keys(c.previousData.GetMetrics())
+
+	for _, metricKey := range met {
+		prevMetric := prevMat.GetMetric(metricKey)
+		curMetric := curMat.GetMetric(metricKey)
+		for key, currInstance := range curMat.GetInstances() {
+			prevInstance := prevMat.GetInstance(key)
+			if prevInstance == nil {
+				continue
+			}
+			prevIndex := prevInstance.GetIndex()
+			currIndex := currInstance.GetIndex()
+			curVal := curMetric.GetValues()[currIndex]
+			prevVal := prevMetric.GetValues()[prevIndex]
+			if curVal != prevVal {
+				if _, ok := metricChanges[key]; !ok {
+					metricChanges[key] = make(map[string]struct{})
+				}
+				metName := curMat.Object + "_" + curMetric.GetName()
+				metricChanges[key][metName] = struct{}{}
+			}
+		}
+	}
+	return metricChanges
+}
+
 // copyPreviousData creates a copy of the previous data for comparison
 func (c *ChangeLog) copyPreviousData(cur *matrix.Matrix) {
 	labels := c.changeLogConfig.PublishLabels
-	labels = append(labels, c.changeLogConfig.Track...)
+	var met []string
+	for _, t := range c.changeLogConfig.Track {
+		mKey := cur.DisplayMetricKey(t)
+		if mKey == "" {
+			labels = append(labels, t)
+		} else {
+			met = append(met, mKey)
+		}
+	}
 	labels = append(labels, "uuid")
-	c.previousData = cur.Clone(matrix.With{Data: true, Metrics: false, Instances: true, ExportInstances: false, Labels: labels})
+	withMetrics := len(met) > 0
+	c.previousData = cur.Clone(matrix.With{Data: true, Metrics: withMetrics, Instances: true, ExportInstances: false, Labels: labels, MetricsNames: met})
 }
 
 // createChangeLogInstance creates a new ChangeLog instance with the given change data

cmd/poller/plugin/changelog/change_log_test.go → cmd/poller/plugin/changelog/changelog_test.go

@@ -39,14 +39,28 @@ func createChangeLog(params, parentParams *node.Node) *ChangeLog {
 
 func newChangeLogUnsupportedTrack(object string) *ChangeLog {
 	params := node.NewS("ChangeLog")
-	t := params.NewChildS("Track", "")
+	t := params.NewChildS("track", "")
 	t.NewChildS("", "abcd")
 	parentParams := node.NewS("parent")
 	parentParams.NewChildS("object", object)
 
 	return createChangeLog(params, parentParams)
 }
 
+func newChangeLogMetricTrack(object string) *ChangeLog {
+	params := node.NewS("ChangeLog")
+	t := params.NewChildS("track", "")
+	t.NewChildS("", "svm")
+	t.NewChildS("", "type")
+	t.NewChildS("", "cpu_usage")
+	t1 := params.NewChildS("publish_labels", "")
+	t1.NewChildS("", "svm")
+	parentParams := node.NewS("parent")
+	parentParams.NewChildS("object", object)
+
+	return createChangeLog(params, parentParams)
+}
+
 func checkChangeLogInstances(t *testing.T, o []*matrix.Matrix, expectedInstances, expectedLabels int, expectedOpLabel, opLabel string) {
 	if len(o) == 1 {
 		cl := o[0]
@@ -91,7 +105,42 @@ func TestChangeLogModified(t *testing.T) {
 
 	o, _, _ := p.Run(data1)
 
-	checkChangeLogInstances(t, o, 2, 9, update, opLabel)
+	checkChangeLogInstances(t, o, 2, 10, update, opLabel)
+}
+
+func TestChangeLogModifiedWithMetrics(t *testing.T) {
+	p := newChangeLogMetricTrack("svm")
+	m := matrix.New("TestChangeLog", "svm", "svm")
+	data := map[string]*matrix.Matrix{
+		"svm": m,
+	}
+	instance, _ := m.NewInstance("0")
+	instance.SetLabel("uuid", "u1")
+	instance.SetLabel("svm", "s1")
+	instance.SetLabel("type", "t1")
+
+	// Add a metric to the instance
+	metric, _ := m.NewMetricFloat64("cpu_usage")
+	_ = metric.SetValueFloat64(instance, 10.0)
+
+	_, _, _ = p.Run(data)
+
+	m1 := matrix.New("TestChangeLog", "svm", "svm")
+	data1 := map[string]*matrix.Matrix{
+		"svm": m1,
+	}
+	instance1, _ := m1.NewInstance("0")
+	instance1.SetLabel("uuid", "u1")
+	instance1.SetLabel("svm", "s2")
+	instance1.SetLabel("type", "t2")
+
+	// Modify the metric value
+	metric1, _ := m1.NewMetricFloat64("cpu_usage")
+	_ = metric1.SetValueFloat64(instance1, 20.0)
+
+	o, _, _ := p.Run(data1)
+
+	checkChangeLogInstances(t, o, 3, 8, update, opLabel)
 }
 
 func TestChangeLogCreated(t *testing.T) {

docs/plugins.md

@@ -637,7 +637,7 @@ The ChangeLog plugin is a feature of Harvest, designed to detect and track chang
 
 Please note that the ChangeLog plugin requires the `uuid` label, which is unique, to be collected by the template. Without the `uuid` label, the plugin will not function.
 
-The ChangeLog feature only detects changes when Harvest is up and running. It does not detect changes that occur when Harvest is down. Additionally, the plugin does not detect changes in metric values.
+The ChangeLog feature only detects changes when Harvest is up and running. It does not detect changes that occur when Harvest is down. Additionally, the plugin does not detect changes in metric values by default, but it can be configured to do so.
 
 ## Enabling the Plugin
 
@@ -675,7 +675,7 @@ By default, the plugin tracks changes in the following labels for svm, node, and
 
 Other objects are not tracked by default.
 
-These default settings can be overwritten as needed in the relevant templates. For instance, if you want to track `junction_path` labels for Volume, you can overwrite this in the volume template.
+These default settings can be overwritten as needed in the relevant templates. For instance, if you want to track `junction_path` label and `size_total` metric for Volume, you can overwrite this in the volume template.
 
 ```yaml
 plugins:
@@ -690,6 +690,7 @@ plugins:
         - state
         - status
         - junction_path
+        - size_total
 ```
 
 ## Change Types and Metrics
@@ -718,24 +719,30 @@ When an existing object is modified, the ChangeLog plugin will publish a metric
 
 | Label        | Description                                                                 |
 |--------------|-----------------------------------------------------------------------------|
-| object       | name of the ONTAP object that was changed                                   |
-| op           | type of change that was made                                                |
-| track        | property of the object which was modified                                   |
-| new_value    | new value of the object after the change                                    |
-| old_value    | previous value of the object before the change                              |
-| metric value | timestamp when Harvest captured the change. 1698735677 in the example below |
+| `object`     | Name of the ONTAP object that was changed                                   |
+| `op`         | Type of change that was made                                                |
+| `track`      | Property of the object which was modified                                   |
+| `new_value`  | New value of the object after the change (only available for label changes and not for metric changes) |
+| `old_value`  | Previous value of the object before the change (only available for label changes and not for metric changes) |
+| `metric value` | Timestamp when Harvest captured the change. `1698735677` in the example below |
+| `category`   | Type of the change, indicating whether it is a `metric` or a `label` change     |
 
-Example of metric shape for object modification:
+Example of metric shape for object modification for label:
 
 ```
-change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter="u2", index="1", instance="localhost:12993", job="prometheus", new_value="offline", node="umeng-aff300-01", object="volume", old_value="online", op="update", style="flexvol", svm="harvest", track="state", volume="harvest_demo"} 1698735677
+change_log{aggr="umeng_aff300_aggr2", category="label", cluster="umeng-aff300-01-02", datacenter="u2", index="1", instance="localhost:12993", job="prometheus", new_value="offline", node="umeng-aff300-01", object="volume", old_value="online", op="update", style="flexvol", svm="harvest", track="state", volume="harvest_demo"} 1698735677
+```
+
+Example of metric shape for metric value change:
+
+```
+change_log{aggr="umeng_aff300_aggr2", category="metric", cluster="umeng-aff300-01-02", datacenter="u2", index="3", instance="localhost:12993", job="prometheus", node="umeng-aff300-01", object="volume", op="metric_change", track="volume_size_total", svm="harvest", volume="harvest_demo"} 1698735800
 ```
 
 ### Object Deletion
 
 When an object is deleted, the ChangeLog plugin will publish a metric with the following labels:
 
-
 | Label        | Description                                                                 |
 |--------------|-----------------------------------------------------------------------------|
 | object       | name of the ONTAP object that was changed                                   |

grafana/dashboards/cmode/changelogMonitor.json

@@ -304,7 +304,7 @@
         },
         {
           "datasource": "${DS_PROMETHEUS}",
-          "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node)  \n\n`OP:` The type of change that was made (e.g., create, update, delete)  \n\n`Track:` Property of the object which was modified  \n\n`New Value:` The new value of the object after the change was made  \n\n`Old Value:` The previous value of the object before the change was made.",
+          "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node)  \n\n`OP:` The type of change that was made (e.g., create, update, delete)  \n\n`Track:` Property of the object which was modified  \n\n`New Value:` The new value of the property after the change was made  \n\n`Old Value:` The previous value of the property before the change was made \n\n*Note:* `Old Value` and `New Value` are only available for label changes and not for metric changes.",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -718,7 +718,7 @@
         },
         {
           "datasource": "${DS_PROMETHEUS}",
-          "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node)  \n\n`OP:` The type of change that was made (e.g., create, update, delete)  \n\n`Track:` Property of the object which was modified  \n\n`New Value:` The new value of the object after the change was made  \n\n`Old Value:` The previous value of the object before the change was made.",
+          "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node)  \n\n`OP:` The type of change that was made (e.g., create, update, delete)  \n\n`Track:` Property of the object which was modified  \n\n`New Value:` The new value of the property after the change was made  \n\n`Old Value:` The previous value of the property before the change was made \n\n*Note:* `Old Value` and `New Value` are only available for label changes and not for metric changes.",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -1158,7 +1158,7 @@
         },
         {
           "datasource": "${DS_PROMETHEUS}",
-          "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node)  \n\n`OP:` The type of change that was made (e.g., create, update, delete)  \n\n`Track:` Property of the object which was modified  \n\n`New Value:` The new value of the object after the change was made  \n\n`Old Value:` The previous value of the object before the change was made.",
+          "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node)  \n\n`OP:` The type of change that was made (e.g., create, update, delete)  \n\n`Track:` Property of the object which was modified  \n\n`New Value:` The new value of the property after the change was made  \n\n`Old Value:` The previous value of the property before the change was made \n\n*Note:* `Old Value` and `New Value` are only available for label changes and not for metric changes.",
           "fieldConfig": {
             "defaults": {
               "color": {