diff --git a/cmd/exporters/prometheus/prometheus.go b/cmd/exporters/prometheus/prometheus.go index bc3e9ba3d..7b25546cb 100644 --- a/cmd/exporters/prometheus/prometheus.go +++ b/cmd/exporters/prometheus/prometheus.go @@ -24,6 +24,7 @@ package prometheus import ( "fmt" "github.com/netapp/harvest/v2/cmd/poller/exporter" + "github.com/netapp/harvest/v2/cmd/poller/plugin/changelog" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/set" @@ -337,6 +338,24 @@ func (p *Prometheus) render(data *matrix.Matrix) ([][]byte, exporter.Stats) { instanceLabels := make([]string, 0) instanceLabelsSet := make(map[string]struct{}) + // The ChangeLog plugin tracks metric values and publishes the names of metrics that have changed. + // For example, it might indicate that 'volume_size_total' has been updated. + // If a global prefix for the exporter is defined, we need to amend the metric name with this prefix. + if p.globalPrefix != "" && data.Object == changelog.ObjectChangeLog { + categoryIsMetric := false + for label, value := range instance.GetLabels() { + if label == changelog.Category && value == changelog.Metric { + categoryIsMetric = true + break + } + } + if categoryIsMetric { + if value, ok := instance.GetLabels()[changelog.Track]; ok { + instance.GetLabels()[changelog.Track] = p.globalPrefix + value + } + } + } + if includeAllLabels { for label, value := range instance.GetLabels() { // temporary fix for the rarely happening duplicate labels diff --git a/cmd/poller/plugin/changelog/changelog.go b/cmd/poller/plugin/changelog/changelog.go index e1ad5d62d..a527d83be 100644 --- a/cmd/poller/plugin/changelog/changelog.go +++ b/cmd/poller/plugin/changelog/changelog.go @@ -19,16 +19,19 @@ The shape of the change_log is specific to each label change and is only applica // Constants for ChangeLog metrics and labels const ( - changeLog = "change" - objectLabel = "object" - opLabel = "op" - create = "create" - update = "update" - del = "delete" - track = "track" - oldValue = "old_value" - newValue = "new_value" - indexLabel = "index" + ObjectChangeLog = "change" + objectLabel = "object" + opLabel = "op" + create = "create" + update = "update" + del = "delete" + Track = "track" + oldValue = "old_value" + newValue = "new_value" + indexLabel = "index" + Metric = "metric" + Label = "label" + Category = "category" ) // Metrics to be used in ChangeLog @@ -56,6 +59,7 @@ type Change struct { oldValue string newValue string time int64 + category string } // New initializes a new instance of the ChangeLog plugin @@ -72,7 +76,7 @@ func (c *ChangeLog) Init() error { } object := c.ParentParams.GetChildS("object") - c.matrixName = object.GetContentS() + "_" + changeLog + c.matrixName = object.GetContentS() + "_" + ObjectChangeLog return c.populateChangeLogConfig() } @@ -95,7 +99,7 @@ func (c *ChangeLog) populateChangeLogConfig() error { // initMatrix initializes a new matrix with the given name func (c *ChangeLog) initMatrix() (map[string]*matrix.Matrix, error) { changeLogMap := make(map[string]*matrix.Matrix) - changeLogMap[c.matrixName] = matrix.New(c.Parent+c.matrixName, changeLog, c.matrixName) + changeLogMap[c.matrixName] = matrix.New(c.Parent+c.matrixName, ObjectChangeLog, c.matrixName) for _, changeLogMatrix := range changeLogMap { changeLogMatrix.SetExportOptions(matrix.DefaultExportOptions()) } @@ -189,13 +193,15 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u op: update, labels: make(map[string]string), track: currentLabel, + category: Label, oldValue: old[currentLabel], newValue: nVal, time: currentTime, } c.updateChangeLogLabels(object, instance, change) - // add changed track and its old, new value - change.labels[track] = currentLabel + // add changed Track and its old, new value + change.labels[Category] = change.category + change.labels[Track] = currentLabel change.labels[oldValue] = change.oldValue change.labels[newValue] = nVal c.createChangeLogInstance(changeMat, change) @@ -206,17 +212,19 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u if changes, ok := metricChanges[key]; ok { for metricName := range changes { change := &Change{ - key: uuid + "_" + object + "_" + metricName, - object: object, - op: update, - labels: make(map[string]string), - track: metricName, + key: uuid + "_" + object + "_" + metricName, + object: object, + op: update, + labels: make(map[string]string), + track: metricName, + category: Metric, // Enabling tracking of both old and new values results in the creation of a new time series each time the pair of values changes. For metrics tracking, it is not suitable. time: currentTime, } c.updateChangeLogLabels(object, instance, change) - // add changed track and its old, new value - change.labels[track] = metricName + // add changed Track and its old, new value + change.labels[Category] = change.category + change.labels[Track] = metricName change.labels[oldValue] = change.oldValue change.labels[newValue] = change.newValue c.createChangeLogInstance(changeMat, change) @@ -266,10 +274,9 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *u } // CompareMetrics compares the metrics of the current and previous instances -func (c *ChangeLog) CompareMetrics(curMat *matrix.Matrix) map[string]map[string][2]float64 { - metricChanges := make(map[string]map[string][2]float64) +func (c *ChangeLog) CompareMetrics(curMat *matrix.Matrix) map[string]map[string]struct{} { + metricChanges := make(map[string]map[string]struct{}) prevMat := c.previousData - met := maps2.Keys(c.previousData.GetMetrics()) for _, metricKey := range met { @@ -277,18 +284,19 @@ func (c *ChangeLog) CompareMetrics(curMat *matrix.Matrix) map[string]map[string] curMetric := curMat.GetMetric(metricKey) for key, currInstance := range curMat.GetInstances() { prevInstance := prevMat.GetInstance(key) + if prevInstance == nil { + continue + } prevIndex := prevInstance.GetIndex() currIndex := currInstance.GetIndex() curVal := curMetric.GetValues()[currIndex] prevVal := prevMetric.GetValues()[prevIndex] - if prevInstance != nil { - if curVal != prevVal { - if _, ok := metricChanges[key]; !ok { - metricChanges[key] = make(map[string][2]float64) - } - metName := curMat.Object + "_" + curMetric.GetName() - metricChanges[key][metName] = [2]float64{prevVal, curVal} + if curVal != prevVal { + if _, ok := metricChanges[key]; !ok { + metricChanges[key] = make(map[string]struct{}) } + metName := curMat.Object + "_" + curMetric.GetName() + metricChanges[key][metName] = struct{}{} } } } diff --git a/cmd/poller/plugin/changelog/changelog_test.go b/cmd/poller/plugin/changelog/changelog_test.go index b60a9333f..2f503d5b3 100644 --- a/cmd/poller/plugin/changelog/changelog_test.go +++ b/cmd/poller/plugin/changelog/changelog_test.go @@ -105,7 +105,7 @@ func TestChangeLogModified(t *testing.T) { o, _, _ := p.Run(data1) - checkChangeLogInstances(t, o, 2, 9, update, opLabel) + checkChangeLogInstances(t, o, 2, 10, update, opLabel) } func TestChangeLogModifiedWithMetrics(t *testing.T) { @@ -140,7 +140,7 @@ func TestChangeLogModifiedWithMetrics(t *testing.T) { o, _, _ := p.Run(data1) - checkChangeLogInstances(t, o, 3, 7, update, opLabel) + checkChangeLogInstances(t, o, 3, 8, update, opLabel) } func TestChangeLogCreated(t *testing.T) { diff --git a/docs/plugins.md b/docs/plugins.md index 89eed4e0d..1a52aca48 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -725,17 +725,18 @@ When an existing object is modified, the ChangeLog plugin will publish a metric | `new_value` | New value of the object after the change (only available for label changes and not for metric changes) | | `old_value` | Previous value of the object before the change (only available for label changes and not for metric changes) | | `metric value` | Timestamp when Harvest captured the change. `1698735677` in the example below | +| `category` | Type of the change, indicating whether it is a `metric` or a `label` change | Example of metric shape for object modification for label: ``` -change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter="u2", index="1", instance="localhost:12993", job="prometheus", new_value="offline", node="umeng-aff300-01", object="volume", old_value="online", op="update", style="flexvol", svm="harvest", track="state", volume="harvest_demo"} 1698735677 +change_log{aggr="umeng_aff300_aggr2", category="label", cluster="umeng-aff300-01-02", datacenter="u2", index="1", instance="localhost:12993", job="prometheus", new_value="offline", node="umeng-aff300-01", object="volume", old_value="online", op="update", style="flexvol", svm="harvest", track="state", volume="harvest_demo"} 1698735677 ``` Example of metric shape for metric value change: ``` -change_log{aggr="umeng_aff300_aggr2", cluster="umeng-aff300-01-02", datacenter="u2", index="3", instance="localhost:12993", job="prometheus", node="umeng-aff300-01", object="volume", op="metric_change", track="volume_size_total", svm="harvest", volume="harvest_demo"} 1698735800 +change_log{aggr="umeng_aff300_aggr2", category="metric", cluster="umeng-aff300-01-02", datacenter="u2", index="3", instance="localhost:12993", job="prometheus", node="umeng-aff300-01", object="volume", op="metric_change", track="volume_size_total", svm="harvest", volume="harvest_demo"} 1698735800 ``` ### Object Deletion