recon: fix replication age metric

collector/fixtures/recon_failed_collect.prom

@@ -3,7 +3,7 @@
 swift_cluster_accounts_quarantined{storage_ip="10.0.0.1"} 0
 # HELP swift_cluster_accounts_replication_age Account replication age reported by the swift-recon tool.
 # TYPE swift_cluster_accounts_replication_age gauge
-swift_cluster_accounts_replication_age{storage_ip="10.0.0.1"} 1.579007237099724e+09
+swift_cluster_accounts_replication_age{storage_ip="10.0.0.1"} -1.579007236099724e+09
 # HELP swift_cluster_accounts_replication_duration Account replication duration reported by the swift-recon tool.
 # TYPE swift_cluster_accounts_replication_duration gauge
 swift_cluster_accounts_replication_duration{storage_ip="10.0.0.1"} 23.422847032546997
@@ -12,7 +12,7 @@ swift_cluster_accounts_replication_duration{storage_ip="10.0.0.1"} 23.4228470325
 swift_cluster_containers_quarantined{storage_ip="10.0.0.1"} 0
 # HELP swift_cluster_containers_replication_age Container replication age reported by the swift-recon tool.
 # TYPE swift_cluster_containers_replication_age gauge
-swift_cluster_containers_replication_age{storage_ip="10.0.0.1"} 1.579007236617117e+09
+swift_cluster_containers_replication_age{storage_ip="10.0.0.1"} -1.579007235617117e+09
 # HELP swift_cluster_containers_replication_duration Container replication duration reported by the swift-recon tool.
 # TYPE swift_cluster_containers_replication_duration gauge
 swift_cluster_containers_replication_duration{storage_ip="10.0.0.1"} 98.37576985359192
@@ -30,7 +30,7 @@ swift_cluster_drives_unmounted{storage_ip="10.0.0.1"} 0
 swift_cluster_objects_quarantined{storage_ip="10.0.0.1"} 0
 # HELP swift_cluster_objects_replication_age Object replication age reported by the swift-recon tool.
 # TYPE swift_cluster_objects_replication_age gauge
-swift_cluster_objects_replication_age{storage_ip="10.0.0.1"} 1.57900646181673e+09
+swift_cluster_objects_replication_age{storage_ip="10.0.0.1"} -1.57900646081673e+09
 # HELP swift_cluster_objects_replication_duration Object replication duration reported by the swift-recon tool.
 # TYPE swift_cluster_objects_replication_duration gauge
 swift_cluster_objects_replication_duration{storage_ip="10.0.0.1"} 5.449508202075958

collector/fixtures/recon_successful_collect.prom

@@ -4,8 +4,8 @@ swift_cluster_accounts_quarantined{storage_ip="10.0.0.1"} 0
 swift_cluster_accounts_quarantined{storage_ip="10.0.0.2"} 0
 # HELP swift_cluster_accounts_replication_age Account replication age reported by the swift-recon tool.
 # TYPE swift_cluster_accounts_replication_age gauge
-swift_cluster_accounts_replication_age{storage_ip="10.0.0.1"} 1.577664676578959e+09
-swift_cluster_accounts_replication_age{storage_ip="10.0.0.2"} 1.5776646689851e+09
+swift_cluster_accounts_replication_age{storage_ip="10.0.0.1"} -1.577664675578959e+09
+swift_cluster_accounts_replication_age{storage_ip="10.0.0.2"} -1.5776646679851e+09
 # HELP swift_cluster_accounts_replication_duration Account replication duration reported by the swift-recon tool.
 # TYPE swift_cluster_accounts_replication_duration gauge
 swift_cluster_accounts_replication_duration{storage_ip="10.0.0.1"} 13.002140045166016
@@ -16,8 +16,8 @@ swift_cluster_containers_quarantined{storage_ip="10.0.0.1"} 0
 swift_cluster_containers_quarantined{storage_ip="10.0.0.2"} 0
 # HELP swift_cluster_containers_replication_age Container replication age reported by the swift-recon tool.
 # TYPE swift_cluster_containers_replication_age gauge
-swift_cluster_containers_replication_age{storage_ip="10.0.0.1"} 1.577664528691438e+09
-swift_cluster_containers_replication_age{storage_ip="10.0.0.2"} 1.577664555743305e+09
+swift_cluster_containers_replication_age{storage_ip="10.0.0.1"} -1.577664527691438e+09
+swift_cluster_containers_replication_age{storage_ip="10.0.0.2"} -1.577664554743305e+09
 # HELP swift_cluster_containers_replication_duration Container replication duration reported by the swift-recon tool.
 # TYPE swift_cluster_containers_replication_duration gauge
 swift_cluster_containers_replication_duration{storage_ip="10.0.0.1"} 83.79213690757751
@@ -56,8 +56,8 @@ swift_cluster_objects_quarantined{storage_ip="10.0.0.1"} 0
 swift_cluster_objects_quarantined{storage_ip="10.0.0.2"} 0
 # HELP swift_cluster_objects_replication_age Object replication age reported by the swift-recon tool.
 # TYPE swift_cluster_objects_replication_age gauge
-swift_cluster_objects_replication_age{storage_ip="10.0.0.1"} 1.577664310620143e+09
-swift_cluster_objects_replication_age{storage_ip="10.0.0.2"} 1.577664316719913e+09
+swift_cluster_objects_replication_age{storage_ip="10.0.0.1"} -1.577664309620143e+09
+swift_cluster_objects_replication_age{storage_ip="10.0.0.2"} -1.577664315719913e+09
 # HELP swift_cluster_objects_replication_duration Object replication duration reported by the swift-recon tool.
 # TYPE swift_cluster_objects_replication_duration gauge
 swift_cluster_objects_replication_duration{storage_ip="10.0.0.1"} 4.6007425824801125

collector/recon.go

@@ -26,16 +26,18 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sapcc/go-bits/logg"
+	"github.com/sapcc/swift-health-exporter/test/cmd/utils"
 )
 
 // ReconCollector implements the prometheus.Collector interface.
 type ReconCollector struct {
 	taskExitCode typedDesc
 	tasks        []collectorTask
+	isTest       bool
 }
 
 // NewReconCollector creates a new ReconCollector.
-func NewReconCollector(pathToExecutable string) *ReconCollector {
+func NewReconCollector(pathToExecutable string, isTest bool) *ReconCollector {
 	return &ReconCollector{
 		taskExitCode: typedDesc{
 			desc: prometheus.NewDesc("swift_recon_task_exit_code",
@@ -48,7 +50,7 @@ func NewReconCollector(pathToExecutable string) *ReconCollector {
 			newReconDriveAuditTask(pathToExecutable),
 			newReconMD5Task(pathToExecutable),
 			newReconQuarantinedTask(pathToExecutable),
-			newReconReplicationTask(pathToExecutable),
+			newReconReplicationTask(pathToExecutable, isTest),
 			newReconUnmountedTask(pathToExecutable),
 			newReconUpdaterSweepTask(pathToExecutable),
 		},
@@ -364,6 +366,7 @@ func (t *reconUpdaterSweepTask) collectMetrics(ch chan<- prometheus.Metric, exit
 // reconReplicationTask implements the collector.collectorTask interface.
 type reconReplicationTask struct {
 	pathToReconExecutable string
+	isTest                bool
 
 	accountReplicationAge        typedDesc
 	accountReplicationDuration   typedDesc
@@ -373,9 +376,10 @@ type reconReplicationTask struct {
 	objectReplicationDuration    typedDesc
 }
 
-func newReconReplicationTask(pathToReconExecutable string) collectorTask {
+func newReconReplicationTask(pathToReconExecutable string, isTest bool) collectorTask {
 	return &reconReplicationTask{
 		pathToReconExecutable: pathToReconExecutable,
+		isTest:                isTest,
 		accountReplicationAge: typedDesc{
 			desc: prometheus.NewDesc("swift_cluster_accounts_replication_age",
 				"Account replication age reported by the swift-recon tool.",
@@ -459,8 +463,15 @@ func (t *reconReplicationTask) collectMetrics(ch chan<- prometheus.Metric, exitC
 					continue // to next host
 				}
 
-				ch <- ageTypedDesc.mustNewConstMetric(data.ReplicationLast, hostname)
-				ch <- durTypedDesc.mustNewConstMetric(data.ReplicationTime, hostname)
+				if data.ReplicationLast > 0 {
+					now := float64(time.Now().Unix())
+					if t.isTest {
+						now = float64(utils.TimeNow().Second())
+					}
+					tDiff := now - data.ReplicationLast
+					ch <- ageTypedDesc.mustNewConstMetric(tDiff, hostname)
+				}
+				ch <- durTypedDesc.mustNewConstMetric(data.ReplicationTime, hostname) // good
 			}
 		} else {
 			exitCode = 1

collector/recon_test.go

@@ -30,7 +30,7 @@ func TestReconCollector(t *testing.T) {
 	}
 
 	registry := prometheus.NewPedanticRegistry()
-	registry.MustRegister(NewReconCollector(pathToExecutable))
+	registry.MustRegister(NewReconCollector(pathToExecutable, true))
 	assert.HTTPRequest{
 		Method:       "GET",
 		Path:         "/metrics",
@@ -46,7 +46,7 @@ func TestReconCollectorWithErrors(t *testing.T) {
 	}
 
 	registry := prometheus.NewPedanticRegistry()
-	registry.MustRegister(NewReconCollector(pathToExecutable))
+	registry.MustRegister(NewReconCollector(pathToExecutable, true))
 	assert.HTTPRequest{
 		Method:       "GET",
 		Path:         "/metrics",

main.go

@@ -34,7 +34,7 @@ func main() {
 	swiftReconPath := getExecutablePath("SWIFT_RECON_PATH", "swift-recon")
 
 	prometheus.MustRegister(collector.NewDispersionCollector(swiftDispersionReportPath))
-	prometheus.MustRegister(collector.NewReconCollector(swiftReconPath))
+	prometheus.MustRegister(collector.NewReconCollector(swiftReconPath, false))
 
 	// this port has been allocated for Swift health exporter
 	// See: https://github.com/prometheus/prometheus/wiki/Default-port-allocations

test/cmd/utils/time.go

@@ -0,0 +1,10 @@
+package utils
+
+import "time"
+
+var clockSeconds int64 = 1
+
+//TimeNow replaces time.Now in unit tests.
+func TimeNow() time.Time {
+	return time.Unix(clockSeconds, 0).UTC()
+}