Skip to content

Commit

Permalink
Add support for extra_labels
Browse files Browse the repository at this point in the history 
`extra_labels` can be applied to both `host_defaults` and
`service_defaults` and define additional fixed labels that can be
applied to the metrics emitted by the collectors.

host extra_labels apply to host metrics and are inherited by services,
service extra_labels apply to service metrics only.
  • Loading branch information
@wrouesnel
wrouesnel committed Oct 20, 2022
1 parent 24c2211 commit 2b81c58
Show file tree
Hide file tree
Showing 9 changed files with 200 additions and 20 deletions.
5 changes: 4 additions & 1 deletion cmd/poller_exporter/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,10 @@ func main() {
// We don't allow duplicate hosts, but also don't want to panic just due
// to a typo, so keep track and skip duplicates here.
seenHosts := make(map[string]bool)
statusCollecter := pollers.NewServiceStatusMetrics()

// The status checker needs to know which labels can appear so it can set them.
customServiceLabels := lo.Union(lo.Keys(cfg.HostDefault.ExtraLabels), lo.Keys(cfg.HostDefault.ServiceDefaults.ExtraLabels))
statusCollecter := pollers.NewServiceStatusMetrics(customServiceLabels)
prometheus.MustRegister(statusCollecter)
realIdx := 0
for _, hostCfg := range cfg.Hosts {
Expand Down
12 changes: 8 additions & 4 deletions pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,13 @@ type CollectorConfig struct {
// HostSettings contains the poller configuration which is applied per hostname
// (as opposed to per service).
type HostSettings struct {
PollFrequency model.Duration `mapstructure:"poll_frequency,omitempty"` // Frequency to poll this specific host
PingDisable bool `mapstructure:"disable_ping,omitempty"` // Disable ping checks for this host
PingTimeout model.Duration `mapstructure:"ping_timeout,omitempty"` // Maximum ping timeout
PingCount uint64 `mapstructure:"ping_count,omitempty"` // Number of pings to send each poll
PollFrequency model.Duration `mapstructure:"poll_frequency,omitempty"` // Frequency to poll this specific host
PingDisable bool `mapstructure:"disable_ping,omitempty"` // Disable ping checks for this host
PingTimeout model.Duration `mapstructure:"ping_timeout,omitempty"` // Maximum ping timeout
PingCount uint64 `mapstructure:"ping_count,omitempty"` // Number of pings to send each poll

ExtraLabels map[string]string `mapstructure:"extra_labels,omitempty"` // Extra Prometheus Metrics to add to collected metrics

ServiceDefaults ServiceSettings `mapstructure:"service_defaults,omitempty"`
}

Expand Down Expand Up @@ -96,6 +99,7 @@ type BasicServiceSettings struct {
TLSCACerts TLSCertificatePool `mapstructure:"tls_cacerts,omitempty"` // Path to CAfile to verify the service TLS with
Proxy string `mapstructure:"proxy,omitempty"` // Proxy configuration for the service
ProxyAuth *BasicAuthConfig `mapstructure:"proxy_auth,omitempty"` // Authentication for the proxy service
ExtraLabels map[string]string `mapstructure:"extra_labels,omitempty"` // Extra Prometheus Metrics to add to collected metrics
}

type ChallengeResponseServiceSettings struct {
Expand Down
32 changes: 31 additions & 1 deletion pkg/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,11 +213,19 @@ func (ce *ConfigExpected) TestCompleteConfig(c *C) {
c.Check(conf.HostDefault.PingTimeout, Equals, model.Duration(time.Second))
c.Check(conf.HostDefault.PingCount, Equals, uint64(3))

c.Check(conf.HostDefault.ExtraLabels, Not(IsNil))
c.Check(conf.HostDefault.ExtraLabels["host_label1"], Equals, "label1-value")
c.Check(conf.HostDefault.ExtraLabels["host_label2"], Equals, "label2-value")

c.Check(conf.HostDefault.ServiceDefaults.Timeout, Equals, model.Duration(time.Second*10))
c.Check(conf.HostDefault.ServiceDefaults.MaxBytes, Equals, uint64(4096))
c.Check(conf.HostDefault.ServiceDefaults.TLSEnable, Equals, false)
c.Check(conf.HostDefault.ServiceDefaults.TLSVerifyFailOk, Equals, false)

c.Check(conf.HostDefault.ServiceDefaults.ExtraLabels, Not(IsNil))
c.Check(conf.HostDefault.ServiceDefaults.ExtraLabels["service_label1"], Equals, "label1-value")
c.Check(conf.HostDefault.ServiceDefaults.ExtraLabels["service_label2"], Equals, "label2-value")

c.Check(len(GetPoolCertificates(conf.HostDefault.ServiceDefaults.TLSCACerts.CertPool)), Equals, len(systemPool)+2, Commentf("Check TLS CA certs looks like the system pool + 2 extra certs"))

// Start checking hosts - convert to map up front
Expand All @@ -236,7 +244,14 @@ func (ce *ConfigExpected) TestCompleteConfig(c *C) {
c.Check(hostConf.ServiceDefaults.Timeout, Equals, model.Duration(time.Second*9))
c.Check(hostConf.ServiceDefaults.MaxBytes, Equals, uint64(1024))
c.Check(hostConf.ServiceDefaults.TLSEnable, Equals, true)
c.Check(len(GetPoolCertificates(hostConf.ServiceDefaults.TLSCACerts.CertPool)), Equals, 1, Commentf("Check sngle default cert"))
c.Check(len(GetPoolCertificates(hostConf.ServiceDefaults.TLSCACerts.CertPool)), Equals, 1, Commentf("Check single default cert"))

c.Check(hostConf.ServiceDefaults.ExtraLabels, Not(IsNil))
c.Check(hostConf.ExtraLabels["host_label1"], Equals, "some-other-value")
c.Check(hostConf.ExtraLabels["host_label2"], Equals, "label2-value")

c.Check(hostConf.ServiceDefaults.ExtraLabels["service_label1"], Equals, "Changed")
c.Check(hostConf.ServiceDefaults.ExtraLabels["service_label2"], Equals, "label2-value")

basicChecks := hostConf.BasicChecks[0]
c.Check(basicChecks.Name, Equals, "SMTP")
Expand All @@ -256,6 +271,11 @@ func (ce *ConfigExpected) TestCompleteConfig(c *C) {
serviceCerts := GetPoolCertificates(basicChecks.TLSCACerts.CertPool)
c.Check(len(serviceCerts), Equals, 1)

c.Check(basicChecks.ExtraLabels["host_label1"], Equals, "This label is reinstated despite the service setting")
//c.Check(basicChecks.ExtraLabels["host_label2"], Equals, "label2-value")
c.Check(basicChecks.ExtraLabels["service_label1"], Equals, "Changed")
c.Check(basicChecks.ExtraLabels["service_label2"], Equals, "Changed on the service")

crChecks := hostConf.ChallengeResponseChecks[0]
c.Check(crChecks.Name, Equals, "CustomDaemon")
c.Check(crChecks.Protocol, Equals, "tcp")
Expand All @@ -264,6 +284,11 @@ func (ce *ConfigExpected) TestCompleteConfig(c *C) {
c.Check(crChecks.TLSEnable, Equals, false)
c.Check(len(GetPoolCertificates(crChecks.TLSCACerts.CertPool)), Equals, 2, Commentf("challenge_response has more certs"))

c.Check(crChecks.ExtraLabels["host_label1"], Equals, "You can do this, but shouldn't")
//c.Check(crChecks.ExtraLabels["host_label2"], Equals, "label2-value")
c.Check(crChecks.ExtraLabels["service_label1"], Equals, "Changed")
c.Check(crChecks.ExtraLabels["service_label2"], Equals, "CR service")

c.Check(*crChecks.ChallengeString, Equals, "MY_UNIQUE_HEADER")
c.Check([]byte(crChecks.ChallengeBinary), DeepEquals, []byte{114, 149, 9, 49, 56, 189, 30, 220, 186, 59, 139, 28, 127, 66, 178, 97})
c.Check(crChecks.ResponseRegex.String(), Equals, regexp.MustCompile("RESPONSE_HEADER").String())
Expand All @@ -281,6 +306,11 @@ func (ce *ConfigExpected) TestCompleteConfig(c *C) {
httpServiceCerts := GetPoolCertificates(httpChecks.TLSCACerts.CertPool)
c.Check(len(httpServiceCerts), Equals, 1)

//c.Check(httpChecks.ExtraLabels["host_label1"], Equals, "label1-value")
c.Check(httpChecks.ExtraLabels["host_label2"], Equals, "You can do this, but shouldn't")
c.Check(httpChecks.ExtraLabels["service_label1"], Equals, "HTTP service")
c.Check(httpChecks.ExtraLabels["service_label2"], Equals, "label2-value")

c.Check(*httpChecks.ChallengeString, Equals, "some-data")
c.Check([]byte(httpChecks.ChallengeBinary), DeepEquals, []byte{114, 149, 9, 49, 56, 189, 30, 220, 186, 59, 139, 28, 127, 66, 178, 97})
c.Check(httpChecks.ResponseRegex.String(), Equals, regexp.MustCompile("^<field-tag>").String())
Expand Down
62 changes: 61 additions & 1 deletion pkg/config/loading.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"io/ioutil"
"reflect"

"github.com/samber/lo"

"go.uber.org/zap"

"github.com/mitchellh/mapstructure"
Expand All @@ -13,6 +15,7 @@ import (

var (
ErrMapStructureDecode = errors.New("MapStructureDecode function failed")
ErrInconsistentLabels = errors.New("Extra Prometheus labels found without defaults set")
)

// loadDefaultConfigMap returns the config file which is embedded in the binary
Expand Down Expand Up @@ -100,7 +103,7 @@ func LoadAndSanitizeConfig(configData []byte) (string, error) {
}

// Load loads a configuration file from the supplied bytes.
//nolint:forcetypeassert,funlen,cyclop
//nolint:forcetypeassert,funlen,cyclop,gocognit
func Load(configData []byte) (*Config, error) {
defaultMap := loadDefaultConfigMap()
configMap, err := loadConfigMap(configData)
Expand Down Expand Up @@ -166,6 +169,63 @@ func Load(configData []byte) (*Config, error) {
if err := decoder.Decode(configMap); err != nil {
return nil, errors.Wrap(err, "Load: second-pass config map decoding failed")
}

// Check that Prometheus extra label keys are consistent.
hostDefaultKeys := lo.Keys(cfg.HostDefault.ExtraLabels)
serviceDefaultKeys := lo.Union(hostDefaultKeys, lo.Keys(cfg.HostDefault.ServiceDefaults.ExtraLabels))
for _, host := range cfg.Hosts {
hostKeys := lo.Keys(host.ExtraLabels)
if _, right := lo.Difference(hostDefaultKeys, hostKeys); len(right) > 0 {
zap.L().Error("Extra keys must have default values from host_defaults, and this host has more then allowed",
zap.Strings("allowed_keys", hostDefaultKeys),
zap.Strings("undefined_keys", right),
zap.String("hostname", host.Hostname))
return nil, errors.Wrapf(ErrInconsistentLabels, "Load: undefined labels found for host: %v", host.Hostname)
}

hostServiceDefaultKeys := lo.Keys(host.ServiceDefaults.ExtraLabels)
if _, right := lo.Difference(serviceDefaultKeys, hostServiceDefaultKeys); len(right) > 0 {
zap.L().Error("Extra keys must have default values from host_defaults.service_defaults, and this host has more then allowed",
zap.Strings("allowed_keys", hostServiceDefaultKeys),
zap.Strings("undefined_keys", right),
zap.String("hostname", host.Hostname))
return nil, errors.Wrapf(ErrInconsistentLabels, "Load: undefined labels found for host: %v", host.Hostname)
}

for _, service := range host.BasicChecks {
if _, right := lo.Difference(serviceDefaultKeys, lo.Keys(service.ExtraLabels)); len(right) > 0 {
zap.L().Error("Extra keys must have default values from host_defaults.service_defaults, and this host has more then allowed",
zap.Strings("allowed_keys", hostServiceDefaultKeys),
zap.Strings("undefined_keys", right),
zap.String("hostname", host.Hostname),
zap.String("service", service.Name))
return nil, errors.Wrapf(ErrInconsistentLabels, "Load: undefined labels found for host: %v", host.Hostname)
}
}

for _, service := range host.ChallengeResponseChecks {
if _, right := lo.Difference(serviceDefaultKeys, lo.Keys(service.ExtraLabels)); len(right) > 0 {
zap.L().Error("Extra keys must have default values from host_defaults.service_defaults, and this host has more then allowed",
zap.Strings("allowed_keys", hostServiceDefaultKeys),
zap.Strings("undefined_keys", right),
zap.String("hostname", host.Hostname),
zap.String("service", service.Name))
return nil, errors.Wrapf(ErrInconsistentLabels, "Load: undefined labels found for host: %v", host.Hostname)
}
}

for _, service := range host.HTTPChecks {
if _, right := lo.Difference(serviceDefaultKeys, lo.Keys(service.ExtraLabels)); len(right) > 0 {
zap.L().Error("Extra keys must have default values from host_defaults.service_defaults, and this host has more then allowed",
zap.Strings("allowed_keys", hostServiceDefaultKeys),
zap.Strings("undefined_keys", right),
zap.String("hostname", host.Hostname),
zap.String("service", service.Name))
return nil, errors.Wrapf(ErrInconsistentLabels, "Load: undefined labels found for host: %v", host.Hostname)
}
}
}

return cfg, nil
}

Expand Down
13 changes: 13 additions & 0 deletions pkg/pollers/basic_service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,19 @@ func NewBasicService(host *Host, opts config.BasicServiceConfig, constantLabels
}
}

// Append additional labels from host and service
if host.ExtraLabels != nil {
for labelKey, labelValue := range host.ExtraLabels {
constantLabels[labelKey] = labelValue
}
}

if opts.ExtraLabels != nil {
for labelKey, labelValue := range opts.ExtraLabels {
constantLabels[labelKey] = labelValue
}
}

newBasicService := &BasicService{
labels: constantLabels,
host: host,
Expand Down
26 changes: 17 additions & 9 deletions pkg/pollers/host.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,14 @@ func NewHost(opts *config.HostConfig) *Host {
zap.L().Error("Received nil hostConfig specification")
return nil
}

hostLabels := prometheus.Labels{"hostname": opts.Hostname}
if opts.ExtraLabels != nil {
for labelKey, labelValue := range opts.ExtraLabels {
hostLabels[labelKey] = labelValue
}
}

// Setup the host
newHost := Host{
IP: "", // Initially unresolved
Expand All @@ -57,35 +65,35 @@ func NewHost(opts *config.HostConfig) *Host {
Subsystem: "host",
Name: "polls_total",
Help: "Number of times this host has been polled by the exporter",
ConstLabels: prometheus.Labels{"hostname": opts.Hostname},
ConstLabels: hostLabels,
}),
LastPollTime: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: Namespace,
Subsystem: "host",
Name: "last_poll_time",
Help: "Last time this host was polled by the exporter",
ConstLabels: prometheus.Labels{"hostname": opts.Hostname},
ConstLabels: hostLabels,
}),
Resolvable: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: Namespace,
Subsystem: "host",
Name: "resolvable_boolean",
Help: "Did the last attempt to DNS resolve this host succeed?",
ConstLabels: prometheus.Labels{"hostname": opts.Hostname},
ConstLabels: hostLabels,
}),
PathReachable: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: Namespace,
Subsystem: "host",
Name: "routable_boolean",
Help: "Is the resolved IP address routable on this hosts network",
ConstLabels: prometheus.Labels{"hostname": opts.Hostname},
ConstLabels: hostLabels,
}),
PingLatency: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: Namespace,
Subsystem: "host",
Name: "latency_microseconds",
Help: "service latency in microseconds",
ConstLabels: prometheus.Labels{"hostname": opts.Hostname},
ConstLabels: hostLabels,
}),
// Cumulative counters
ResolvableCount: prometheus.NewCounterVec(
Expand All @@ -94,7 +102,7 @@ func NewHost(opts *config.HostConfig) *Host {
Subsystem: "host",
Name: "resolvable_total",
Help: "cumulative successful DNS resolutions",
ConstLabels: prometheus.Labels{"hostname": opts.Hostname},
ConstLabels: hostLabels,
},
[]string{"result"},
),
Expand All @@ -104,7 +112,7 @@ func NewHost(opts *config.HostConfig) *Host {
Subsystem: "host",
Name: "routable_total",
Help: "cumulative successful network route resolutions",
ConstLabels: prometheus.Labels{"hostname": opts.Hostname},
ConstLabels: hostLabels,
},
[]string{"result"},
),
Expand All @@ -114,7 +122,7 @@ func NewHost(opts *config.HostConfig) *Host {
Subsystem: "host",
Name: "ping_count_total",
Help: "cumulative number of pings sent to the host",
ConstLabels: prometheus.Labels{"hostname": opts.Hostname},
ConstLabels: hostLabels,
},
[]string{"result"},
),
Expand All @@ -124,7 +132,7 @@ func NewHost(opts *config.HostConfig) *Host {
Subsystem: "host",
Name: "latency_seconds_total",
Help: "cumulative service latency in seconds",
ConstLabels: prometheus.Labels{"hostname": opts.Hostname},
ConstLabels: hostLabels,
},
),

Expand Down
11 changes: 9 additions & 2 deletions pkg/pollers/status_collector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ package pollers
import (
"sync"

"github.com/samber/lo"

"github.com/prometheus/client_golang/prometheus"
"go.uber.org/zap"
)
Expand All @@ -17,7 +19,12 @@ type ServiceStatusMetricCollector struct {
}

// NewServiceStatusMetrics creates a new service status metric collector.
func NewServiceStatusMetrics() *ServiceStatusMetricCollector {
func NewServiceStatusMetrics(labels []string) *ServiceStatusMetricCollector {
if labels == nil {
labels = []string{}
}
fullLabels := lo.Union([]string{"poller_type", "hostname", "name", "protocol", "port"}, labels)

return &ServiceStatusMetricCollector{
ServiceStatus: prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Expand All @@ -26,7 +33,7 @@ func NewServiceStatusMetrics() *ServiceStatusMetricCollector {
Name: "status_boolean",
Help: "whether the poller succeeded by its current configuration - 1 means true, 0 false, NaN unknown",
},
[]string{"poller_type", "hostname", "name", "protocol", "port"},
fullLabels,
),
locker: &sync.Mutex{},
l: zap.L(),
Expand Down

0 comments on commit 2b81c58

Please sign in to comment.