From 3159ad88cfa895efe42625a5fcf1e0baf4bb0197 Mon Sep 17 00:00:00 2001 From: Hristo Voyvodov Date: Wed, 20 Dec 2023 09:06:07 +0200 Subject: [PATCH 1/2] update docs with minions health functionality --- README.md | 5 +++++ docs/docs/salt-exporter/configuration.md | 17 ++++++++++++++++ docs/docs/salt-exporter/metrics.md | 26 ++++++++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/README.md b/README.md index ae2d22f..b0b87cf 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,11 @@ salt_responses_total{minion="local",success="true"} 6 salt_responses_total{minion="node1",success="true"} 6 salt_scheduled_job_return_total{function="state.sls",minion="local",state="test",success="true"} 2 + +salt_health_last_heartbeat{minion="local"} 1703053536 +salt_health_last_heartbeat{minion="node1"} 1703053536 + +salt_health_minions_total{} 2 ``` ### Deprecation notice diff --git a/docs/docs/salt-exporter/configuration.md b/docs/docs/salt-exporter/configuration.md index a3ea6d9..dfad931 100644 --- a/docs/docs/salt-exporter/configuration.md +++ b/docs/docs/salt-exporter/configuration.md @@ -26,6 +26,9 @@ log-level: "info" listen-address: "" listen-port: 2112 +pki-dir: /etc/salt/pki/master +ipc-file: /var/run/salt/master/master_event_pub.ipc + tls: enabled: true key: "/path/to/key" @@ -70,6 +73,7 @@ metrics: | log-level | `info` | log level can be: debug, info, warn, error, fatal, panic, disabled | | listen-address | `0.0.0.0` | listening address | | listen-port | `2112` | listening port | +| pki-dir | `/etc/salt/pki/master` | path to Salt master's PKI directory | ### TLS settings @@ -101,6 +105,19 @@ All parameters below are in the `metrics` section of the configuration. | salt_function_status.filters.function | `state.highstate` | updates the metric only if the event function matches the filter | | salt_function_status.filters.states | `highstate` | updates the metric only if the event state matches the filter | +### Minions health detection + +In most of the cases all that you need to configure is to enable [`status` beacon](https://docs.saltproject.io/en/latest/ref/beacons/all/salt.beacons.status.html#:~:text=salt.-,beacons.,presence%20to%20be%20set%20up.) on Salt minions. +However, if you change the [pki directory](https://docs.saltproject.io/en/latest/ref/configuration/master.html#pki-dir) for Salt master, you'll need to make a change in the exporter side too by changing it in the configuration +```yaml +log-level: "info" + +pki-dir: /path/as/set/in/master/config + +tls: +... +``` + ## Alternative methods ### Environment variables diff --git a/docs/docs/salt-exporter/metrics.md b/docs/docs/salt-exporter/metrics.md index 569f6c6..b664f93 100644 --- a/docs/docs/salt-exporter/metrics.md +++ b/docs/docs/salt-exporter/metrics.md @@ -23,6 +23,8 @@ title: Metrics | `salt_scheduled_job_return_total` | `function`, `state`, `success`
(opt: `minion`) | Counter incremented each time a minion sends a scheduled job result | | `salt_responses_total` | `minion`, `success` | Total number of job responses
_including scheduled_job responses_ | | `salt_function_status` | `function`, `state`, `minion` | Last status of a job execution* | +| `salt_health_last_heartbeat` | `minion` | Last heartbeat from minion in UNIX timestamp +| `salt_health_minions_total` | | Total number of registered minions \* more details in the section below. @@ -55,6 +57,22 @@ You can find an example of Prometheus alerts that could be used [here](https://g See the [configuration page](./configuration.md) if you want to watch other functions/states, or if you want to disable this metric. +## Minions health + +The exporter is supporting "hearbeat"-ing detection from minions which can be used to monitor for non-responding/dead minions. Under the hood it depends on Salt's beacons. +To ensure that all required minions are reported (even if there is no heartbeat from them yet), exporter needs access to the PKI directory of the Salt Master (by default `/etc/salt/pki/master`) where it watches for accepted minion's public keys (located under `/etc/salt/pki/master/minions`). +On startup, all currently accepted minions are added with last heartbeat set to current time. From this point forward, exporter is using __fsnotify__ to detect added or removed minions. This will ensure that once minion is added, it will be monitored for heartbeat and metric will be removed once minion is deleted from Salt master. + +To use this functionality you'll need to add [`status` beacon](https://docs.saltproject.io/en/latest/ref/beacons/all/salt.beacons.status.html#:~:text=salt.-,beacons.,presence%20to%20be%20set%20up.) to each minion. It doesn't mater what functions will returned or the period. Exporter will just detect such events (in the format `salt/beacon//status`) and register the timestamp as last heartbeat. + +### Detecting dead minions + +The most simple way is (e.g. no heartbeat in last hour): + ``` { .promql .copy } + (time() - salt_health_last_heartbeat) > 3600 + ``` +> __NOTE__: Above is assuming beacon interval is set to < 3600 seconds + ## How to estimate missing responses Simple way: @@ -126,3 +144,11 @@ More advanced: salt_scheduled_job_return_total{function="state.sls",minion="local",state="test",success="true"} 3 ``` +??? example "Minions heartbeat" + + ```promql + salt_health_last_heartbeat{minion="local"} 1703053536 + salt_health_last_heartbeat{minion="node1"} 1703052536 + + salt_health_minions_total{} 2 + ``` \ No newline at end of file From 95981af914c04656f8588480bcdf2bb1cc0e3dc7 Mon Sep 17 00:00:00 2001 From: Hristo Voyvodov Date: Wed, 20 Dec 2023 09:06:41 +0200 Subject: [PATCH 2/2] fix PKI path to use master's one, not generic --- cmd/salt-exporter/config_test.go | 2 +- pkg/listener/pkiwatcher.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/salt-exporter/config_test.go b/cmd/salt-exporter/config_test.go index 202a3a1..34f9159 100644 --- a/cmd/salt-exporter/config_test.go +++ b/cmd/salt-exporter/config_test.go @@ -120,7 +120,7 @@ func TestReadConfigFlagOnly(t *testing.T) { ListenAddress: "127.0.0.1", ListenPort: 8080, IPCFile: "/dev/null", - PKIDir: "/etc/salt/pki", + PKIDir: "/etc/salt/pki/master", TLS: struct { Enabled bool Key string diff --git a/pkg/listener/pkiwatcher.go b/pkg/listener/pkiwatcher.go index 23f0a35..8400d38 100644 --- a/pkg/listener/pkiwatcher.go +++ b/pkg/listener/pkiwatcher.go @@ -12,7 +12,7 @@ import ( "github.com/rs/zerolog/log" ) -const DefaultPKIDirpath = "/etc/salt/pki" +const DefaultPKIDirpath = "/etc/salt/pki/master" type PKIWatcher struct { ctx context.Context @@ -62,7 +62,7 @@ func (w *PKIWatcher) open() { default: } - minionsDir := path.Join(w.pkiDirPath, "master/minions") + minionsDir := path.Join(w.pkiDirPath, "minions") log.Info().Msg("loading currently accepted minions") entries, err := os.ReadDir(minionsDir)