Merge 4d4f772 into b4e483d

cmd/boulder-observer/README.md

@@ -0,0 +1,86 @@
+# boulder-observer
+A modular config driven approach to black box monitoring with Prometheus
+
+
+## Usage
+### Starting the `observer` daemon
+```shell
+$ ./observer/plugins/build.sh && go run ./cmd/boulder-observer/main.go -config test/config-next/observer.yaml
+Building plugins:
+⚙️ observer/plugins/dns.so
+✅dns.so
+⚙️ observer/plugins/http.so
+✅http.so
+OK
+I191418 main ksKu7w4 Versions: main=(Unspecified Unspecified) Golang=(go1.15.7) BuildHost=(Unspecified)
+I191418 main o9me0QI Initializing boulder-observer daemon from config: test/config-next/observer.yaml
+I191420 main wv7tug0 HTTP monitor "https://letsencrypt.org-200" succeeded while taking:=120.900665ms
+I191422 main ss-hzQ8 HTTP monitor "https://letsencrypt.org-200" succeeded while taking:=23.051998ms
+I191424 main -fD46gg HTTP monitor "https://letsencrypt.org-200" succeeded while taking:=23.419121ms
+I191426 main urmy8AM HTTP monitor "https://letsencrypt.org-200" succeeded while taking:=23.875478ms
+I191428 main qaGe0Qc DNS monitor "udp-8.8.8.8:53-google.com-A" succeeded while taking:=5.088261ms
+I191428 main i677rw0 DNS monitor "tcp-8.8.8.8:53-google.com-A" succeeded while taking:=5.156114ms
+I191428 main ooyq_Qo DNS monitor "udp-owen.ns.cloudflare.com:53-letsencrypt.org-A" succeeded while taking:=15.858563ms
+```
+
+### Help
+```shell
+$ go run ./cmd/boulder-observer/main.go -help
+main:
+  -config string
+        Path to boulder-observer configuration file (default "config.yaml")
+```
+## Configuration
+```yaml
+debugAddr: 8040
+syslog: 
+  stdoutlevel: 6
+  sysloglevel: 6
+timeout: 5
+monitors: []
+```
+
+### Monitors
+
+#### Using the DNS plugin
+```yaml
+monitors:
+  - 
+    enabled: true
+    period: 1
+    plugin:
+      name: DNS
+      path: "./cmd/boulder-observer/observer/plugins/dns.so"
+    settings:
+      qproto: udp
+      qrecurse: false
+      qname: letsencrypt.org
+      qtype: A
+      qserver: "owen.ns.cloudflare.com:53"
+```
+
+#### Using the HTTP plugin
+```yaml
+monitors:
+  - 
+    enabled: true
+    period: 1
+    plugin:
+      name: HTTP
+      path: "./cmd/boulder-observer/observer/plugins/http.so"
+    settings: 
+      url: https://letsencrypt.org
+      rcode: 200
+```
+
+### Plugins
+**Building plugins**
+```shell
+$ ./observer/plugins/build.sh
+Building plugins:
+⚙️ observer/plugins/dns.so
+✅dns.so
+⚙️ observer/plugins/http.so
+✅http.so
+OK
+```

cmd/boulder-observer/main.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"flag"
+	"io/ioutil"
+
+	"github.com/letsencrypt/boulder/cmd"
+	"github.com/letsencrypt/boulder/observer"
+	"gopkg.in/yaml.v2"
+)
+
+func main() {
+	configPath := flag.String(
+		"config", "config.yaml", "Path to boulder-observer configuration file")
+	flag.Parse()
+
+	configYAML, err := ioutil.ReadFile(*configPath)
+	cmd.FailOnError(err, "failed to read config file")
+
+	// parse YAML config
+	var config observer.ObsConf
+	err = yaml.Unmarshal(configYAML, &config)
+	if err != nil {
+		cmd.FailOnError(err, "failed to parse YAML config")
+	}
+
+	// validate config
+	err = config.Validate()
+	if err != nil {
+		cmd.FailOnError(err, "YAML config failed validation")
+	}
+
+	// start monitoring and logging
+	prom, logger := cmd.StatsAndLogging(config.Syslog, config.DebugAddr)
+	defer logger.AuditPanic()
+	logger.Info(cmd.VersionString())
+
+	// start daemon
+	logger.Infof("Initializing boulder-observer daemon from config: %s", *configPath)
+	logger.Debugf("Using config: %+v", config)
+	observer := observer.New(config, logger, prom)
+	observer.Start()
+}

observer/mon_conf.go

@@ -0,0 +1,44 @@
+package observer
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/letsencrypt/boulder/observer/plugins"
+)
+
+var (
+	errNewMonEmpty   = errors.New("monitor config is empty")
+	errNewMonInvalid = errors.New("monitor config is invalid")
+)
+
+// MonConf is exported to recieve the supplied monitor config
+type MonConf struct {
+	Enabled  bool                   `yaml:"enabled"`
+	Period   int                    `yaml:"period"`
+	Timeout  int                    `yaml:"timeout"`
+	Plugin   plugins.Info           `yaml:"plugin"`
+	Settings map[string]interface{} `yaml:"settings"`
+}
+
+func (c MonConf) normalize() {
+	c.Plugin.Name = strings.ToLower(c.Plugin.Name)
+	c.Plugin.Path = strings.ToLower(c.Plugin.Path)
+}
+
+// validate normalizes and validates the received monitor config
+func (c MonConf) validate() error {
+	c.normalize()
+	pluginConf, err := plugins.GetPluginConf(c.Settings, c.Plugin.Path, c.Plugin.Name)
+	if err != nil {
+		if err != nil {
+			return fmt.Errorf("failed to get plugin: %w", err)
+		}
+	}
+	err = pluginConf.Validate()
+	if err != nil {
+		return fmt.Errorf("failed to validate plugin settings: %w", err)
+	}
+	return nil
+}

observer/monitor.go

@@ -0,0 +1,54 @@
+package observer
+
+import (
+	"time"
+
+	blog "github.com/letsencrypt/boulder/log"
+	"github.com/letsencrypt/boulder/observer/plugins"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type monitor struct {
+	name     string
+	period   time.Duration
+	timeout  time.Duration
+	pluginIs string
+	probe    plugins.Plugin
+	logger   blog.Logger
+	metric   prometheus.Registerer
+}
+
+func (m monitor) start() *time.Ticker {
+	ticker := time.NewTicker(m.period)
+	go func() {
+		for {
+			select {
+			case tick := <-ticker.C:
+				success, took := m.probe.Do(tick, m.timeout)
+				statTotalObservations.WithLabelValues(m.pluginIs, m.name).Add(1)
+				if !success {
+					statTotalErrors.WithLabelValues(m.pluginIs, m.name).Add(1)
+					m.logger.Infof("%s monitor %q failed while taking:=%s", m.pluginIs, m.name, took.String())
+					return
+				}
+				m.logger.Infof("%s monitor %q succeeded while taking:=%s", m.pluginIs, m.name, took.String())
+			}
+		}
+	}()
+	return ticker
+}
+
+func (m monitor) New(c MonConf, log blog.Logger, prom prometheus.Registerer, t int) *monitor {
+	if c.Timeout == 0 {
+		c.Timeout = t
+	}
+	plugin, _ := plugins.GetPluginConf(c.Settings, c.Plugin.Path, c.Plugin.Name)
+	m.name = plugin.GetMonitorName()
+	m.period = time.Duration(c.Period * 1000000000)
+	m.timeout = time.Duration(c.Timeout * 1000000000)
+	m.pluginIs = c.Plugin.Name
+	m.probe = plugin.AsProbe()
+	m.logger = log
+	m.metric = prom
+	return &m
+}

observer/obs_conf.go

@@ -0,0 +1,58 @@
+package observer
+
+import (
+	"errors"
+
+	"github.com/letsencrypt/boulder/cmd"
+	"github.com/letsencrypt/boulder/observer/plugins"
+)
+
+var (
+	errNewObsNoMons  = errors.New("observer config is invalid, 0 monitors configured")
+	errNewObsEmpty   = errors.New("observer config is empty")
+	errNewObsInvalid = errors.New("observer config is invalid")
+)
+
+// ObsConf is exported to recieve the supplied observer config
+type ObsConf struct {
+	Syslog    cmd.SyslogConfig `yaml:"syslog"`
+	DebugAddr string           `yaml:"debugAddr"`
+	Modules   []plugins.Conf   `yaml:"modules"`
+	Timeout   int              `yaml:"timeout"`
+	NewMons   []MonConf        `yaml:"monitors"`
+}
+
+func (n *ObsConf) validateMonConfs() error {
+	i := 0
+	for _, m := range n.NewMons {
+		if !m.Enabled {
+			continue
+		}
+		err := m.validate()
+		if err != nil {
+			return err
+		}
+		n.NewMons[i] = m
+		i++
+	}
+	n.NewMons = n.NewMons[:i]
+	return nil
+}
+
+// Validate normalizes and validates the received monitor config
+func (n *ObsConf) Validate() error {
+	if n == nil {
+		return errNewObsEmpty
+	}
+	if n.DebugAddr == "" {
+		return errNewObsInvalid
+	}
+	err := n.validateMonConfs()
+	if err != nil {
+		return err
+	}
+	if len(n.NewMons) == 0 {
+		return errNewObsNoMons
+	}
+	return nil
+}

observer/observer.go

@@ -0,0 +1,72 @@
+package observer
+
+import (
+	"time"
+
+	blog "github.com/letsencrypt/boulder/log"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	statTotalMonitors = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "obs_monitors",
+			Help: "count of configured monitors",
+		},
+		[]string{"plugin", "name"},
+	)
+	statTotalErrors = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "obs_errors",
+			Help: "count of errors encountered by all monitors",
+		},
+		[]string{"plugin", "name"},
+	)
+	statTotalObservations = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "obs_oberservations",
+			Help: "count of observations performed by all monitors",
+		},
+		[]string{"plugin", "name"},
+	)
+)
+
+// Observer acts as the
+type Observer struct {
+	Timeout  time.Duration
+	Logger   blog.Logger
+	Metric   prometheus.Registerer
+	Monitors []*monitor
+}
+
+// Start acts as the supervisor for all monitor goroutines
+func (o Observer) Start() {
+	runningChan := make(chan bool)
+
+	// register metrics
+	o.Metric.MustRegister(statTotalErrors)
+	o.Metric.MustRegister(statTotalMonitors)
+	o.Metric.MustRegister(statTotalObservations)
+
+	// start each monitor
+	for _, mon := range o.Monitors {
+		statTotalMonitors.WithLabelValues(mon.pluginIs, mon.name).Inc()
+		go mon.start()
+	}
+
+	// run forever
+	<-runningChan
+}
+
+// New initializes new Observer objects
+func New(c ObsConf, l blog.Logger, p prometheus.Registerer) *Observer {
+	var o Observer
+	o.Timeout = time.Duration(c.Timeout * 1000000000)
+	o.Logger = l
+	o.Metric = p
+	for _, monConf := range c.NewMons {
+		var mon monitor
+		o.Monitors = append(o.Monitors, mon.New(monConf, l, p, c.Timeout))
+	}
+	return &o
+}