Merge 4ef18f8 into b4e483d

cmd/boulder-observer/README.md

@@ -0,0 +1,86 @@
+# boulder-observer
+A modular config driven approach to black box monitoring with Prometheus
+
+
+## Usage
+### Starting the `observer` daemon
+```shell
+$ ./observer/plugins/build.sh && go run ./cmd/boulder-observer/main.go -config test/config-next/observer.yaml
+Building plugins:
+⚙️ observer/plugins/dns.so
+✅dns.so
+⚙️ observer/plugins/http.so
+✅http.so
+OK
+I191418 main ksKu7w4 Versions: main=(Unspecified Unspecified) Golang=(go1.15.7) BuildHost=(Unspecified)
+I191418 main o9me0QI Initializing boulder-observer daemon from config: test/config-next/observer.yaml
+I191420 main wv7tug0 HTTP monitor "https://letsencrypt.org-200" succeeded while taking:=120.900665ms
+I191422 main ss-hzQ8 HTTP monitor "https://letsencrypt.org-200" succeeded while taking:=23.051998ms
+I191424 main -fD46gg HTTP monitor "https://letsencrypt.org-200" succeeded while taking:=23.419121ms
+I191426 main urmy8AM HTTP monitor "https://letsencrypt.org-200" succeeded while taking:=23.875478ms
+I191428 main qaGe0Qc DNS monitor "udp-8.8.8.8:53-google.com-A" succeeded while taking:=5.088261ms
+I191428 main i677rw0 DNS monitor "tcp-8.8.8.8:53-google.com-A" succeeded while taking:=5.156114ms
+I191428 main ooyq_Qo DNS monitor "udp-owen.ns.cloudflare.com:53-letsencrypt.org-A" succeeded while taking:=15.858563ms
+```
+
+### Help
+```shell
+$ go run ./cmd/boulder-observer/main.go -help
+main:
+  -config string
+        Path to boulder-observer configuration file (default "config.yaml")
+```
+## Configuration
+```yaml
+debugAddr: 8040
+syslog: 
+  stdoutlevel: 6
+  sysloglevel: 6
+timeout: 5
+monitors: []
+```
+
+### Monitors
+
+#### Using the DNS plugin
+```yaml
+monitors:
+  - 
+    enabled: true
+    period: 1
+    plugin:
+      name: DNS
+      path: "./cmd/boulder-observer/observer/plugins/dns.so"
+    settings:
+      qproto: udp
+      qrecurse: false
+      qname: letsencrypt.org
+      qtype: A
+      qserver: "owen.ns.cloudflare.com:53"
+```
+
+#### Using the HTTP plugin
+```yaml
+monitors:
+  - 
+    enabled: true
+    period: 1
+    plugin:
+      name: HTTP
+      path: "./cmd/boulder-observer/observer/plugins/http.so"
+    settings: 
+      url: https://letsencrypt.org
+      rcode: 200
+```
+
+### Plugins
+**Building plugins**
+```shell
+$ ./observer/plugins/build.sh
+Building plugins:
+⚙️ observer/plugins/dns.so
+✅dns.so
+⚙️ observer/plugins/http.so
+✅http.so
+OK
+```

cmd/boulder-observer/main.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"io/ioutil"
+
+	"github.com/letsencrypt/boulder/cmd"
+	"github.com/letsencrypt/boulder/observer"
+	"gopkg.in/yaml.v2"
+)
+
+func main() {
+	configPath := flag.String(
+		"config", "config.yaml", "Path to boulder-observer configuration file")
+	flag.Parse()
+
+	configYAML, err := ioutil.ReadFile(*configPath)
+	cmd.FailOnError(err, "failed to read config file")
+
+	// parse YAML config
+	var config observer.ObsConf
+	err = yaml.Unmarshal(configYAML, &config)
+	if err != nil {
+		cmd.FailOnError(err, "failed to parse YAML config")
+	}
+
+	if config.DebugAddr == "" {
+		cmd.FailOnError(errors.New(""), "debugaddr is not defined")
+	}
+
+	// start monitoring and logging
+	prom, logger := cmd.StatsAndLogging(config.Syslog, config.DebugAddr)
+	defer logger.AuditPanic()
+	logger.Info(cmd.VersionString())
+
+	// validate config
+	err = config.Validate(logger)
+	if err != nil {
+		cmd.FailOnError(err, "config failed validation")
+	}
+
+	// start daemon
+	logger.Infof("Initializing boulder-observer daemon from config: %s", *configPath)
+	logger.Debugf("Using config: %+v", config)
+	observer := observer.New(config, logger, prom)
+	observer.Start()
+}

observer/mon_conf.go

@@ -0,0 +1,63 @@
+package observer
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/letsencrypt/boulder/cmd"
+	p "github.com/letsencrypt/boulder/observer/probes"
+	"gopkg.in/yaml.v2"
+)
+
+type settings map[string]interface{}
+
+// MonConf is exported to receive yaml configuration
+type MonConf struct {
+	Valid    bool
+	Period   cmd.ConfigDuration `yaml:"period"`
+	Timeout  int                `yaml:"timeout"`
+	Probe    string             `yaml:"type"`
+	Settings settings           `yaml:"settings"`
+}
+
+func (c MonConf) normalize() {
+	c.Probe = strings.ToLower(c.Probe)
+}
+
+func (c MonConf) unmashalProbeSettings() (p.Configurer, error) {
+	probeConf, err := p.GetProbeConf(c.Probe, c.Settings)
+	if err != nil {
+		return nil, err
+	}
+	s, _ := yaml.Marshal(c.Settings)
+	probeConf, err = probeConf.UnmarshalSettings(s)
+	if err != nil {
+		return nil, err
+	}
+	return probeConf, nil
+}
+
+// validate normalizes and validates the received monitor config
+func (c *MonConf) validate() error {
+	c.normalize()
+	probeConf, err := c.unmashalProbeSettings()
+	if err != nil {
+		return err
+	}
+	err = probeConf.Validate()
+	if err != nil {
+		return fmt.Errorf("failed to validate probe: %s with settings: %+v due to: %w", c.Probe, probeConf, err)
+	}
+	c.Valid = true
+	return nil
+}
+
+func (c MonConf) getProber() p.Prober {
+	probeConf, _ := c.unmashalProbeSettings()
+	return probeConf.AsProbe()
+}
+
+func (c MonConf) getName() string {
+	probeConf, _ := c.unmashalProbeSettings()
+	return probeConf.GetMonitorName()
+}

observer/monitor.go

@@ -0,0 +1,58 @@
+package observer
+
+import (
+	"time"
+
+	blog "github.com/letsencrypt/boulder/log"
+	p "github.com/letsencrypt/boulder/observer/probes"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// monitor contains the parsed, normalized, and validated configuration
+// describing a given oberver monitor
+type monitor struct {
+	name   string
+	period time.Duration
+	probe  string
+	prober p.Prober
+	logger blog.Logger
+	metric prometheus.Registerer
+}
+
+// start creates a ticker channel then spins off a prober goroutine for
+// each period specified in the monitor config and a timeout inferred
+// from that period. This is not perfect, it means that the effective
+// deadline for a prober goroutine will be TTL + time-to-schedule, but
+// it's close enough for our purposes
+func (m monitor) start() *time.Ticker {
+	ticker := time.NewTicker(m.period)
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
+				success, took := m.prober.Do(m.period)
+				statTotalObservations.WithLabelValues(m.probe, m.name).Add(1)
+				if !success {
+					statTotalErrors.WithLabelValues(m.probe, m.name).Add(1)
+					m.logger.Infof(
+						"%s monitor %q failed while taking: %s",
+						m.probe, m.name, took.String())
+				}
+				m.logger.Infof(
+					"%s monitor %q succeeded while taking: %s",
+					m.probe, m.name, took.String())
+			}
+		}
+	}()
+	return ticker
+}
+
+func (m monitor) New(c MonConf, log blog.Logger, prom prometheus.Registerer) *monitor {
+	m.name = c.getName()
+	m.period = c.Period.Duration
+	m.probe = c.Probe
+	m.prober = c.getProber()
+	m.logger = log
+	m.metric = prom
+	return &m
+}

observer/obs_conf.go

@@ -0,0 +1,76 @@
+package observer
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/letsencrypt/boulder/cmd"
+	blog "github.com/letsencrypt/boulder/log"
+	p "github.com/letsencrypt/boulder/observer/probes"
+)
+
+var (
+	errNewObsNoMons = errors.New("observer config is invalid, 0 monitors configured")
+	errNewObsEmpty  = errors.New("observer config is empty")
+)
+
+// ObsConf is exported to receive yaml configuration
+type ObsConf struct {
+	Syslog    cmd.SyslogConfig `yaml:"syslog"`
+	DebugAddr string           `yaml:"debugAddr"`
+	Modules   []p.Configurer   `yaml:"modules"`
+	MonConfs  []*MonConf       `yaml:"monitors"`
+}
+
+func (n *ObsConf) validateMonConfs() ([]error, bool) {
+	var validationErrs []error
+	for _, m := range n.MonConfs {
+		err := m.validate()
+		if err != nil {
+			validationErrs = append(validationErrs, err)
+		}
+	}
+
+	// all configured monitors are invalid, cannot continue
+	if len(n.MonConfs) == len(validationErrs) {
+		return validationErrs, false
+	}
+	return validationErrs, true
+}
+
+// Validate normalizes and validates the observer config as well as each
+// monitor config. If there is at least one valid monitor configured. If,
+// after validation, no valid monitor configs remain, Validate will return
+// an error indicating that observer cannot be started. In all instances
+// the the rationale for invalidating a monitor will logged to stderr
+func (n *ObsConf) Validate(log blog.Logger) error {
+	if n == nil {
+		return errNewObsEmpty
+	}
+
+	if len(n.MonConfs) == 0 {
+		return errNewObsNoMons
+	}
+
+	logErrs := func(errs []error, lenMons int) {
+		log.Errf("%d of %d monitors failed validation", len(errs), lenMons)
+		for _, err := range errs {
+			log.Errf("invalid monitor: %s", err)
+		}
+	}
+
+	errs, ok := n.validateMonConfs()
+
+	// if no valid mons remain, log validation errors, and return in
+	// error
+	if len(errs) != 0 && !ok {
+		logErrs(errs, len(n.MonConfs))
+		return fmt.Errorf("no valid mons, cannot continue")
+	}
+
+	// if at least 1 valid monitor remains, only log validation errors
+	if len(errs) != 0 && ok {
+		logErrs(errs, len(n.MonConfs))
+	}
+	return nil
+}