Skip to content

Commit

Permalink
Squashing Fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
beautifulentropy committed Mar 7, 2021
1 parent b2688b1 commit 9b0c800
Show file tree
Hide file tree
Showing 22 changed files with 755 additions and 568 deletions.
88 changes: 35 additions & 53 deletions cmd/boulder-observer/README.md
Expand Up @@ -3,84 +3,66 @@ A modular config driven approach to black box monitoring with Prometheus


## Usage
### Starting the `observer` daemon
```shell
$ ./observer/plugins/build.sh && go run ./cmd/boulder-observer/main.go -config test/config-next/observer.yaml
Building plugins:
⚙️ observer/plugins/dns.so
✅dns.so
⚙️ observer/plugins/http.so
✅http.so
OK
I191418 main ksKu7w4 Versions: main=(Unspecified Unspecified) Golang=(go1.15.7) BuildHost=(Unspecified)
I191418 main o9me0QI Initializing boulder-observer daemon from config: test/config-next/observer.yaml
I191420 main wv7tug0 HTTP monitor "https://letsencrypt.org-200" succeeded while taking:=120.900665ms
I191422 main ss-hzQ8 HTTP monitor "https://letsencrypt.org-200" succeeded while taking:=23.051998ms
I191424 main -fD46gg HTTP monitor "https://letsencrypt.org-200" succeeded while taking:=23.419121ms
I191426 main urmy8AM HTTP monitor "https://letsencrypt.org-200" succeeded while taking:=23.875478ms
I191428 main qaGe0Qc DNS monitor "udp-8.8.8.8:53-google.com-A" succeeded while taking:=5.088261ms
I191428 main i677rw0 DNS monitor "tcp-8.8.8.8:53-google.com-A" succeeded while taking:=5.156114ms
I191428 main ooyq_Qo DNS monitor "udp-owen.ns.cloudflare.com:53-letsencrypt.org-A" succeeded while taking:=15.858563ms
```

### Help
```shell
$ go run ./cmd/boulder-observer/main.go -help
main:
-config string
Path to boulder-observer configuration file (default "config.yaml")
Path to boulder-observer configuration file (default "config.yml")
```

### Starting the boulder-observer daemon
```shell
$ go run ./cmd/boulder-observer/main.go -config test/config-next/observer.yml
I181830 main ksKu7w4 Versions: main=(Unspecified Unspecified) Golang=(go1.15.7) BuildHost=(Unspecified)
I181830 main q_D84gk Initializing boulder-observer daemon from config: test/config-next/observer.yml
I181832 main 34Ccpgs status=[success] probe=[HTTP] duration=[128.386914ms] monitor=[http://letsencrypt.org-200]
I181832 main 0buu-wI status=[success] probe=[HTTP] duration=[148.592537ms] monitor=[https://letsencrypt.org-200]
I181834 main 1bL9-A0 status=[success] probe=[HTTP] duration=[24.501939ms] monitor=[https://letsencrypt.org-200]
I181834 main z-m2mAc status=[success] probe=[HTTP] duration=[48.078282ms] monitor=[http://letsencrypt.org-200]
I181835 main 1I-QuwM status=[success] probe=[DNS] duration=[5.318966ms] monitor=[udp-1.1.1.1:53-google.com-A]
I181835 main puyqZgA status=[success] probe=[DNS] duration=[8.766023ms] monitor=[tcp-1.1.1.1:53-google.com-A]
I181835 main 5cWimwc status=[success] probe=[DNS] duration=[15.923062ms] monitor=[udp-owen.ns.cloudflare.com:53-letsencrypt.org-A]
I181835 main jsjXxQ0 status=[success] probe=[DNS] duration=[19.95004ms] monitor=[tcp-owen.ns.cloudflare.com:53-letsencrypt.org-A]
```

## Configuration

### Observer
```yaml
debugAddr: 8040
syslog:
debugAddr: :8040
syslog:
stdoutlevel: 6
sysloglevel: 6
timeout: 5
monitors: []
monitors:
-
...
```

### Monitors

#### Using the DNS plugin
#### Configuring a DNS monitor
```yaml
monitors:
-
enabled: true
period: 1
plugin:
name: DNS
path: "./cmd/boulder-observer/observer/plugins/dns.so"
period: 10s
type: DNS
settings:
qproto: udp
qrecurse: false
qname: letsencrypt.org
qtype: A
qserver: "owen.ns.cloudflare.com:53"
protocol: tcp
server: 8.8.8.8:53
recurse: true
query_name: google.com
query_type: A
```

#### Using the HTTP plugin
#### Configuring an HTTP monitor
```yaml
monitors:
-
enabled: true
period: 1
plugin:
name: HTTP
path: "./cmd/boulder-observer/observer/plugins/http.so"
period: 2s
type: HTTP
settings:
url: https://letsencrypt.org
rcode: 200
```
### Plugins
**Building plugins**
```shell
$ ./observer/plugins/build.sh
Building plugins:
⚙️ observer/plugins/dns.so
✅dns.so
⚙️ observer/plugins/http.so
✅http.so
OK
```
15 changes: 10 additions & 5 deletions cmd/boulder-observer/main.go
@@ -1,6 +1,7 @@
package main

import (
"errors"
"flag"
"io/ioutil"

Expand All @@ -21,20 +22,24 @@ func main() {
var config observer.ObsConf
err = yaml.Unmarshal(configYAML, &config)
if err != nil {
cmd.FailOnError(err, "failed to parse YAML config")
cmd.FailOnError(err, "failed to parse yaml config")
}

// validate config
err = config.Validate()
if err != nil {
cmd.FailOnError(err, "YAML config failed validation")
if config.DebugAddr == "" {
cmd.FailOnError(errors.New(""), "debugaddr is not defined")
}

// start monitoring and logging
prom, logger := cmd.StatsAndLogging(config.Syslog, config.DebugAddr)
defer logger.AuditPanic()
logger.Info(cmd.VersionString())

// validate config
err = config.Validate(logger)
if err != nil {
cmd.FailOnError(err, "config failed validation")
}

// start daemon
logger.Infof("Initializing boulder-observer daemon from config: %s", *configPath)
logger.Debugf("Using config: %+v", config)
Expand Down
61 changes: 40 additions & 21 deletions observer/mon_conf.go
@@ -1,44 +1,63 @@
package observer

import (
"errors"
"fmt"
"strings"

"github.com/letsencrypt/boulder/observer/plugins"
"github.com/letsencrypt/boulder/cmd"
p "github.com/letsencrypt/boulder/observer/probes"
"gopkg.in/yaml.v2"
)

var (
errNewMonEmpty = errors.New("monitor config is empty")
errNewMonInvalid = errors.New("monitor config is invalid")
)
type settings map[string]interface{}

// MonConf is exported to receive the supplied monitor config
// MonConf is exported to receive yaml configuration
type MonConf struct {
Enabled bool `yaml:"enabled"`
Period int `yaml:"period"`
Timeout int `yaml:"timeout"`
Plugin plugins.Info `yaml:"plugin"`
Settings map[string]interface{} `yaml:"settings"`
Valid bool
Period cmd.ConfigDuration `yaml:"period"`
Timeout int `yaml:"timeout"`
Probe string `yaml:"type"`
Settings settings `yaml:"settings"`
}

func (c MonConf) normalize() {
c.Plugin.Name = strings.ToLower(c.Plugin.Name)
c.Plugin.Path = strings.ToLower(c.Plugin.Path)
c.Probe = strings.ToLower(c.Probe)
}

func (c MonConf) unmashalProbeSettings() (p.Configurer, error) {
probeConf, err := p.GetProbeConf(c.Probe, c.Settings)
if err != nil {
return nil, err
}
s, _ := yaml.Marshal(c.Settings)
probeConf, err = probeConf.UnmarshalSettings(s)
if err != nil {
return nil, err
}
return probeConf, nil
}

// validate normalizes and validates the received monitor config
func (c MonConf) validate() error {
func (c *MonConf) validate() error {
c.normalize()
pluginConf, err := plugins.GetPluginConf(c.Settings, c.Plugin.Path, c.Plugin.Name)
probeConf, err := c.unmashalProbeSettings()
if err != nil {
if err != nil {
return fmt.Errorf("failed to get plugin: %w", err)
}
return err
}
err = pluginConf.Validate()
err = probeConf.Validate()
if err != nil {
return fmt.Errorf("failed to validate plugin settings: %w", err)
return fmt.Errorf("failed to validate probe: %s with settings: %+v due to: %w", c.Probe, probeConf, err)
}
c.Valid = true
return nil
}

func (c MonConf) getProber() p.Prober {
probeConf, _ := c.unmashalProbeSettings()
return probeConf.AsProbe()
}

func (c MonConf) getName() string {
probeConf, _ := c.unmashalProbeSettings()
return probeConf.GetMonitorName()
}
55 changes: 30 additions & 25 deletions observer/monitor.go
Expand Up @@ -4,50 +4,55 @@ import (
"time"

blog "github.com/letsencrypt/boulder/log"
"github.com/letsencrypt/boulder/observer/plugins"
p "github.com/letsencrypt/boulder/observer/probes"
"github.com/prometheus/client_golang/prometheus"
)

// monitor contains the parsed, normalized, and validated configuration
// describing a given oberver monitor
type monitor struct {
name string
period time.Duration
timeout time.Duration
pluginIs string
probe plugins.Plugin
logger blog.Logger
metric prometheus.Registerer
name string
valid bool
period time.Duration
prober p.Prober
logger blog.Logger
metric prometheus.Registerer
}

// start creates a ticker channel then spins off a prober goroutine for
// each period specified in the monitor config and a timeout inferred
// from that period. This is not perfect, it means that the effective
// deadline for a prober goroutine will be TTL + time-to-schedule, but
// it's close enough for our purposes
func (m monitor) start() *time.Ticker {
ticker := time.NewTicker(m.period)
logInfo := func(status string, took string) {
m.logger.Infof(
"status=[%s] type=[%s] duration=[%s] monitor=[%s]",
status, m.prober.Name(), took, m.name)
}
go func() {
for {
select {
case tick := <-ticker.C:
success, took := m.probe.Do(tick, m.timeout)
statTotalObservations.WithLabelValues(m.pluginIs, m.name).Add(1)
case <-ticker.C:
success, took := m.prober.Do(m.period)
if !success {
statTotalErrors.WithLabelValues(m.pluginIs, m.name).Add(1)
m.logger.Infof("%s monitor %q failed while taking:=%s", m.pluginIs, m.name, took.String())
return
statObservations.WithLabelValues(m.name, m.prober.Name(), "false").Add(1)
logInfo("failure", took.String())
}
m.logger.Infof("%s monitor %q succeeded while taking:=%s", m.pluginIs, m.name, took.String())
statObservations.WithLabelValues(m.name, m.prober.Name(), "true").Add(1)
logInfo("success", took.String())
}
}
}()
return ticker
}

func (m monitor) New(c MonConf, log blog.Logger, prom prometheus.Registerer, t int) *monitor {
if c.Timeout == 0 {
c.Timeout = t
}
plugin, _ := plugins.GetPluginConf(c.Settings, c.Plugin.Path, c.Plugin.Name)
m.name = plugin.GetMonitorName()
m.period = time.Duration(c.Period * 1000000000)
m.timeout = time.Duration(c.Timeout * 1000000000)
m.pluginIs = c.Plugin.Name
m.probe = plugin.AsProbe()
func (m monitor) New(c MonConf, log blog.Logger, prom prometheus.Registerer) *monitor {
m.name = c.getName()
m.valid = c.Valid
m.period = c.Period.Duration
m.prober = c.getProber()
m.logger = log
m.metric = prom
return &m
Expand Down

0 comments on commit 9b0c800

Please sign in to comment.