Systemd's Watchdog

admin/beehive.service

@@ -8,6 +8,9 @@ User=beehive
 Group=users
 RestartSec=5
 Restart=on-failure
+# Enable the software watchdog.
+# See https://github.com/muesli/beehive/tree/master/docs/watchdog.md
+WatchdogSec=30s
 
 [Install]
 WantedBy=multi-user.target

docs/watchdog.md

@@ -0,0 +1,32 @@
+# Systemd's Watchdog integration
+
+Integrates Beehive with the Systemd's watchdog.
+
+See http://0pointer.de/blog/projects/watchdog.html
+
+## Configuration
+
+As a system service:
+
+* Add the service unit to `/etc/systemd/system/beehive.conf`
+
+Sample service unit:
+
+```
+[Unit]
+Description=Beehive with Systemd's watchdog
+[Service]
+Type=simple
+ExecStart=/usr/bin/beehive --config /path/to/beehive.conf
+Restart=on-failure
+WatchdogSec=30s
+
+[Install]
+WantedBy=multi-user.target
+```
+*Note: change `/path/to/beehive.conf` to a real path pointing to Beehive's config*
+
+* Enable the new service: `systemctl enable beehive`
+* Start the service: `systemctl start beehive`
+
+Beehive will automatically detect it's running under Systemd and notify Systemd's watchdog every WatchdogSec/3 seconds (10 seconds in this particular case). Note the notification interval is hardcoded to WatchdogSec/3.

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/bwmarrin/discordgo v0.19.0
 	github.com/carlosdp/twiliogo v0.0.0-20161027183705-b26045ebb9d1
 	github.com/cloudflare/cloudflare-go v0.10.6
+	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
 	github.com/deckarep/gosx-notifier v0.0.0-20180201035817-e127226297fb
 	github.com/dustin/go-jsonpointer v0.0.0-20160814072949-ba0abeacc3dc // indirect
 	github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad // indirect

go.sum

@@ -32,6 +32,8 @@ github.com/carlosdp/twiliogo v0.0.0-20161027183705-b26045ebb9d1 h1:hXakhQtPnXH83
 github.com/carlosdp/twiliogo v0.0.0-20161027183705-b26045ebb9d1/go.mod h1:pAxCBpjl/0JxYZlWGP/Dyi8f/LQSCQD2WAsG/iNzqQ8=
 github.com/cloudflare/cloudflare-go v0.10.6 h1:mbv0IrcrrLlPLxAzCdW6aQ/CPlqhyXrXTjviU0Tb+34=
 github.com/cloudflare/cloudflare-go v0.10.6/go.mod h1:dcRl7AXBH5Bf7QFTBVc3TRzwvotSeO4AlnMhuxORAX8=
+github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c=
+github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

watchdog_linux.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/coreos/go-systemd/daemon"
+	"github.com/muesli/beehive/api"
+	log "github.com/sirupsen/logrus"
+)
+
+// Notifies Systemd's watchdog every WatchdogSec/3 seconds when running
+// under Systemd and the watchdog feature has been enabled in
+// the service unit.
+//
+// This will no-op when not running under Systemd.
+//
+// See http://0pointer.de/blog/projects/watchdog.html
+// and https://www.freedesktop.org/software/systemd/man/systemd.service.html
+//
+func init() {
+	// returns the configured WatchdogSec in the service unit as time.Duration
+	interval, err := daemon.SdWatchdogEnabled(false)
+	if err != nil || interval == 0 {
+		log.Printf("Systemd watchdog not enabled")
+		return
+	}
+
+	// We want to notify the watchdog every WatchdogSec/3, that is, if WatchdogSec is
+	// set to 30 seconds, we'll send a notification to systemd every 10 seconds.
+	runEvery := interval / 3
+	log.Printf("Systemd watchdog notifications every %.2f seconds", runEvery.Seconds())
+
+	go func() {
+		for {
+			select {
+			case <-time.After(runEvery):
+				resp, err := http.Get(api.CanonicalURL().String())
+				if err == nil {
+					resp.Body.Close()
+					log.Debugf("Systemd watchdog notify")
+					daemon.SdNotify(false, daemon.SdNotifyWatchdog)
+				}
+			}
+		}
+	}()
+}