Handle SIGHUP, SIGTERM, SIGINT signals

Dockerfile

@@ -1,10 +1,12 @@
-FROM centurylink/ca-certs
-COPY hound /
-COPY index.html /
-COPY alert.html /
+FROM golang:1.8
+WORKDIR /go/src/app
+COPY . .
+RUN go-wrapper install
+
 ENV HOUND_HTTP_PORT=9998
-ENV HOUND_TEMPLATE_FILE=/index.html
+ENV HOUND_TEMPLATE_FILE=/go/src/app/index.html
 ENV HOUND_SMTP_SERVER=postfix
 ENV HOUND_SMTP_PORT=25
 EXPOSE 9998
-CMD ["/hound", "-config=/config.json"]
+CMD ["go-wrapper", "run", "-config", "/config.json"]
+

Makefile

@@ -20,7 +20,7 @@ coverage.html: coverage.out
 	go tool cover -html=coverage.out -o coverage.html
 
 build:
-	docker run --rm -v $(ROOT_DIR):/src -v /var/run/docker.sock:/var/run/docker.sock centurylink/golang-builder ccnmtl/hound
+	docker build -t ccnmtl/hound .
 
 push: build
 	docker push ccnmtl/hound

alertscollection.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"net"
 	"time"
@@ -123,11 +124,15 @@ func intmin(a, b int) int {
 	return b
 }
 
-func (ac *alertsCollection) Run() {
+func (ac *alertsCollection) Run(ctx context.Context) {
 	for {
-		ac.processAll()
-		ac.DisplayAll()
-		time.Sleep(time.Duration(checkInterval) * time.Minute)
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(time.Duration(checkInterval) * time.Minute):
+			ac.processAll()
+			ac.DisplayAll()
+		}
 	}
 }
 

hound.go

@@ -1,14 +1,18 @@
 package main // import "github.com/ccnmtl/hound"
 
 import (
+	"context"
 	"encoding/json"
 	"expvar"
 	"flag"
 	"fmt"
 	"html/template"
 	"io/ioutil"
 	"net/http"
+	"os"
+	"os/signal"
 	"strings"
+	"syscall"
 	"time"
 
 	log "github.com/Sirupsen/logrus"
@@ -71,19 +75,8 @@ func main() {
 	flag.StringVar(&configfile, "config", "./config.json", "JSON config file")
 	flag.Parse()
 
-	file, err := ioutil.ReadFile(configfile)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	f := configData{}
-	err = json.Unmarshal(file, &f)
-	if err != nil {
-		log.Fatal(err)
-	}
-
 	var c config
-	err = envconfig.Process("hound", &c)
+	err := envconfig.Process("hound", &c)
 	if err != nil {
 		log.Fatal(err.Error())
 	}
@@ -140,20 +133,65 @@ func main() {
 		}
 	}()
 
-	// initialize all the alerts
-	ac := newAlertsCollection(smtpEmailer{})
-	for _, a := range f.Alerts {
-		emailTo := a.EmailTo
-		if emailTo == "" {
-			emailTo = c.EmailTo
+	f := loadConfig(configfile)
+
+	bgcontext := context.Background()
+	s, alertscancel := startServices(bgcontext, f, c)
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP)
+
+	for {
+		// wait for a signal
+		signal := <-sigs
+
+		// shut everything down nicely
+		// then gracefully shut everything down.
+		alertscancel()
+
+		// giving the http server 1 second to close its connections
+		ctx, cancel := context.WithTimeout(bgcontext, 1*time.Second)
+
+		if err = s.Shutdown(ctx); err != nil {
+			log.WithFields(
+				log.Fields{
+					"error": fmt.Sprintf("%v", err),
+				}).Fatal("graceful shutdown failed")
+		} else {
+			log.Info("successful graceful shutdown")
+		}
+		cancel()
+		if signal == syscall.SIGHUP {
+			// reload config and restart services
+			f = loadConfig(configfile)
+			log.Info("re-read config")
+			s, alertscancel = startServices(bgcontext, f, c)
+			log.Info("restarted services")
+		} else {
+			// SIGINT or SIGTERM. We're done.
+			log.Info("exiting")
+			return
 		}
-		ac.addAlert(newAlert(a.Name, a.Metric, a.Type, a.Threshold, a.Direction, httpFetcher{}, emailTo, a.RunBookLink))
 	}
+}
 
-	// kick it off in the background
-	go ac.Run()
+func loadConfig(configfile string) configData {
+	file, err := ioutil.ReadFile(configfile)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	f := configData{}
+	err = json.Unmarshal(file, &f)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return f
+}
 
-	http.HandleFunc("/",
+func registerHandlers(ac *alertsCollection, c config) *http.ServeMux {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/",
 		func(w http.ResponseWriter, r *http.Request) {
 			pr := ac.MakePageResponse()
 
@@ -166,7 +204,7 @@ func main() {
 			t.Execute(w, pr)
 		})
 
-	http.HandleFunc("/alert/",
+	mux.HandleFunc("/alert/",
 		func(w http.ResponseWriter, r *http.Request) {
 			stringIdx := strings.Split(r.URL.String(), "/")[2]
 			pr := ac.MakeindivPageResponse(stringIdx)
@@ -182,10 +220,41 @@ func main() {
 			}
 			t.Execute(w, pr)
 		})
+	return mux
+}
+
+func startAlertsCollection(ctx context.Context, f configData, c config) (*alertsCollection, context.CancelFunc) {
+	// initialize all the alerts
+	ac := newAlertsCollection(smtpEmailer{})
+	for _, a := range f.Alerts {
+		emailTo := a.EmailTo
+		if emailTo == "" {
+			emailTo = c.EmailTo
+		}
+		ac.addAlert(newAlert(a.Name, a.Metric, a.Type, a.Threshold, a.Direction, httpFetcher{}, emailTo, a.RunBookLink))
+	}
+	alertsctx, alertscancel := context.WithCancel(ctx)
+
+	// kick off alerts in the background
+	go ac.Run(alertsctx)
+
+	return ac, alertscancel
+}
+
+func startServices(ctx context.Context, f configData, c config) (*http.Server, context.CancelFunc) {
+	ac, alertscancel := startAlertsCollection(ctx, f, c)
+	mux := registerHandlers(ac, c)
 	s := &http.Server{
 		Addr:         ":" + c.HTTPPort,
+		Handler:      mux,
 		ReadTimeout:  time.Duration(c.ReadTimeout) * time.Second,
 		WriteTimeout: time.Duration(c.WriteTimeout) * time.Second,
 	}
-	log.Fatal(s.ListenAndServe())
+
+	// and the http server in the background
+	go func() {
+		s.ListenAndServe()
+	}()
+
+	return s, alertscancel
 }