diff --git a/Dockerfile b/Dockerfile index 4e6285a..7eaa488 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,12 @@ -FROM centurylink/ca-certs -COPY hound / -COPY index.html / -COPY alert.html / +FROM golang:1.8 +WORKDIR /go/src/app +COPY . . +RUN go-wrapper install + ENV HOUND_HTTP_PORT=9998 -ENV HOUND_TEMPLATE_FILE=/index.html +ENV HOUND_TEMPLATE_FILE=/go/src/app/index.html ENV HOUND_SMTP_SERVER=postfix ENV HOUND_SMTP_PORT=25 EXPOSE 9998 -CMD ["/hound", "-config=/config.json"] +CMD ["go-wrapper", "run", "-config", "/config.json"] + diff --git a/Makefile b/Makefile index cb84a32..381e219 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ coverage.html: coverage.out go tool cover -html=coverage.out -o coverage.html build: - docker run --rm -v $(ROOT_DIR):/src -v /var/run/docker.sock:/var/run/docker.sock centurylink/golang-builder ccnmtl/hound + docker build -t ccnmtl/hound . push: build docker push ccnmtl/hound diff --git a/alertscollection.go b/alertscollection.go index f5598a4..0022557 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "fmt" "net" "time" @@ -123,11 +124,15 @@ func intmin(a, b int) int { return b } -func (ac *alertsCollection) Run() { +func (ac *alertsCollection) Run(ctx context.Context) { for { - ac.processAll() - ac.DisplayAll() - time.Sleep(time.Duration(checkInterval) * time.Minute) + select { + case <-ctx.Done(): + return + case <-time.After(time.Duration(checkInterval) * time.Minute): + ac.processAll() + ac.DisplayAll() + } } } diff --git a/hound.go b/hound.go index f2299b6..6bc5764 100644 --- a/hound.go +++ b/hound.go @@ -1,6 +1,7 @@ package main // import "github.com/ccnmtl/hound" import ( + "context" "encoding/json" "expvar" "flag" @@ -8,7 +9,10 @@ import ( "html/template" "io/ioutil" "net/http" + "os" + "os/signal" "strings" + "syscall" "time" log "github.com/Sirupsen/logrus" @@ -71,19 +75,8 @@ func main() { flag.StringVar(&configfile, "config", "./config.json", "JSON config file") flag.Parse() - file, err := ioutil.ReadFile(configfile) - if err != nil { - log.Fatal(err) - } - - f := configData{} - err = json.Unmarshal(file, &f) - if err != nil { - log.Fatal(err) - } - var c config - err = envconfig.Process("hound", &c) + err := envconfig.Process("hound", &c) if err != nil { log.Fatal(err.Error()) } @@ -140,20 +133,65 @@ func main() { } }() - // initialize all the alerts - ac := newAlertsCollection(smtpEmailer{}) - for _, a := range f.Alerts { - emailTo := a.EmailTo - if emailTo == "" { - emailTo = c.EmailTo + f := loadConfig(configfile) + + bgcontext := context.Background() + s, alertscancel := startServices(bgcontext, f, c) + + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP) + + for { + // wait for a signal + signal := <-sigs + + // shut everything down nicely + // then gracefully shut everything down. + alertscancel() + + // giving the http server 1 second to close its connections + ctx, cancel := context.WithTimeout(bgcontext, 1*time.Second) + + if err = s.Shutdown(ctx); err != nil { + log.WithFields( + log.Fields{ + "error": fmt.Sprintf("%v", err), + }).Fatal("graceful shutdown failed") + } else { + log.Info("successful graceful shutdown") + } + cancel() + if signal == syscall.SIGHUP { + // reload config and restart services + f = loadConfig(configfile) + log.Info("re-read config") + s, alertscancel = startServices(bgcontext, f, c) + log.Info("restarted services") + } else { + // SIGINT or SIGTERM. We're done. + log.Info("exiting") + return } - ac.addAlert(newAlert(a.Name, a.Metric, a.Type, a.Threshold, a.Direction, httpFetcher{}, emailTo, a.RunBookLink)) } +} - // kick it off in the background - go ac.Run() +func loadConfig(configfile string) configData { + file, err := ioutil.ReadFile(configfile) + if err != nil { + log.Fatal(err) + } + + f := configData{} + err = json.Unmarshal(file, &f) + if err != nil { + log.Fatal(err) + } + return f +} - http.HandleFunc("/", +func registerHandlers(ac *alertsCollection, c config) *http.ServeMux { + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { pr := ac.MakePageResponse() @@ -166,7 +204,7 @@ func main() { t.Execute(w, pr) }) - http.HandleFunc("/alert/", + mux.HandleFunc("/alert/", func(w http.ResponseWriter, r *http.Request) { stringIdx := strings.Split(r.URL.String(), "/")[2] pr := ac.MakeindivPageResponse(stringIdx) @@ -182,10 +220,41 @@ func main() { } t.Execute(w, pr) }) + return mux +} + +func startAlertsCollection(ctx context.Context, f configData, c config) (*alertsCollection, context.CancelFunc) { + // initialize all the alerts + ac := newAlertsCollection(smtpEmailer{}) + for _, a := range f.Alerts { + emailTo := a.EmailTo + if emailTo == "" { + emailTo = c.EmailTo + } + ac.addAlert(newAlert(a.Name, a.Metric, a.Type, a.Threshold, a.Direction, httpFetcher{}, emailTo, a.RunBookLink)) + } + alertsctx, alertscancel := context.WithCancel(ctx) + + // kick off alerts in the background + go ac.Run(alertsctx) + + return ac, alertscancel +} + +func startServices(ctx context.Context, f configData, c config) (*http.Server, context.CancelFunc) { + ac, alertscancel := startAlertsCollection(ctx, f, c) + mux := registerHandlers(ac, c) s := &http.Server{ Addr: ":" + c.HTTPPort, + Handler: mux, ReadTimeout: time.Duration(c.ReadTimeout) * time.Second, WriteTimeout: time.Duration(c.WriteTimeout) * time.Second, } - log.Fatal(s.ListenAndServe()) + + // and the http server in the background + go func() { + s.ListenAndServe() + }() + + return s, alertscancel }