Support graceful shutdown and hot-deploying

.travis.yml

@@ -1,7 +1,6 @@
 language: go
 go:
-  - 1.7.4
-  - 1.8rc1
+  - 1.8
   - tip
 
 os:
@@ -15,4 +14,4 @@ install:
 script:
   - make bundle
   - make check
-
+

README.md

@@ -25,7 +25,7 @@ There are two way to install Gaurun; using a precompiled binary or install from
 
 To install a precompiled binary, download the appropriate zip package for your OS and architecture from [here](https://github.com/mercari/gaurun/releases). Once the zip is downloaded, unzip it and place the binary where you want to use (if you want to access it from the command-line, make sure to put it on `$PATH`).
 
-To compile from source, you need Go1.7.3 or later (including `$GOPATH` setup) and [glide](https://github.com/Masterminds/glide) for dependency management. After setup, then clone the source code by running the following command,
+To compile from source, you need Go1.8 or later (including `$GOPATH` setup) and [glide](https://github.com/Masterminds/glide) for dependency management. After setup, then clone the source code by running the following command,
 
 ```bash
 $ mkdir -p $GOPATH/src/github.com/mercari

cmd/gaurun/gaurun.go

@@ -1,12 +1,15 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"github.com/mercari/gaurun/gaurun"
 )
@@ -85,8 +88,8 @@ func main() {
 		}
 	}
 
-	sigChan := make(chan os.Signal, 1)
-	signal.Notify(sigChan, syscall.SIGHUP)
+	sigHUPChan := make(chan os.Signal, 1)
+	signal.Notify(sigHUPChan, syscall.SIGHUP)
 
 	sighupHandler := func() {
 		if err := accessLogReopener.Reopen(); err != nil {
@@ -97,16 +100,50 @@ func main() {
 		}
 	}
 
-	go signalHandler(sigChan, sighupHandler)
+	go signalHandler(sigHUPChan, sighupHandler)
 
 	if err := gaurun.InitHttpClient(); err != nil {
 		gaurun.LogSetupFatal(fmt.Errorf("failed to init http client"))
 	}
 	gaurun.InitStat()
 	gaurun.StartPushWorkers(gaurun.ConfGaurun.Core.WorkerNum, gaurun.ConfGaurun.Core.QueueNum)
 
-	gaurun.RegisterHTTPHandlers()
-	gaurun.RunHTTPServer()
+	mux := http.NewServeMux()
+	gaurun.RegisterHandlers(mux)
+
+	server := &http.Server{
+		Handler: mux,
+	}
+	go func() {
+		gaurun.LogError.Info("start server")
+		if err := gaurun.RunServer(server, &gaurun.ConfGaurun); err != nil {
+			gaurun.LogError.Info(fmt.Sprintf("failed to serve: %s", err))
+		}
+	}()
+
+	// Graceful shutdown (kicked by TERM ).
+	//
+	// First, it shutdowns server and stops accepting new requests.
+	// Then wait until all remaining queues in buffer are flushed.
+	sigTERMChan := make(chan os.Signal, 1)
+	signal.Notify(sigTERMChan, syscall.SIGTERM)
+
+	<-sigTERMChan
+	gaurun.LogError.Info("shutdown server")
+	timeout := time.Duration(conf.Core.ShutdownTimeout) * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	if err := server.Shutdown(ctx); err != nil {
+		gaurun.LogError.Error(fmt.Sprintf("failed to shutdown server: %v", err))
+	}
+
+	gaurun.LogError.Info("wait until queue is empty")
+	flushWaitInterval := 1 * time.Second
+	for len(gaurun.QueueNotification) > 0 {
+		time.Sleep(flushWaitInterval)
+	}
+
+	gaurun.LogError.Info("successfully shutdown")
 }
 
 func signalHandler(ch <-chan os.Signal, sighupFn func()) {

gaurun/conf.go

@@ -25,6 +25,7 @@ type SectionCore struct {
 	QueueNum        int64  `toml:"queues"`
 	NotificationMax int64  `toml:"notification_max"`
 	PusherMax       int64  `toml:"pusher_max"`
+	ShutdownTimeout int64  `toml:"shutdown_timeout"`
 }
 
 type SectionAndroid struct {
@@ -64,6 +65,7 @@ func BuildDefaultConf() ConfToml {
 	conf.Core.QueueNum = 8192
 	conf.Core.NotificationMax = 100
 	conf.Core.PusherMax = 0
+	conf.Core.ShutdownTimeout = 10
 	// Android
 	conf.Android.ApiKey = ""
 	conf.Android.Enabled = true

gaurun/server.go

@@ -1,47 +1,83 @@
 package gaurun
 
 import (
-	"log"
+	"fmt"
 	"net"
 	"net/http"
 	"os"
 	"strconv"
 	"strings"
 
 	statsGo "github.com/fukata/golang-stats-api-handler"
+	"github.com/lestrrat/go-server-starter/listener"
 )
 
-func RegisterHTTPHandlers() {
-	http.HandleFunc("/push", PushNotificationHandler)
-	http.HandleFunc("/stat/app", StatsHandler)
-	http.HandleFunc("/config/app", ConfigHandler)
+func RegisterHandlers(mux *http.ServeMux) {
+	mux.HandleFunc("/push", PushNotificationHandler)
+	mux.HandleFunc("/stat/app", StatsHandler)
+	mux.HandleFunc("/config/app", ConfigHandler)
+	mux.HandleFunc("/config/pushers", ConfigPushersHandler)
+
 	statsGo.PrettyPrintEnabled()
-	http.HandleFunc("/stat/go", statsGo.Handler)
-	http.HandleFunc("/config/pushers", ConfigPushersHandler)
+	mux.HandleFunc("/stat/go", statsGo.Handler)
 }
 
-func RunHTTPServer() {
-	// Listen TCP Port
-	if _, err := strconv.Atoi(ConfGaurun.Core.Port); err == nil {
-		http.ListenAndServe(":"+ConfGaurun.Core.Port, nil)
+// getListener returns a listener.
+func getListener(conf *ConfToml) (net.Listener, error) {
+	// By default, it starts to listen a listener provided
+	// by `go-server-starter`. If not, then check port defined
+	// in configuration file.
+	listeners, err := listener.ListenAll()
+	if err != nil && err != listener.ErrNoListeningTarget {
+		return nil, err
+	}
+
+	if len(listeners) > 0 {
+		return listeners[0], nil
+	}
+
+	// If port is empty, nothing to listen so returns error.
+	port := conf.Core.Port
+	if len(port) == 0 {
+		return nil, fmt.Errorf("no port to listen")
+	}
+
+	// Try to listen as TCP port, first
+	if _, err := strconv.Atoi(port); err == nil {
+		l, err := net.Listen("tcp", ":"+port)
+		if err != nil {
+			return nil, err
+		}
+		return l, nil
 	}
 
-	// Listen UNIX Socket
-	if strings.HasPrefix(ConfGaurun.Core.Port, "unix:/") {
-		sockPath := ConfGaurun.Core.Port[5:]
+	// Try to listen as UNIX socket.
+	if strings.HasPrefix(port, "unix:/") {
+		sockPath := port[5:]
+
 		fi, err := os.Lstat(sockPath)
 		if err == nil && (fi.Mode()&os.ModeSocket) == os.ModeSocket {
-			err := os.Remove(sockPath)
-			if err != nil {
-				log.Fatal("failed to remove " + sockPath)
+			if err := os.Remove(sockPath); err != nil {
+				return nil, fmt.Errorf("failed to remove socket path: %s", err)
 			}
 		}
+
 		l, err := net.Listen("unix", sockPath)
 		if err != nil {
-			log.Fatal("failed to listen: " + sockPath)
+			return nil, fmt.Errorf("failed to listen unix socket: %s", err)
 		}
-		http.Serve(l, nil)
+
+		return l, nil
+	}
+
+	return nil, fmt.Errorf("invalid port %s (it must be number or path start with 'unix:/')", port)
+}
+
+func RunServer(server *http.Server, conf *ConfToml) error {
+	l, err := getListener(conf)
+	if err != nil {
+		return err
 	}
 
-	log.Fatal("port parameter is invalid: " + ConfGaurun.Core.Port)
+	return server.Serve(l)
 }

glide.yaml

@@ -13,3 +13,4 @@ import:
 - package: github.com/stretchr/testify
 - package: github.com/uber-go/zap
 - package: github.com/client9/reopen
+- package: github.com/lestrrat/go-server-starter