Merge pull request QubitProducts#33 from sttts/reload-race-condition

Synchronize haproxy reloads
xperimental · Oct 2, 2014 · df8d324 · df8d324
2 parents 7ab7ff4 + a6e29c8
commit df8d324
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 3 deletions.
diff --git a/README.md b/README.md
@@ -87,7 +87,7 @@ This section tries to explain usage in code comment style:
   "HAProxy": {
     "TemplatePath": "/var/bamboo/haproxy_template.cfg",
     "OutputPath": "/etc/haproxy/haproxy.cfg",
-    "ReloadCommand": "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -sf $(cat /var/run/haproxy.pid)"
+    "ReloadCommand": "read PIDS < /var/run/haproxy.pid; haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -sf $PIDS && while ps -p $PIDS; do sleep 0.2; done"
   },
 
   // Enable or disable StatsD event tracking

diff --git a/config/development.json b/config/development.json
@@ -15,7 +15,7 @@
   "HAProxy": {
     "TemplatePath": "config/haproxy_template.cfg",
     "OutputPath": "/etc/haproxy/haproxy.cfg",
-    "ReloadCommand": "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -sf $(cat /var/run/haproxy.pid)"
+    "ReloadCommand": "read PIDS < /var/run/haproxy.pid; haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -sf $PIDS && while ps -p $PIDS; do sleep 0.2; done"
   },
 
   "StatsD": {

diff --git a/config/production.example.json b/config/production.example.json
@@ -15,7 +15,7 @@
   "HAProxy": {
     "TemplatePath": "/var/bamboo/haproxy_template.cfg",
     "OutputPath": "/etc/haproxy/haproxy.cfg",
-    "ReloadCommand": "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -sf $(cat /var/run/haproxy.pid)"
+    "ReloadCommand": "read PIDS < /var/run/haproxy.pid; haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -sf $PIDS && while ps -p $PIDS; do sleep 0.2; done"
   },
 
   "StatsD": {

diff --git a/main/bamboo/bamboo.go b/main/bamboo/bamboo.go
@@ -6,6 +6,8 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"os/signal"
+	"syscall"
 	"time"
 
 	lumberjack "github.com/natefinch/lumberjack"
@@ -41,6 +43,19 @@ func main() {
 
 	eventBus := event_bus.New()
 
+	// Wait for died children to avoid zombies
+	signalChannel := make(chan os.Signal, 2)
+	signal.Notify(signalChannel, os.Interrupt, syscall.SIGCHLD)
+	go func() {
+		for {
+			sig := <-signalChannel
+			if sig == syscall.SIGCHLD {
+				r := syscall.Rusage {}
+				syscall.Wait4( -1,  nil, 0, &r)
+			}
+		}
+	}()
+
 	// Create StatsD client
 	conf.StatsD.CreateClient()
 

diff --git a/services/event_bus/event_handler.go b/services/event_bus/event_handler.go
@@ -43,6 +43,8 @@ func (h *Handlers) ServiceEventHandler(event ServiceEvent) {
 	h.Conf.StatsD.Increment(1.0, "reload.domain", 1)
 }
 
+var execSem = make(chan int, 1)
+
 func handleHAPUpdate(conf *configuration.Configuration, conn *zk.Conn) bool {
 	currentContent, _ := ioutil.ReadFile(conf.HAProxy.OutputPath)
 
@@ -59,7 +61,10 @@ func handleHAPUpdate(conf *configuration.Configuration, conn *zk.Conn) bool {
 		err := ioutil.WriteFile(conf.HAProxy.OutputPath, []byte(newContent), 0666)
 		if err != nil { log.Fatalf("Failed to write template on path: %s", err) }
 
+		execSem <- 1
 		execCommand(conf.HAProxy.ReloadCommand)
+		<-execSem
+
 		log.Println("HAProxy: Configuration updated")
 		return true
 	} else {