Merge pull request #70 from hashicorp/sethvargo/splay

Add splay option

README.md

@@ -46,7 +46,8 @@ Usage
 | `retry`           | The amount of time to wait if Consul returns an error when communicating with the API. The default value is 5 seconds.
 | `prefix`          | A prefix to watch in Consul. This may be specified multiple times.
 | `secret`          | A secret to watch in Vault. This may be specified multiple times.
-| `sanitize`        | Replace invalid characters in keys to underscores .
+| `sanitize`        | Replace invalid characters in keys to underscores.
+| `splay`           | The maximum time to wait before restarting the program, from which a random value is chosen.
 | `upcase`          | Convert all environment variable keys to uppercase.
 | `config`          | The path to a configuration file or directory of configuration files on disk, relative to the current working directory. Values specified on the CLI take precedence over values specified in the configuration file. There is no default value.
 | `log-level`       | The log level for output. This applies to the stdout/stderr logging as well as syslog logging (if enabled). Valid values are "debug", "info", "warn", and "err". The default value is "warn".
@@ -120,6 +121,7 @@ max_stale = "10m"
 timeout   = "5s"
 retry     = "10s"
 sanitize  = true
+splay     = "5s"
 
 kill_signal = "SIGHUP"
 

cli.go

@@ -294,6 +294,12 @@ func (cli *CLI) parseFlags(args []string) (*Config, []string, bool, bool, error)
 		return nil
 	}), "sanitize", "")
 
+	flags.Var((funcDurationVar)(func(d time.Duration) error {
+		config.Splay = d
+		config.set("splay")
+		return nil
+	}), "splay", "")
+
 	flags.Var((funcBoolVar)(func(b bool) error {
 		config.Upcase = b
 		config.set("upcase")
@@ -408,6 +414,8 @@ Options:
                            result taking precedence, including any values
                            specified with -prefix
   -sanitize                Replace invalid characters in keys to underscores
+  -splay                   The maximum time to wait before restarting the
+                           program, from which a random value is chosen
   -upcase                  Convert all environment variable keys to uppercase
   -kill-signal             The signal to send to kill the process
 

cli_test.go

@@ -364,6 +364,24 @@ func TestParseFlags_sanitize(t *testing.T) {
 	}
 }
 
+func TestParseFlags_splay(t *testing.T) {
+	cli := NewCLI(ioutil.Discard, ioutil.Discard)
+	config, _, _, _, err := cli.parseFlags([]string{
+		"-splay", "10s",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := 10 * time.Second
+	if config.Splay != expected {
+		t.Errorf("expected %v to be %v", config.Splay, expected)
+	}
+	if !config.WasSet("splay") {
+		t.Errorf("expected splay to be set")
+	}
+}
+
 func TestParseFlags_upcase(t *testing.T) {
 	cli := NewCLI(ioutil.Discard, ioutil.Discard)
 	config, _, _, _, err := cli.parseFlags([]string{

config.go

@@ -64,6 +64,11 @@ type Config struct {
 	// Sanitize converts any "bad" characters in key values to underscores
 	Sanitize bool `json:"sanitize" mapstructure:"sanitize"`
 
+	// Splay is the maximum time in seconds to wait before restarting the program,
+	// from which a random value is chosen. This is designed to prevent the
+	// "thundering herd" problem.
+	Splay time.Duration `json:"splay" mapstructure:"splay"`
+
 	// Upcase converts environment variables to uppercase
 	Upcase bool `json:"upcase" mapstructure:"upcase"`
 
@@ -220,6 +225,10 @@ func (c *Config) Merge(config *Config) {
 		c.Sanitize = config.Sanitize
 	}
 
+	if config.WasSet("splay") {
+		c.Splay = config.Splay
+	}
+
 	if config.WasSet("upcase") {
 		c.Upcase = config.Upcase
 	}

config_test.go

@@ -282,6 +282,7 @@ func TestParseConfig_correctValues(t *testing.T) {
 		wait = "5s:10s"
 		retry = "10s"
 		log_level = "warn"
+		splay = "10s"
 
 		// Deprecated
 		prefixes = ["old/syntax", "deprecated/soon"]
@@ -339,6 +340,7 @@ func TestParseConfig_correctValues(t *testing.T) {
 		MaxStale: time.Second * 5,
 		Upcase:   false,
 		Sanitize: false,
+		Splay:    10 * time.Second,
 		Timeout:  5 * time.Second,
 		Auth: &AuthConfig{
 			Enabled:  true,

runner.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"math/rand"
 	"net/http"
 	"os"
 	"os/exec"
@@ -535,6 +536,21 @@ func (r *Runner) killProcess() {
 	// Kill the process
 	exited := false
 
+	// If a splay value was given, sleep for a random amount of time up to the
+	// splay.
+	if r.config.Splay > 0 {
+		nanoseconds := r.config.Splay.Nanoseconds()
+		offset := rand.Int63n(nanoseconds)
+
+		log.Printf("[INFO] (runner) waiting %.2fs for random splay",
+			time.Duration(offset).Seconds())
+
+		select {
+		case <-time.After(time.Duration(offset)):
+		case <-r.ExitCh:
+		}
+	}
+
 	if err := r.cmd.Process.Signal(r.killSignal); err == nil {
 		// Wait a few seconds for it to exit
 		killCh := make(chan struct{})