5-second Raft election timeout during CI testing

CI testing may suffer from lack of CPU resources, meaning leader
elections may continually occur. This gives leader nodes more time to
send out heartbeats.

CHANGELOG.md

@@ -8,7 +8,7 @@
 
 ### Features
 - [#2398](https://github.com/influxdb/influxdb/pull/2398) Track more stats and report errors for shards.
-- [#2410](https://github.com/influxdb/influxdb/pull/2410) Allow Raft election timeout to be configured.
+- [#2410](https://github.com/influxdb/influxdb/pull/2410) Allow Raft time configuration
 
 ### Bugfixes
 - [#2370](https://github.com/influxdb/influxdb/pull/2370): Fix data race in openTSDB endpoint.

cmd/influxd/config.go

@@ -59,9 +59,18 @@ const (
 	// DefaultMaxTopicSize is the default maximum size in bytes a segment can consume on disk of a broker.
 	DefaultBrokerMaxSegmentSize = 10 * 1024 * 1024
 
+	// DefaultRaftApplyInterval is the period between applying commited Raft log entries.
+	DefaultRaftApplyInterval = 10 * time.Millisecond
+
 	// DefaultRaftElectionTimeout is the default Leader Election timeout.
 	DefaultRaftElectionTimeout = 1 * time.Second
 
+	// DefaultRaftHeartbeatInterval is the interval between leader heartbeats.
+	DefaultRaftHeartbeatInterval = 100 * time.Millisecond
+
+	// DefaultRaftReconnectTimeout is the default wait time between reconnections.
+	DefaultRaftReconnectTimeout = 10 * time.Millisecond
+
 	// DefaultGraphiteDatabaseName is the default Graphite database if none is specified
 	DefaultGraphiteDatabaseName = "graphite"
 
@@ -118,7 +127,10 @@ type Broker struct {
 
 // Raft represents the Raft configuration for broker nodes.
 type Raft struct {
-	ElectionTimeout Duration `toml:"election-timeout"`
+	ApplyInterval     Duration `toml:"apply-interval"`
+	ElectionTimeout   Duration `toml:"election-timeout"`
+	HeartbeatInterval Duration `toml:"heartbeat-interval"`
+	ReconnectTimeout  Duration `toml:"reconnect-timeout"`
 }
 
 // Snapshot represents the configuration for a snapshot service. Snapshot configuration
@@ -269,7 +281,10 @@ func NewConfig() *Config {
 	c.Broker.MaxTopicSize = DefaultBrokerMaxTopicSize
 	c.Broker.MaxSegmentSize = DefaultBrokerMaxSegmentSize
 
+	c.Raft.ApplyInterval = Duration(DefaultRaftApplyInterval)
 	c.Raft.ElectionTimeout = Duration(DefaultRaftElectionTimeout)
+	c.Raft.HeartbeatInterval = Duration(DefaultRaftHeartbeatInterval)
+	c.Raft.ReconnectTimeout = Duration(DefaultRaftReconnectTimeout)
 
 	// FIX(benbjohnson): Append where the udp servers are actually used.
 	// config.UDPServers = append(config.UDPServers, UDPInputConfig{
@@ -295,7 +310,10 @@ func NewTestConfig() (*Config, error) {
 	c.Broker.Enabled = true
 	c.Broker.Dir = filepath.Join(u.HomeDir, ".influxdb/broker")
 
+	c.Raft.ApplyInterval = Duration(DefaultRaftApplyInterval)
 	c.Raft.ElectionTimeout = Duration(DefaultRaftElectionTimeout)
+	c.Raft.HeartbeatInterval = Duration(DefaultRaftHeartbeatInterval)
+	c.Raft.ReconnectTimeout = Duration(DefaultRaftReconnectTimeout)
 
 	c.Data.Enabled = true
 	c.Data.Dir = filepath.Join(u.HomeDir, ".influxdb/data")

cmd/influxd/config_test.go

@@ -105,6 +105,7 @@ dir  = "/tmp/influxdb/development/broker"
 
 # Raft distributed consensus
 [raft]
+apply-interval = "10ms"
 election-timeout = "1s"
 
 [data]
@@ -254,6 +255,10 @@ func TestParseConfig(t *testing.T) {
 		t.Fatalf("broker disabled mismatch: %v, got: %v", false, c.Broker.Enabled)
 	}
 
+	if c.Raft.ApplyInterval != main.Duration(10*time.Millisecond) {
+		t.Fatalf("Raft apply interval mismatch: %v, got %v", 10*time.Millisecond, c.Raft.ApplyInterval)
+	}
+
 	if c.Data.Dir != "/tmp/influxdb/development/db" {
 		t.Fatalf("data dir mismatch: %v", c.Data.Dir)
 	}

cmd/influxd/run.go

@@ -546,7 +546,10 @@ func (cmd *RunCommand) openBroker(brokerURLs []url.URL, h *Handler) {
 
 	// Create Raft clock.
 	clk := raft.NewClock()
+	clk.ApplyInterval = time.Duration(cmd.config.Raft.ApplyInterval)
 	clk.ElectionTimeout = time.Duration(cmd.config.Raft.ElectionTimeout)
+	clk.HeartbeatInterval = time.Duration(cmd.config.Raft.HeartbeatInterval)
+	clk.ReconnectTimeout = time.Duration(cmd.config.Raft.ReconnectTimeout)
 	l.Clock = clk
 
 	// Open broker so it can feed last index data to the log.

etc/config.sample.toml

@@ -81,7 +81,11 @@ max-topic-size = 1073741824
 max-segment-size = 10485760
 
 # Raft configuration. Controls the distributed consensus system.
+[raft]
+apply-interval = "10ms"
 election-timeout = "1s"
+heartbeat-interval = "100ms"
+reconnect-timeout = "10ms"
 
 # Data node configuration. Data nodes are where the time-series data, in the form of
 # shards, is stored.