Add number_of_routing_shards config set to 30

With elastic/elasticsearch#26931 the possibility for splitting shards was introduced. To make use of this feature for indices created with ES >=6.1 the config option `index.number_of_routing_shards` is required. This adds this config option currently set to 30 as it's a multiple of 1, 3 and 5, our current number of default shards in Beats and ES. This allows users with default configs to scale their split their shards.

The `number_of_routing_shards` can also be overwritten in the config file.

CHANGELOG.asciidoc

@@ -94,6 +94,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Refactor add_kubernetes_metadata to support autodiscovery {pull}5434[5434]
 - Improve custom flag handling and CLI flags usage message. {pull}5543[5543]
 - Update to Golang 1.9.2
+- Add number_of_routing_shards config set to 30 {pull}5570[5570]
 
 *Auditbeat*
 

auditbeat/auditbeat.reference.yml

@@ -755,6 +755,7 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
+    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

filebeat/filebeat.reference.yml

@@ -1177,6 +1177,7 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
+    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

heartbeat/heartbeat.reference.yml

@@ -875,6 +875,7 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
+    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

libbeat/_meta/config.reference.yml

@@ -661,6 +661,7 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
+    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

libbeat/template/template.go

@@ -11,8 +11,9 @@ import (
 
 var (
 	// Defaults used in the template
-	defaultDateDetection    = false
-	defaultTotalFieldsLimit = 10000
+	defaultDateDetection         = false
+	defaultTotalFieldsLimit      = 10000
+	defaultNumberOfRoutingShards = 30
 
 	// Array to store dynamicTemplate parts in
 	dynamicTemplates []common.MapStr
@@ -147,6 +148,13 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.
 			},
 		},
 	}
+
+	// number_of_routing shards is only supported for ES version >= 6.1
+	version61, _ := common.NewVersion("6.1.0")
+	if !t.esVersion.LessThan(version61) {
+		indexSettings.Put("number_of_routing_shards", defaultNumberOfRoutingShards)
+	}
+
 	indexSettings.DeepUpdate(t.settings.Index)
 
 	var mappingName string

libbeat/template/template_test.go

@@ -0,0 +1,56 @@
+// +build !integration
+
+package template
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNumberOfRoutingShards(t *testing.T) {
+
+	beatVersion := "6.1.0"
+	beatName := "testbeat"
+	config := TemplateConfig{}
+
+	// Test it exists in 6.1
+	template, err := New(beatVersion, beatName, "6.1.0", config)
+	assert.NoError(t, err)
+
+	data := template.generate(nil, nil)
+	shards, err := data.GetValue("settings.index.number_of_routing_shards")
+	assert.NoError(t, err)
+
+	assert.Equal(t, 30, shards.(int))
+
+	// Test it does not exist in 6.0
+	template, err = New(beatVersion, beatName, "6.0.0", config)
+	assert.NoError(t, err)
+
+	data = template.generate(nil, nil)
+	shards, err = data.GetValue("settings.index.number_of_routing_shards")
+	assert.Error(t, err)
+	assert.Equal(t, nil, shards)
+}
+
+func TestNumberOfRoutingShardsOverwrite(t *testing.T) {
+
+	beatVersion := "6.1.0"
+	beatName := "testbeat"
+	config := TemplateConfig{
+		Settings: TemplateSettings{
+			Index: map[string]interface{}{"number_of_routing_shards": 5},
+		},
+	}
+
+	// Test it exists in 6.1
+	template, err := New(beatVersion, beatName, "6.1.0", config)
+	assert.NoError(t, err)
+
+	data := template.generate(nil, nil)
+	shards, err := data.GetValue("settings.index.number_of_routing_shards")
+	assert.NoError(t, err)
+
+	assert.Equal(t, 5, shards.(int))
+}

metricbeat/metricbeat.reference.yml

@@ -1142,6 +1142,7 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
+    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

packetbeat/packetbeat.reference.yml

@@ -1125,6 +1125,7 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
+    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

winlogbeat/winlogbeat.reference.yml

@@ -690,6 +690,7 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
+    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html