Cherry-pick #16038 to 7.x: Template generation maintenance + r… (#16109)

* Template generation maintenance + remove number of routing sha… (#16038)

* remove number of routing shards from default template
* Add more template version tests

(cherry picked from commit 9d99b89)

CHANGELOG.next.asciidoc

@@ -70,6 +70,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Do not load dashboards where not available. {pull}15802[15802]
 - Fix issue where default go logger is not discarded when either * or stdout is selected. {issue}10251[10251] {pull}15708[15708]
 - Fix issue where TLS settings would be ignored when a forward proxy was in use. {pull}15516{15516}
+- Remove superfluous use of number_of_routing_shards setting from the default template. {pull}16038[16038]
 
 *Auditbeat*
 

auditbeat/auditbeat.reference.yml

@@ -1060,7 +1060,6 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
-    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

filebeat/filebeat.reference.yml

@@ -1757,7 +1757,6 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
-    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

heartbeat/heartbeat.reference.yml

@@ -1204,7 +1204,6 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
-    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

journalbeat/journalbeat.reference.yml

@@ -998,7 +998,6 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
-    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

libbeat/_meta/config.reference.yml.tmpl

@@ -941,7 +941,6 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
-    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

libbeat/template/template.go

@@ -312,8 +312,9 @@ func buildIdxSettings(ver common.Version, userSettings common.MapStr) common.Map
 	}
 
 	// number_of_routing shards is only supported for ES version >= 6.1
+	// If ES >= 7.0 we can exclude this setting as well.
 	version61, _ := common.NewVersion("6.1.0")
-	if !ver.LessThan(version61) {
+	if !ver.LessThan(version61) && ver.Major < 7 {
 		indexSettings.Put("number_of_routing_shards", defaultNumberOfRoutingShards)
 	}
 

libbeat/template/template_test.go

@@ -20,75 +20,163 @@
 package template
 
 import (
+	"encoding/json"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/version"
 )
 
-func TestNumberOfRoutingShards(t *testing.T) {
-
-	beatVersion := "6.1.0"
-	beatName := "testbeat"
-	config := TemplateConfig{}
+type testTemplate struct {
+	t    *testing.T
+	tmpl *Template
+	data common.MapStr
+}
 
-	// Test it exists in 6.1
-	ver := common.MustNewVersion("6.1.0")
-	template, err := New(beatVersion, beatName, *ver, config, false)
-	assert.NoError(t, err)
+func TestNumberOfRoutingShards(t *testing.T) {
+	const notPresent = 0 // setting missing indicator
+	const settingKey = "number_of_routing_shards"
+	const fullKey = "settings.index." + settingKey
+
+	cases := map[string]struct {
+		esVersion string
+		set       int
+		want      int
+	}{
+		"Do not set default for older version than 6.1": {
+			esVersion: "6.0.0",
+			want:      notPresent,
+		},
+		"Default present if ES 6.1.0 is used": {
+			esVersion: "6.1.0",
+			want:      30,
+		},
+		"Default present for newer ES 6.x version": {
+			esVersion: "6.8.2",
+			want:      30,
+		},
+		"Can overwrite default for ES 6.x": {
+			esVersion: "6.1.0",
+			set:       1024,
+			want:      1024,
+		},
+		"Do not set by default for ES 7.x": {
+			esVersion: "7.0.0",
+			want:      notPresent,
+		},
+		"Still configurable for ES 7.x": {
+			esVersion: "7.0.0",
+			set:       1024,
+			want:      1024,
+		},
+		"Do not set with current version": {
+			want: notPresent,
+		},
+		"Still configurable with current version": {
+			set:  1024,
+			want: 1024,
+		},
+	}
 
-	data := template.Generate(nil, nil)
-	shards, err := data.GetValue("settings.index.number_of_routing_shards")
-	assert.NoError(t, err)
+	for name, test := range cases {
+		t.Run(name, func(t *testing.T) {
+			beatVersion := getVersion("")
+			esVersion := getVersion(test.esVersion)
+
+			indexSettings := map[string]interface{}{}
+			if test.set > 0 {
+				indexSettings[settingKey] = test.set
+			}
+
+			template := createTestTemplate(t, beatVersion, esVersion, TemplateConfig{
+				Settings: TemplateSettings{
+					Index: indexSettings,
+				},
+			})
+
+			if test.want == notPresent {
+				template.AssertMissing(fullKey)
+			} else {
+				template.Assert(fullKey, test.want)
+			}
+		})
+	}
+}
 
-	assert.Equal(t, 30, shards.(int))
+func TestTemplate(t *testing.T) {
+	currentVersion := getVersion("")
+
+	t.Run("for ES 6.x", func(t *testing.T) {
+		template := createTestTemplate(t, currentVersion, "6.4.0", DefaultConfig())
+		template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"})
+		template.Assert("order", 1)
+		template.Assert("mappings.doc._meta", common.MapStr{"beat": "testbeat", "version": currentVersion})
+	})
+
+	t.Run("for ES 7.x", func(t *testing.T) {
+		template := createTestTemplate(t, currentVersion, "7.2.0", DefaultConfig())
+		template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"})
+		template.Assert("order", 1)
+		template.Assert("mappings._meta", common.MapStr{"beat": "testbeat", "version": currentVersion})
+	})
+
+	t.Run("for ES 8.x", func(t *testing.T) {
+		template := createTestTemplate(t, currentVersion, "8.0.0", DefaultConfig())
+		template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"})
+		template.Assert("order", 1)
+		template.Assert("mappings._meta", common.MapStr{"beat": "testbeat", "version": currentVersion})
+	})
+}
 
-	// Test it does not exist in 6.0
-	ver = common.MustNewVersion("6.0.0")
-	template, err = New(beatVersion, beatName, *ver, config, false)
-	assert.NoError(t, err)
+func createTestTemplate(t *testing.T, beatVersion, esVersion string, config TemplateConfig) *testTemplate {
+	beatVersion = getVersion(beatVersion)
+	esVersion = getVersion(esVersion)
+	ver := common.MustNewVersion(esVersion)
+	template, err := New(beatVersion, "testbeat", *ver, config, false)
+	if err != nil {
+		t.Fatalf("Failed to create the template: %+v", err)
+	}
 
-	data = template.Generate(nil, nil)
-	shards, err = data.GetValue("settings.index.number_of_routing_shards")
-	assert.Error(t, err)
-	assert.Equal(t, nil, shards)
+	return &testTemplate{t: t, tmpl: template, data: template.Generate(nil, nil)}
 }
 
-func TestNumberOfRoutingShardsOverwrite(t *testing.T) {
-
-	beatVersion := "6.1.0"
-	beatName := "testbeat"
-	config := TemplateConfig{
-		Settings: TemplateSettings{
-			Index: map[string]interface{}{"number_of_routing_shards": 5},
-		},
+func (t *testTemplate) Has(path string) bool {
+	t.t.Helper()
+	has, err := t.data.HasKey(path)
+	if err != nil && err != common.ErrKeyNotFound {
+		serialized, _ := json.MarshalIndent(t.data, "", "    ")
+		t.t.Fatalf("error accessing '%v': %v\ntemplate: %s", path, err, serialized)
 	}
+	return has
+}
 
-	// Test it exists in 6.1
-	ver := common.MustNewVersion("6.1.0")
-	template, err := New(beatVersion, beatName, *ver, config, false)
-	assert.NoError(t, err)
+func (t *testTemplate) Get(path string) interface{} {
+	t.t.Helper()
+	val, err := t.data.GetValue(path)
+	if err != nil {
+		serialized, _ := json.MarshalIndent(t.data, "", "    ")
+		t.t.Fatalf("error accessing '%v': %v\ntemplate: %s", path, err, serialized)
+	}
+	return val
+}
 
-	data := template.Generate(nil, nil)
-	shards, err := data.GetValue("settings.index.number_of_routing_shards")
-	assert.NoError(t, err)
+func (t *testTemplate) AssertMissing(path string) {
+	t.t.Helper()
+	if t.Has(path) {
+		t.t.Fatalf("Expected '%v' to be missing", path)
+	}
+}
 
-	assert.Equal(t, 5, shards.(int))
+func (t *testTemplate) Assert(path string, val interface{}) {
+	t.t.Helper()
+	assert.Equal(t.t, val, t.Get(path))
 }
 
-func TestTemplate(t *testing.T) {
-	beatVersion := "6.6.0"
-	beatName := "testbeat"
-	ver := common.MustNewVersion("6.6.0")
-	template, err := New(beatVersion, beatName, *ver, DefaultConfig(), false)
-	require.NoError(t, err)
-
-	data := template.Generate(common.MapStr{}, nil)
-	assert.Equal(t, []string{"testbeat-6.6.0-*"}, data["index_patterns"])
-	assert.Equal(t, 1, data["order"])
-	meta, err := data.GetValue("mappings.doc._meta")
-	require.NoError(t, err)
-	assert.Equal(t, common.MapStr{"beat": "testbeat", "version": "6.6.0"}, meta)
+func getVersion(in string) string {
+	if in == "" {
+		return version.GetDefaultVersion()
+	}
+	return in
 }

metricbeat/metricbeat.reference.yml

@@ -1748,7 +1748,6 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
-    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

packetbeat/packetbeat.reference.yml

@@ -1482,7 +1482,6 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
-    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

winlogbeat/winlogbeat.reference.yml

@@ -983,7 +983,6 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
-    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

x-pack/auditbeat/auditbeat.reference.yml

@@ -1111,7 +1111,6 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
-    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

x-pack/filebeat/filebeat.reference.yml

@@ -2253,7 +2253,6 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
-    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

x-pack/functionbeat/functionbeat.reference.yml

@@ -1009,7 +1009,6 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
-    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

x-pack/metricbeat/metricbeat.reference.yml

@@ -1981,7 +1981,6 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
-    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html

x-pack/winlogbeat/winlogbeat.reference.yml

@@ -986,7 +986,6 @@ setup.template.settings:
   #index:
     #number_of_shards: 1
     #codec: best_compression
-    #number_of_routing_shards: 30
 
   # A dictionary of settings for the _source field. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html