From af01fb7c91029e512d3fc71621289b9e0276341c Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Thu, 6 Feb 2020 17:25:14 +0100 Subject: [PATCH] =?UTF-8?q?Cherry-pick=20#16038=20to=207.x:=20Template=20g?= =?UTF-8?q?eneration=20maintenance=20+=20r=E2=80=A6=20(#16109)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Template generation maintenance + remove number of routing sha… (#16038) * remove number of routing shards from default template * Add more template version tests (cherry picked from commit 9d99b894fc300de19c50f4c546e44db928ee83e0) --- CHANGELOG.next.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 1 - filebeat/filebeat.reference.yml | 1 - heartbeat/heartbeat.reference.yml | 1 - journalbeat/journalbeat.reference.yml | 1 - libbeat/_meta/config.reference.yml.tmpl | 1 - libbeat/template/template.go | 3 +- libbeat/template/template_test.go | 190 +++++++++++++----- metricbeat/metricbeat.reference.yml | 1 - packetbeat/packetbeat.reference.yml | 1 - winlogbeat/winlogbeat.reference.yml | 1 - x-pack/auditbeat/auditbeat.reference.yml | 1 - x-pack/filebeat/filebeat.reference.yml | 1 - .../functionbeat/functionbeat.reference.yml | 1 - x-pack/metricbeat/metricbeat.reference.yml | 1 - x-pack/winlogbeat/winlogbeat.reference.yml | 1 - 16 files changed, 142 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 395187449c0..7f63b034853 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -70,6 +70,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Do not load dashboards where not available. {pull}15802[15802] - Fix issue where default go logger is not discarded when either * or stdout is selected. {issue}10251[10251] {pull}15708[15708] - Fix issue where TLS settings would be ignored when a forward proxy was in use. {pull}15516{15516} +- Remove superfluous use of number_of_routing_shards setting from the default template. {pull}16038[16038] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index cf1b6de5c40..b285e5a2ed7 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -1060,7 +1060,6 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression - #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index e0fc67776a6..e653f5de622 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1757,7 +1757,6 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression - #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 8f579e900b8..af10819b81a 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -1204,7 +1204,6 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression - #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index 797b670c690..9270a862925 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -998,7 +998,6 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression - #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/libbeat/_meta/config.reference.yml.tmpl b/libbeat/_meta/config.reference.yml.tmpl index 79a093f3bd5..ede5934fd24 100644 --- a/libbeat/_meta/config.reference.yml.tmpl +++ b/libbeat/_meta/config.reference.yml.tmpl @@ -941,7 +941,6 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression - #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 1bd23009927..fa4ee37a6e5 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -312,8 +312,9 @@ func buildIdxSettings(ver common.Version, userSettings common.MapStr) common.Map } // number_of_routing shards is only supported for ES version >= 6.1 + // If ES >= 7.0 we can exclude this setting as well. version61, _ := common.NewVersion("6.1.0") - if !ver.LessThan(version61) { + if !ver.LessThan(version61) && ver.Major < 7 { indexSettings.Put("number_of_routing_shards", defaultNumberOfRoutingShards) } diff --git a/libbeat/template/template_test.go b/libbeat/template/template_test.go index 8bb9eed8ab8..70cd3e14ad7 100644 --- a/libbeat/template/template_test.go +++ b/libbeat/template/template_test.go @@ -20,75 +20,163 @@ package template import ( + "encoding/json" "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/version" ) -func TestNumberOfRoutingShards(t *testing.T) { - - beatVersion := "6.1.0" - beatName := "testbeat" - config := TemplateConfig{} +type testTemplate struct { + t *testing.T + tmpl *Template + data common.MapStr +} - // Test it exists in 6.1 - ver := common.MustNewVersion("6.1.0") - template, err := New(beatVersion, beatName, *ver, config, false) - assert.NoError(t, err) +func TestNumberOfRoutingShards(t *testing.T) { + const notPresent = 0 // setting missing indicator + const settingKey = "number_of_routing_shards" + const fullKey = "settings.index." + settingKey + + cases := map[string]struct { + esVersion string + set int + want int + }{ + "Do not set default for older version than 6.1": { + esVersion: "6.0.0", + want: notPresent, + }, + "Default present if ES 6.1.0 is used": { + esVersion: "6.1.0", + want: 30, + }, + "Default present for newer ES 6.x version": { + esVersion: "6.8.2", + want: 30, + }, + "Can overwrite default for ES 6.x": { + esVersion: "6.1.0", + set: 1024, + want: 1024, + }, + "Do not set by default for ES 7.x": { + esVersion: "7.0.0", + want: notPresent, + }, + "Still configurable for ES 7.x": { + esVersion: "7.0.0", + set: 1024, + want: 1024, + }, + "Do not set with current version": { + want: notPresent, + }, + "Still configurable with current version": { + set: 1024, + want: 1024, + }, + } - data := template.Generate(nil, nil) - shards, err := data.GetValue("settings.index.number_of_routing_shards") - assert.NoError(t, err) + for name, test := range cases { + t.Run(name, func(t *testing.T) { + beatVersion := getVersion("") + esVersion := getVersion(test.esVersion) + + indexSettings := map[string]interface{}{} + if test.set > 0 { + indexSettings[settingKey] = test.set + } + + template := createTestTemplate(t, beatVersion, esVersion, TemplateConfig{ + Settings: TemplateSettings{ + Index: indexSettings, + }, + }) + + if test.want == notPresent { + template.AssertMissing(fullKey) + } else { + template.Assert(fullKey, test.want) + } + }) + } +} - assert.Equal(t, 30, shards.(int)) +func TestTemplate(t *testing.T) { + currentVersion := getVersion("") + + t.Run("for ES 6.x", func(t *testing.T) { + template := createTestTemplate(t, currentVersion, "6.4.0", DefaultConfig()) + template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"}) + template.Assert("order", 1) + template.Assert("mappings.doc._meta", common.MapStr{"beat": "testbeat", "version": currentVersion}) + }) + + t.Run("for ES 7.x", func(t *testing.T) { + template := createTestTemplate(t, currentVersion, "7.2.0", DefaultConfig()) + template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"}) + template.Assert("order", 1) + template.Assert("mappings._meta", common.MapStr{"beat": "testbeat", "version": currentVersion}) + }) + + t.Run("for ES 8.x", func(t *testing.T) { + template := createTestTemplate(t, currentVersion, "8.0.0", DefaultConfig()) + template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"}) + template.Assert("order", 1) + template.Assert("mappings._meta", common.MapStr{"beat": "testbeat", "version": currentVersion}) + }) +} - // Test it does not exist in 6.0 - ver = common.MustNewVersion("6.0.0") - template, err = New(beatVersion, beatName, *ver, config, false) - assert.NoError(t, err) +func createTestTemplate(t *testing.T, beatVersion, esVersion string, config TemplateConfig) *testTemplate { + beatVersion = getVersion(beatVersion) + esVersion = getVersion(esVersion) + ver := common.MustNewVersion(esVersion) + template, err := New(beatVersion, "testbeat", *ver, config, false) + if err != nil { + t.Fatalf("Failed to create the template: %+v", err) + } - data = template.Generate(nil, nil) - shards, err = data.GetValue("settings.index.number_of_routing_shards") - assert.Error(t, err) - assert.Equal(t, nil, shards) + return &testTemplate{t: t, tmpl: template, data: template.Generate(nil, nil)} } -func TestNumberOfRoutingShardsOverwrite(t *testing.T) { - - beatVersion := "6.1.0" - beatName := "testbeat" - config := TemplateConfig{ - Settings: TemplateSettings{ - Index: map[string]interface{}{"number_of_routing_shards": 5}, - }, +func (t *testTemplate) Has(path string) bool { + t.t.Helper() + has, err := t.data.HasKey(path) + if err != nil && err != common.ErrKeyNotFound { + serialized, _ := json.MarshalIndent(t.data, "", " ") + t.t.Fatalf("error accessing '%v': %v\ntemplate: %s", path, err, serialized) } + return has +} - // Test it exists in 6.1 - ver := common.MustNewVersion("6.1.0") - template, err := New(beatVersion, beatName, *ver, config, false) - assert.NoError(t, err) +func (t *testTemplate) Get(path string) interface{} { + t.t.Helper() + val, err := t.data.GetValue(path) + if err != nil { + serialized, _ := json.MarshalIndent(t.data, "", " ") + t.t.Fatalf("error accessing '%v': %v\ntemplate: %s", path, err, serialized) + } + return val +} - data := template.Generate(nil, nil) - shards, err := data.GetValue("settings.index.number_of_routing_shards") - assert.NoError(t, err) +func (t *testTemplate) AssertMissing(path string) { + t.t.Helper() + if t.Has(path) { + t.t.Fatalf("Expected '%v' to be missing", path) + } +} - assert.Equal(t, 5, shards.(int)) +func (t *testTemplate) Assert(path string, val interface{}) { + t.t.Helper() + assert.Equal(t.t, val, t.Get(path)) } -func TestTemplate(t *testing.T) { - beatVersion := "6.6.0" - beatName := "testbeat" - ver := common.MustNewVersion("6.6.0") - template, err := New(beatVersion, beatName, *ver, DefaultConfig(), false) - require.NoError(t, err) - - data := template.Generate(common.MapStr{}, nil) - assert.Equal(t, []string{"testbeat-6.6.0-*"}, data["index_patterns"]) - assert.Equal(t, 1, data["order"]) - meta, err := data.GetValue("mappings.doc._meta") - require.NoError(t, err) - assert.Equal(t, common.MapStr{"beat": "testbeat", "version": "6.6.0"}, meta) +func getVersion(in string) string { + if in == "" { + return version.GetDefaultVersion() + } + return in } diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 8c6821fa906..246a8d71bc3 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1748,7 +1748,6 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression - #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index fa495209f4b..1edcbc4109f 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1482,7 +1482,6 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression - #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 4e995b0504b..fbf716b157d 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -983,7 +983,6 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression - #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index ef9c19cecfe..01afe40ab21 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -1111,7 +1111,6 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression - #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 6d4d4d09aeb..2f295720c8b 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -2253,7 +2253,6 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression - #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index 461df86d12f..ab6a5e5bd44 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -1009,7 +1009,6 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression - #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 5579d6185fc..b010788aa1b 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1981,7 +1981,6 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression - #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index d7b25e5c204..1745ebdb730 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -986,7 +986,6 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression - #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html