diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 29ad47761ba..854ef8161f5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -11,6 +11,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Affecting all Beats* - Update to Golang 1.12.1. {pull}11330[11330] +- Running `setup` cmd respects `setup.ilm.overwrite` setting for improved support of custom policies. {pull}14741[14741] *Auditbeat* diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 1131bc56954..af07e50b561 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -498,7 +498,7 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er loadTemplate = idxmgmt.LoadModeOverwrite } if setup.IndexManagement || setup.ILMPolicy { - loadILM = idxmgmt.LoadModeOverwrite + loadILM = idxmgmt.LoadModeEnabled } m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields)) if ok, warn := m.VerifySetup(loadTemplate, loadILM); !ok { diff --git a/libbeat/idxmgmt/ilm/ilm.go b/libbeat/idxmgmt/ilm/ilm.go index 84ca92c432e..d9f6bbe6a06 100644 --- a/libbeat/idxmgmt/ilm/ilm.go +++ b/libbeat/idxmgmt/ilm/ilm.go @@ -36,15 +36,20 @@ type SupportFactory func(*logp.Logger, beat.Info, *common.Config) (Supporter, er // Supporter implements ILM support. For loading the policies and creating // write alias a manager instance must be generated. type Supporter interface { + // Query settings Mode() Mode Alias() Alias Policy() Policy + Overwrite() bool + + // Manager creates a new Manager instance for checking and installing + // resources. Manager(h ClientHandler) Manager } // Manager uses a ClientHandler to install a policy. type Manager interface { - Enabled() (bool, error) + CheckEnabled() (bool, error) EnsureAlias() error diff --git a/libbeat/idxmgmt/ilm/ilm_test.go b/libbeat/idxmgmt/ilm/ilm_test.go index 8a7e960745e..8167c164c1e 100644 --- a/libbeat/idxmgmt/ilm/ilm_test.go +++ b/libbeat/idxmgmt/ilm/ilm_test.go @@ -139,11 +139,11 @@ func TestDefaultSupport_Init(t *testing.T) { func TestDefaultSupport_Manager_Enabled(t *testing.T) { cases := map[string]struct { - calls []onCall - cfg map[string]interface{} - b bool - fail error - err bool + calls []onCall + cfg map[string]interface{} + enabled bool + fail error + err bool }{ "disabled via config": { cfg: map[string]interface{}{"enabled": false}, @@ -157,14 +157,14 @@ func TestDefaultSupport_Manager_Enabled(t *testing.T) { calls: []onCall{ onCheckILMEnabled(ModeAuto).Return(true, nil), }, - b: true, + enabled: true, }, "handler confirms enabled flag": { calls: []onCall{ onCheckILMEnabled(ModeEnabled).Return(true, nil), }, - cfg: map[string]interface{}{"enabled": true}, - b: true, + cfg: map[string]interface{}{"enabled": true}, + enabled: true, }, "fail enabled": { calls: []onCall{ @@ -191,7 +191,7 @@ func TestDefaultSupport_Manager_Enabled(t *testing.T) { h := newMockHandler(test.calls...) m := createManager(t, h, test.cfg) - b, err := m.Enabled() + enabled, err := m.CheckEnabled() if test.fail == nil && !test.err { require.NoError(t, err) @@ -203,7 +203,7 @@ func TestDefaultSupport_Manager_Enabled(t *testing.T) { assert.Equal(t, test.fail, ErrReason(err)) } - assert.Equal(t, test.b, b) + assert.Equal(t, test.enabled, enabled) h.AssertExpectations(t) }) } diff --git a/libbeat/idxmgmt/ilm/noop.go b/libbeat/idxmgmt/ilm/noop.go index eda57f24622..7acbafbb645 100644 --- a/libbeat/idxmgmt/ilm/noop.go +++ b/libbeat/idxmgmt/ilm/noop.go @@ -34,8 +34,9 @@ func NewNoopSupport(info beat.Info, config *common.Config) (Supporter, error) { func (*noopSupport) Mode() Mode { return ModeDisabled } func (*noopSupport) Alias() Alias { return Alias{} } func (*noopSupport) Policy() Policy { return Policy{} } +func (*noopSupport) Overwrite() bool { return false } func (*noopSupport) Manager(_ ClientHandler) Manager { return (*noopManager)(nil) } -func (*noopManager) Enabled() (bool, error) { return false, nil } +func (*noopManager) CheckEnabled() (bool, error) { return false, nil } func (*noopManager) EnsureAlias() error { return errOf(ErrOpNotAvailable) } func (*noopManager) EnsurePolicy(_ bool) (bool, error) { return false, errOf(ErrOpNotAvailable) } diff --git a/libbeat/idxmgmt/ilm/std.go b/libbeat/idxmgmt/ilm/std.go index b4f0c41b66b..fbb497857f9 100644 --- a/libbeat/idxmgmt/ilm/std.go +++ b/libbeat/idxmgmt/ilm/std.go @@ -67,9 +67,10 @@ func NewStdSupport( } } -func (s *stdSupport) Mode() Mode { return s.mode } -func (s *stdSupport) Alias() Alias { return s.alias } -func (s *stdSupport) Policy() Policy { return s.policy } +func (s *stdSupport) Mode() Mode { return s.mode } +func (s *stdSupport) Alias() Alias { return s.alias } +func (s *stdSupport) Policy() Policy { return s.policy } +func (s *stdSupport) Overwrite() bool { return s.overwrite } func (s *stdSupport) Manager(h ClientHandler) Manager { return &stdManager{ @@ -78,7 +79,7 @@ func (s *stdSupport) Manager(h ClientHandler) Manager { } } -func (m *stdManager) Enabled() (bool, error) { +func (m *stdManager) CheckEnabled() (bool, error) { if m.mode == ModeDisabled { return false, nil } @@ -120,7 +121,7 @@ func (m *stdManager) EnsureAlias() error { func (m *stdManager) EnsurePolicy(overwrite bool) (bool, error) { log := m.log - overwrite = overwrite || m.overwrite + overwrite = overwrite || m.Overwrite() exists := true if m.checkExists && !overwrite { diff --git a/libbeat/idxmgmt/mockilm_test.go b/libbeat/idxmgmt/mockilm_test.go index db4c60493c1..a36e17bd421 100644 --- a/libbeat/idxmgmt/mockilm_test.go +++ b/libbeat/idxmgmt/mockilm_test.go @@ -69,12 +69,17 @@ func (m *mockILMSupport) Policy() ilm.Policy { return args.Get(0).(ilm.Policy) } +func onOverwrite() onCall { return makeOnCall("Overwrite") } +func (m *mockILMSupport) Overwrite() bool { + return m.Called().Bool(0) +} + func (m *mockILMSupport) Manager(_ ilm.ClientHandler) ilm.Manager { return m } -func onEnabled() onCall { return makeOnCall("Enabled") } -func (m *mockILMSupport) Enabled() (bool, error) { +func onCheckEnabled() onCall { return makeOnCall("CheckEnabled") } +func (m *mockILMSupport) CheckEnabled() (bool, error) { args := m.Called() return args.Bool(0), args.Error(1) } diff --git a/libbeat/idxmgmt/std.go b/libbeat/idxmgmt/std.go index f6951d729ad..bb961712436 100644 --- a/libbeat/idxmgmt/std.go +++ b/libbeat/idxmgmt/std.go @@ -218,7 +218,7 @@ func (s *indexSupport) BuildSelector(cfg *common.Config) (outputs.IndexSelector, } func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string) { - ilmComponent := newFeature(componentILM, m.support.enabled(componentILM), false, loadILM) + ilmComponent := newFeature(componentILM, m.support.enabled(componentILM), m.support.ilm.Overwrite(), loadILM) templateComponent := newFeature(componentTemplate, m.support.enabled(componentTemplate), m.support.templateCfg.Overwrite, loadTemplate) @@ -236,10 +236,12 @@ func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string var warn string if !ilmComponent.load { - warn += "ILM policy and write alias loading not enabled. " + warn += "ILM policy and write alias loading not enabled.\n" + } else if !ilmComponent.overwrite { + warn += "Overwriting ILM policy is disabled. Set `setup.ilm.overwrite:true` for enabling.\n" } if !templateComponent.load { - warn += "Template loading not enabled." + warn += "Template loading not enabled.\n" } return warn == "", warn } @@ -256,7 +258,7 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { log.Info("Auto ILM enable success.") } - ilmComponent := newFeature(componentILM, withILM, false, loadILM) + ilmComponent := newFeature(componentILM, withILM, m.support.ilm.Overwrite(), loadILM) templateComponent := newFeature(componentTemplate, m.support.enabled(componentTemplate), m.support.templateCfg.Overwrite, loadTemplate) @@ -312,7 +314,7 @@ func (m *indexManager) setupWithILM() (bool, error) { var err error withILM := m.support.st.withILM.Load() if !withILM { - withILM, err = m.ilm.Enabled() + withILM, err = m.ilm.CheckEnabled() if err != nil { return false, err } diff --git a/libbeat/idxmgmt/std_test.go b/libbeat/idxmgmt/std_test.go index ac0cf47ae8b..bbbd17f53ba 100644 --- a/libbeat/idxmgmt/std_test.go +++ b/libbeat/idxmgmt/std_test.go @@ -218,17 +218,17 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { func TestIndexManager_VerifySetup(t *testing.T) { for name, setup := range map[string]struct { - tmpl, ilm bool - loadTmpl, loadILM LoadMode - ok bool - warn string + tmplEnabled, ilmEnabled, ilmOverwrite bool + loadTmpl, loadILM LoadMode + ok bool + warn string }{ "load template with ilm without loading ilm": { - ilm: true, tmpl: true, loadILM: LoadModeDisabled, + ilmEnabled: true, tmplEnabled: true, loadILM: LoadModeDisabled, warn: "whithout loading ILM policy and alias", }, "load ilm without template": { - ilm: true, loadILM: LoadModeUnset, + ilmEnabled: true, loadILM: LoadModeUnset, warn: "without loading template is not recommended", }, "template disabled but loading enabled": { @@ -236,26 +236,33 @@ func TestIndexManager_VerifySetup(t *testing.T) { warn: "loading not enabled", }, "ilm disabled but loading enabled": { - loadILM: LoadModeEnabled, tmpl: true, + loadILM: LoadModeEnabled, tmplEnabled: true, warn: "loading not enabled", }, "ilm enabled but loading disabled": { - ilm: true, loadILM: LoadModeDisabled, + ilmEnabled: true, loadILM: LoadModeDisabled, warn: "loading not enabled", }, "template enabled but loading disabled": { - tmpl: true, loadTmpl: LoadModeDisabled, + tmplEnabled: true, loadTmpl: LoadModeDisabled, warn: "loading not enabled", }, + "ilm enabled but overwrite disabled": { + tmplEnabled: true, + ilmEnabled: true, ilmOverwrite: false, loadILM: LoadModeEnabled, + warn: "Overwriting ILM policy is disabled", + }, "everything enabled": { - tmpl: true, loadTmpl: LoadModeUnset, ilm: true, loadILM: LoadModeUnset, + tmplEnabled: true, + ilmEnabled: true, ilmOverwrite: true, ok: true, }, } { t.Run(name, func(t *testing.T) { cfg, err := common.NewConfigFrom(common.MapStr{ - "setup.ilm.enabled": setup.ilm, - "setup.template.enabled": setup.tmpl, + "setup.ilm.enabled": setup.ilmEnabled, + "setup.ilm.overwrite": setup.ilmOverwrite, + "setup.template.enabled": setup.tmplEnabled, }) require.NoError(t, err) support, err := MakeDefaultSupport(ilm.StdSupport)(nil, beat.Info{}, cfg) diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py index 52162ae5521..26d3fc66555 100644 --- a/libbeat/tests/system/test_cmd_setup_index_management.py +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -127,6 +127,53 @@ def test_setup_policy_name(self): self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.custom_policy, self.alias_name) self.idxmgmt.assert_policy_created(self.custom_policy) + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_ilm_policy_no_overwrite(self): + """ + Test setup --index-management respects overwrite configuration + """ + policy_name = "mockbeat-test" + # update policy to verify overwrite behaviour + self.es.transport.perform_request('PUT', '/_ilm/policy/' + policy_name, + body={ + "policy": { + "phases": { + "delete": { + "actions": { + "delete": {} + } + } + } + } + }) + resp = self.es.transport.perform_request('GET', '/_ilm/policy/' + policy_name) + assert "delete" in resp[policy_name]["policy"]["phases"] + assert "hot" not in resp[policy_name]["policy"]["phases"] + + # ensure ilm policy is not overwritten + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.ilm.enabled=true", + "-E", "setup.ilm.overwrite=false", + "-E", "setup.ilm.policy_name="+policy_name]) + assert exit_code == 0 + resp = self.es.transport.perform_request('GET', '/_ilm/policy/' + policy_name) + assert "delete" in resp[policy_name]["policy"]["phases"] + assert "hot" not in resp[policy_name]["policy"]["phases"] + + # ensure ilm policy is overwritten + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.ilm.enabled=true", + "-E", "setup.ilm.overwrite=true", + "-E", "setup.ilm.policy_name="+policy_name]) + assert exit_code == 0 + resp = self.es.transport.perform_request('GET', '/_ilm/policy/' + policy_name) + assert "delete" not in resp[policy_name]["policy"]["phases"] + assert "hot" in resp[policy_name]["policy"]["phases"] + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @attr('integration') def test_setup_rollover_alias(self):