[7.5] [ILM] respect overwrite config in setup (#14741)

CHANGELOG.next.asciidoc

@@ -11,6 +11,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Affecting all Beats*
 
 - Update to Golang 1.12.1. {pull}11330[11330]
+- Running `setup` cmd respects `setup.ilm.overwrite` setting for improved support of custom policies. {pull}14741[14741]
 
 *Auditbeat*
 

libbeat/cmd/instance/beat.go

@@ -498,7 +498,7 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er
 				loadTemplate = idxmgmt.LoadModeOverwrite
 			}
 			if setup.IndexManagement || setup.ILMPolicy {
-				loadILM = idxmgmt.LoadModeOverwrite
+				loadILM = idxmgmt.LoadModeEnabled
 			}
 			m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields))
 			if ok, warn := m.VerifySetup(loadTemplate, loadILM); !ok {

libbeat/idxmgmt/ilm/ilm.go

@@ -36,15 +36,20 @@ type SupportFactory func(*logp.Logger, beat.Info, *common.Config) (Supporter, er
 // Supporter implements ILM support. For loading the policies and creating
 // write alias a manager instance must be generated.
 type Supporter interface {
+	// Query settings
 	Mode() Mode
 	Alias() Alias
 	Policy() Policy
+	Overwrite() bool
+
+	// Manager creates a new Manager instance for checking and installing
+	// resources.
 	Manager(h ClientHandler) Manager
 }
 
 // Manager uses a ClientHandler to install a policy.
 type Manager interface {
-	Enabled() (bool, error)
+	CheckEnabled() (bool, error)
 
 	EnsureAlias() error
 

libbeat/idxmgmt/ilm/ilm_test.go

@@ -139,11 +139,11 @@ func TestDefaultSupport_Init(t *testing.T) {
 
 func TestDefaultSupport_Manager_Enabled(t *testing.T) {
 	cases := map[string]struct {
-		calls []onCall
-		cfg   map[string]interface{}
-		b     bool
-		fail  error
-		err   bool
+		calls   []onCall
+		cfg     map[string]interface{}
+		enabled bool
+		fail    error
+		err     bool
 	}{
 		"disabled via config": {
 			cfg: map[string]interface{}{"enabled": false},
@@ -157,14 +157,14 @@ func TestDefaultSupport_Manager_Enabled(t *testing.T) {
 			calls: []onCall{
 				onCheckILMEnabled(ModeAuto).Return(true, nil),
 			},
-			b: true,
+			enabled: true,
 		},
 		"handler confirms enabled flag": {
 			calls: []onCall{
 				onCheckILMEnabled(ModeEnabled).Return(true, nil),
 			},
-			cfg: map[string]interface{}{"enabled": true},
-			b:   true,
+			cfg:     map[string]interface{}{"enabled": true},
+			enabled: true,
 		},
 		"fail enabled": {
 			calls: []onCall{
@@ -191,7 +191,7 @@ func TestDefaultSupport_Manager_Enabled(t *testing.T) {
 
 			h := newMockHandler(test.calls...)
 			m := createManager(t, h, test.cfg)
-			b, err := m.Enabled()
+			enabled, err := m.CheckEnabled()
 
 			if test.fail == nil && !test.err {
 				require.NoError(t, err)
@@ -203,7 +203,7 @@ func TestDefaultSupport_Manager_Enabled(t *testing.T) {
 				assert.Equal(t, test.fail, ErrReason(err))
 			}
 
-			assert.Equal(t, test.b, b)
+			assert.Equal(t, test.enabled, enabled)
 			h.AssertExpectations(t)
 		})
 	}

libbeat/idxmgmt/ilm/noop.go

@@ -34,8 +34,9 @@ func NewNoopSupport(info beat.Info, config *common.Config) (Supporter, error) {
 func (*noopSupport) Mode() Mode                      { return ModeDisabled }
 func (*noopSupport) Alias() Alias                    { return Alias{} }
 func (*noopSupport) Policy() Policy                  { return Policy{} }
+func (*noopSupport) Overwrite() bool                 { return false }
 func (*noopSupport) Manager(_ ClientHandler) Manager { return (*noopManager)(nil) }
 
-func (*noopManager) Enabled() (bool, error)            { return false, nil }
+func (*noopManager) CheckEnabled() (bool, error)       { return false, nil }
 func (*noopManager) EnsureAlias() error                { return errOf(ErrOpNotAvailable) }
 func (*noopManager) EnsurePolicy(_ bool) (bool, error) { return false, errOf(ErrOpNotAvailable) }

libbeat/idxmgmt/ilm/std.go

@@ -67,9 +67,10 @@ func NewStdSupport(
 	}
 }
 
-func (s *stdSupport) Mode() Mode     { return s.mode }
-func (s *stdSupport) Alias() Alias   { return s.alias }
-func (s *stdSupport) Policy() Policy { return s.policy }
+func (s *stdSupport) Mode() Mode      { return s.mode }
+func (s *stdSupport) Alias() Alias    { return s.alias }
+func (s *stdSupport) Policy() Policy  { return s.policy }
+func (s *stdSupport) Overwrite() bool { return s.overwrite }
 
 func (s *stdSupport) Manager(h ClientHandler) Manager {
 	return &stdManager{
@@ -78,7 +79,7 @@ func (s *stdSupport) Manager(h ClientHandler) Manager {
 	}
 }
 
-func (m *stdManager) Enabled() (bool, error) {
+func (m *stdManager) CheckEnabled() (bool, error) {
 	if m.mode == ModeDisabled {
 		return false, nil
 	}
@@ -120,7 +121,7 @@ func (m *stdManager) EnsureAlias() error {
 
 func (m *stdManager) EnsurePolicy(overwrite bool) (bool, error) {
 	log := m.log
-	overwrite = overwrite || m.overwrite
+	overwrite = overwrite || m.Overwrite()
 
 	exists := true
 	if m.checkExists && !overwrite {

libbeat/idxmgmt/mockilm_test.go

@@ -69,12 +69,17 @@ func (m *mockILMSupport) Policy() ilm.Policy {
 	return args.Get(0).(ilm.Policy)
 }
 
+func onOverwrite() onCall { return makeOnCall("Overwrite") }
+func (m *mockILMSupport) Overwrite() bool {
+	return m.Called().Bool(0)
+}
+
 func (m *mockILMSupport) Manager(_ ilm.ClientHandler) ilm.Manager {
 	return m
 }
 
-func onEnabled() onCall { return makeOnCall("Enabled") }
-func (m *mockILMSupport) Enabled() (bool, error) {
+func onCheckEnabled() onCall { return makeOnCall("CheckEnabled") }
+func (m *mockILMSupport) CheckEnabled() (bool, error) {
 	args := m.Called()
 	return args.Bool(0), args.Error(1)
 }

libbeat/idxmgmt/std.go

@@ -218,7 +218,7 @@ func (s *indexSupport) BuildSelector(cfg *common.Config) (outputs.IndexSelector,
 }
 
 func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string) {
-	ilmComponent := newFeature(componentILM, m.support.enabled(componentILM), false, loadILM)
+	ilmComponent := newFeature(componentILM, m.support.enabled(componentILM), m.support.ilm.Overwrite(), loadILM)
 
 	templateComponent := newFeature(componentTemplate, m.support.enabled(componentTemplate),
 		m.support.templateCfg.Overwrite, loadTemplate)
@@ -236,10 +236,12 @@ func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string
 
 	var warn string
 	if !ilmComponent.load {
-		warn += "ILM policy and write alias loading not enabled. "
+		warn += "ILM policy and write alias loading not enabled.\n"
+	} else if !ilmComponent.overwrite {
+		warn += "Overwriting ILM policy is disabled. Set `setup.ilm.overwrite:true` for enabling.\n"
 	}
 	if !templateComponent.load {
-		warn += "Template loading not enabled."
+		warn += "Template loading not enabled.\n"
 	}
 	return warn == "", warn
 }
@@ -256,7 +258,7 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error {
 		log.Info("Auto ILM enable success.")
 	}
 
-	ilmComponent := newFeature(componentILM, withILM, false, loadILM)
+	ilmComponent := newFeature(componentILM, withILM, m.support.ilm.Overwrite(), loadILM)
 	templateComponent := newFeature(componentTemplate, m.support.enabled(componentTemplate),
 		m.support.templateCfg.Overwrite, loadTemplate)
 
@@ -312,7 +314,7 @@ func (m *indexManager) setupWithILM() (bool, error) {
 	var err error
 	withILM := m.support.st.withILM.Load()
 	if !withILM {
-		withILM, err = m.ilm.Enabled()
+		withILM, err = m.ilm.CheckEnabled()
 		if err != nil {
 			return false, err
 		}

libbeat/idxmgmt/std_test.go

@@ -218,44 +218,51 @@ func TestDefaultSupport_BuildSelector(t *testing.T) {
 
 func TestIndexManager_VerifySetup(t *testing.T) {
 	for name, setup := range map[string]struct {
-		tmpl, ilm         bool
-		loadTmpl, loadILM LoadMode
-		ok                bool
-		warn              string
+		tmplEnabled, ilmEnabled, ilmOverwrite bool
+		loadTmpl, loadILM                     LoadMode
+		ok                                    bool
+		warn                                  string
 	}{
 		"load template with ilm without loading ilm": {
-			ilm: true, tmpl: true, loadILM: LoadModeDisabled,
+			ilmEnabled: true, tmplEnabled: true, loadILM: LoadModeDisabled,
 			warn: "whithout loading ILM policy and alias",
 		},
 		"load ilm without template": {
-			ilm: true, loadILM: LoadModeUnset,
+			ilmEnabled: true, loadILM: LoadModeUnset,
 			warn: "without loading template is not recommended",
 		},
 		"template disabled but loading enabled": {
 			loadTmpl: LoadModeEnabled,
 			warn:     "loading not enabled",
 		},
 		"ilm disabled but loading enabled": {
-			loadILM: LoadModeEnabled, tmpl: true,
+			loadILM: LoadModeEnabled, tmplEnabled: true,
 			warn: "loading not enabled",
 		},
 		"ilm enabled but loading disabled": {
-			ilm: true, loadILM: LoadModeDisabled,
+			ilmEnabled: true, loadILM: LoadModeDisabled,
 			warn: "loading not enabled",
 		},
 		"template enabled but loading disabled": {
-			tmpl: true, loadTmpl: LoadModeDisabled,
+			tmplEnabled: true, loadTmpl: LoadModeDisabled,
 			warn: "loading not enabled",
 		},
+		"ilm enabled but overwrite disabled": {
+			tmplEnabled: true,
+			ilmEnabled:  true, ilmOverwrite: false, loadILM: LoadModeEnabled,
+			warn: "Overwriting ILM policy is disabled",
+		},
 		"everything enabled": {
-			tmpl: true, loadTmpl: LoadModeUnset, ilm: true, loadILM: LoadModeUnset,
+			tmplEnabled: true,
+			ilmEnabled:  true, ilmOverwrite: true,
 			ok: true,
 		},
 	} {
 		t.Run(name, func(t *testing.T) {
 			cfg, err := common.NewConfigFrom(common.MapStr{
-				"setup.ilm.enabled":      setup.ilm,
-				"setup.template.enabled": setup.tmpl,
+				"setup.ilm.enabled":      setup.ilmEnabled,
+				"setup.ilm.overwrite":    setup.ilmOverwrite,
+				"setup.template.enabled": setup.tmplEnabled,
 			})
 			require.NoError(t, err)
 			support, err := MakeDefaultSupport(ilm.StdSupport)(nil, beat.Info{}, cfg)

libbeat/tests/system/test_cmd_setup_index_management.py

@@ -127,6 +127,53 @@ def test_setup_policy_name(self):
         self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.custom_policy, self.alias_name)
         self.idxmgmt.assert_policy_created(self.custom_policy)
 
+    @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
+    @attr('integration')
+    def test_setup_ilm_policy_no_overwrite(self):
+        """
+        Test setup --index-management respects overwrite configuration
+        """
+        policy_name = "mockbeat-test"
+        # update policy to verify overwrite behaviour
+        self.es.transport.perform_request('PUT', '/_ilm/policy/' + policy_name,
+                                          body={
+                                              "policy": {
+                                                 "phases": {
+                                                     "delete": {
+                                                         "actions": {
+                                                             "delete": {}
+                                                         }
+                                                     }
+                                                 }
+                                              }
+                                          })
+        resp = self.es.transport.perform_request('GET', '/_ilm/policy/' + policy_name)
+        assert "delete" in resp[policy_name]["policy"]["phases"]
+        assert "hot" not in resp[policy_name]["policy"]["phases"]
+
+        # ensure ilm policy is not overwritten
+        self.render_config()
+        exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
+                                  extra_args=["setup", self.cmd,
+                                              "-E", "setup.ilm.enabled=true",
+                                              "-E", "setup.ilm.overwrite=false",
+                                              "-E", "setup.ilm.policy_name="+policy_name])
+        assert exit_code == 0
+        resp = self.es.transport.perform_request('GET', '/_ilm/policy/' + policy_name)
+        assert "delete" in resp[policy_name]["policy"]["phases"]
+        assert "hot" not in resp[policy_name]["policy"]["phases"]
+
+        # ensure ilm policy is overwritten
+        exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
+                                  extra_args=["setup", self.cmd,
+                                              "-E", "setup.ilm.enabled=true",
+                                              "-E", "setup.ilm.overwrite=true",
+                                              "-E", "setup.ilm.policy_name="+policy_name])
+        assert exit_code == 0
+        resp = self.es.transport.perform_request('GET', '/_ilm/policy/' + policy_name)
+        assert "delete" not in resp[policy_name]["policy"]["phases"]
+        assert "hot" in resp[policy_name]["policy"]["phases"]
+
     @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
     @attr('integration')
     def test_setup_rollover_alias(self):