diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 1162a583575..d4bbd0d5eae 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -38,3 +38,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Reduce idxmgmt.Supporter interface and rework export commands to reuse logic. {pull}11777[11777], {pull}12065[12065], {pull}12067[12067] - Update urllib3 version to 1.24.2 {pull}11930[11930] - Add libbeat/common/cleanup package. {pull}12134[12134] +- Only Load minimal template if no fields are provided. {pull}12103[12103] \ No newline at end of file diff --git a/libbeat/template/load.go b/libbeat/template/load.go index 2d8fdbf6607..33b9886dea8 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -175,6 +175,9 @@ func buildBody(tmpl *Template, config TemplateConfig, fields []byte) (common.Map if config.Fields != "" { return buildBodyFromFile(tmpl, config) } + if fields == nil { + return buildMinimalTemplate(tmpl) + } return buildBodyFromFields(tmpl, fields) } @@ -216,6 +219,15 @@ func buildBodyFromFields(tmpl *Template, fields []byte) (common.MapStr, error) { return body, nil } +func buildMinimalTemplate(tmpl *Template) (common.MapStr, error) { + logp.Debug("template", "Load minimal template") + body, err := tmpl.LoadMinimal() + if err != nil { + return nil, fmt.Errorf("error creating mimimal template: %v", err) + } + return body, nil +} + func esVersionParams(ver common.Version) map[string]string { if ver.Major == 6 && ver.Minor == 7 { return map[string]string{ diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index e33f8aae294..ae393010f0d 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -22,245 +22,225 @@ package template import ( "encoding/json" "fmt" + "io/ioutil" + "math/rand" "path/filepath" "strconv" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/outputs/elasticsearch" "github.com/elastic/beats/libbeat/outputs/elasticsearch/estest" "github.com/elastic/beats/libbeat/version" ) +func init() { + rand.Seed(time.Now().UnixNano()) +} + type testTemplate struct { t *testing.T client ESClient common.MapStr } -var ( - beatInfo = beat.Info{ - Beat: "testbeat", - IndexPrefix: "testbeatidx", - Version: version.GetDefaultVersion(), - } - - templateName = "testbeatidx-" + version.GetDefaultVersion() -) +type testSetup struct { + t *testing.T + client ESClient + loader *ESLoader + config TemplateConfig +} -func defaultESLoader(t *testing.T) *ESLoader { +func newTestSetup(t *testing.T, cfg TemplateConfig) *testSetup { + if cfg.Name == "" { + cfg.Name = fmt.Sprintf("load-test-%+v", rand.Int()) + } client := estest.GetTestingElasticsearch(t) if err := client.Connect(); err != nil { t.Fatal(err) } - - return NewESLoader(client) + s := testSetup{t: t, client: client, loader: NewESLoader(client), config: cfg} + client.Request("DELETE", "/_template/"+cfg.Name, "", nil, nil) + require.False(t, s.loader.templateExists(cfg.Name)) + return &s } - -func TestCheckTemplate(t *testing.T) { - loader := defaultESLoader(t) - - // Check for non existent template - assert.False(t, loader.templateExists("libbeat-notexists")) +func (ts *testSetup) loadFromFile(fileElems []string) error { + ts.config.Fields = path(ts.t, fileElems) + beatInfo := beat.Info{Version: version.GetDefaultVersion()} + return ts.loader.Load(ts.config, beatInfo, nil, false) } -func TestLoadTemplate(t *testing.T) { - // Setup ES - loader := defaultESLoader(t) - client := loader.client - - // Load template - absPath, err := filepath.Abs("../") - assert.NotNil(t, absPath) - assert.Nil(t, err) - - fieldsPath := absPath + "/fields.yml" - index := "testbeat" - - tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{}, false) - require.NoError(t, err) - content, err := tmpl.LoadFile(fieldsPath) - require.NoError(t, err) - - // Load template - err = loader.loadTemplate(tmpl.GetName(), content) - require.NoError(t, err) +func (ts *testSetup) load(fields []byte) error { + beatInfo := beat.Info{Version: version.GetDefaultVersion()} + return ts.loader.Load(ts.config, beatInfo, fields, false) +} - // Make sure template was loaded - assert.True(t, loader.templateExists(tmpl.GetName())) +func (ts *testSetup) mustLoad(fields []byte) { + require.NoError(ts.t, ts.load(fields)) + require.True(ts.t, ts.loader.templateExists(ts.config.Name)) +} - // Delete template again to clean up - client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) +func TestESLoader_Load(t *testing.T) { + t.Run("failure", func(t *testing.T) { + t.Run("loading disabled", func(t *testing.T) { + setup := newTestSetup(t, TemplateConfig{Enabled: false}) + + setup.load(nil) + assert.False(t, setup.loader.templateExists(setup.config.Name)) + }) + + t.Run("invalid version", func(t *testing.T) { + setup := newTestSetup(t, TemplateConfig{Enabled: true}) + + beatInfo := beat.Info{Version: "invalid"} + err := setup.loader.Load(setup.config, beatInfo, nil, false) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "version is not semver") + } + }) + }) + + t.Run("overwrite", func(t *testing.T) { + // Setup create template with source enabled + setup := newTestSetup(t, TemplateConfig{Enabled: true}) + setup.mustLoad(nil) + + // Add custom settings + setup.config.Settings = TemplateSettings{Source: map[string]interface{}{"enabled": false}} + + t.Run("disabled", func(t *testing.T) { + setup.load(nil) + tmpl := getTemplate(t, setup.client, setup.config.Name) + assert.Equal(t, true, tmpl.SourceEnabled()) + }) + + t.Run("enabled", func(t *testing.T) { + setup.config.Overwrite = true + setup.load(nil) + tmpl := getTemplate(t, setup.client, setup.config.Name) + assert.Equal(t, false, tmpl.SourceEnabled()) + }) + }) + + t.Run("json.name", func(t *testing.T) { + nameJSON := "bar" + + setup := newTestSetup(t, TemplateConfig{Enabled: true}) + setup.mustLoad(nil) + + // Load Template with same name, but different JSON.name and ensure it is used + setup.config.JSON = struct { + Enabled bool `config:"enabled"` + Path string `config:"path"` + Name string `config:"name"` + }{Enabled: true, Path: path(t, []string{"testdata", "fields.json"}), Name: nameJSON} + setup.load(nil) + assert.True(t, setup.loader.templateExists(nameJSON)) + }) + + t.Run("load template successful", func(t *testing.T) { + fields, err := ioutil.ReadFile(path(t, []string{"testdata", "default_fields.yml"})) + require.NoError(t, err) + for run, data := range map[string]struct { + cfg TemplateConfig + fields []byte + fieldsPath string + properties []string + }{ + "default config with fields": { + cfg: TemplateConfig{Enabled: true}, + fields: fields, + properties: []string{"foo", "bar"}, + }, + "minimal template": { + cfg: TemplateConfig{Enabled: true}, + fields: nil, + }, + "fields from file": { + cfg: TemplateConfig{Enabled: true, Fields: path(t, []string{"testdata", "fields.yml"})}, + fields: fields, + properties: []string{"object", "keyword", "alias", "migration_alias_false", "object_disabled"}, + }, + "fields from json": { + cfg: TemplateConfig{Enabled: true, Name: "json-template", JSON: struct { + Enabled bool `config:"enabled"` + Path string `config:"path"` + Name string `config:"name"` + }{Enabled: true, Path: path(t, []string{"testdata", "fields.json"}), Name: "json-template"}}, + fields: fields, + properties: []string{"host_name"}, + }, + } { + t.Run(run, func(t *testing.T) { + setup := newTestSetup(t, data.cfg) + setup.mustLoad(data.fields) + + // Fetch properties + tmpl := getTemplate(t, setup.client, setup.config.Name) + val, err := tmpl.GetValue("mappings.properties") + if data.properties == nil { + assert.Error(t, err) + } else { + require.NoError(t, err) + p, ok := val.(map[string]interface{}) + require.True(t, ok) + var properties []string + for k := range p { + properties = append(properties, k) + } + assert.ElementsMatch(t, properties, data.properties) + } + }) + } + }) +} - // Make sure it was removed - assert.False(t, loader.templateExists(tmpl.GetName())) +func TestTemplate_LoadFile(t *testing.T) { + setup := newTestSetup(t, TemplateConfig{Enabled: true}) + assert.NoError(t, setup.loadFromFile([]string{"..", "fields.yml"})) + assert.True(t, setup.loader.templateExists(setup.config.Name)) } func TestLoadInvalidTemplate(t *testing.T) { - // Invalid Template - template := map[string]interface{}{ - "json": "invalid", - } - - // Setup ES - loader := defaultESLoader(t) - - templateName := "invalidtemplate" + setup := newTestSetup(t, TemplateConfig{}) // Try to load invalid template - err := loader.loadTemplate(templateName, template) + template := map[string]interface{}{"json": "invalid"} + err := setup.loader.loadTemplate(setup.config.Name, template) assert.Error(t, err) - - // Make sure template was not loaded - assert.False(t, loader.templateExists(templateName)) + assert.False(t, setup.loader.templateExists(setup.config.Name)) } // Tests loading the templates for each beat -func TestLoadBeatsTemplate(t *testing.T) { +func TestLoadBeatsTemplate_fromFile(t *testing.T) { beats := []string{ "libbeat", } for _, beat := range beats { - // Load template - absPath, err := filepath.Abs("../../" + beat) - assert.NotNil(t, absPath) - assert.Nil(t, err) - - // Setup ES - loader := defaultESLoader(t) - client := loader.client - - fieldsPath := absPath + "/fields.yml" - index := beat - - tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{}, false) - assert.NoError(t, err) - content, err := tmpl.LoadFile(fieldsPath) - assert.NoError(t, err) - - // Load template - err = loader.loadTemplate(tmpl.GetName(), content) - assert.Nil(t, err) - - // Make sure template was loaded - assert.True(t, loader.templateExists(tmpl.GetName())) - - // Delete template again to clean up - client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) - - // Make sure it was removed - assert.False(t, loader.templateExists(tmpl.GetName())) + setup := newTestSetup(t, TemplateConfig{Name: beat, Enabled: true}) + assert.NoError(t, setup.loadFromFile([]string{"..", "..", beat, "fields.yml"})) + assert.True(t, setup.loader.templateExists(setup.config.Name)) } } func TestTemplateSettings(t *testing.T) { - // Setup ES - loader := defaultESLoader(t) - client := loader.client - - // Load template - absPath, err := filepath.Abs("../") - assert.NotNil(t, absPath) - assert.Nil(t, err) - - fieldsPath := absPath + "/fields.yml" - settings := TemplateSettings{ - Index: common.MapStr{ - "number_of_shards": 1, - }, - Source: common.MapStr{ - "enabled": false, - }, + Index: common.MapStr{"number_of_shards": 1}, + Source: common.MapStr{"enabled": false}, } - config := TemplateConfig{ - Settings: settings, - } - tmpl, err := New(version.GetDefaultVersion(), "testbeat", client.GetVersion(), config, false) - assert.NoError(t, err) - content, err := tmpl.LoadFile(fieldsPath) - assert.NoError(t, err) - - // Load template - err = loader.loadTemplate(tmpl.GetName(), content) - assert.Nil(t, err) + setup := newTestSetup(t, TemplateConfig{Settings: settings, Enabled: true}) + require.NoError(t, setup.loadFromFile([]string{"..", "fields.yml"})) // Check that it contains the mapping - templateJSON := getTemplate(t, client, tmpl.GetName()) + templateJSON := getTemplate(t, setup.client, setup.config.Name) assert.Equal(t, 1, templateJSON.NumberOfShards()) assert.Equal(t, false, templateJSON.SourceEnabled()) - - // Delete template again to clean up - client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) - - // Make sure it was removed - assert.False(t, loader.templateExists(tmpl.GetName())) -} - -func TestOverwrite(t *testing.T) { - // Setup ES - loader := defaultESLoader(t) - client := loader.client - - templateName := "testbeatidx-" + version.GetDefaultVersion() - - absPath, err := filepath.Abs("../") - assert.NotNil(t, absPath) - assert.Nil(t, err) - - // make sure no template is already there - client.Request("DELETE", "/_template/"+templateName, "", nil, nil) - - // Load template - config := TemplateConfig{ - Enabled: true, - Fields: absPath + "/fields.yml", - } - err = loader.Load(config, beatInfo, nil, false) - assert.NoError(t, err) - - // Load template again, this time with custom settings - config = TemplateConfig{ - Enabled: true, - Fields: absPath + "/fields.yml", - Settings: TemplateSettings{ - Source: map[string]interface{}{ - "enabled": false, - }, - }, - } - - err = loader.Load(config, beatInfo, nil, false) - assert.NoError(t, err) - - // Overwrite was not enabled, so the first version should still be there - templateJSON := getTemplate(t, client, templateName) - assert.Equal(t, true, templateJSON.SourceEnabled()) - - // Load template again, this time with custom settings AND overwrite: true - config = TemplateConfig{ - Enabled: true, - Overwrite: true, - Fields: absPath + "/fields.yml", - Settings: TemplateSettings{ - Source: map[string]interface{}{ - "enabled": false, - }, - }, - } - err = loader.Load(config, beatInfo, nil, false) - assert.NoError(t, err) - - // Overwrite was enabled, so the custom setting should be there - templateJSON = getTemplate(t, client, templateName) - assert.Equal(t, false, templateJSON.SourceEnabled()) - - // Delete template again to clean up - client.Request("DELETE", "/_template/"+templateName, "", nil, nil) } var dataTests = []struct { @@ -307,31 +287,12 @@ var dataTests = []struct { // Tests if data can be loaded into elasticsearch with right types func TestTemplateWithData(t *testing.T) { - fieldsPath, err := filepath.Abs("./testdata/fields.yml") - assert.NotNil(t, fieldsPath) - assert.Nil(t, err) - - // Setup ES - client := estest.GetTestingElasticsearch(t) - if err := client.Connect(); err != nil { - t.Fatal(err) - } - loader := NewESLoader(client) - - tmpl, err := New(version.GetDefaultVersion(), "testindex", client.GetVersion(), TemplateConfig{}, false) - assert.NoError(t, err) - content, err := tmpl.LoadFile(fieldsPath) - assert.NoError(t, err) - - // Load template - err = loader.loadTemplate(tmpl.GetName(), content) - assert.Nil(t, err) - - // Make sure template was loaded - assert.True(t, loader.templateExists(tmpl.GetName())) - + setup := newTestSetup(t, TemplateConfig{Enabled: true}) + require.NoError(t, setup.loadFromFile([]string{"testdata", "fields.yml"})) + require.True(t, setup.loader.templateExists(setup.config.Name)) + esClient := setup.client.(*elasticsearch.Client) for _, test := range dataTests { - _, _, err = client.Index(tmpl.GetName(), "_doc", "", nil, test.data) + _, _, err := esClient.Index(setup.config.Name, "_doc", "", nil, test.data) if test.error { assert.NotNil(t, err) @@ -339,22 +300,16 @@ func TestTemplateWithData(t *testing.T) { assert.Nil(t, err) } } - - // Delete template again to clean up - client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) - - // Make sure it was removed - assert.False(t, loader.templateExists(tmpl.GetName())) } func getTemplate(t *testing.T, client ESClient, templateName string) testTemplate { status, body, err := client.Request("GET", "/_template/"+templateName, "", nil, nil) - assert.NoError(t, err) - assert.Equal(t, status, 200) + require.NoError(t, err) + require.Equal(t, status, 200) var response common.MapStr err = json.Unmarshal(body, &response) - assert.NoError(t, err) + require.NoError(t, err) return testTemplate{ t: t, @@ -389,3 +344,9 @@ func (tt *testTemplate) NumberOfShards() int { require.NoError(tt.t, err) return i } + +func path(t *testing.T, fileElems []string) string { + fieldsPath, err := filepath.Abs(filepath.Join(fileElems...)) + require.NoError(t, err) + return fieldsPath +} diff --git a/libbeat/template/load_test.go b/libbeat/template/load_test.go index 07f0ca7ad5b..ff0d6bfece4 100644 --- a/libbeat/template/load_test.go +++ b/libbeat/template/load_test.go @@ -23,7 +23,6 @@ import ( "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/version" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -33,62 +32,70 @@ func TestFileLoader_Load(t *testing.T) { ver := "7.0.0" prefix := "mock" info := beat.Info{Version: ver, IndexPrefix: prefix} + tmplName := fmt.Sprintf("%s-%s", prefix, ver) for name, test := range map[string]struct { - cfg TemplateConfig - fields []byte - migration bool - - name string + settings TemplateSettings + body common.MapStr }{ - "default config": { - cfg: DefaultConfig(), - name: fmt.Sprintf("%s-%s", prefix, ver), - migration: false, + "load minimal config info": { + body: common.MapStr{ + "index_patterns": []string{"mock-7.0.0-*"}, + "order": 0, + "settings": common.MapStr{"index": nil}}, + }, + "load minimal config with index settings": { + settings: TemplateSettings{Index: common.MapStr{"code": "best_compression"}}, + body: common.MapStr{ + "index_patterns": []string{"mock-7.0.0-*"}, + "order": 0, + "settings": common.MapStr{"index": common.MapStr{"code": "best_compression"}}}, }, - "default config with migration": { - cfg: DefaultConfig(), - name: fmt.Sprintf("%s-%s", prefix, ver), - migration: true, + "load minimal config with source settings": { + settings: TemplateSettings{Source: common.MapStr{"enabled": false}}, + body: common.MapStr{ + "index_patterns": []string{"mock-7.0.0-*"}, + "order": 0, + "settings": common.MapStr{"index": nil}, + "mappings": common.MapStr{ + "_source": common.MapStr{"enabled": false}, + "_meta": common.MapStr{"beat": prefix, "version": ver}, + "date_detection": false, + "dynamic_templates": nil, + "properties": nil, + }}, }, } { t.Run(name, func(t *testing.T) { fc, err := newFileClient(ver) require.NoError(t, err) fl := NewFileLoader(fc) - err = fl.Load(test.cfg, info, test.fields, false) - require.NoError(t, err) - tmpl, err := New(ver, prefix, *common.MustNewVersion(ver), test.cfg, test.migration) - require.NoError(t, err) - body, err := buildBody(tmpl, test.cfg, test.fields) + cfg := DefaultConfig() + cfg.Settings = test.settings + + err = fl.Load(cfg, info, nil, false) require.NoError(t, err) - assert.Equal(t, body.StringToPrint()+"\n", fc.body) + assert.Equal(t, "template", fc.component) + assert.Equal(t, tmplName, fc.name) + assert.Equal(t, test.body.StringToPrint()+"\n", fc.body) }) } } type fileClient struct { - ver common.Version - kind, name, body string + component, name, body, ver string } func newFileClient(ver string) (*fileClient, error) { - if ver == "" { - ver = version.GetDefaultVersion() - } - v, err := common.NewVersion(ver) - if err != nil { - return nil, err - } - return &fileClient{ver: *v}, nil + return &fileClient{ver: ver}, nil } func (c *fileClient) GetVersion() common.Version { - return c.ver + return *common.MustNewVersion(c.ver) } func (c *fileClient) Write(component string, name string, body string) error { - c.kind, c.name, c.body = component, name, body + c.component, c.name, c.body = component, name, body return nil } diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 092280cdd6a..ecdc38d0d5f 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -183,6 +183,25 @@ func (t *Template) LoadBytes(data []byte) (common.MapStr, error) { return t.load(fields) } +// LoadMinimal loads the template only with the given configuration +func (t *Template) LoadMinimal() (common.MapStr, error) { + keyPattern, patterns := buildPatternSettings(t.esVersion, t.GetPattern()) + m := common.MapStr{ + keyPattern: patterns, + "order": t.order, + "settings": common.MapStr{ + "index": t.config.Settings.Index, + }, + } + if t.config.Settings.Source != nil { + m["mappings"] = buildMappings( + t.beatVersion, t.esVersion, t.beatName, + nil, nil, + common.MapStr(t.config.Settings.Source)) + } + return m, nil +} + // GetName returns the name of the template func (t *Template) GetName() string { return t.name diff --git a/libbeat/template/testdata/default_fields.yml b/libbeat/template/testdata/default_fields.yml new file mode 100644 index 00000000000..370f3199340 --- /dev/null +++ b/libbeat/template/testdata/default_fields.yml @@ -0,0 +1,7 @@ +- key: test + title: Test default fieldds + fields: + - name: foo + type: keyword + - name: bar + type: keyword diff --git a/libbeat/template/testdata/fields.json b/libbeat/template/testdata/fields.json new file mode 100644 index 00000000000..d95b7a7dabe --- /dev/null +++ b/libbeat/template/testdata/fields.json @@ -0,0 +1,16 @@ +{ + "index_patterns": ["foo"], + "settings": { + "number_of_shards": 1 + }, + "mappings": { + "_source": { + "enabled": false + }, + "properties": { + "host_name": { + "type": "keyword" + } + } + } +}