template.go

package template

import (
	"fmt"
	"time"

	"github.com/elastic/beats/libbeat/beat"
	"github.com/elastic/beats/libbeat/common"
	"github.com/elastic/beats/libbeat/common/fmtstr"
)

var (
	// Defaults used in the template
	defaultDateDetection         = false
	defaultTotalFieldsLimit      = 10000
	defaultNumberOfRoutingShards = 30

	// Array to store dynamicTemplate parts in
	dynamicTemplates []common.MapStr
)

type Template struct {
	name        string
	pattern     string
	beatVersion common.Version
	esVersion   common.Version
	settings    TemplateSettings
}

// New creates a new template instance
func New(beatVersion string, beatName string, esVersion string, config TemplateConfig) (*Template, error) {
	bV, err := common.NewVersion(beatVersion)
	if err != nil {
		return nil, err
	}

	name := config.Name
	if name == "" {
		name = fmt.Sprintf("%s-%s", beatName, bV.String())
	}

	pattern := config.Pattern
	if pattern == "" {
		pattern = name + "-*"
	}

	event := &beat.Event{
		Fields: common.MapStr{
			"beat": common.MapStr{
				"name":    beatName,
				"version": bV.String(),
			},
		},
		Timestamp: time.Now(),
	}

	nameFormatter, err := fmtstr.CompileEvent(name)
	if err != nil {
		return nil, err
	}
	name, err = nameFormatter.Run(event)
	if err != nil {
		return nil, err
	}

	patternFormatter, err := fmtstr.CompileEvent(pattern)
	if err != nil {
		return nil, err
	}
	pattern, err = patternFormatter.Run(event)
	if err != nil {
		return nil, err
	}

	// In case no esVersion is set, it is assumed the same as beat version
	if esVersion == "" {
		esVersion = beatVersion
	}

	esV, err := common.NewVersion(esVersion)
	if err != nil {
		return nil, err
	}

	return &Template{
		pattern:     pattern,
		name:        name,
		beatVersion: *bV,
		esVersion:   *esV,
		settings:    config.Settings,
	}, nil
}

// Load the given input and generates the input based on it
func (t *Template) Load(file string) (common.MapStr, error) {
	fields, err := common.LoadFieldsYaml(file)
	if err != nil {
		return nil, err
	}

	// Start processing at the root
	properties := common.MapStr{}
	processor := Processor{EsVersion: t.esVersion}
	if err := processor.process(fields, "", properties); err != nil {
		return nil, err
	}
	output := t.generate(properties, dynamicTemplates)

	return output, nil
}

// GetName returns the name of the template
func (t *Template) GetName() string {
	return t.name
}

// GetPattern returns the pattern of the template
func (t *Template) GetPattern() string {
	return t.pattern
}

// generate generates the full template
// The default values are taken from the default variable.
func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.MapStr) common.MapStr {
	// Add base dynamic template
	var dynamicTemplateBase = common.MapStr{
		"strings_as_keyword": common.MapStr{
			"mapping": common.MapStr{
				"ignore_above": 1024,
				"type":         "keyword",
			},
			"match_mapping_type": "string",
		},
	}

	if t.esVersion.IsMajor(2) {
		dynamicTemplateBase.Put("strings_as_keyword.mapping.type", "string")
		dynamicTemplateBase.Put("strings_as_keyword.mapping.index", "not_analyzed")
	}

	dynamicTemplates = append(dynamicTemplates, dynamicTemplateBase)

	indexSettings := common.MapStr{
		"refresh_interval": "5s",
		"mapping": common.MapStr{
			"total_fields": common.MapStr{
				"limit": defaultTotalFieldsLimit,
			},
		},
	}

	// number_of_routing shards is only supported for ES version >= 6.1
	version61, _ := common.NewVersion("6.1.0")
	if !t.esVersion.LessThan(version61) {
		indexSettings.Put("number_of_routing_shards", defaultNumberOfRoutingShards)
	}

	indexSettings.DeepUpdate(t.settings.Index)

	var mappingName string
	if t.esVersion.Major >= 6 {
		mappingName = "doc"
	} else {
		mappingName = "_default_"
	}

	// Load basic structure
	basicStructure := common.MapStr{
		"mappings": common.MapStr{
			mappingName: common.MapStr{
				"_meta": common.MapStr{
					"version": t.beatVersion.String(),
				},
				"date_detection":    defaultDateDetection,
				"dynamic_templates": dynamicTemplates,
				"properties":        properties,
			},
		},
		"order": 1,
		"settings": common.MapStr{
			"index": indexSettings,
		},
	}

	if len(t.settings.Source) > 0 {
		key := fmt.Sprintf("mappings.%s._source", mappingName)
		basicStructure.Put(key, t.settings.Source)
	}

	// ES 6 moved from template to index_patterns: https://github.com/elastic/elasticsearch/pull/21009
	if t.esVersion.Major >= 6 {
		basicStructure.Put("index_patterns", []string{t.GetPattern()})
	} else {
		basicStructure.Put("template", t.GetPattern())
	}

	if t.esVersion.IsMajor(2) {
		basicStructure.Put("mappings._default_._all.norms.enabled", false)
	}

	return basicStructure
}