Implement index template generation in Golang (#3603)

* Implement index template generation in Golang

The index template generation is moved to golang.

Changes
* Use `go run` to execute command
* Template generation is part of libbeat, only execution command is outside as it as has some additional logic on how to read the files.
* Add assumption that $ES_BEATS is always a relative path
* Add support for more fine grained versioning in template generation
* Remove python script as not needed anymore

* implement review feedback

dev-tools/cmd/index_template/index_template.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/elastic/beats/libbeat/beat"
+	"github.com/elastic/beats/libbeat/template"
+)
+
+// main generates index templates for the beats
+func main() {
+
+	beatName := flag.String("beat.name", "", ": Base index name. Normally {beat_name} (required)")
+	output := flag.String("output", "", "Required: Full path to the output file (required)")
+	version := flag.String("es.version", beat.GetDefaultVersion(), "Elasticsearch version")
+
+	flag.Parse()
+
+	var existingFiles []string
+	files := flag.Args()
+
+	if len(files) == 0 {
+		fmt.Fprintf(os.Stderr, "No fields.yml files provided. At least one file must be added.")
+		os.Exit(1)
+	}
+
+	if *beatName == "" {
+		fmt.Fprintf(os.Stderr, "beat.name is empty. It must be set.")
+		os.Exit(1)
+	}
+
+	// Skip some of the passed files, as not all beats have the same files
+	for _, f := range files {
+		if _, err := os.Stat(f); err != nil {
+			fmt.Printf("Skipping file because it does not exist: %s", f)
+			continue
+		}
+		existingFiles = append(existingFiles, f)
+	}
+
+	// Make it compatible with the sem versioning
+	if *version == "2x" {
+		*version = "2.0.0"
+	}
+
+	templateString, err := template.GetTemplate(*version, *beatName, existingFiles)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error generating template: %+v", err)
+		os.Exit(1)
+	}
+
+	err = ioutil.WriteFile(*output, []byte(templateString.StringToPrint()), 0644)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error writing output: %s", err)
+		os.Exit(1)
+	}
+}

filebeat/filebeat.template.json

@@ -455,7 +455,13 @@
   },
   "order": 0,
   "settings": {
-    "index.mapping.total_fields.limit": 10000,
+    "index": {
+      "mapping": {
+        "total_fields": {
+          "limit": 10000
+        }
+      }
+    },
     "index.refresh_interval": "5s"
   },
   "template": "filebeat-6.0.0-alpha1-*"

heartbeat/heartbeat.template.json

@@ -185,7 +185,13 @@
   },
   "order": 0,
   "settings": {
-    "index.mapping.total_fields.limit": 10000,
+    "index": {
+      "mapping": {
+        "total_fields": {
+          "limit": 10000
+        }
+      }
+    },
     "index.refresh_interval": "5s"
   },
   "template": "heartbeat-6.0.0-alpha1-*"

libbeat/scripts/Makefile

@@ -8,7 +8,7 @@ BEAT_DOC_URL?=https://www.elastic.co/guide/en/beats/${BEAT_NAME}/current/index.h
 BEAT_LICENSE?=ASL 2.0 ## @packaging Software license of the application
 BEAT_VENDOR?=Elastic ## @packaging Name of the vendor of the application
 BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
-ES_BEATS?=..## @community_beat Must be set to ./vendor/github.com/elastic/beats
+ES_BEATS?=..## @community_beat Must be set to ./vendor/github.com/elastic/beats. It must always be a relative path.
 GOPACKAGES?=${BEAT_PATH}/...## @community_beat Must be set to $(shell glide novendor)
 PACKER_TEMPLATES_DIR?=${ES_BEATS}/dev-tools/packer ## @Building Directory of templates that are used by "make package"
 NOTICE_FILE?=../NOTICE
@@ -249,8 +249,9 @@ update: python-env collect
 	. ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_fields_docs.py $(PWD) ${BEAT_NAME} ${ES_BEATS}
 
 	# Generate index templates
-	. ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_template.py $(PWD) ${BEAT_NAME} ${ES_BEATS}
-	. ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_template.py --es2x $(PWD) ${BEAT_NAME} ${ES_BEATS}
+	go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -beat.name ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template.json ${PWD}/${ES_BEATS}/libbeat/_meta/fields.generated.yml ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.yml ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.generated.yml
+
+	go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -es.version 2.0.0 -beat.name ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template-es2x.json ${PWD}/${ES_BEATS}/libbeat/_meta/fields.generated.yml ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.yml ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.generated.yml
 
 	# Generate index-pattern
 	echo "Generate index pattern"