Add support for loading the raw json template file

CHANGELOG.asciidoc

@@ -136,6 +136,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Add Indexer indexing by pod uid. Enable pod uid metadata gathering in add_kubernetes_metadata. Extended Matcher log_path matching to support volume mounts {pull}7072[7072]
 - Add Kibana module with log fileset. {pull}7052[7052]
 - Add default_fields to Elasticsearch template when connecting to Elasticsearch >= 7.0. {pull}7015[7015]
+- Add support for loading a template.json file directly instead of using fields.yml. {pull}7039[7039]
 
 *Auditbeat*
 

auditbeat/auditbeat.reference.yml

@@ -854,6 +854,15 @@ output.elasticsearch:
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"
 
+# Enable json template loading. If this is enabled, the fields.yml is ignored.
+#setup.template.json.enabled: false
+
+# Path to the json template file
+#setup.template.json.path: "${path.config}/template.json"
+
+# Name under which the template is stored in Elasticsearch
+#setup.template.json.name: ""
+
 # Overwrite existing template
 #setup.template.overwrite: false
 

filebeat/filebeat.reference.yml

@@ -1420,6 +1420,15 @@ output.elasticsearch:
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"
 
+# Enable json template loading. If this is enabled, the fields.yml is ignored.
+#setup.template.json.enabled: false
+
+# Path to the json template file
+#setup.template.json.path: "${path.config}/template.json"
+
+# Name under which the template is stored in Elasticsearch
+#setup.template.json.name: ""
+
 # Overwrite existing template
 #setup.template.overwrite: false
 

heartbeat/heartbeat.reference.yml

@@ -963,6 +963,15 @@ output.elasticsearch:
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"
 
+# Enable json template loading. If this is enabled, the fields.yml is ignored.
+#setup.template.json.enabled: false
+
+# Path to the json template file
+#setup.template.json.path: "${path.config}/template.json"
+
+# Name under which the template is stored in Elasticsearch
+#setup.template.json.name: ""
+
 # Overwrite existing template
 #setup.template.overwrite: false
 

libbeat/_meta/config.reference.yml

@@ -749,6 +749,15 @@ output.elasticsearch:
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"
 
+# Enable json template loading. If this is enabled, the fields.yml is ignored.
+#setup.template.json.enabled: false
+
+# Path to the json template file
+#setup.template.json.path: "${path.config}/template.json"
+
+# Name under which the template is stored in Elasticsearch
+#setup.template.json.name: ""
+
 # Overwrite existing template
 #setup.template.overwrite: false
 

libbeat/docs/template-config.asciidoc

@@ -86,4 +86,17 @@ ifeval::["{beatname_lc}"!="apm-server"]
 *`setup.template.append_fields`*:: A list of of fields to be added to the template and Kibana index pattern. experimental[]
 
 NOTE: With append_fields only new fields can be added an no existing one overwritten or changed. This is especially useful if data is collected through the http/json metricset where the data structure is not known in advance. Changing the config of append_fields means the template has to be overwritten and only applies to new indices. If there are 2 Beats with different append_fields configs the last one writing the template will win. Any changes will also have an affect on the Kibana Index pattern.
+
+*`setup.template.json.enabled`*:: Set to true to load a json based template file. Specify the path to your Elasticsearch
+index template file and set the name of the template. experimental[]
+
+["source","yaml",subs="attributes"]
+----------------------------------------------------------------------
+setup.template.json.enabled: true
+setup.template.json.path: "template.json"
+setup.template.json.name: "template-name
+----------------------------------------------------------------------
+
+NOTE: If the JSON template is used, the fields.yml is skipped for the template generation.
+
 endif::[]

libbeat/template/config.go

@@ -3,10 +3,15 @@ package template
 import "github.com/elastic/beats/libbeat/common"
 
 type TemplateConfig struct {
-	Enabled      bool             `config:"enabled"`
-	Name         string           `config:"name"`
-	Pattern      string           `config:"pattern"`
-	Fields       string           `config:"fields"`
+	Enabled bool   `config:"enabled"`
+	Name    string `config:"name"`
+	Pattern string `config:"pattern"`
+	Fields  string `config:"fields"`
+	JSON    struct {
+		Enabled bool   `config:"enabled"`
+		Path    string `config:"path"`
+		Name    string `config:"name"`
+	} `config:"json"`
 	AppendFields common.Fields    `config:"append_fields"`
 	Overwrite    bool             `config:"overwrite"`
 	Settings     TemplateSettings `config:"settings"`

libbeat/template/load.go

@@ -1,7 +1,10 @@
 package template
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"os"
 
 	"github.com/elastic/beats/libbeat/beat"
 	"github.com/elastic/beats/libbeat/common"
@@ -51,40 +54,60 @@ func (l *Loader) Load() error {
 		return fmt.Errorf("error creating template instance: %v", err)
 	}
 
+	templateName := tmpl.GetName()
+	if l.config.JSON.Enabled {
+		templateName = l.config.JSON.Name
+	}
 	// Check if template already exist or should be overwritten
-	exists := l.CheckTemplate(tmpl.GetName())
+	exists := l.CheckTemplate(templateName)
 	if !exists || l.config.Overwrite {
 
 		logp.Info("Loading template for Elasticsearch version: %s", l.client.GetVersion())
-
 		if l.config.Overwrite {
 			logp.Info("Existing template will be overwritten, as overwrite is enabled.")
 		}
 
-		var output common.MapStr
+		var template map[string]interface{}
+		if l.config.JSON.Enabled {
+			jsonPath := paths.Resolve(paths.Config, l.config.JSON.Path)
+			if _, err := os.Stat(jsonPath); err != nil {
+				return fmt.Errorf("error checking for json template: %s", err)
+			}
+
+			logp.Info("Loading json template from file %s", jsonPath)
+
+			content, err := ioutil.ReadFile(jsonPath)
+			if err != nil {
+				return fmt.Errorf("error reading file. Path: %s, Error: %s", jsonPath, err)
 
-		// Load fields from path
-		if l.config.Fields != "" {
+			}
+			err = json.Unmarshal(content, &template)
+			if err != nil {
+				return fmt.Errorf("could not unmarshal json template: %s", err)
+			}
+			// Load fields from path
+		} else if l.config.Fields != "" {
 			logp.Debug("template", "Load fields.yml from file: %s", l.config.Fields)
 
 			fieldsPath := paths.Resolve(paths.Config, l.config.Fields)
 
-			output, err = tmpl.LoadFile(fieldsPath)
+			template, err = tmpl.LoadFile(fieldsPath)
 			if err != nil {
 				return fmt.Errorf("error creating template from file %s: %v", fieldsPath, err)
 			}
 		} else {
 			logp.Debug("template", "Load default fields.yml")
-			output, err = tmpl.LoadBytes(l.fields)
+			template, err = tmpl.LoadBytes(l.fields)
 			if err != nil {
 				return fmt.Errorf("error creating template: %v", err)
 			}
 		}
 
-		err = l.LoadTemplate(tmpl.GetName(), output)
+		err = l.LoadTemplate(templateName, template)
 		if err != nil {
-			return fmt.Errorf("could not load template. Elasticsearh returned: %v. Template is: %s", err, output)
+			return fmt.Errorf("could not load template. Elasticsearh returned: %v. Template is: %s", err, template)
 		}
+
 	} else {
 		logp.Info("Template already exists and will not be overwritten.")
 	}

libbeat/tests/files/template.json

@@ -0,0 +1,22 @@
+{
+    "index_patterns": ["bla"],
+    "settings": {
+        "number_of_shards": 1
+    },
+    "mappings": {
+        "example": {
+            "_source": {
+                "enabled": false
+            },
+            "properties": {
+                "host_name": {
+                    "type": "keyword"
+                },
+                "created_at": {
+                    "type": "date",
+                    "format": "EEE MMM dd HH:mm:ss Z YYYY"
+                }
+            }
+        }
+    }
+}

libbeat/tests/system/config/mockbeat.yml.j2

@@ -73,6 +73,12 @@ setup.template:
     index.codec: best_compression
   name: {{ es_template_name }}
   pattern: {{ es_template_pattern }}
+  overwrite: {{ template_overwrite|default("false") }}
+  json:
+    enabled: {{ template_json_enabled|default("false") }}
+    path: {{ template_json_path }}
+    name: {{ template_json_name }}
+
 #================================ Logging =====================================
 
 {% if metrics_period -%}

libbeat/tests/system/test_template.py

@@ -1,4 +1,10 @@
 from base import BaseTest
+import os
+from elasticsearch import Elasticsearch, TransportError
+from nose.plugins.attrib import attr
+import unittest
+
+INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False)
 
 
 class Test(BaseTest):
@@ -72,3 +78,36 @@ def test_index_with_pattern_name(self):
         proc = self.start_beat()
         self.wait_until(lambda: self.log_contains("mockbeat start running."))
         proc.check_kill_and_wait()
+
+    @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
+    @attr('integration')
+    def test_json_template(self):
+        """
+        Test loading of json based template
+        """
+
+        self.copy_files(["template.json"])
+
+        path = os.path.join(self.working_dir, "template.json")
+
+        print path
+        self.render_config_template(
+            elasticsearch={"hosts": self.get_host()},
+            template_overwrite="true",
+            template_json_enabled="true",
+            template_json_path=path,
+            template_json_name="bla",
+        )
+
+        proc = self.start_beat()
+        self.wait_until(lambda: self.log_contains("mockbeat start running."))
+        self.wait_until(lambda: self.log_contains("Loading json template from file"))
+        self.wait_until(lambda: self.log_contains("Elasticsearch template with name 'bla' loaded"))
+        proc.check_kill_and_wait()
+
+        es = Elasticsearch([self.get_elasticsearch_url()])
+        result = es.transport.perform_request('GET', '/_template/bla')
+        assert len(result) == 1
+
+    def get_host(self):
+        return os.getenv('ES_HOST', 'localhost') + ':' + os.getenv('ES_PORT', '9200')

metricbeat/metricbeat.reference.yml

@@ -1363,6 +1363,15 @@ output.elasticsearch:
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"
 
+# Enable json template loading. If this is enabled, the fields.yml is ignored.
+#setup.template.json.enabled: false
+
+# Path to the json template file
+#setup.template.json.path: "${path.config}/template.json"
+
+# Name under which the template is stored in Elasticsearch
+#setup.template.json.name: ""
+
 # Overwrite existing template
 #setup.template.overwrite: false
 

packetbeat/packetbeat.reference.yml

@@ -1226,6 +1226,15 @@ output.elasticsearch:
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"
 
+# Enable json template loading. If this is enabled, the fields.yml is ignored.
+#setup.template.json.enabled: false
+
+# Path to the json template file
+#setup.template.json.path: "${path.config}/template.json"
+
+# Name under which the template is stored in Elasticsearch
+#setup.template.json.name: ""
+
 # Overwrite existing template
 #setup.template.overwrite: false
 

winlogbeat/winlogbeat.reference.yml

@@ -778,6 +778,15 @@ output.elasticsearch:
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"
 
+# Enable json template loading. If this is enabled, the fields.yml is ignored.
+#setup.template.json.enabled: false
+
+# Path to the json template file
+#setup.template.json.path: "${path.config}/template.json"
+
+# Name under which the template is stored in Elasticsearch
+#setup.template.json.name: ""
+
 # Overwrite existing template
 #setup.template.overwrite: false