Build config files with Go text/template (#18148)

Rather than relying on file concatenation followed by Go text/template we'll just
use Go text/template. This makes is easier to customize a template without having
to modify libbeat since you can just overwrite a template section (like inject your own
processors section).

This works by loading all of the templates from `libbeat/_meta/config/*.tmpl` followed
by templates from `$mybeat/_meta/config/*.tmpl`. Files loaded last take precedence
(it uses go text/template ParseGlob). Finally each template is created from
`libbeat/_meta/config/default{., reference, docker}.yml.tmpl` (which can be changed
if needed, see elastic/agent).

CHANGELOG-developer.next.asciidoc

@@ -41,6 +41,11 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Rename `queue.BufferConfig.Events` to `queue.BufferConfig.MaxEvents`. {pull}17622[17622]
 - Remove `queue.Feature` and replace `queue.RegisterType` with `queue.RegisterQueueType`. {pull}17666[17666]
 - Introduce APM libbeat instrumentation. `Publish` method on `Client` interface now takes a Context as first argument. {pull}17938[17938]
+- The way configuration files are generated has changed to make it easier to customize parts
+  of the config without requiring changes to libbeat config templates. Generation is now
+  fully based on Go text/template and no longer uses file concatenation to generate the config.
+  Your magefile.go will require a change to adapt the devtool API. See the pull request for
+  more details. {pull}18148[18148]
 
 ==== Bugfixes
 

auditbeat/_meta/config/auditbeat.config.modules.yml.tmpl

@@ -0,0 +1,18 @@
+{{header "Config Reloading"}}
+
+# Config reloading allows to dynamically load modules. Each file which is
+# monitored must contain one or multiple modules as a list.
+auditbeat.config.modules:
+
+  # Glob pattern for configuration reloading
+  path: ${path.config}/modules.d/*.yml
+
+  # Period on which files under path should be checked for changes
+  reload.period: 10s
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+# Maximum amount of time to randomly delay the start of a dataset. Use 0 to
+# disable startup delay.
+auditbeat.max_start_delay: 10s

auditbeat/_meta/config/auditbeat.modules.yml.tmpl

@@ -0,0 +1,2 @@
+{{header "Modules configuration"}}
+auditbeat.modules:

auditbeat/_meta/config/beat.reference.yml.tmpl

@@ -0,0 +1,4 @@
+{{template "header.reference.yml.tmpl" .}}
+{{template "auditbeat.config.modules.yml.tmpl" .}}
+{{template "auditbeat.modules.yml.tmpl" .}}
+{{template "config.modules.yml.tmpl" .}}

auditbeat/_meta/config/beat.yml.tmpl

@@ -0,0 +1,4 @@
+{{template "header.yml.tmpl" .}}
+{{template "auditbeat.modules.yml.tmpl" .}}
+{{template "config.modules.yml.tmpl" .}}
+{{template "setup.template.yml.tmpl" .}}

auditbeat/_meta/config/header.reference.yml.tmpl

@@ -0,0 +1,8 @@
+########################## Auditbeat Configuration #############################
+
+# This is a reference configuration file documenting all non-deprecated options
+# in comments. For a shorter configuration example that contains only the most
+# common options, please see auditbeat.yml in the same directory.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/auditbeat/index.html

auditbeat/_meta/common.p1.yml → auditbeat/_meta/config/header.yml.tmpl

@@ -6,7 +6,3 @@
 #
 # You can find the full configuration reference here:
 # https://www.elastic.co/guide/en/beats/auditbeat/index.html
-
-#==========================  Modules configuration =============================
-auditbeat.modules:
-

auditbeat/_meta/config/setup.template.yml.tmpl

@@ -0,0 +1,5 @@
+{{header "Elasticsearch template setting"}}
+setup.template.settings:
+  index.number_of_shards: 1
+  #index.codec: best_compression
+  #_source.enabled: false

auditbeat/auditbeat.docker.yml

@@ -12,6 +12,7 @@ auditbeat.modules:
     - /sbin
     - /usr/sbin
     - /etc
+
 processors:
 - add_cloud_metadata: ~
 - add_docker_metadata: ~

auditbeat/auditbeat.reference.yml

@@ -7,7 +7,7 @@
 # You can find the full configuration reference here:
 # https://www.elastic.co/guide/en/beats/auditbeat/index.html
 
-#============================  Config Reloading ================================
+# ============================== Config Reloading ==============================
 
 # Config reloading allows to dynamically load modules. Each file which is
 # monitored must contain one or multiple modules as a list.
@@ -26,7 +26,7 @@ auditbeat.config.modules:
 # disable startup delay.
 auditbeat.max_start_delay: 10s
 
-#==========================  Modules configuration =============================
+# =========================== Modules configuration ============================
 auditbeat.modules:
 
 # The auditd module collects events from the audit framework in the Linux
@@ -118,7 +118,8 @@ auditbeat.modules:
   #keep_null: false
 
 
-#================================ General ======================================
+
+# ================================== General ===================================
 
 # The name of the shipper that publishes the network data. It can be used to group
 # all the transactions sent by a single shipper in the web interface.
@@ -226,7 +227,7 @@ auditbeat.modules:
 # default is the number of logical CPUs available in the system.
 #max_procs:
 
-#================================ Processors ===================================
+# ================================= Processors =================================
 
 # Processors are used to reduce the number of fields in the exported event or to
 # enhance the event with external metadata. This section defines a list of
@@ -389,7 +390,7 @@ auditbeat.modules:
 #      ignore_missing: false
 #      fail_on_error: true
 
-#============================= Elastic Cloud ==================================
+# =============================== Elastic Cloud ================================
 
 # These settings simplify using Auditbeat with the Elastic Cloud (https://cloud.elastic.co/).
 
@@ -402,11 +403,11 @@ auditbeat.modules:
 # `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
 #cloud.auth:
 
-#================================ Outputs ======================================
+# ================================== Outputs ===================================
 
 # Configure what output to use when sending the data collected by the beat.
 
-#-------------------------- Elasticsearch output -------------------------------
+# ---------------------------- Elasticsearch Output ----------------------------
 output.elasticsearch:
   # Boolean flag to enable or disable the output module.
   #enabled: true
@@ -547,7 +548,7 @@ output.elasticsearch:
   # Kerberos realm.
   #kerberos.realm: ELASTIC
 
-#----------------------------- Logstash output ---------------------------------
+# ------------------------------ Logstash Output -------------------------------
 #output.logstash:
   # Boolean flag to enable or disable the output module.
   #enabled: true
@@ -661,7 +662,7 @@ output.elasticsearch:
   # timing out. The default is 30s.
   #timeout: 30s
 
-#------------------------------- Kafka output ----------------------------------
+# -------------------------------- Kafka Output --------------------------------
 #output.kafka:
   # Boolean flag to enable or disable the output module.
   #enabled: true
@@ -840,7 +841,7 @@ output.elasticsearch:
   # Kerberos realm.
   #kerberos.realm: ELASTIC
 
-#------------------------------- Redis output ----------------------------------
+# -------------------------------- Redis Output --------------------------------
 #output.redis:
   # Boolean flag to enable or disable the output module.
   #enabled: true
@@ -958,7 +959,7 @@ output.elasticsearch:
   # never, once, and freely. Default is never.
   #ssl.renegotiation: never
 
-#------------------------------- File output -----------------------------------
+# -------------------------------- File Output ---------------------------------
 #output.file:
   # Boolean flag to enable or disable the output module.
   #enabled: true
@@ -992,7 +993,7 @@ output.elasticsearch:
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
 
-#----------------------------- Console output ---------------------------------
+# ------------------------------- Console Output -------------------------------
 #output.console:
   # Boolean flag to enable or disable the output module.
   #enabled: true
@@ -1005,7 +1006,7 @@ output.elasticsearch:
     # Configure escaping HTML symbols in strings.
     #escape_html: false
 
-#================================= Paths ======================================
+# =================================== Paths ====================================
 
 # The home path for the Auditbeat installation. This is the default base path
 # for all other path settings and for miscellaneous files that come with the
@@ -1031,11 +1032,13 @@ output.elasticsearch:
 # the default for the logs path is a logs subdirectory inside the home path.
 #path.logs: ${path.home}/logs
 
-#================================ Keystore ==========================================
+# ================================== Keystore ==================================
+
 # Location of the Keystore containing the keys and their sensitive values.
 #keystore.path: "${path.config}/beats.keystore"
 
-#============================== Dashboards =====================================
+# ================================= Dashboards =================================
+
 # These settings control loading the sample dashboards to the Kibana index. Loading
 # the dashboards are disabled by default and can be enabled either by setting the
 # options here, or by using the `-setup` CLI flag or the `setup` command.
@@ -1079,8 +1082,7 @@ output.elasticsearch:
 # Maximum number of retries before exiting with an error, 0 for unlimited retrying.
 #setup.dashboards.retry.maximum: 0
 
-
-#============================== Template =====================================
+# ================================== Template ==================================
 
 # A template is used to set the mapping in Elasticsearch
 # By default template loading is enabled and the template is loaded.
@@ -1134,7 +1136,7 @@ setup.template.settings:
   #_source:
     #enabled: false
 
-#============================== Setup ILM =====================================
+# ====================== Index Lifecycle Management (ILM) ======================
 
 # Configure index lifecycle management (ILM). These settings create a write
 # alias and add additional settings to the index template. When ILM is enabled,
@@ -1148,13 +1150,13 @@ setup.template.settings:
 
 # Set the prefix used in the index lifecycle write alias name. The default alias
 # name is 'auditbeat-%{[agent.version]}'.
-#setup.ilm.rollover_alias: "auditbeat"
+#setup.ilm.rollover_alias: 'auditbeat'
 
 # Set the rollover index pattern. The default is "%{now/d}-000001".
 #setup.ilm.pattern: "{now/d}-000001"
 
 # Set the lifecycle policy name. The default policy name is
-# 'auditbeat'.
+# 'beatname'.
 #setup.ilm.policy_name: "mypolicy"
 
 # The path to a JSON file that contains a lifecycle policy configuration. Used
@@ -1169,7 +1171,7 @@ setup.template.settings:
 # Overwrite the lifecycle policy at startup. The default is false.
 #setup.ilm.overwrite: false
 
-#============================== Kibana =====================================
+# =================================== Kibana ===================================
 
 # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
 # This requires a Kibana endpoint configuration.
@@ -1224,9 +1226,8 @@ setup.kibana:
   # Configure curve types for ECDHE-based cipher suites
   #ssl.curve_types: []
 
+# ================================== Logging ===================================
 
-
-#================================ Logging ======================================
 # There are four options for the log output: file, stderr, syslog, eventlog
 # The file output is the default.
 
@@ -1293,8 +1294,7 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
-
-#============================== X-Pack Monitoring ===============================
+# ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The
 # reporting is disabled by default.
@@ -1436,7 +1436,8 @@ logging.files:
 # and `monitoring.elasticsearch.password` settings. The format is `<user>:<pass>`.
 #monitoring.cloud.auth:
 
-#================================ HTTP Endpoint ======================================
+# =============================== HTTP Endpoint ================================
+
 # Each beat can expose internal metrics through a HTTP endpoint. For security
 # reasons the endpoint is disabled by default. This feature is currently experimental.
 # Stats can be access through http://localhost:5066/stats . For pretty JSON output
@@ -1460,12 +1461,13 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
-#============================= Process Security ================================
+# ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
 #seccomp.enabled: true
 
-#================================= Migration ==================================
+# ================================= Migration ==================================
 
 # This allows to enable 6.7 migration aliases
 #migration.6_to_7.enabled: false
+