.github/workflows/pdk-test-unit.yml

@@ -0,0 +1,22 @@
+name: Run pdk test unit
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Clone repository
+      uses: actions/checkout@v3
+
+    - name: Run unit tests
+      uses: puppets-epic-show-theatre/action-pdk-test-unit@v1
+      with:
+        puppet-version: ""
+        # [optional]
+        # A string indicating the Puppet version to validate against, such as "5.4.2" or "5.5".
+        pe-version: ""
+        # [optional]
+        # A string indicating the PE version to validate against, such as "2017.3.5" or "2018.1".

.github/workflows/pdk-validate.yml

@@ -0,0 +1,22 @@
+name: Run pdk static validation
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Clone repository
+      uses: actions/checkout@v3
+
+    - name: Run pdk validate
+      uses: puppets-epic-show-theatre/action-pdk-validate@v1
+      with:
+        puppet-version: ""
+        # [optional]
+        # A string indicating the Puppet version to validate against, such as "5.4.2" or "5.5".
+        pe-version: ""
+        # [optional]
+        # A string indicating the PE version to validate against, such as "2017.3.5" or "2018.1".

CHANGELOG.md

@@ -4,10 +4,18 @@ Changelog
 ## Unreleased
 [Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.13.0...HEAD)
 
+## [v4.14.0](https://github.com/pcfens/puppet-filebeat/tree/v4.14.0)
+[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.13.0...v4.14.0)
+
+- Template SSL Correction [\#322](https://github.com/pcfens/puppet-filebeat/pull/322)
+- Avoid deprecated has_ssl method [\#324](https://github.com/pcfens/puppet-filebeat/pull/324)
+- Add missing options to JSON decoding config [\#326](https://github.com/pcfens/puppet-filebeat/pull/326)
+
 ## [v4.13.0](https://github.com/pcfens/puppet-filebeat/tree/v4.13.0)
 [Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.12.0...v4.13.0)
 
-- Support the filebeat version 8 validation[\#318]- Support queue configuration [\#313](https://github.com/pcfens/puppet-filebeat/pull/318)
+- Support the filebeat version 8 validation[\#318](https://github.com/pcfens/puppet-filebeat/pull/318)
+- Support queue configuration [\#313](https://github.com/pcfens/puppet-filebeat/pull/313)
 - Support Windows 2016 [\#316](https://github.com/pcfens/puppet-filebeat/pull/316)
 - Support queue configuration [\#313](https://github.com/pcfens/puppet-filebeat/pull/313)
 - Support file exclusion with filestream [\#311](https://github.com/pcfens/puppet-filebeat/pull/311)

README.md

@@ -17,6 +17,7 @@
       - [Multiline Logs](#multiline-logs)
       - [JSON Logs](#json-logs)
     - [Inputs in Hiera](#inputs-in-hiera)
+    - [Usage of filebeat modules](#usage-of-filebeat-modules)
     - [Usage on Windows](#usage-on-windows)
     - [Processors](#processors)
       - [Processors in Hiera](#processors-in-hiera)
@@ -34,6 +35,7 @@
       - [Class: `filebeat::install::windows`](#class-filebeatinstallwindows)
     - [Public Defines](#public-defines)
       - [Define: `filebeat::input`](#define-filebeatinput)
+      - [Define: `filebeat::module`](#define-filebeatmodule)
   - [Limitations](#limitations)
     - [Generic template](#generic-template)
     - [Debian Systems](#debian-systems)
@@ -143,8 +145,11 @@ as documented in the filebeat [configuration documentation](https://www.elastic.
 #### JSON Logs
 
 Filebeat inputs (versions >= 5.0) can natively decode JSON objects if they are stored one per line. The `json`
-parameter accepts a hash containing `message_key`, `keys_under_root`, `overwrite_keys`, and `add_error_key`
-as documented in the filebeat [configuration documentation](https://www.elastic.co/guide/en/beats/filebeat/5.5/configuration-filebeat-options.html#config-json).
+parameter accepts a hash containing `message_key`, `keys_under_root`, `overwrite_keys`, and `add_error_key`.
+
+Depending on the version, `expand_keys`, `document_id` and `ignore_decoding_error` may be supported as well.
+
+See the filebeat [configuration documentation](https://www.elastic.co/guide/en/beats/filebeat/7.11/filebeat-input-log.html#filebeat-input-log-config-json) for details.
 
 ### Inputs in Hiera
 
@@ -156,6 +161,35 @@ flag.
 `inputs` can be a Hash that will follow all the parameters listed on this documentation or an
 Array that will output as is to the input config file.
 
+### Usage of filebeat modules
+
+Filebeat ships with modules which contain pipelines and dashboards for common software. Filebeat needs to be setup to ship directly into elasticsearch that
+it's possible that filebeat will setup pipelines and dashboards automatically.
+
+If your setup includes logstash or some other service between filebeat and elasticsearch the following settings might not work as expected.
+
+The following should be a minimal example to get `filebeat::module::*` to create the required config and push pipeline and dashboards into your elasticsearch & kibana.
+
+```puppet
+class { 'filebeat::module::system':
+  syslog_enabled => true,
+  auth_enabled => true,
+}
+
+class { 'filebeat':
+  enable_conf_modules => true,
+  overwrite_pipelines => true,
+  setup => {
+    dashboards => {
+      enabled => true
+    },
+    kibana => {
+      host => 'http://kibana.example.com:5601',
+    }
+  }
+}
+```
+
 ### Usage on Windows
 
 When installing on Windows, this module will download the windows version of Filebeat from
@@ -245,19 +279,43 @@ filebeat::setup:
 ```
 
 ## Reference
- - [**Public Classes**](#public-classes)
-    - [Class: filebeat](#class-filebeat)
- - [**Private Classes**](#private-classes)
-    - [Class: filebeat::config](#class-filebeatconfig)
-    - [Class: filebeat::install](#class-filebeatinstall)
-    - [Class: filebeat::params](#class-filebeatparams)
-    - [Class: filebeat::repo](#class-filebeatrepo)
-    - [Class: filebeat::service](#class-filebeatservice)
-    - [Class: filebeat::install::linux](#class-filebeatinstalllinux)
-    - [Class: filebeat::install::windows](#class-filebeatinstallwindows)
- - [**Public Defines**](#public-defines)
-    - [Define: filebeat::input](#define-filebeatinput)
-    - [Define: filebeat::processors](#define-filebeatprocessor)
+- [puppet-filebeat](#puppet-filebeat)
+      - [Table of Contents](#table-of-contents)
+  - [Description](#description)
+  - [Setup](#setup)
+    - [What filebeat affects](#what-filebeat-affects)
+    - [Upgrading to Filebeat 7.x](#upgrading-to-filebeat-7x)
+    - [Setup Requirements](#setup-requirements)
+    - [Beginning with filebeat](#beginning-with-filebeat)
+  - [Usage](#usage)
+    - [Adding an Input](#adding-an-input)
+      - [Multiline Logs](#multiline-logs)
+      - [JSON Logs](#json-logs)
+    - [Inputs in Hiera](#inputs-in-hiera)
+    - [Usage on Windows](#usage-on-windows)
+    - [Processors](#processors)
+      - [Processors in Hiera](#processors-in-hiera)
+    - [Index Lifecycle Management](#index-lifecycle-management)
+  - [Reference](#reference)
+    - [Public Classes](#public-classes)
+      - [Class: `filebeat`](#class-filebeat)
+    - [Private Classes](#private-classes)
+      - [Class: `filebeat::config`](#class-filebeatconfig)
+      - [Class: `filebeat::install`](#class-filebeatinstall)
+      - [Class: `filebeat::params`](#class-filebeatparams)
+      - [Class: `filebeat::repo`](#class-filebeatrepo)
+      - [Class: `filebeat::service`](#class-filebeatservice)
+      - [Class: `filebeat::install::linux`](#class-filebeatinstalllinux)
+      - [Class: `filebeat::install::windows`](#class-filebeatinstallwindows)
+    - [Public Defines](#public-defines)
+      - [Define: `filebeat::input`](#define-filebeatinput)
+      - [Define: `filebeat::module`](#define-filebeatmodule)
+  - [Limitations](#limitations)
+    - [Generic template](#generic-template)
+    - [Debian Systems](#debian-systems)
+    - [Using config\_file](#using-config_file)
+    - [Logging on systems with Systemd and with version filebeat 7.0+ installed](#logging-on-systems-with-systemd-and-with-version-filebeat-70-installed)
+  - [Development](#development)
 
 ### Public Classes
 
@@ -294,6 +352,7 @@ Installs and configures filebeat.
 - `enable_conf_modules`: [Boolean] Should filebeat.config.modules be enabled
 - `modules_dir`: [String] The directory where module configurations should be defined (default: /etc/filebeat/modules.d)
 - `cloud`: [Hash] Will be converted to YAML for the optional cloud.id and cloud.auth of the configuration (see documentation, and above)
+- `features`: [Hash] Will be converted to YAML for the optional features section of the configuration (see documentation, and above)
 - `queue`: [Hash] Will be converted to YAML for the optional queue.mem and queue.disk of the configuration (see documentation, and above)
 - `outputs`: [Hash] Will be converted to YAML for the required outputs section of the configuration (see documentation, and above)
 - `shipper`: [Hash] Will be converted to YAML to create the optional shipper section of the filebeat config (see documentation)
@@ -371,7 +430,8 @@ to fully understand what these parameters do.
   - `syslog_host`: [String] Host to listen for syslog messages (default: localhost:5140)
   - `exclude_files`: [Array] Files that match any regex in the list are excluded from filebeat (default: [])
   - `encoding`: [String] The file encoding. (default: plain)
-  - `input_type`: [String] where filebeat reads the log from (default:log)
+  - `input_type`: [String] where filebeat reads the log from (default: filestream)
+  - `take_over` : [Boolean] Optionally enable [`take_over`](https://www.elastic.co/guide/en/beats/filebeat/8.11/filebeat-input-filestream.html#filebeat-input-filestream-take-over) when switchting from the deprecated input type `log` to the new input type `filestream`. This avoids re-ingesting already logfiles Filebeat already read when switching to `filestream`. This feature requires Filebeat 8.x.
   - `fields`: [Hash] Optional fields to add information to the output (default: {})
   - `fields_under_root`: [Boolean] Should the `fields` parameter fields be stored at the top level of indexed documents.
   - `ignore_older`: [String] Files older than this field will be ignored by filebeat (default: ignore nothing)
@@ -405,6 +465,15 @@ to fully understand what these parameters do.
   - `include_matches`: [Array] Journald input only, A collection of filter expressions used to match fields. The format of the expression is field=value (default: [])
   - `seek`: [Enum] Journald input only, The position to start reading the journal from (default: undef)
   - `index`: [String] If present, this formatted string overrides the index for events from this input (for elasticsearch outputs), or sets the raw_index field of the event’s metadata (for other outputs) (default: undef)
+  - `publisher_pipeline_disable_host`: [Boolean] This disables the "host.name" attribute being added to events. See [filebeat input configuration reference](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-log.html#_publisher_pipeline_disable_host_13) (default: false)
+
+#### Define: `filebeat::module`
+
+Base resource used to implement filebeat module support in this puppet module and can be useful if you have custom filebeat modules.
+
+**Parameters for `filebeat::module`**
+  - `ensure`: The ensure parameter on the module configuration file. (default: present)
+  - `config`: [Hash] Full hash representation of the module configuration
 
 ## Limitations
 This module doesn't load the [elasticsearch index template](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-getting-started.html#filebeat-template) into elasticsearch (required when shipping

lib/facter/filebeat_version.rb

@@ -3,29 +3,29 @@
   confine 'kernel' => ['FreeBSD', 'OpenBSD', 'Linux', 'Windows', 'SunOS']
   if File.executable?('/usr/bin/filebeat')
     filebeat_version = Facter::Util::Resolution.exec('/usr/bin/filebeat version')
-    if filebeat_version.empty?
+    if filebeat_version.empty? or filebeat_version.include? 'error'
       filebeat_version = Facter::Util::Resolution.exec('/usr/bin/filebeat --version')
     end
   elsif File.executable?('/usr/local/bin/filebeat')
     filebeat_version = Facter::Util::Resolution.exec('/usr/local/bin/filebeat version')
-    if filebeat_version.empty?
+    if filebeat_version.empty? or filebeat_version.include? 'error'
       filebeat_version = Facter::Util::Resolution.exec('/usr/local/bin/filebeat --version')
     end
   elsif File.executable?('/opt/local/bin/filebeat')
     filebeat_version = Facter::Util::Resolution.exec('/opt/local/bin/filebeat version')
-    if filebeat_version.empty?
+    if filebeat_version.empty? or filebeat_version.include? 'error'
       filebeat_version = Facter::Util::Resolution.exec('/opt/local/bin/filebeat --version')
     end
   elsif File.executable?('/usr/share/filebeat/bin/filebeat')
     filebeat_version = Facter::Util::Resolution.exec('/usr/share/filebeat/bin/filebeat --version')
   elsif File.executable?('/usr/local/sbin/filebeat')
     filebeat_version = Facter::Util::Resolution.exec('/usr/local/sbin/filebeat version')
-    if filebeat_version.empty?
+    if filebeat_version.empty? or filebeat_version.include? 'error'
       filebeat_version = Facter::Util::Resolution.exec('/usr/local/sbin/filebeat --version')
     end
   elsif File.exist?('c:\Program Files\Filebeat\filebeat.exe')
     filebeat_version = Facter::Util::Resolution.exec('"c:\Program Files\Filebeat\filebeat.exe" version')
-    if filebeat_version.empty?
+    if filebeat_version.empty? or filebeat_version.include? 'error'
       filebeat_version = Facter::Util::Resolution.exec('"c:\Program Files\Filebeat\filebeat.exe" --version')
     end
   end

manifests/config.pp

@@ -6,7 +6,7 @@
 class filebeat::config {
   $major_version = $filebeat::major_version
 
-  if has_key($filebeat::setup, 'ilm.policy') {
+  if 'ilm.policy' in $filebeat::setup {
     file { "${filebeat::config_dir}/ilm_policy.json":
       content => to_json({ 'policy' => $filebeat::setup['ilm.policy'] }),
       notify  => Service['filebeat'],
@@ -56,10 +56,17 @@
     })
     # Add the 'xpack' section if supported (version >= 6.1.0) and not undef
     if $filebeat::xpack and versioncmp($filebeat::package_ensure, '6.1.0') >= 0 {
-      $filebeat_config = deep_merge($filebeat_config_temp, { 'xpack' => $filebeat::xpack })
+      $filebeat_config_xpack = deep_merge($filebeat_config_temp, { 'xpack' => $filebeat::xpack })
     }
     else {
-      $filebeat_config = $filebeat_config_temp
+      $filebeat_config_xpack = $filebeat_config_temp
+    }
+    # Add the 'features' section if supported (version >= 8.7.0) and not undef
+    if $filebeat::features and versioncmp($filebeat::package_ensure, '8.7.0') >= 0 {
+      $filebeat_config = deep_merge($filebeat_config_xpack, { 'features' => $filebeat::features })
+    }
+    else {
+      $filebeat_config = $filebeat_config_xpack
     }
   } else {
     $filebeat_config_temp = delete_undef_values({
@@ -136,6 +143,19 @@
         force   => true,
         notify  => Service['filebeat'],
       }
+
+      file { 'filebeat-modules-dir':
+        ensure  => $filebeat::directory_ensure,
+        path    => $filebeat::modules_dir,
+        owner   => $filebeat::config_dir_owner,
+        group   => $filebeat::config_dir_group,
+        mode    => $filebeat::config_dir_mode,
+        recurse => $filebeat::purge_conf_dir,
+        purge   => $filebeat::purge_conf_dir,
+        force   => true,
+        notify  => Service['filebeat'],
+        require => File['filebeat-config-dir'],
+      }
     } # end Linux
 
     'SunOS'   : {
@@ -166,6 +186,18 @@
         purge   => $filebeat::purge_conf_dir,
         force   => true,
       }
+
+      file { 'filebeat-modules-dir':
+        ensure  => $filebeat::directory_ensure,
+        path    => $filebeat::modules_dir,
+        owner   => $filebeat::config_dir_owner,
+        group   => $filebeat::config_dir_group,
+        mode    => $filebeat::config_dir_mode,
+        recurse => $filebeat::purge_conf_dir,
+        purge   => $filebeat::purge_conf_dir,
+        force   => true,
+        require => File['filebeat-config-dir'],
+      }
     } # end SunOS
 
     'FreeBSD'   : {
@@ -200,6 +232,19 @@
         force   => true,
         notify  => Service['filebeat'],
       }
+
+      file { 'filebeat-modules-dir':
+        ensure  => $filebeat::directory_ensure,
+        path    => $filebeat::modules_dir,
+        owner   => $filebeat::config_dir_owner,
+        group   => $filebeat::config_dir_group,
+        mode    => $filebeat::config_dir_mode,
+        recurse => $filebeat::purge_conf_dir,
+        purge   => $filebeat::purge_conf_dir,
+        force   => true,
+        notify  => Service['filebeat'],
+        require => File['filebeat-config-dir'],
+      }
     } # end FreeBSD
 
     'OpenBSD'   : {
@@ -234,6 +279,19 @@
         force   => true,
         notify  => Service['filebeat'],
       }
+
+      file { 'filebeat-modules-dir':
+        ensure  => $filebeat::directory_ensure,
+        path    => $filebeat::modules_dir,
+        owner   => $filebeat::config_dir_owner,
+        group   => $filebeat::config_dir_group,
+        mode    => $filebeat::config_dir_mode,
+        recurse => $filebeat::purge_conf_dir,
+        purge   => $filebeat::purge_conf_dir,
+        force   => true,
+        notify  => Service['filebeat'],
+        require => File['filebeat-config-dir'],
+      }
     } # end OpenBSD
 
     'Windows' : {
@@ -264,6 +322,15 @@
         purge   => $filebeat::purge_conf_dir,
         force   => true,
       }
+
+      file { 'filebeat-modules-dir':
+        ensure  => $filebeat::directory_ensure,
+        path    => $filebeat::modules_dir,
+        recurse => $filebeat::purge_conf_dir,
+        purge   => $filebeat::purge_conf_dir,
+        force   => true,
+        require => File['filebeat-config-dir'],
+      }
     } # end Windows
 
     default : {

manifests/init.pp

@@ -39,6 +39,7 @@
 # @param modules_dir [String] The directory where module configurations should be defined (default: /etc/filebeat/modules.d)
 # @param http [Hash] A hash of the http section of configuration
 # @param cloud [Hash] Will be converted to YAML for the optional cloud of the configuration (see documentation, and above)
+# @param features [Hash] Will be converted to YAML to create the optional features section of the filebeat config (see documentation)
 # @param queue [Hash] Will be converted to YAML for the optional queue of the configuration (see documentation, and above)
 # @param outputs [Hash] Will be converted to YAML for the required outputs section of the configuration (see documentation, and above)
 # @param shipper [Hash] Will be converted to YAML to create the optional shipper section of the filebeat config (see documentation)
@@ -91,6 +92,7 @@
   Boolean $enable_conf_modules                                        = $filebeat::params::enable_conf_modules,
   Hash    $http                                                       = $filebeat::params::http,
   Hash    $cloud                                                      = $filebeat::params::cloud,
+  Hash    $features                                                   = $filebeat::params::features,
   Hash    $queue                                                      = $filebeat::params::queue,
   Hash    $outputs                                                    = $filebeat::params::outputs,
   Hash    $shipper                                                    = $filebeat::params::shipper,