.gitlab-ci.yml

@@ -10,7 +10,10 @@ cache:
 before_script:
   - bundle -v
   - rm Gemfile.lock || true
-  - gem update --system $RUBYGEMS_VERSION
+  - "# Update system gems if requested. This is useful to temporarily workaround troubles in the test runner"
+  - "# Set `rubygems_version` in the .sync.yml to set a value"
+  - "# Ignore exit code of SIGPIPE'd yes to not fail with shell's pipefail set"
+  - '[ -z "$RUBYGEMS_VERSION" ] || (yes || true) | gem update --system $RUBYGEMS_VERSION'
   - gem --version
   - bundle -v
   - bundle install --without system_tests --path vendor/bundle --jobs $(nproc)

.sync.yml

@@ -0,0 +1,5 @@
+mock_with: ':mocha'
+
+.travis.yml:
+  user: pcfens
+  secure: "z1SbP/Hisr5k66XL/ACLsZ/fG7cCpwl8apjZzt/YciWizwReioU2EkLr5tvXdUC10aIH6H7XBdA9XwPqwXa81cIqcdIHlRMIbosMUGYaXcUm1xhctB3GvEDqsxFqdZSHYXax+IR6Wt507Eop+iU3S5pf/zJcp4uSKQVapCMoeVCEQYLRwllgeaqtEUZwqOUwPk31C4YZxwrzmgbIVyXmPrp3SDToXaQm4S4RkayOqHH2lYi8isz3IPPQvDZY5681TBpo35AbsIRbhiLzGlBHbgRaE2dz7J1Gs8MBGFyrtDaPtc9UpbgEmyxgmaPs3NIeZkmfVoosjt2AHRsoMZB7ntaPAQ20mk44ugMhxd5HX8t7QdLPiYQqgA3O4QfKraxPzdEjYVs9Pf7BBgY4JpGSOAD3dlWNK0U40MzKe74cj6dshg9SfIdyf3M3MmI0KIIvdKhpgl8mSIL8MCWjnYYNpQMQDFgyrXvePnkPVlt7zlBxn+LJFFx3VLGNfSWbKavITM/nrvjpFkQZ34mPHTtTUYnT6HVehtwPd5x6ILqYcppEeeiloa4uLWhW/vg0wAOdOBv2IALdAqRMC56ODPK33gFRkX+CclsegtOh2In407njbXXZBQrY5h3SXuEVxZcFhGVTxJIV29viuWFSm7VF0a7IUmEbVrM23bqeaM+aOgs="

.travis.yml

@@ -5,7 +5,10 @@ cache: bundler
 before_install:
   - bundle -v
   - rm -f Gemfile.lock
-  - gem update --system $RUBYGEMS_VERSION
+  - "# Update system gems if requested. This is useful to temporarily workaround troubles in the test runner"
+  - "# See https://github.com/puppetlabs/pdk-templates/commit/705154d5c437796b821691b707156e1b056d244f for an example of how this was used"
+  - "# Ignore exit code of SIGPIPE'd yes to not fail with shell's pipefail set"
+  - '[ -z "$RUBYGEMS_VERSION" ] || (yes || true) | gem update --system $RUBYGEMS_VERSION'
   - gem --version
   - bundle -v
 script:
@@ -51,4 +54,4 @@ deploy:
   on:
     tags: true
     all_branches: true
-    condition: "$DEPLOY_TO_FORGE = yes"
+    condition: "$DEPLOY_TO_FORGE = yes"

CHANGELOG.md

@@ -4,28 +4,36 @@ Changelog
 ## Unreleased
 [Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.2.0...HEAD)
 
+## [v4.3.0](https://github.com/pcfens/puppet-filebeat/tree/v4.3.0)
+[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.2.0...v4.3.0)
+
+- Support setting index index lifecycle policy [\#238](https://github.com/pcfens/puppet-filebeat/pull/238)
+- Support logging overrides with Puppet older than 6.1 [\#241](https://github.com/pcfens/puppet-filebeat/pull/241)
+- Allow inputs to be defined using just a hash (supports open ended inputs) [\#236](https://github.com/pcfens/puppet-filebeat/pull/236)
+
+
 ## [v4.2.0](https://github.com/pcfens/puppet-filebeat/tree/v4.2.0)
 [Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.1.2...v4.2.0)
 
 - Fix regression of processors type
-- Add support for syslog inputs [#\232](https://github.com/pcfens/puppet-filebeat/pull/233)
+- Add support for syslog inputs [\#232](https://github.com/pcfens/puppet-filebeat/pull/233)
 - Upgrade PDK to 1.15.0
 
 ## [v4.1.2](https://github.com/pcfens/puppet-filebeat/tree/v4.1.2)
 [Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.1.1...v4.1.2)
 
-- Fix hardocded path to ym [#\229](https://github.com/pcfens/puppet-filebeat/pull/229)
+- Fix hardocded path to yum [\#229](https://github.com/pcfens/puppet-filebeat/pull/229)
 
 ## [v4.1.1](https://github.com/pcfens/puppet-filebeat/tree/v4.1.1)
 [Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.1.0...v4.1.1)
 
-- Support old versions of filebeat for a version fact only [#\227](https://github.com/pcfens/puppet-filebeat/pull/227)
-- Fix the processor input data type [#\228](https://github.com/pcfens/puppet-filebeat/issues/228)
+- Support old versions of filebeat for a version fact only [\#227](https://github.com/pcfens/puppet-filebeat/pull/227)
+- Fix the processor input data type [\#228](https://github.com/pcfens/puppet-filebeat/issues/228)
 
 ## [v4.1.0](https://github.com/pcfens/puppet-filebeat/tree/v4.1.0)
 [Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.0.5...v4.1.0)
 
-- Allow for override of the default systemd logging options [#\223](https://github.com/pcfens/puppet-filebeat/pull/223)
+- Allow for override of the default systemd logging options [\#223](https://github.com/pcfens/puppet-filebeat/pull/223)
 
 ## [v4.0.5](https://github.com/pcfens/puppet-filebeat/tree/v4.0.5)
 [Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.0.4...v4.0.5)

Gemfile

@@ -24,10 +24,10 @@ group :development do
   gem "json", '= 2.0.4',                                         require: false if Gem::Requirement.create('~> 2.4.2').satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
   gem "json", '= 2.1.0',                                         require: false if Gem::Requirement.create(['>= 2.5.0', '< 2.7.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
   gem "rb-readline", '= 0.5.5',                                  require: false, platforms: [:mswin, :mingw, :x64_mingw]
-  gem "puppet-module-posix-default-r#{minor_version}", '~> 0.3', require: false, platforms: [:ruby]
-  gem "puppet-module-posix-dev-r#{minor_version}", '~> 0.3',     require: false, platforms: [:ruby]
-  gem "puppet-module-win-default-r#{minor_version}", '~> 0.3',   require: false, platforms: [:mswin, :mingw, :x64_mingw]
-  gem "puppet-module-win-dev-r#{minor_version}", '~> 0.3',       require: false, platforms: [:mswin, :mingw, :x64_mingw]
+  gem "puppet-module-posix-default-r#{minor_version}", '~> 0.4', require: false, platforms: [:ruby]
+  gem "puppet-module-posix-dev-r#{minor_version}", '~> 0.4',     require: false, platforms: [:ruby]
+  gem "puppet-module-win-default-r#{minor_version}", '~> 0.4',   require: false, platforms: [:mswin, :mingw, :x64_mingw]
+  gem "puppet-module-win-dev-r#{minor_version}", '~> 0.4',       require: false, platforms: [:mswin, :mingw, :x64_mingw]
 end
 
 puppet_version = ENV['PUPPET_GEM_VERSION']

README.md

@@ -16,6 +16,7 @@
     - [Inputs in hiera](#inputs-in-hiera)
     - [Usage on Windows](#usage-on-windows)
     - [Processors](#processors)
+    - [Index Lifecycle Management](#index-lifecycle-management)
 4. [Reference](#reference)
     - [Public Classes](#public-classes)
     - [Private Classes](#private-classes)
@@ -137,6 +138,9 @@ input declarations down the hiera hierarchy. That behavior can be changed by con
 [lookup_options](https://docs.puppet.com/puppet/latest/reference/lookup_quick.html#setting-lookupoptions-in-data)
 flag.
 
+`inputs` can be a Hash that will follow all the parameters listed on this documentation or an
+Array that will output as is to the input config file.
+
 ### Usage on Windows
 
 When installing on Windows, this module will download the windows version of Filebeat from
@@ -201,6 +205,22 @@ processor declarations down the hiera hierarchy. That behavior can be changed by
 [lookup_options](https://docs.puppet.com/puppet/latest/reference/lookup_quick.html#setting-lookupoptions-in-data)
 flag.
 
+### Index Lifecycle Management
+
+You can override the default filebeat ILM policy by specifying `ilm.policy` hash in `filebeat::setup` parameter:
+
+```
+filebeat::setup:
+  ilm.policy:
+    phases:
+      hot:
+        min_age: "0ms"
+        actions:
+          rollover:
+            max_size: "10gb"
+            max_age: "1d"
+```
+
 ## Reference
  - [**Public Classes**](#public-classes)
     - [Class: filebeat](#class-filebeat)
@@ -266,7 +286,7 @@ Installs and configures filebeat.
 - `fields_under_root`: [Boolean] If set to true, custom fields are stored in the top level instead of under fields
 - `disable_config_test`: [Boolean] If set to true, configuration tests won't be run on config files before writing them.
 - `processors`: [Hash] Processors that should be configured.
-- `inputs`: [Hash] Inputs that will be created. Commonly used to create inputs using hiera
+- `inputs`: [Hash] or [Array] Inputs that will be created. Commonly used to create inputs using hiera
 - `setup`: [Hash] Setup that will be created. Commonly used to create setup using hiera
 - `xpack`: [Hash] XPack configuration to pass to filebeat
 

Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'puppet_litmus/rake_tasks' if Bundler.rubygems.find_name('puppet_litmus').any?
 require 'puppetlabs_spec_helper/rake_tasks'
 require 'puppet-syntax/tasks/puppet-syntax'

lib/facter/filebeat_version.rb

@@ -7,7 +7,10 @@
       filebeat_version = Facter::Util::Resolution.exec('/usr/bin/filebeat --version')
     end
   elsif File.executable?('/usr/local/bin/filebeat')
-    filebeat_version = Facter::Util::Resolution.exec('/usr/local/bin/filebeat --version')
+    filebeat_version = Facter::Util::Resolution.exec('/usr/local/bin/filebeat version')
+    if filebeat_version.empty?
+      filebeat_version = Facter::Util::Resolution.exec('/usr/local/bin/filebeat --version')
+    end
   elsif File.executable?('/usr/share/filebeat/bin/filebeat')
     filebeat_version = Facter::Util::Resolution.exec('/usr/share/filebeat/bin/filebeat --version')
   elsif File.executable?('/usr/local/sbin/filebeat')

manifests/config.pp

@@ -6,6 +6,17 @@
 class filebeat::config {
   $major_version = $filebeat::major_version
 
+  if has_key($filebeat::setup, 'ilm.policy') {
+    file {"${filebeat::config_dir}/ilm_policy.json":
+      content => to_json({'policy' => $filebeat::setup['ilm.policy']}),
+      notify  => Service['filebeat'],
+      require => File['filebeat-config-dir'],
+    }
+    $setup = $filebeat::setup - 'ilm.policy' + {'ilm.policy_file' => "${filebeat::config_dir}/ilm_policy.json"}
+  } else {
+    $setup = $filebeat::setup
+  }
+
   if versioncmp($major_version, '6') >= 0 {
     $filebeat_config_temp = delete_undef_values({
       'shutdown_timeout'  => $filebeat::shutdown_timeout,
@@ -32,7 +43,7 @@
       'logging'           => $filebeat::logging,
       'runoptions'        => $filebeat::run_options,
       'processors'        => $filebeat::processors,
-      'setup'             => $filebeat::setup,
+      'setup'             => $setup,
     })
     # Add the 'xpack' section if supported (version >= 6.1.0) and not undef
     if $filebeat::xpack and versioncmp($filebeat::package_ensure, '6.1.0') >= 0 {

manifests/init.pp

@@ -42,7 +42,7 @@
 # @param fields [Hash] Optional fields that should be added to each event output
 # @param fields_under_root [Boolean] If set to true, custom fields are stored in the top level instead of under fields
 # @param processors [Array] Processors that will be added. Commonly used to create processors using hiera.
-# @param inputs [Hash] Inputs that will be created. Commonly used to create inputs using hiera
+# @param inputs [Hash] or [Array] Inputs that will be created. Commonly used to create inputs using hiera
 # @param setup [Hash] setup that will be created. Commonly used to create setup using hiera
 # @param inputs_merge [Boolean] Whether $inputs should merge all hiera sources, or use simple automatic parameter lookup
 # proxy_address [String] Proxy server to use for downloading files
@@ -87,7 +87,7 @@
   Boolean $fields_under_root                                          = $filebeat::params::fields_under_root,
   Boolean $disable_config_test                                        = $filebeat::params::disable_config_test,
   Array   $processors                                                 = [],
-  Hash    $inputs                                                     = {},
+  Variant[Hash, Array] $inputs                                        = {},
   Hash    $setup                                                      = {},
   Array   $modules                                                    = [],
   Optional[Variant[Stdlib::HTTPUrl, Stdlib::HTTPSUrl]] $proxy_address = undef, # lint:ignore:140chars
@@ -144,7 +144,11 @@
 
   if $package_ensure != 'absent' {
     if !empty($inputs) {
-      create_resources('filebeat::input', $inputs)
+      if $inputs =~ Array {
+        create_resources('filebeat::input', { 'inputs' => { pure_array => true } })
+      } else {
+        create_resources('filebeat::input', $inputs)
+      }
     }
   }
 }

manifests/input.pp

@@ -48,11 +48,12 @@
   Boolean $symlinks                        = false,
   Optional[String] $pipeline               = undef,
   Array $processors                        = [],
+  Boolean $pure_array                      = false,
 ) {
 
   $input_template = $filebeat::major_version ? {
-    '5'     => 'prospector5.yml.erb',
-    default => 'prospector.yml.erb',
+    '5'     => 'prospector.yml.erb',
+    default => 'input.yml.erb',
   }
 
   if 'filebeat_version' in $facts and $facts['filebeat_version'] != false {

manifests/service.pp

@@ -40,6 +40,26 @@
       }
 
     } else {
+
+      unless $systemd_beat_log_opts_override == undef {
+        $ensure_overide = 'present'
+      } else {
+        $ensure_overide = 'absent'
+      }
+
+      if !defined(File[$filebeat::systemd_override_dir]) {
+        file{$filebeat::systemd_override_dir:
+          ensure => 'directory',
+        }
+      }
+
+      file { "${filebeat::systemd_override_dir}/logging.conf":
+        ensure  => $ensure_overide,
+        content => template($filebeat::systemd_beat_log_opts_template),
+        require => File[$filebeat::systemd_override_dir],
+        notify  => Service['filebeat'],
+      }
+
       unless defined('systemd') {
         warning('You\'ve specified an $systemd_beat_log_opts_override varible on a system running puppet version < 6.1 and not declared "systemd" resource See README.md for more information') # lint:ignore:140chars
       }

metadata.json

@@ -1,6 +1,6 @@
 {
   "name": "pcfens-filebeat",
-  "version": "4.2.0",
+  "version": "4.3.0",
   "author": "pcfens",
   "summary": "A module to install and manage the filebeat log shipper",
   "license": "Apache-2.0",
@@ -106,7 +106,7 @@
     "elasticsearch",
     "elastic"
   ],
-  "pdk-version": "1.15.0",
-  "template-url": "pdk-default#1.15.0",
-  "template-ref": "tags/1.15.0-0-g0bc522e"
+  "pdk-version": "1.16.0",
+  "template-url": "pdk-default#1.16.0",
+  "template-ref": "tags/1.16.0-0-gaf44904"
 }

spec/defines/input_spec.rb

@@ -10,6 +10,21 @@
             ],
           },
         },
+        inputs => [
+          {
+            "type" => "logs",
+            "paths" => [
+              "/var/log/auth.log",
+              "/var/log/syslog",
+            ],
+          },
+          {
+            "type" => "syslog",
+            "protocol.tcp" => {
+              "host" => "0.0.0.0:514",
+            },
+          },
+        ],
       }'
   end
 
@@ -65,6 +80,33 @@
     end
   end
 
+  on_supported_os(facterversion: '2.4').each do |os, os_facts|
+    context "with array input support on #{os}" do
+      let(:facts) { os_facts }
+
+      # Docker Support
+      let(:title) { 'test-array' }
+      let(:params) do
+        {
+          'pure_array' => true,
+        }
+      end
+
+      if os_facts[:kernel] == 'Linux'
+        it { is_expected.to compile }
+
+        it {
+          is_expected.to contain_file('filebeat-test-array').with(
+            notify: 'Service[filebeat]',
+          )
+          is_expected.to contain_file('filebeat-test-array').with_content(
+            %r{- type: logs\n\s{2}paths:\n\s{2}- "/var/log/auth.log"\n\s{2}- "/var/log/syslog"\n- type: syslog\n\s{2}protocol.tcp:\n\s{4}host: 0.0.0.0:514\n},
+          )
+        }
+      end
+    end
+  end
+
   context 'with no parameters' do
     let(:title) { 'test-logs' }
     let(:params) do

spec/spec_helper.rb

@@ -1,3 +1,9 @@
+# frozen_string_literal: true
+
+RSpec.configure do |c|
+  c.mock_with :mocha
+end
+
 require 'puppetlabs_spec_helper/module_spec_helper'
 require 'rspec-puppet-facts'