.travis.yml

@@ -7,9 +7,16 @@ script:
 rvm:
   - 1.9.3
   - 2.0.0
+  - 2.1.8
 env:
   - PUPPET_GEM_VERSION="~> 3.5.1"
   - PUPPET_GEM_VERSION="~> 3.8.6"
   - PUPPET_GEM_VERSION="~> 4.4.0"
+matrix:
+  allow_failures:
+    - rvm: 2.1.8
+      env: PUPPET_GEM_VERSION="~> 3.5.1"
+    - rvm: 2.1.8
+      env: PUPPET_GEM_VERSION="~> 3.8.6"
 notifications:
   email: false

CHANGELOG.md

@@ -2,7 +2,17 @@ Changelog
 =========
 
 ## Unreleased
-[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.5.8...HEAD)
+[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.6.0...HEAD)
+
+## [v0.6.0](https://github.com/pcfens/puppet-filebeat/tree/v0.6.0)
+[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.5.8...v0.6.0)
+
+- Add the `close_older` parameter to support the option of the same name in filebeat 1.2.0
+- Add support for the `publish_async` parameter.
+
+**Fixed Bugs**
+- Added limited, but improved support for Ruby versions pre-1.9.1 by fixing the hash sort issue
+[\#20](https://github.com/pcfens/puppet-filebeat/issues/20)
 
 ## [v0.5.8](https://github.com/pcfens/puppet-filebeat/tree/v0.5.8)
 [Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.5.7...v0.5.8)

README.md

@@ -15,6 +15,7 @@
     - [Private Classes](#private-classes)
     - [Public Defines](#public-defines)
 5. [Limitations - OS compatibility, etc.](#limitations)
+    - [Pre-1.9.1 Ruby](#pre-191-ruby)
 6. [Development - Guide for contributing to the module](#development)
 
 ## Description
@@ -131,8 +132,10 @@ Installs and configures filebeat.
 - `manage_repo`: [Boolean] Whether or not the upstream (elastic) repo should be configured or not (default: true)
 - `service_ensure`: [String] The ensure parameter on the filebeat service (default: running)
 - `service_enable`: [String] The enable parameter on the filebeat service (default: true)
+- `service_provider`: [String] The provier parameter on the filebeat service (default: on RedHat based systems use redhat, otherwise undefined)
 - `spool_size`: [Integer] How large the spool should grow before being flushed to the network (default: 1024)
 - `idle_timeout`: [String] How often the spooler should be flushed even if spool size isn't reached (default: 5s)
+- `publish_async`: [Boolean] If set to true filebeat will publish while preparing the next batch of lines to transmit (defualt: false)
 - `registry_file`: [String] The registry file used to store positions, absolute or relative to working directory (default .filebeat)
 - `config_dir`: [String] The directory where prospectors should be defined (default: /etc/filebeat/conf.d)
 - `config_dir_mode`: [String] The permissions mode set on the configuration directory (default: 0755)
@@ -194,7 +197,10 @@ to fully understand what these parameters do.
   - `input_type`: [String] log or stdin - where filebeat reads the log from (default:log)
   - `fields`: [Hash] Optional fields to add information to the output (default: {})
   - `fields_under_root`: [Boolean] Should the `fields` parameter fields be stored at the top level of indexed documents.
-  - `ignore_older`: [String] Files older than this field will be ignored by filebeat (default: 24h)
+  - `ignore_older`: [String] Files older than this field will be ignored by filebeat (default: 24h in filebeat < 1.2.0, infinite in filebeat >= 1.2.0)
+  - `close_older`: [String] Files that haven't been modified since `close_older`, they'll be closed. New
+  modifications will be read when files are scanned again according to `scan_frequency`. Introduced in
+  filebeat 1.2.0 (default: 1h)
   - `log_type`: [String] \(Deprecated - use `doc_type`\) The document_type setting (optional - default: log)
   - `doc_type`: [String] The event type to used for published lines, used as type field in logstash
     and elasticsearch (optional - default: log)
@@ -218,13 +224,26 @@ to fully understand what these parameters do.
 
 
 ## Limitations
-
 This module doesn't load the [elasticsearch index template](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-getting-started.html#filebeat-template) into elasticsearch (required when shipping
 directly to elasticsearch).
 
 Only filebeat versions after 1.0.0-rc1 are supported. 1.0.0-rc1 and older don't
 support YAML like the ruby template can easily generate.
 
+### Pre-1.9.1 Ruby
+If you're on a system running a Ruby pre-1.9.1, hashes aren't sorted consistently, causing puppet runs to
+not be idempotent. To fix this, a limited template is used if the rubyversion is pre-1.9.1.
+The limited template only supports elasticsearch, logstash, file, and console outputs, and not all options
+may be supported (there is no warning when an option is omitted). Unlike with newer rubies, as new versions
+of filebeat are released, this template may not work until it's updated, even if you're using an updated
+configuration hash.
+
+If you don't care about keeping puppet idempotent, this can be overridden by setting the `conf_template`
+parameter to 'filebeat/filebeat.yml.erb'.
+
+See [templates/filebeat.yml.ruby18.erb](https://github.com/pcfens/puppet-filebeat/blob/master/templates/filebeat.yml.ruby18.erb)
+for the all of the details.
+
 ## Development
 
 Pull requests and bug reports are welcome. If you're sending a pull request, please consider

manifests/config.pp

@@ -4,6 +4,7 @@
       'spool_size'    => $filebeat::spool_size,
       'idle_timeout'  => $filebeat::idle_timeout,
       'registry_file' => $filebeat::registry_file,
+      'publish_async' => $filebeat::publish_async,
       'config_dir'    => $filebeat::config_dir,
     },
     'output'     => $filebeat::outputs,

manifests/init.pp

@@ -18,6 +18,7 @@
 # @param service_enable [String] The enable parameter on the filebeat service (default: true)
 # @param spool_size [Integer] How large the spool should grow before being flushed to the network (default: 1024)
 # @param idle_timeout [String] How often the spooler should be flushed even if spool size isn't reached (default: 5s)
+# @param publish_async [Boolean] If set to true filebeat will publish while preparing the next batch of lines to send (defualt: false)
 # @param registry_file [String] The registry file used to store positions, absolute or relative to working directory (default .filebeat)
 # @param config_dir [String] The directory where prospectors should be defined (default: /etc/filebeat/conf.d)
 # @param config_dir_mode [String] The unix permissions mode set on the configuration directory (default: 0755)
@@ -39,6 +40,7 @@
   $service_provider = $filebeat::params::service_provider,
   $spool_size       = $filebeat::params::spool_size,
   $idle_timeout     = $filebeat::params::idle_timeout,
+  $publish_async    = $filebeat::params::publish_async,
   $registry_file    = $filebeat::params::registry_file,
   $config_dir       = $filebeat::params::config_dir,
   $config_dir_mode  = $filebeat::params::config_dir_mode,

manifests/params.pp

@@ -5,6 +5,7 @@
   $service_enable = true
   $spool_size     = 1024
   $idle_timeout   = '5s'
+  $publish_async  = false
   $registry_file  = '.filebeat'
   $config_dir_mode = '0755'
   $config_file_mode = '0644'
@@ -13,7 +14,12 @@
   $shipper        = {}
   $logging        = {}
   $run_options    = {}
-  $conf_template  = "${module_name}/filebeat.yml.erb"
+
+  if versioncmp('1.9.1', $::rubyversion) > 0 {
+    $conf_template = "${module_name}/filebeat.yml.ruby18.erb"
+  } else {
+    $conf_template = "${module_name}/filebeat.yml.erb"
+  }
 
   case $::kernel {
     'Linux'   : {

manifests/prospector.pp

@@ -6,7 +6,8 @@
   $input_type            = 'log',
   $fields                = {},
   $fields_under_root     = false,
-  $ignore_older          = '24h',
+  $ignore_older          = undef,
+  $close_older           = undef,
   $log_type              = undef,
   $doc_type              = 'log',
   $scan_frequency        = '10s',

metadata.json

@@ -1,6 +1,6 @@
 {
   "name": "pcfens-filebeat",
-  "version": "0.5.8",
+  "version": "0.6.0",
   "author": "pcfens",
   "license": "Apache-2.0",
   "summary": "A module to install and manage the filebeat log shipper",

spec/classes/filebeat_spec.rb

@@ -6,6 +6,7 @@
       :kernel => 'Linux',
       :osfamily => 'Debian',
       :lsbdistid => 'Ubuntu',
+      :rubyversion => '1.9.3',
     }
   end
 
@@ -42,6 +43,7 @@
       {
         :kernel => 'Linux',
         :osfamily => 'RedHat',
+        :rubyversion => '1.8.7',
       }
     end
 
@@ -62,6 +64,7 @@
     let :facts do
       {
         :kernel => 'Windows',
+        :rubyversion => '1.9.3',
       }
     end
 

spec/defines/prospector_spec.rb

@@ -24,6 +24,7 @@
     let :facts do {
       :kernel => 'Linux',
       :osfamily => 'Linux',
+      :rubyversion => '1.9.3',
     }
     end
 
@@ -47,7 +48,6 @@
       encoding: plain
       fields_under_root: false
       input_type: log
-      ignore_older: 24h
       document_type: apache
       scan_frequency: 10s
       harvester_buffer_size: 16384
@@ -98,7 +98,6 @@
       encoding: plain
       fields_under_root: false
       input_type: log
-      ignore_older: 24h
       document_type: java_app
       scan_frequency: 10s
       harvester_buffer_size: 16384
@@ -126,6 +125,7 @@
   context 'On Windows' do
     let :facts do {
       :kernel => 'Windows',
+      :rubyversion => '1.9.3',
     }
     end
 
@@ -148,7 +148,6 @@
       encoding: plain
       fields_under_root: false
       input_type: log
-      ignore_older: 24h
       document_type: apache
       scan_frequency: 10s
       harvester_buffer_size: 16384

templates/filebeat.yml.erb

@@ -7,10 +7,23 @@
 <%- @filebeat_config['output']['file']['number_of_files'] = Integer(@filebeat_config['output']['file']['number_of_files']) -%>
 <%- end -%>
 <%- end -%>
+<%- if @filebeat_config['output']['elasticsearch'] -%>
+<%- if @filebeat_config['output']['elasticsearch']['port'] -%>
+<%- @filebeat_config['output']['elasticsearch']['port'] = Integer(@filebeat_config['output']['elasticsearch']['port']) -%>
+<%- end -%>
+<%- end -%>
 <%- if @filebeat_config['output']['logstash'] -%>
 <%- if @filebeat_config['output']['logstash']['bulk_max_size'] -%>
 <%- @filebeat_config['output']['logstash']['bulk_max_size'] = Integer(@filebeat_config['output']['logstash']['bulk_max_size']) -%>
 <%- end -%>
+<%- if @filebeat_config['output']['logstash']['port'] -%>
+<%- @filebeat_config['output']['logstash']['port'] = Integer(@filebeat_config['output']['logstash']['port']) -%>
+<%- end -%>
+<%- end -%>
+<%- if @filebeat_config['output']['redis'] -%>
+<%- if @filebeat_config['output']['redis']['port'] -%>
+<%- @filebeat_config['output']['redis']['port'] = Integer(@filebeat_config['output']['redis']['port']) -%>
+<%- end -%>
 <%- end -%>
 <%- if @filebeat_config['shipper'] -%>
 <%- if @filebeat_config['shipper']['refresh_topology_freq'] -%>