CHANGELOG.md

@@ -2,7 +2,21 @@ Changelog
 =========
 
 ## Unreleased
-[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.6.3...HEAD)
+[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.7.0...HEAD)
+
+## [v0.7.0](https://github.com/pcfens/puppet-filebeat/tree/v0.7.0)
+[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.6.3...v0.7.0)
+
+- Setting the `prospectors_merge` parameter to true will create prospectors across multiple hiera levels
+  using `hiera_hash()` [\#25](https://github.com/pcfens/puppet-filebeat/pull/25)
+- No longer manage the windows temp directory where the Filebeat download is kept. The assumption is made
+  that the directory exists and is writable by puppet.
+- Update the default windows download to Filebeat version 1.2.3
+- Add redis output to the Ruby 1.8 template
+- Wrap include_lines and exclude_lines array elements in quotes [\#28](https://github.com/pcfens/puppet-filebeat/issues/28)
+
+**Fixed Bugs**
+- SLES repository and metaparameters didn't match [\#25](https://github.com/pcfens/puppet-filebeat/pull/25)
 
 ## [v0.6.3](https://github.com/pcfens/puppet-filebeat/tree/v0.6.3)
 [Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.6.2...v0.6.3)

README.md

@@ -10,6 +10,10 @@
     - [Setup requirements](#setup-requirements)
     - [Beginning with filebeat](#beginning-with-filebeat)
 3. [Usage - Configuration options and additional functionality](#usage)
+    - [Adding a prospector](#adding-a-prospector)
+      - [Multiline Logs](#multiline-logs)
+    - [Prospectors in hiera](#prospectors-in-hiera)
+    - [Usage on Windows](#usage-on-windows)
 4. [Reference](#reference)
     - [Public Classes](#public-classes)
     - [Private Classes](#private-classes)
@@ -107,6 +111,24 @@ Filebeat prospectors (versions >= 1.1) can handle multiline log entries. The `mu
 parameter accepts a hash containing `pattern`, `negate`, `match`, `max_lines`, and `timeout`
 as documented in the filebeat [configuration documentation](https://www.elastic.co/guide/en/beats/filebeat/1.1/filebeat-configuration-details.html#multiline).
 
+### Prospectors in Hiera
+
+Prospectors can be declared in hiera using the `prospectors` parameter. By default, hiera will not merge
+prospector declarations down the hiera hierarchy. To change the behavior in puppet 3 use the `prospectors_merge`
+parameter. In puppet 4, you can use `prospectors_merge`, but can also use the
+[lookup_options](https://docs.puppet.com/puppet/latest/reference/lookup_quick.html#setting-lookupoptions-in-data)
+flag.
+
+When `prospectors_merge` is set to true, `prospectors` will be replaced by the output of
+`hiera_hash('filebeat::prospectors')`.
+
+### Usage on Windows
+
+When installing on Windows, this module will download the windows version of Filebeat from
+[elastic](https://www.elastic.co/downloads/beats/filebeat) to `C:\Temp` by default. The directory
+can be overridden using the `tmp_dir` parameter. `tmp_dir` is not managed by this module,
+but is expected to exist as a directory that puppet can write to.
+
 ## Reference
  - [**Public Classes**](#public-classes)
     - [Class: filebeat](#class-filebeat)
@@ -149,6 +171,7 @@ Installs and configures filebeat.
 - `install_dir`: [String] Where filebeat should be installed (windows only)
 - `tmp_dir`: [String] Where filebeat should be temporarily downloaded to so it can be installed (windows only)
 - `prospectors`: [Hash] Prospectors that will be created. Commonly used to create prospectors using hiera
+- `prospectors_merge`: [Boolean] If true, `hiera_hash()` will be used to build the the `prospectors` parameter (default: false)
 
 ### Private Classes
 
@@ -230,6 +253,10 @@ directly to elasticsearch).
 Only filebeat versions after 1.0.0-rc1 are supported. 1.0.0-rc1 and older don't
 support YAML like the ruby template can easily generate.
 
+When installing on Windows, there's an expectation that `C:\Temp` already exists, or an alternative
+location specified in the `tmp_dir` parameter exists and is writable by puppet. The temp directory
+is used to store the downloaded installer only.
+
 ### Pre-1.9.1 Ruby
 If you're on a system running a Ruby pre-1.9.1, hashes aren't sorted consistently, causing puppet runs to
 not be idempotent. To fix this, a limited template is used if the rubyversion is pre-1.9.1.

manifests/init.pp

@@ -32,35 +32,44 @@
 # @param install_dir [String] Where filebeat should be installed (windows only)
 # @param tmp_dir [String] Where filebeat should be temporarily downloaded to so it can be installed (windows only)
 # @param prospectors [Hash] Prospectors that will be created. Commonly used to create prospectors using hiera
+# @param prospectors_merge [Boolean] Whether $prospectors should merge all hiera sources, or use simple automatic parameter lookup
 class filebeat (
-  $package_ensure   = $filebeat::params::package_ensure,
-  $manage_repo      = $filebeat::params::manage_repo,
-  $service_ensure   = $filebeat::params::service_ensure,
-  $service_enable   = $filebeat::params::service_enable,
-  $service_provider = $filebeat::params::service_provider,
-  $spool_size       = $filebeat::params::spool_size,
-  $idle_timeout     = $filebeat::params::idle_timeout,
-  $publish_async    = $filebeat::params::publish_async,
-  $registry_file    = $filebeat::params::registry_file,
-  $config_dir       = $filebeat::params::config_dir,
-  $config_dir_mode  = $filebeat::params::config_dir_mode,
-  $config_file_mode = $filebeat::params::config_file_mode,
-  $purge_conf_dir   = $filebeat::params::purge_conf_dir,
-  $outputs          = $filebeat::params::outputs,
-  $shipper          = $filebeat::params::shipper,
-  $logging          = $filebeat::params::logging,
-  $run_options      = $filebeat::params::run_options,
-  $conf_template    = $filebeat::params::conf_template,
-  $download_url     = $filebeat::params::download_url,
-  $install_dir      = $filebeat::params::install_dir,
-  $tmp_dir          = $filebeat::params::tmp_dir,
-  $prospectors      = {},
+  $package_ensure    = $filebeat::params::package_ensure,
+  $manage_repo       = $filebeat::params::manage_repo,
+  $service_ensure    = $filebeat::params::service_ensure,
+  $service_enable    = $filebeat::params::service_enable,
+  $service_provider  = $filebeat::params::service_provider,
+  $spool_size        = $filebeat::params::spool_size,
+  $idle_timeout      = $filebeat::params::idle_timeout,
+  $publish_async     = $filebeat::params::publish_async,
+  $registry_file     = $filebeat::params::registry_file,
+  $config_dir        = $filebeat::params::config_dir,
+  $config_dir_mode   = $filebeat::params::config_dir_mode,
+  $config_file_mode  = $filebeat::params::config_file_mode,
+  $purge_conf_dir    = $filebeat::params::purge_conf_dir,
+  $outputs           = $filebeat::params::outputs,
+  $shipper           = $filebeat::params::shipper,
+  $logging           = $filebeat::params::logging,
+  $run_options       = $filebeat::params::run_options,
+  $conf_template     = $filebeat::params::conf_template,
+  $download_url      = $filebeat::params::download_url,
+  $install_dir       = $filebeat::params::install_dir,
+  $tmp_dir           = $filebeat::params::tmp_dir,
+  $prospectors       = {},
+  $prospectors_merge = false,
 ) inherits filebeat::params {
 
   $kernel_fail_message = "${::kernel} is not supported by filebeat."
 
-  validate_bool($manage_repo)
-  validate_hash($outputs, $logging, $prospectors)
+  validate_bool($manage_repo, $prospectors_merge)
+
+  if $prospectors_merge {
+    $prospectors_final = hiera_hash('filebeat::prospectors', $prospectors)
+  } else {
+    $prospectors_final = $prospectors
+  }
+
+  validate_hash($outputs, $logging, $prospectors_final)
   validate_string($idle_timeout, $registry_file, $config_dir, $package_ensure)
 
   if $package_ensure == '1.0.0-beta4' or $package_ensure == '1.0.0-rc1' {
@@ -73,7 +82,7 @@
   class { 'filebeat::service': } ->
   anchor { 'filebeat::end': }
 
-  if !empty($prospectors) {
-    create_resources('filebeat::prospector', $prospectors)
+  if !empty($prospectors_final) {
+    create_resources('filebeat::prospector', $prospectors_final)
   }
 }

manifests/install/windows.pp

@@ -2,18 +2,13 @@
   $filename = regsubst($filebeat::download_url, '^https.*\/([^\/]+)\.[^.].*', '\1')
   $foldername = 'Filebeat'
 
-  file { $filebeat::tmp_dir:
-    ensure => directory
-  }
-
   file { $filebeat::install_dir:
     ensure => directory
   }
 
   remote_file {"${filebeat::tmp_dir}/${filename}.zip":
     ensure      => present,
     source      => $filebeat::download_url,
-    require     => File[$filebeat::tmp_dir],
     verify_peer => false,
   }
 

manifests/params.pp

@@ -41,7 +41,7 @@
 
     'Windows' : {
       $config_dir       = 'C:/Program Files/Filebeat/conf.d'
-      $download_url     = 'https://download.elastic.co/beats/filebeat/filebeat-1.0.1-windows.zip'
+      $download_url     = 'https://download.elastic.co/beats/filebeat/filebeat-1.2.3-windows.zip'
       $install_dir      = 'C:/Program Files'
       $tmp_dir          = 'C:/Temp'
       $service_provider = undef

metadata.json

@@ -1,6 +1,6 @@
 {
   "name": "pcfens-filebeat",
-  "version": "0.6.3",
+  "version": "0.7.0",
   "author": "pcfens",
   "license": "Apache-2.0",
   "summary": "A module to install and manage the filebeat log shipper",
@@ -21,7 +21,8 @@
       "operatingsystem": "Ubuntu",
       "operatingsystemrelease": [
         "12.04",
-        "14.04"
+        "14.04",
+        "16.04"
       ]
     },
     {

spec/defines/prospector_spec.rb

@@ -113,10 +113,10 @@
         negate: true
         match: after
       include_lines:
-        - ^ERROR
-        - ^WARN
+        - "^ERROR"
+        - "^WARN"
       exclude_lines:
-        - ^DEBUG
+        - "^DEBUG"
 ',
       )}
     end

templates/filebeat.yml.ruby18.erb

@@ -154,6 +154,36 @@
         <%- end -%>
       <%- end -%>
     <%- end -%>
+    <%- if @filebeat_config['output']['redis'] != nil -%>
+    redis:
+      <%- if @filebeat_config['output']['redis']['host'] != nil -%>
+      host: "<%= @filebeat_config['output']['redis']['host'] %>"
+      <%- end -%>
+      <%- if @filebeat_config['output']['redis']['port'] != nil -%>
+      port: <%= @filebeat_config['output']['redis']['port'] %>
+      <%- end -%>
+      <%- if @filebeat_config['output']['redis']['save_topology'] != nil -%>
+      save_topology: <%= @filebeat_config['output']['redis']['save_topology'] %>
+      <%- end -%>
+      <%- if @filebeat_config['output']['redis']['index'] != nil -%>
+      index: "<%= @filebeat_config['output']['redis']['index'] %>"
+      <%- end -%>
+      <%- if @filebeat_config['output']['redis']['db'] != nil -%>
+      db: <%= @filebeat_config['output']['redis']['db'] %>
+      <%- end -%>
+      <%- if @filebeat_config['output']['redis']['db_topology'] != nil -%>
+      db_topology: <%= @filebeat_config['output']['redis']['db_topology'] %>
+      <%- end -%>
+      <%- if @filebeat_config['output']['redis']['password'] != nil -%>
+      password: "<%= @filebeat_config['output']['redis']['password'] %>"
+      <%- end -%>
+      <%- if @filebeat_config['output']['redis']['timeout'] != nil -%>
+      timeout: <%= @filebeat_config['output']['redis']['timeout'] %>
+      <%- end -%>
+      <%- if @filebeat_config['output']['redis']['reconnect_interval'] != nil -%>
+      reconnect_interval: <%= @filebeat_config['output']['redis']['reconnect_interval'] %>
+      <%- end -%>
+    <%- end -%>
     <%- if @filebeat_config['output']['file'] != nil -%>
     file:
       <%- if @filebeat_config['output']['file']['path'] != nil -%>

templates/prospector.yml.erb

@@ -76,12 +76,12 @@ filebeat:
       <%- if @include_lines.length > 0 -%>
       include_lines:
         <%- @include_lines.each do |include_line| -%>
-        - <%= include_line %>
+        - "<%= include_line %>"
         <%- end -%>
       <%- end -%>
       <%- if @exclude_lines.length > 0 -%>
       exclude_lines:
         <%- @exclude_lines.each do |exclude_line| -%>
-        - <%= exclude_line %>
+        - "<%= exclude_line %>"
         <%- end -%>
       <%- end -%>