CHANGELOG.md

@@ -2,7 +2,13 @@ Changelog
 =========
 
 ## Unreleased
-[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.9.1...HEAD)
+[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.10.0...HEAD)
+
+
+## [v0.10.0](https://github.com/pcfens/puppet-filebeat/tree/v0.10.0)
+[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.9.2...v0.10.0)
+
+- Add support for JSON decoding [\72](https://github.com/pcfens/puppet-filebeat/pull/72)
 
 ## [v0.9.2](https://github.com/pcfens/puppet-filebeat/tree/v0.9.2)
 [Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.9.1...v0.9.2)

README.md

@@ -13,6 +13,7 @@
 3. [Usage - Configuration options and additional functionality](#usage)
     - [Adding a prospector](#adding-a-prospector)
       - [Multiline Logs](#multiline-logs)
+      - [JSON logs](#json-logs)
     - [Prospectors in hiera](#prospectors-in-hiera)
     - [Usage on Windows](#usage-on-windows)
 4. [Reference](#reference)
@@ -126,6 +127,12 @@ Filebeat prospectors (versions >= 1.1) can handle multiline log entries. The `mu
 parameter accepts a hash containing `pattern`, `negate`, `match`, `max_lines`, and `timeout`
 as documented in the filebeat [configuration documentation](https://www.elastic.co/guide/en/beats/filebeat/1.1/filebeat-configuration-details.html#multiline).
 
+#### JSON Logs
+
+Filebeat prospectors (versions >= 5.0) can natively decode JSON objects if they are stored one per line. The `json`
+parameter accepts a hash containing `message_key`, `keys_under_root`, `overwrite_keys`, and `add_error_key`
+as documented in the filebeat [configuration documentation](https://www.elastic.co/guide/en/beats/filebeat/5.0/configuration-filebeat-options.html#config-json).
+
 ### Prospectors in Hiera
 
 Prospectors can be declared in hiera using the `prospectors` parameter. By default, hiera will not merge
@@ -320,6 +327,8 @@ to fully understand what these parameters do.
   - `exclude_lines`: [Array] A list of regular expressions to match the files that you want to exclude.
     Ignored if empty (default: [])
   - `max_bytes`: [Integer] The maximum number of bytes that a single log message can have (default: 10485760)
+  - `json`: [Hash] Options that control how filebeat handles decoding of log messages in JSON format
+    [See above](#json-logs). (default: {})
   - `multiline`: [Hash] Options that control how Filebeat handles log messages that span multiple lines.
     [See above](#multiline-logs). (default: {})
 

manifests/prospector.pp

@@ -27,10 +27,11 @@
   $exclude_lines         = [],
   $max_bytes             = '10485760',
   $multiline             = {},
+  $json                  = {},
   $tags                  = [],
 ) {
 
-  validate_hash($fields, $multiline)
+  validate_hash($fields, $multiline, $json)
   validate_array($paths, $exclude_files, $include_lines, $exclude_lines, $tags)
   validate_bool($tail_files, $close_renamed, $close_removed, $close_eof, $clean_removed)
 

metadata.json

@@ -1,6 +1,6 @@
 {
   "name": "pcfens-filebeat",
-  "version": "0.9.2",
+  "version": "0.10.0",
   "author": "pcfens",
   "license": "Apache-2.0",
   "summary": "A module to install and manage the filebeat log shipper",

templates/prospector5.yml.erb

@@ -56,27 +56,37 @@ filebeat:
       max_bytes: <%= @max_bytes %>
       <%- end -%>
 
+      <%- if @json.length > 0 -%>
       ### JSON configuration
-
-      # Decode JSON options. Enable this if your logs are structured in JSON.
-      # JSON key on which to apply the line filtering and multiline settings. This key
-      # must be top level and its value must be string, otherwise it is ignored. If
-      # no text key is defined, the line filtering and multiline features cannot be used.
-      #json.message_key:
-
-      # By default, the decoded JSON is placed under a "json" key in the output document.
-      # If you enable this setting, the keys are copied top level in the output document.
-      #json.keys_under_root: false
-
-      # If keys_under_root and this setting are enabled, then the values from the decoded
-      # JSON object overwrite the fields that Filebeat normally adds (type, source, offset, etc.)
-      # in case of conflicts.
-      #json.overwrite_keys: false
-
-      # If this setting is enabled, Filebeat adds a "json_error" key in case of JSON
-      # unmarshaling errors or when a text key is defined in the configuration but cannot
-      # be used.
-      #json.add_error_key: false
+      json:
+          # Decode JSON options. Enable this if your logs are structured in JSON.
+          # JSON key on which to apply the line filtering and multiline settings. This key
+          # must be top level and its value must be string, otherwise it is ignored. If
+          # no text key is defined, the line filtering and multiline features cannot be used.
+          <%- if @json['message_key'] -%>
+          message_key: '<%= @json['message_key'] %>'
+          <%- end -%>
+
+          # By default, the decoded JSON is placed under a "json" key in the output document.
+          # If you enable this setting, the keys are copied top level in the output document.
+          <%- if @json['keys_under_root'] -%>
+          keys_under_root: <%= @json['keys_under_root'] %>
+          <%- end -%>
+
+          # If keys_under_root and this setting are enabled, then the values from the decoded
+          # JSON object overwrite the fields that Filebeat normally adds (type, source, offset, etc.)
+          # in case of conflicts.
+          <%- if @json['overwrite_keys'] -%>
+          overwrite_keys: <%= @json['overwrite_keys'] %>
+          <%- end -%>
+
+          # If this setting is enabled, Filebeat adds a "json_error" key in case of JSON
+          # unmarshaling errors or when a text key is defined in the configuration but cannot
+          # be used.
+          <%- if @json['add_error_key'] -%>
+          add_error_key: <%= @json['add_error_key'] %>
+          <%- end -%>
+      <%- end -%>
 
       <%- if @multiline.length > 0 -%>
       multiline: