diff --git a/.fixtures.yml b/.fixtures.yml index c9bf040e..d9c4a044 100644 --- a/.fixtures.yml +++ b/.fixtures.yml @@ -1,8 +1,10 @@ fixtures: repositories: - "stdlib": "https://github.com/puppetlabs/puppetlabs-stdlib.git" - "apt": "https://github.com/puppetlabs/puppetlabs-apt.git" - "powershell": "https://github.com/puppetlabs/puppetlabs-powershell.git" - "remotefile": "https://github.com/lwf/puppet-remote_file.git" + stdlib: "https://github.com/puppetlabs/puppetlabs-stdlib.git" + apt: + repo: "https://github.com/puppetlabs/puppetlabs-apt.git" + ref: '4.2.0' + powershell: "https://github.com/puppetlabs/puppetlabs-powershell.git" + archive: "https://github.com/voxpupuli/puppet-archive.git" symlinks: - "filebeat": "#{source_dir}" + filebeat: "#{source_dir}" diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..543dd6ad --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +*.rb eol=lf +*.erb eol=lf +*.pp eol=lf +*.sh eol=lf diff --git a/.gitignore b/.gitignore index ef01482d..78d9ade0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,21 @@ -pkg/ -Gemfile.lock -vendor/ -spec/fixtures/ -.vagrant/ -.bundle/ -coverage/ -.idea/ +.*.sw[op] +.metadata +.yardoc +.yardwarns *.iml -*.swp +/.bundle/ +/.idea/ +/.vagrant/ +/coverage/ +/bin/ +/doc/ +/Gemfile.local +/Gemfile.lock +/junit/ +/log/ +/log/ +/pkg/ +/spec/fixtures/manifests/ +/spec/fixtures/modules/ +/tmp/ +/vendor/ diff --git a/.pmtignore b/.pmtignore new file mode 100644 index 00000000..fb589575 --- /dev/null +++ b/.pmtignore @@ -0,0 +1,20 @@ +docs/ +pkg/ +Gemfile.lock +Gemfile.local +vendor/ +.vendor/ +spec/fixtures/manifests/ +spec/fixtures/modules/ +.vagrant/ +.bundle/ +.ruby-version +coverage/ +log/ +.idea/ +.dependencies/ +.librarian/ +Puppetfile.lock +*.iml +.*.sw? +.yardoc/ diff --git a/.rspec b/.rspec new file mode 100644 index 00000000..16f9cdb0 --- /dev/null +++ b/.rspec @@ -0,0 +1,2 @@ +--color +--format documentation diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 00000000..72d2f75e --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,104 @@ +--- +require: rubocop-rspec +AllCops: + TargetRubyVersion: '2.1' + Include: + - "./**/*.rb" + Exclude: + - bin/* + - ".vendor/**/*" + - Gemfile + - Rakefile + - pkg/**/* + - spec/fixtures/**/* + - vendor/**/* +Metrics/LineLength: + Description: People have wide screens, use them. + Max: 200 +RSpec/BeforeAfterAll: + Description: Beware of using after(:all) as it may cause state to leak between tests. + A necessary evil in acceptance testing. + Exclude: + - spec/acceptance/**/*.rb +RSpec/HookArgument: + Description: Prefer explicit :each argument, matching existing module's style + EnforcedStyle: each +Style/BlockDelimiters: + Description: Prefer braces for chaining. Mostly an aesthetical choice. Better to + be consistent then. + EnforcedStyle: braces_for_chaining +Style/ClassAndModuleChildren: + Description: Compact style reduces the required amount of indentation. + EnforcedStyle: compact +Style/EmptyElse: + Description: Enforce against empty else clauses, but allow `nil` for clarity. + EnforcedStyle: empty +Style/FormatString: + Description: Following the main puppet project's style, prefer the % format format. + EnforcedStyle: percent +Style/FormatStringToken: + Description: Following the main puppet project's style, prefer the simpler template + tokens over annotated ones. + EnforcedStyle: template +Style/Lambda: + Description: Prefer the keyword for easier discoverability. + EnforcedStyle: literal +Style/RegexpLiteral: + Description: Community preference. See https://github.com/voxpupuli/modulesync_config/issues/168 + EnforcedStyle: percent_r +Style/TernaryParentheses: + Description: Checks for use of parentheses around ternary conditions. Enforce parentheses + on complex expressions for better readability, but seriously consider breaking + it up. + EnforcedStyle: require_parentheses_when_complex +Style/TrailingCommaInArguments: + Description: Prefer always trailing comma on multiline argument lists. This makes + diffs, and re-ordering nicer. + EnforcedStyleForMultiline: comma +Style/TrailingCommaInLiteral: + Description: Prefer always trailing comma on multiline literals. This makes diffs, + and re-ordering nicer. + EnforcedStyleForMultiline: comma +Style/SymbolArray: + Description: Using percent style obscures symbolic intent of array's contents. + EnforcedStyle: brackets +Style/EndOfLine: + Enabled: false +Style/CollectionMethods: + Enabled: true +Style/MethodCalledOnDoEndBlock: + Enabled: true +Style/StringMethods: + Enabled: true +Metrics/AbcSize: + Enabled: false +Metrics/BlockLength: + Enabled: false +Metrics/ClassLength: + Enabled: false +Metrics/CyclomaticComplexity: + Enabled: false +Metrics/MethodLength: + Enabled: false +Metrics/ModuleLength: + Enabled: false +Metrics/ParameterLists: + Enabled: false +Metrics/PerceivedComplexity: + Enabled: false +RSpec/DescribeClass: + Enabled: false +RSpec/ExampleLength: + Enabled: false +RSpec/MessageExpectation: + Enabled: false +RSpec/MultipleExpectations: + Enabled: false +RSpec/NestedGroups: + Enabled: false +Style/AsciiComments: + Enabled: false +Style/IfUnlessModifier: + Enabled: false +Style/SymbolProc: + Enabled: false diff --git a/.travis.yml b/.travis.yml index 287a833b..1271c07e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,15 @@ --- language: ruby cache: bundler +before_install: + - bundle -v + - rm Gemfile.lock || true + - gem update --system + - gem update bundler + - gem --version + - bundle -v script: - - "bundle exec rake validate lint spec SPEC_OPTS='--format documentation'" + - "bundle exec rake validate metadata_lint lint spec SPEC_OPTS='--format documentation'" matrix: fast_finish: true include: @@ -12,29 +19,30 @@ matrix: script: bundle exec rake beaker services: docker sudo: required - - rvm: 2.3.1 - dist: trusty - env: BEAKER_set=ubuntu-16.04 PUPPET_INSTALL_TYPE=agent BEAKER_debug=true - script: bundle exec rake beaker - services: docker - sudo: required - rvm: 2.3.1 dist: trusty env: BEAKER_set=centos-7 PUPPET_INSTALL_TYPE=agent BEAKER_debug=true script: bundle exec rake beaker services: docker sudo: required - - rvm: 2.1.9 - bundler_args: --without system_tests - env: PUPPET_GEM_VERSION="~> 4.5" STRICT_VARIABLES="yes" - - rvm: 2.1.6 + - rvm: 2.3.1 bundler_args: --without system_tests - env: PUPPET_GEM_VERSION="~> 3.5" FUTURE_PARSER="yes" - - rvm: 2.1.6 + env: STRICT_VARIABLES="yes" + script: bundle exec rake rubocop + - rvm: 2.1.9 bundler_args: --without system_tests - env: PUPPET_GEM_VERSION="~> 3.8" - - rvm: 1.9.3 + env: PUPPET_GEM_VERSION="~> 4.0" STRICT_VARIABLES=yes + - rvm: 2.4.0 bundler_args: --without system_tests - env: PUPPET_GEM_VERSION="~> 3.8" + env: PUPPET_GEM_VERSION="~> 5.0" STRICT_VARIABLES=yes DEPLOY_TO_FORGE=yes +deploy: + provider: puppetforge + user: "pcfens" + password: + secure: "z1SbP/Hisr5k66XL/ACLsZ/fG7cCpwl8apjZzt/YciWizwReioU2EkLr5tvXdUC10aIH6H7XBdA9XwPqwXa81cIqcdIHlRMIbosMUGYaXcUm1xhctB3GvEDqsxFqdZSHYXax+IR6Wt507Eop+iU3S5pf/zJcp4uSKQVapCMoeVCEQYLRwllgeaqtEUZwqOUwPk31C4YZxwrzmgbIVyXmPrp3SDToXaQm4S4RkayOqHH2lYi8isz3IPPQvDZY5681TBpo35AbsIRbhiLzGlBHbgRaE2dz7J1Gs8MBGFyrtDaPtc9UpbgEmyxgmaPs3NIeZkmfVoosjt2AHRsoMZB7ntaPAQ20mk44ugMhxd5HX8t7QdLPiYQqgA3O4QfKraxPzdEjYVs9Pf7BBgY4JpGSOAD3dlWNK0U40MzKe74cj6dshg9SfIdyf3M3MmI0KIIvdKhpgl8mSIL8MCWjnYYNpQMQDFgyrXvePnkPVlt7zlBxn+LJFFx3VLGNfSWbKavITM/nrvjpFkQZ34mPHTtTUYnT6HVehtwPd5x6ILqYcppEeeiloa4uLWhW/vg0wAOdOBv2IALdAqRMC56ODPK33gFRkX+CclsegtOh2In407njbXXZBQrY5h3SXuEVxZcFhGVTxJIV29viuWFSm7VF0a7IUmEbVrM23bqeaM+aOgs=" + on: + tags: true + all_branches: true + condition: "$DEPLOY_TO_FORGE = yes" notifications: email: false diff --git a/.yardopts b/.yardopts new file mode 100644 index 00000000..3687f518 --- /dev/null +++ b/.yardopts @@ -0,0 +1,2 @@ +--markup markdown +--output-dir docs/ diff --git a/CHANGELOG.md b/CHANGELOG.md index c2d453c6..0d101099 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,145 @@ Changelog ========= ## Unreleased -[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.8.6...HEAD) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v2.3.0...HEAD) + + +## [v2.3.0](https://github.com/pcfens/puppet-filebeat/tree/v2.3.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v2.2.0...v2.3.0) + +- Add support for Filebeat 6 [\#141](https://github.com/pcfens/puppet-filebeat/pull/141) +- Add Support for hash.random in Kafka output [\#142](https://github.com/pcfens/puppet-filebeat/pull/142) + +## [v2.2.0](https://github.com/pcfens/puppet-filebeat/tree/v2.2.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v2.1.0...v2.2.0) + +- Support pipeline configurations in prospectors [\#134](https://github.com/pcfens/puppet-filebeat/pull/134) +- Fix regex for validating download URL [\#135](https://github.com/pcfens/puppet-filebeat/pull/135) +- Overhaul testing + +## [v2.1.0](https://github.com/pcfens/puppet-filebeat/tree/v2.1.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v2.0.0...v2.1.0) + +- Change beat_name configuration parameter to name [\#126](https://github.com/pcfens/puppet-filebeat/issues/126) +- Make configuration directory/file ownership configurable [\#127](https://github.com/pcfens/puppet-filebeat/issues/127) + +## [v2.0.0](https://github.com/pcfens/puppet-filebeat/tree/v2.0.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v1.0.0...v2.0.0) + +- Drop support for Puppet 3 +- Drop support for Filebeat versions before 5 +- Add support for Puppet 5 +- Use a generic template by default +- Remove processor defined type (create it in the config template) +- Add a flag to disable validating the configuration (`disable_config_test`) + +## [v1.0.0](https://github.com/pcfens/puppet-filebeat/tree/v1.0.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.12.1...v1.0.0) + +- This is the last release with support for Filebeat versions prior to 5 +- Last release with support for Puppet 3 +- Add Logstash SSL support [\#121](https://github.com/pcfens/puppet-filebeat/pull/121) +- Add ES loadbalance support [\#119](https://github.com/pcfens/puppet-filebeat/pull/119) + +The next major release will be a breaking release for anyone using processors. + +## [v0.12.1](https://github.com/pcfens/puppet-filebeat/tree/v0.12.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.12.0...v0.12.1) + +- Add support for SSL in Logstash [\#117](https://github.com/pcfens/puppet-filebeat/pull/117) + +## [v0.12.0](https://github.com/pcfens/puppet-filebeat/tree/v0.12.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.11.2...v0.12.0) + +Windows users: you may see a restart and replacement of your existing filebeat directory. +There's a very brief discussion of the change in the [pull request](https://github.com/pcfens/puppet-filebeat/pull/113#issuecomment-307628477) + +- Support upgrades in Windows [\#113](https://github.com/pcfens/puppet-filebeat/pull/113) +- Add optional repo_priority parameter [\#110](https://github.com/pcfens/puppet-filebeat/pull/110) +- Update external dependencies, including pinning apt version + +## [v0.11.2](https://github.com/pcfens/puppet-filebeat/tree/v0.11.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.11.1...v0.11.2) + +- Explicitly support newer versions of the powershell modules [\#105](https://github.com/pcfens/puppet-filebeat/issues/105) +- Support kafka codec.format module [\#106](https://github.com/pcfens/puppet-filebeat/pull/106) +- The `add_locale` processor doesnt' require parameters [\#104](https://github.com/pcfens/puppet-filebeat/pull/104) + +## [v0.11.1](https://github.com/pcfens/puppet-filebeat/tree/v0.11.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.11.0...v0.11.1) + +- Restore puppet 3.x compatibility regression ([PUP-2523](https://tickets.puppetlabs.com/browse/PUP-2523)) + +## [v0.11.0](https://github.com/pcfens/puppet-filebeat/tree/v0.11.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.10.4...v0.11.0) + +**Breaking Changes** +- Processors are managed by their own defined resource (slight syntax change) [\#85](https://github.com/pcfens/puppet-filebeat/pull/85) +- The registry file has likely moved because of an upstream change. Moving this file + can cause problems (duplicate or missed log entries), so you may want to point it + to your existing file (check in /.filebeat on Linux systems) + +**Normal Enhancements/Bugfixes** +- Support proxy for windows file downloads [\#90](https://github.com/pcfens/puppet-filebeat/pull/90) +- Setting `package_ensure` to absent removes puppet managed files and removes the package +- Add support for index conditional output to elasticsearch [\#97](https://github.com/pcfens/puppet-filebeat/pull/97) +- Add support for a conditional pipeline for elasticsearch [\#98](https://github.com/pcfens/puppet-filebeat/pull/98) +- Template should check for nil instead of undef [\#63](https://github.com/pcfens/puppet-filebeat/issues/63) +- Support for the round_robin and group_events parameters in kafka outputs [\#100](https://github.com/pcfens/puppet-filebeat/pull/100) + +## [v0.10.4](https://github.com/pcfens/puppet-filebeat/tree/v0.10.4) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.10.3...v0.10.4) + +- Add output.console support to the config template [\#91](https://github.com/pcfens/puppet-filebeat/issues/91) +- Support puppet with strict variables enabled [\#92](https://github.com/pcfens/puppet-filebeat/issues/92) + +## [v0.10.3](https://github.com/pcfens/puppet-filebeat/tree/v0.10.3) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.10.2...v0.10.3) + +- Allow non-SSL downloads of windows filebeat zipfile [\#82](https://github.com/pcfens/puppet-filebeat/pull/82) +- Basic support of processors in puppet <4.x [\#79](https://github.com/pcfens/puppet-filebeat/issues/79) (See note above) +- Confine the filebeat_version fact in a way that works in Ruby 1.8.7 [\#88](https://github.com/pcfens/puppet-filebeat/pull/88) + +## [v0.10.2](https://github.com/pcfens/puppet-filebeat/tree/v0.10.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.10.1...v0.10.2) + +- Add close_older and force_close_files within prospector v5 [\#77](https://github.com/pcfens/puppet-filebeat/pull/77) + +## [v0.10.1](https://github.com/pcfens/puppet-filebeat/tree/v0.10.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.10.0...v0.10.1) + +- Support harvesting symlinks [\#74](https://github.com/pcfens/puppet-filebeat/pull/74) +- Fix windows config file validation command [\#75](https://github.com/pcfens/puppet-filebeat/issues/75) + +## [v0.10.0](https://github.com/pcfens/puppet-filebeat/tree/v0.10.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.9.2...v0.10.0) + +- Add support for JSON decoding [\#72](https://github.com/pcfens/puppet-filebeat/pull/72) + +## [v0.9.2](https://github.com/pcfens/puppet-filebeat/tree/v0.9.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.9.1...v0.9.2) + +- Add support for close_* and clean_* parameters in prospectors [\#70](https://github.com/pcfens/puppet-filebeat/pull/70) + +## [v0.9.1](https://github.com/pcfens/puppet-filebeat/tree/v0.9.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.9.0...v0.9.1) + +- Fix yaml syntax around filebeat processors [\#71](https://github.com/pcfens/puppet-filebeat/pull/71) + +## [v0.9.0](https://github.com/pcfens/puppet-filebeat/tree/v0.9.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.8.7...v0.9.0) + +- Add support for tags in prospectors [\#68](https://github.com/pcfens/puppet-filebeat/pull/68) +- Add support for filebeat processors [\#69](https://github.com/pcfens/puppet-filebeat/pull/69) +- Fix the `filebeat_version` fact in Windows [\#59](https://github.com/pcfens/puppet-filebeat/issues/59) +- Validate configuration files before notifying the filebeat service +- Update the Windows install URL to the latest version + +## [v0.8.7](https://github.com/pcfens/puppet-filebeat/tree/v0.8.7) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.8.6...v0.8.7) + +- Update windows URL to the latest 5.x release +- Remove nil values before rendering the template [\#65](https://github.com/pcfens/puppet-filebeat/pull/65) ## [v0.8.6](https://github.com/pcfens/puppet-filebeat/tree/v0.8.6) [Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.8.5...v0.8.6) diff --git a/Gemfile b/Gemfile index 0ef1490c..b44878f7 100644 --- a/Gemfile +++ b/Gemfile @@ -1,47 +1,133 @@ -source ENV['GEM_SOURCE'] || "https://rubygems.org" +source ENV['GEM_SOURCE'] || 'https://rubygems.org' -def location_for(place, version = nil) - if place =~ /^(git[:@][^#]*)#(.*)/ - [version, { :git => $1, :branch => $2, :require => false}].compact - elsif place =~ /^file:\/\/(.*)/ - ['>= 0', { :path => File.expand_path($1), :require => false}] +def location_for(place_or_version, fake_version = nil) + if place_or_version =~ %r{\A(git[:@][^#]*)#(.*)} + [fake_version, { git: Regexp.last_match(1), branch: Regexp.last_match(2), require: false }].compact + elsif place_or_version =~ %r{\Afile:\/\/(.*)} + ['>= 0', { path: File.expand_path(Regexp.last_match(1)), require: false }] else - [place, version, { :require => false}].compact + [place_or_version, { require: false }] end end -group :development, :unit_tests do -gem 'json', :require => false - gem 'metadata-json-lint', :require => false - gem 'puppet_facts', :require => false - gem 'puppetlabs_spec_helper', :require => false - gem 'rspec-puppet', '>= 2.3.2', :require => false - gem 'simplecov', :require => false +def gem_type(place_or_version) + if place_or_version =~ %r{\Agit[:@]} + :git + elsif !place_or_version.nil? && place_or_version.start_with?('file:') + :file + else + :gem + end +end + +ruby_version_segments = Gem::Version.new(RUBY_VERSION.dup).segments +minor_version = ruby_version_segments[0..1].join('.') + +group :development do + gem "fast_gettext", '1.1.0', require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.1.0') + gem "fast_gettext", require: false if Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('2.1.0') + gem "json_pure", '<= 2.0.1', require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.0.0') + gem "json", '= 1.8.1', require: false if Gem::Version.new(RUBY_VERSION.dup) == Gem::Version.new('2.1.9') + gem "puppet-module-posix-default-r#{minor_version}", require: false, platforms: [:ruby] + gem "puppet-module-posix-dev-r#{minor_version}", require: false, platforms: [:ruby] + gem "puppet-module-win-default-r#{minor_version}", require: false, platforms: [:mswin, :mingw, :x64_mingw] + gem "puppet-module-win-dev-r#{minor_version}", require: false, platforms: [:mswin, :mingw, :x64_mingw] end group :system_tests do - gem 'beaker-rspec', *location_for(ENV['BEAKER_RSPEC_VERSION'] || '>= 3.4') - gem 'beaker', *location_for(ENV['BEAKER_VERSION']) - gem 'serverspec', :require => false - gem 'beaker-puppet_install_helper', :require => false - gem 'master_manipulator', :require => false - gem 'beaker-hostgenerator', *location_for(ENV['BEAKER_HOSTGENERATOR_VERSION']) + gem "beaker", *location_for(ENV['BEAKER_VERSION'] || '>= 3') + gem "beaker-rspec", *location_for(ENV['BEAKER_RSPEC_VERSION']) + gem 'beaker-puppet_install_helper' + gem 'beaker-module_install_helper' +end + +puppet_version = ENV['PUPPET_GEM_VERSION'] +puppet_type = gem_type(puppet_version) +facter_version = ENV['FACTER_GEM_VERSION'] +hiera_version = ENV['HIERA_GEM_VERSION'] + +def puppet_older_than?(version) + puppet_version = ENV['PUPPET_GEM_VERSION'] + !puppet_version.nil? && + Gem::Version.correct?(puppet_version) && + Gem::Requirement.new("< #{version}").satisfied_by?(Gem::Version.new(puppet_version.dup)) +end + +gems = {} + +gems['puppet'] = location_for(puppet_version) + +# If facter or hiera versions have been specified via the environment +# variables, use those versions. If not, and if the puppet version is < 3.5.0, +# use known good versions of both for puppet < 3.5.0. +if facter_version + gems['facter'] = location_for(facter_version) +elsif puppet_type == :gem && puppet_older_than?('3.5.0') + gems['facter'] = ['>= 1.6.11', '<= 1.7.5', require: false] +end + +if hiera_version + gems['hiera'] = location_for(ENV['HIERA_GEM_VERSION']) +elsif puppet_type == :gem && puppet_older_than?('3.5.0') + gem['hiera'] = ['>= 1.0.0', '<= 1.3.0', require: false] end -# json_pure 2.0.2 added a requirement on ruby >= 2. We pin to json_pure 2.0.1 -# if using ruby 1.x -gem 'json_pure', '<=2.0.1', :require => false if RUBY_VERSION =~ /^1\./ +if Gem.win_platform? && (puppet_type != :gem || puppet_older_than?('3.5.0')) + # For Puppet gems < 3.5.0 (tested as far back as 3.0.0) on Windows + if puppet_type == :gem + gems['ffi'] = ['1.9.0', require: false] + gems['minitar'] = ['0.5.4', require: false] + gems['win32-eventlog'] = ['0.5.3', '<= 0.6.5', require: false] + gems['win32-process'] = ['0.6.5', '<= 0.7.5', require: false] + gems['win32-security'] = ['~> 0.1.2', '<= 0.2.5', require: false] + gems['win32-service'] = ['0.7.2', '<= 0.8.8', require: false] + else + gems['ffi'] = ['~> 1.9.0', require: false] + gems['minitar'] = ['~> 0.5.4', require: false] + gems['win32-eventlog'] = ['~> 0.5', '<= 0.6.5', require: false] + gems['win32-process'] = ['~> 0.6', '<= 0.7.5', require: false] + gems['win32-security'] = ['~> 0.1', '<= 0.2.5', require: false] + gems['win32-service'] = ['~> 0.7', '<= 0.8.8', require: false] + end + + gems['win32-dir'] = ['~> 0.3', '<= 0.4.9', require: false] -if facterversion = ENV['FACTER_GEM_VERSION'] - gem 'facter', facterversion, :require => false -else - gem 'facter', :require => false + if RUBY_VERSION.start_with?('1.') + gems['win32console'] = ['1.3.2', require: false] + # sys-admin was removed in Puppet 3.7.0 and doesn't compile under Ruby 2.x + gems['sys-admin'] = ['1.5.6', require: false] + end + + # Puppet < 3.7.0 requires these. + # Puppet >= 3.5.0 gem includes these as requirements. + # The following versions are tested to work with 3.0.0 <= puppet < 3.7.0. + gems['win32-api'] = ['1.4.8', require: false] + gems['win32-taskscheduler'] = ['0.2.2', require: false] + gems['windows-api'] = ['0.4.3', require: false] + gems['windows-pr'] = ['1.2.3', require: false] +elsif Gem.win_platform? + # If we're using a Puppet gem on Windows which handles its own win32-xxx gem + # dependencies (>= 3.5.0), set the maximum versions (see PUP-6445). + gems['win32-dir'] = ['<= 0.4.9', require: false] + gems['win32-eventlog'] = ['<= 0.6.5', require: false] + gems['win32-process'] = ['<= 0.7.5', require: false] + gems['win32-security'] = ['<= 0.2.5', require: false] + gems['win32-service'] = ['<= 0.8.8', require: false] end -if puppetversion = ENV['PUPPET_GEM_VERSION'] - gem 'puppet', puppetversion, :require => false -else - gem 'puppet', :require => false +gems.each do |gem_name, gem_params| + gem gem_name, *gem_params end -# vim:ft=ruby +# Evaluate Gemfile.local and ~/.gemfile if they exist +extra_gemfiles = [ + "#{__FILE__}.local", + File.join(Dir.home, '.gemfile'), +] + +extra_gemfiles.each do |gemfile| + if File.file?(gemfile) && File.readable?(gemfile) + eval(File.read(gemfile), binding) + end +end +# vim: syntax=ruby diff --git a/README.md b/README.md index afd2889c..85df8d86 100644 --- a/README.md +++ b/README.md @@ -2,25 +2,20 @@ [![Build Status](https://travis-ci.org/pcfens/puppet-filebeat.svg?branch=master)](https://travis-ci.org/pcfens/puppet-filebeat) -## Warning - -Version 0.8.0 adds support for filebeat version 5. You may notice some changes to your configuration files. - -The changelog has more information. - #### Table of Contents 1. [Description](#description) 2. [Setup - The basics of getting started with filebeat](#setup) - [What filebeat affects](#what-filebeat-affects) - - [Upgrading to Filebeat 5.x](#upgrading-to-filebeat-5x) - [Setup requirements](#setup-requirements) - [Beginning with filebeat](#beginning-with-filebeat) 3. [Usage - Configuration options and additional functionality](#usage) - [Adding a prospector](#adding-a-prospector) - [Multiline Logs](#multiline-logs) + - [JSON logs](#json-logs) - [Prospectors in hiera](#prospectors-in-hiera) - [Usage on Windows](#usage-on-windows) + - [Processors](#processors) 4. [Reference](#reference) - [Public Classes](#public-classes) - [Private Classes](#private-classes) @@ -41,18 +36,10 @@ The `filebeat` module installs and configures the [filebeat log shipper](https:/ By default `filebeat` adds a software repository to your system, and installs filebeat along with required configurations. -### Upgrading to Filebeat 5.x +### Upgrading to Filebeat 6.x -If you use this module on a system with filebeat 1.x installed, and you keep your current parameters -nothing will change. Setting `major_version` to '5' will modify the configuration template and update -package repositories, but won't update the package itself. To update the package set the -`package_ensure` parameter to at least 5.0.0. +To upgrade to Filebeat 6.x, simply set `$filebeat::major_version` to `6` and `$filebeat::package_ensure` to `latest`. -Windows users should set `major_version` to 5 and update the `download_url` parameter to the correct -[download](https://www.elastic.co/downloads/beats/filebeat). - -If you're on a Debian based system, you need to make sure that the apt-transport-https package -is installed if you want this module to manage the repository for you (it does by default). ### Setup Requirements @@ -78,6 +65,7 @@ class { 'filebeat': 'http://localhost:9200', 'http://anotherserver:9200' ], + 'loadbalance' => true, 'index' => 'packetbeat', 'cas' => [ '/etc/pki/root/ca.pem', @@ -104,7 +92,9 @@ class { 'filebeat': ``` -[Shipper](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-configuration-details.html#configuration-shipper) and [logging](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-configuration-details.html#configuration-logging) options can be configured the same way, and are documented on the [elastic website](https://www.elastic.co/guide/en/beats/filebeat/current/index.html). +[Shipper](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-configuration-details.html#configuration-shipper) and +[logging](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-configuration-details.html#configuration-logging) options +can be configured the same way, and are documented on the [elastic website](https://www.elastic.co/guide/en/beats/filebeat/current/index.html). ### Adding a prospector @@ -128,21 +118,23 @@ filebeat::prospector { 'syslogs': #### Multiline Logs -Filebeat prospectors (versions >= 1.1) can handle multiline log entries. The `multiline` +Filebeat prospectors can handle multiline log entries. The `multiline` parameter accepts a hash containing `pattern`, `negate`, `match`, `max_lines`, and `timeout` -as documented in the filebeat [configuration documentation](https://www.elastic.co/guide/en/beats/filebeat/1.1/filebeat-configuration-details.html#multiline). +as documented in the filebeat [configuration documentation](https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html). + +#### JSON Logs + +Filebeat prospectors (versions >= 5.0) can natively decode JSON objects if they are stored one per line. The `json` +parameter accepts a hash containing `message_key`, `keys_under_root`, `overwrite_keys`, and `add_error_key` +as documented in the filebeat [configuration documentation](https://www.elastic.co/guide/en/beats/filebeat/5.5/configuration-filebeat-options.html#config-json). ### Prospectors in Hiera -Prospectors can be declared in hiera using the `prospectors` parameter. By default, hiera will not merge -prospector declarations down the hiera hierarchy. To change the behavior in puppet 3 use the `prospectors_merge` -parameter. In puppet 4, you can use `prospectors_merge`, but can also use the +Prospectors can be defined in hiera using the `prospectors` parameter. By default, hiera will not merge +prospector declarations down the hiera hierarchy. That behavior can be changed by configuring the [lookup_options](https://docs.puppet.com/puppet/latest/reference/lookup_quick.html#setting-lookupoptions-in-data) flag. -When `prospectors_merge` is set to true, `prospectors` will be replaced by the output of -`hiera_hash('filebeat::prospectors')`. - ### Usage on Windows When installing on Windows, this module will download the windows version of Filebeat from @@ -150,6 +142,63 @@ When installing on Windows, this module will download the windows version of Fil can be overridden using the `tmp_dir` parameter. `tmp_dir` is not managed by this module, but is expected to exist as a directory that puppet can write to. +### Processors + +Filebeat 5.0 and greater includes a new libbeat feature for filtering and/or enhancing all +exported data through processors before being sent to the configured output(s). They can be +defined as a hash added to the class declaration (also used for automatically creating +processors using hiera), or as their own defined resources. + +To drop the offset and input_type fields from all events: + +```puppet +class{"filebeat": + processors => { + "drop_fields" => { + "params" => {"fields" => ["input_type", "offset"]} + }, + }, +} +``` + +To drop all events that have the http response code equal to 200: + +```puppet +class{"filebeat": + processors => { + "drop_event" => { + "when" => {"equals" => {"http.code" => 200}} + }, + }, +} +``` + +Now to combine these examples into a single definition: + +```puppet +class{"filebeat": + processors => { + "drop_fields" => { + "params" => {"fields" => ["input_type", "offset"]}, + "priority" => 1, + }, + "drop_event" => { + "when" => {"equals" => {"http.code" => 200}}, + "priority: => 2, + }, + }, +} +``` + +For more information please review the documentation [here](https://www.elastic.co/guide/en/beats/filebeat/5.1/configuration-processors.html). + +#### Processors in Hiera + +Processors can be declared in hiera using the `processors` parameter. By default, hiera will not merge +processor declarations down the hiera hierarchy. That behavior can be changed by configuring the +[lookup_options](https://docs.puppet.com/puppet/latest/reference/lookup_quick.html#setting-lookupoptions-in-data) +flag. + ## Reference - [**Public Classes**](#public-classes) - [Class: filebeat](#class-filebeat) @@ -163,6 +212,7 @@ but is expected to exist as a directory that puppet can write to. - [Class: filebeat::install::windows](#class-filebeatinstallwindows) - [**Public Defines**](#public-defines) - [Define: filebeat::prospector](#define-filebeatprospector) + - [Define: filebeat::processors](#define-filebeatprocessor) ### Public Classes @@ -171,32 +221,37 @@ but is expected to exist as a directory that puppet can write to. Installs and configures filebeat. **Parameters within `filebeat`** -- `major_version`: [String] The major version of filebeat to install. Should be either undef, 1, or 5. (default 5 if 1 not already installed) -- `package_ensure`: [String] The ensure parameter for the filebeat package (default: present) +- `package_ensure`: [String] The ensure parameter for the filebeat package If set to absent, + prospectors and processors passed as parameters are ignored and everything managed by + puppet will be removed. (default: present) - `manage_repo`: [Boolean] Whether or not the upstream (elastic) repo should be configured or not (default: true) +- `major_version`: [Enum] The major version of Filebeat to install. Should be either `5` or `6`. The default value is `5`. - `service_ensure`: [String] The ensure parameter on the filebeat service (default: running) - `service_enable`: [String] The enable parameter on the filebeat service (default: true) +- `param repo_priority`: [Integer] Repository priority. yum and apt supported (default: undef) - `service_provider`: [String] The provider parameter on the filebeat service (default: on RedHat based systems use redhat, otherwise undefined) - `spool_size`: [Integer] How large the spool should grow before being flushed to the network (default: 2048) - `idle_timeout`: [String] How often the spooler should be flushed even if spool size isn't reached (default: 5s) - `publish_async`: [Boolean] If set to true filebeat will publish while preparing the next batch of lines to transmit (defualt: false) -- `registry_file`: [String] The registry file used to store positions, absolute or relative to working directory (default .filebeat) +- `registry_file`: [String] The registry file used to store positions, must be an absolute path (default is OS dependent - see params.pp) - `config_file`: [String] Where the configuration file managed by this module should be placed. If you think you might want to use this, read the [limitations](#using-config_file) first. Defaults to the location that filebeat expects for your operating system. - `config_dir`: [String] The directory where prospectors should be defined (default: /etc/filebeat/conf.d) - `config_dir_mode`: [String] The permissions mode set on the configuration directory (default: 0755) +- `config_dir_owner`: [String] The owner of the configuration directory (default: root). Linux only. +- `config_dir_group`: [String] The group of the configuration directory (default: root). Linux only. - `config_file_mode`: [String] The permissions mode set on configuration files (default: 0644) +- `config_file_owner`: [String] The owner of the configuration files, including prospectors (default: root). Linux only. +- `config_file_group`: [String] The group of the configuration files, including prospectors (default: root). Linux only. - `purge_conf_dir`: [Boolean] Should files in the prospector configuration directory not managed by puppet be automatically purged - `outputs`: [Hash] Will be converted to YAML for the required outputs section of the configuration (see documentation, and above) - `shipper`: [Hash] Will be converted to YAML to create the optional shipper section of the filebeat config (see documentation) - `logging`: [Hash] Will be converted to YAML to create the optional logging section of the filebeat config (see documentation) -- `conf_template`: [String] The configuration template to use to generate the main filebeat.yml config file +- `conf_template`: [String] The configuration template to use to generate the main filebeat.yml config file. - `download_url`: [String] The URL of the zip file that should be downloaded to install filebeat (windows only) - `install_dir`: [String] Where filebeat should be installed (windows only) - `tmp_dir`: [String] Where filebeat should be temporarily downloaded to so it can be installed (windows only) -- `use_generic_template`: [Boolean] Use a more generic version of the configuration template. The generic template is more - future proof (if types are correct), but looks very different than the example file (default: false) - `shutdown_timeout`: [String] How long filebeat waits on shutdown for the publisher to finish sending events - `beat_name`: [String] The name of the beat shipper (default: hostname) - `tags`: [Array] A list of tags that will be included with each published transaction @@ -204,8 +259,9 @@ Installs and configures filebeat. - `max_procs`: [Number] The maximum number of CPUs that can be simultaneously used - `fields`: [Hash] Optional fields that should be added to each event output - `fields_under_root`: [Boolean] If set to true, custom fields are stored in the top level instead of under fields +- `disable_config_test`: [Boolean] If set to true, configuration tests won't be run on config files before writing them. +- `processors`: [Hash] Processors that should be configured. - `prospectors`: [Hash] Prospectors that will be created. Commonly used to create prospectors using hiera -- `prospectors_merge`: [Boolean] If true, `hiera_hash()` will be used to build the the `prospectors` parameter (default: false) ### Private Classes @@ -254,10 +310,9 @@ to fully understand what these parameters do. - `input_type`: [String] log or stdin - where filebeat reads the log from (default:log) - `fields`: [Hash] Optional fields to add information to the output (default: {}) - `fields_under_root`: [Boolean] Should the `fields` parameter fields be stored at the top level of indexed documents. - - `ignore_older`: [String] Files older than this field will be ignored by filebeat (default: 24h in filebeat < 1.2.0, infinite in filebeat >= 1.2.0) + - `ignore_older`: [String] Files older than this field will be ignored by filebeat (default: ignore nothing) - `close_older`: [String] Files that haven't been modified since `close_older`, they'll be closed. New - modifications will be read when files are scanned again according to `scan_frequency`. Introduced in - filebeat 1.2.0 (default: 1h) + modifications will be read when files are scanned again according to `scan_frequency`. (default: 1h) - `log_type`: [String] \(Deprecated - use `doc_type`\) The document_type setting (optional - default: log) - `doc_type`: [String] The event type to used for published lines, used as type field in logstash and elasticsearch (optional - default: log) @@ -269,43 +324,43 @@ to fully understand what these parameters do. - `backoff_factor`: [Integer] `backoff` is multiplied by this parameter until `max_backoff` is reached to determine the actual backoff (default: 2) - `force_close_files`: [Boolean] Should filebeat forcibly close a file when renamed (default: false) + - `pipeline`: [String] Filebeat can be configured for a different ingest pipeline for each prospector (default: undef) - `include_lines`: [Array] A list of regular expressions to match the lines that you want to include. Ignored if empty (default: []) - `exclude_lines`: [Array] A list of regular expressions to match the files that you want to exclude. Ignored if empty (default: []) - `max_bytes`: [Integer] The maximum number of bytes that a single log message can have (default: 10485760) + - `json`: [Hash] Options that control how filebeat handles decoding of log messages in JSON format + [See above](#json-logs). (default: {}) - `multiline`: [Hash] Options that control how Filebeat handles log messages that span multiple lines. [See above](#multiline-logs). (default: {}) - ## Limitations This module doesn't load the [elasticsearch index template](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-getting-started.html#filebeat-template) into elasticsearch (required when shipping directly to elasticsearch). -Only filebeat versions after 1.0.0-rc1 are supported. 1.0.0-rc1 and older don't -support YAML like the ruby template can easily generate. - When installing on Windows, there's an expectation that `C:\Temp` already exists, or an alternative location specified in the `tmp_dir` parameter exists and is writable by puppet. The temp directory is used to store the downloaded installer only. -### Debian Systems +### Generic template -Filebeat 5.x and newer requires apt-transport-https, but this module won't install it for you. +By default, a generic, open ended template is used that simply converts your configuration into +a hash that is produced as YAML on the system. To use a template that is more strict, but possibly +incomplete, set `conf_template` to `filebeat/filebeat.yml.erb`. + +### Registry Path -### Pre-1.9.1 Ruby -If you're on a system running a Ruby pre-1.9.1, hashes aren't sorted consistently, causing puppet runs to -not be idempotent. To fix this, a limited template is used if the rubyversion is pre-1.9.1. -The limited template only supports elasticsearch, logstash, file, and console outputs, and not all options -may be supported (there is no warning when an option is omitted). Unlike with newer rubies, as new versions -of filebeat are released, this template may not work until it's updated, even if you're using an updated -configuration hash. +The default registry file in this module doesn't match the filebeat default, but moving the file +while the filbeat service is running can cause data duplication or data loss. If you're installing +filebeat for the first time you should consider setting `registry_file` to match the +[default](https://www.elastic.co/guide/en/beats/filebeat/current/configuration-global-options.html#_registry_file). -If you don't care about keeping puppet idempotent, this can be overridden by setting the `conf_template` -parameter to 'filebeat/filebeat.yml.erb'. +Be sure to include a path or the file will be put at the root of your filesystem. -See [templates/filebeat.yml.ruby18.erb](https://github.com/pcfens/puppet-filebeat/blob/master/templates/filebeat.yml.ruby18.erb) -for the all of the details. +### Debian Systems + +Filebeat 5.x and newer requires apt-transport-https, but this module won't install it for you. ### Using config_file There are a few very specific use cases where you don't want this module to directly manage the filebeat diff --git a/Rakefile b/Rakefile index f3fdfab3..81381e0c 100644 --- a/Rakefile +++ b/Rakefile @@ -1,17 +1,2 @@ -require 'puppet-lint/tasks/puppet-lint' require 'puppetlabs_spec_helper/rake_tasks' - -PuppetLint.configuration.fail_on_warnings = true -PuppetLint.configuration.send('relative') -PuppetLint.configuration.send('disable_80chars') -PuppetLint.configuration.send('disable_140chars') -PuppetLint.configuration.send('disable_class_inherits_from_params_class') -PuppetLint.configuration.send('disable_documentation') -PuppetLint.configuration.ignore_paths = ["spec/**/*.pp", "pkg/**/*.pp", "vendor/**/*.pp"] - -desc "Parallel spec tests" -task :parallel_spec do - Rake::Task[:spec_prep].invoke - ParallelTests::CLI.new.run('--type test -t rspec spec/classes spec/defines'.split) - Rake::Task[:spec_clean].invoke -end +require 'puppet-syntax/tasks/puppet-syntax' diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000..bb7cc46f --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,59 @@ +version: 1.1.x.{build} +skip_commits: + message: /^\(?doc\)?.*/ +clone_depth: 10 +init: + - SET + - 'mkdir C:\ProgramData\PuppetLabs\code && exit 0' + - 'mkdir C:\ProgramData\PuppetLabs\facter && exit 0' + - 'mkdir C:\ProgramData\PuppetLabs\hiera && exit 0' + - 'mkdir C:\ProgramData\PuppetLabs\puppet\var && exit 0' +environment: + PUPPET_GEM_VERSION: ~> 4.0 + matrix: + - + RUBY_VERSION: 24-x64 + CHECK: syntax lint + - + RUBY_VERSION: 24-x64 + CHECK: metadata_lint + - + RUBY_VERSION: 24-x64 + CHECK: rubocop + - + RUBY_VERSION: 24-x64 + CHECK: spec + - + RUBY_VERSION: 21-x64 + CHECK: spec +matrix: + fast_finish: true +install: + - set PATH=C:\Ruby%RUBY_VERSION%\bin;%PATH% + # Due to a bug in the version of OpenSSL shipped with Ruby 2.4.1 on Windows + # (https://bugs.ruby-lang.org/issues/11033). Errors are ignored because the + # mingw gem calls out to pacman to install OpenSSL which is already + # installed, causing gem to raise a warning that powershell determines to be + # a fatal error. + - ps: | + $ErrorActionPreference = "SilentlyContinue" + if($env:RUBY_VERSION -eq "24-x64") { + gem install openssl "~> 2.0.4" --no-rdoc --no-ri -- --with-openssl-dir=C:\msys64\mingw64 + } + $host.SetShouldExit(0) + - bundle install --jobs 4 --retry 2 --without system_tests + - type Gemfile.lock +build: off +test_script: + - bundle exec puppet -V + - ruby -v + - gem -v + - bundle -v + - bundle exec rake %CHECK% +notifications: + - provider: Email + to: + - nobody@nowhere.com + on_build_success: false + on_build_failure: false + on_build_status_changed: false diff --git a/lib/facter/filebeat_version.rb b/lib/facter/filebeat_version.rb index 905a3406..71b1f2b3 100644 --- a/lib/facter/filebeat_version.rb +++ b/lib/facter/filebeat_version.rb @@ -1,14 +1,14 @@ require 'facter' Facter.add('filebeat_version') do - confine :kernel => %w(Linux Windows) + confine 'kernel' => %w[Linux Windows] + if File.executable?('/usr/bin/filebeat') + filebeat_version = Facter::Util::Resolution.exec('/usr/bin/filebeat --version') + elsif File.executable?('/usr/share/filebeat/bin/filebeat') + filebeat_version = Facter::Util::Resolution.exec('/usr/share/filebeat/bin/filebeat --version') + elsif File.exist?('c:\Program Files\Filebeat\filebeat.exe') + filebeat_version = Facter::Util::Resolution.exec('"c:\Program Files\Filebeat\filebeat.exe" --version') + end setcode do - if File.executable?('/usr/bin/filebeat') - filebeat_version = Facter::Util::Resolution.exec('/usr/bin/filebeat --version') - elsif File.executable?('/usr/share/filebeat/bin/filebeat') - filebeat_version = Facter::Util::Resolution.exec('/usr/share/filebeat/bin/filebeat --version') - elsif File.executable?('C:/Program Files/Filebeat/filebeat.exe --version') - filebeat_version = Facter::Util::Resolution.exec('C:/Program Files/Filebeat/filebeat.exe --version') - end %r{^filebeat version ([^\s]+)?}.match(filebeat_version)[1] unless filebeat_version.nil? end end diff --git a/manifests/config.pp b/manifests/config.pp index 9ac54ae8..cba75f90 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -1,62 +1,115 @@ +# filebeat::config +# +# Manage the configuration files for filebeat +# +# @summary A private class to manage the filebeat config file class filebeat::config { - $filebeat_config = { - 'shutdown_timeout' => $filebeat::shutdown_timeout, - 'beat_name' => $filebeat::beat_name, - 'tags' => $filebeat::tags, - 'queue_size' => $filebeat::queue_size, - 'max_procs' => $filebeat::max_procs, - 'fields' => $filebeat::fields, - 'fields_under_root' => $filebeat::fields_under_root, - 'filebeat' => { - 'spool_size' => $filebeat::spool_size, - 'idle_timeout' => $filebeat::idle_timeout, - 'registry_file' => $filebeat::registry_file, - 'publish_async' => $filebeat::publish_async, - 'config_dir' => $filebeat::config_dir, - 'shutdown_timeout' => $filebeat::shutdown_timeout, - }, - 'output' => $filebeat::outputs, - 'shipper' => $filebeat::shipper, - 'logging' => $filebeat::logging, - 'runoptions' => $filebeat::run_options, + $major_version = $filebeat::major_version + + if versioncmp($major_version, '6') >= 0 { + $filebeat_config = delete_undef_values({ + 'shutdown_timeout' => $filebeat::shutdown_timeout, + 'name' => $filebeat::beat_name, + 'tags' => $filebeat::tags, + 'max_procs' => $filebeat::max_procs, + 'fields' => $filebeat::fields, + 'fields_under_root' => $filebeat::fields_under_root, + 'filebeat' => { + 'registry_file' => $filebeat::registry_file, + 'config_dir' => $filebeat::config_dir, + 'shutdown_timeout' => $filebeat::shutdown_timeout, + }, + 'output' => $filebeat::outputs, + 'shipper' => $filebeat::shipper, + 'logging' => $filebeat::logging, + 'runoptions' => $filebeat::run_options, + 'processors' => $filebeat::processors, + }) + } else { + $filebeat_config = delete_undef_values({ + 'shutdown_timeout' => $filebeat::shutdown_timeout, + 'name' => $filebeat::beat_name, + 'tags' => $filebeat::tags, + 'queue_size' => $filebeat::queue_size, + 'max_procs' => $filebeat::max_procs, + 'fields' => $filebeat::fields, + 'fields_under_root' => $filebeat::fields_under_root, + 'filebeat' => { + 'spool_size' => $filebeat::spool_size, + 'idle_timeout' => $filebeat::idle_timeout, + 'registry_file' => $filebeat::registry_file, + 'publish_async' => $filebeat::publish_async, + 'config_dir' => $filebeat::config_dir, + 'shutdown_timeout' => $filebeat::shutdown_timeout, + }, + 'output' => $filebeat::outputs, + 'shipper' => $filebeat::shipper, + 'logging' => $filebeat::logging, + 'runoptions' => $filebeat::run_options, + 'processors' => $filebeat::processors, + }) } + Filebeat::Prospector <| |> -> File['filebeat.yml'] + case $::kernel { 'Linux' : { + $validate_cmd = $filebeat::disable_config_test ? { + true => undef, + default => $major_version ? { + '5' => '/usr/share/filebeat/bin/filebeat -N -configtest -c %', + default => '/usr/share/filebeat/bin/filebeat -c % test config', + }, + } + file {'filebeat.yml': - ensure => file, - path => $filebeat::config_file, - content => template($filebeat::real_conf_template), - owner => 'root', - group => 'root', - mode => $filebeat::config_file_mode, - notify => Service['filebeat'], + ensure => $filebeat::file_ensure, + path => $filebeat::config_file, + content => template($filebeat::conf_template), + owner => $filebeat::config_file_owner, + group => $filebeat::config_file_group, + mode => $filebeat::config_file_mode, + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + require => File['filebeat-config-dir'], } file {'filebeat-config-dir': - ensure => directory, + ensure => $filebeat::directory_ensure, path => $filebeat::config_dir, - owner => 'root', - group => 'root', + owner => $filebeat::config_dir_owner, + group => $filebeat::config_dir_group, mode => $filebeat::config_dir_mode, recurse => $filebeat::purge_conf_dir, purge => $filebeat::purge_conf_dir, + force => true, } } # end Linux 'Windows' : { + $cmd_install_dir = regsubst($filebeat::install_dir, '/', '\\', 'G') + $filebeat_path = join([$cmd_install_dir, 'Filebeat', 'filebeat.exe'], '\\') + + $validate_cmd = $filebeat::disable_config_test ? { + true => undef, + default => "\"${filebeat_path}\" -N -configtest -c \"%\"", + } + file {'filebeat.yml': - ensure => file, - path => $filebeat::config_file, - content => template($filebeat::real_conf_template), - notify => Service['filebeat'], + ensure => $filebeat::file_ensure, + path => $filebeat::config_file, + content => template($filebeat::conf_template), + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + require => File['filebeat-config-dir'], } file {'filebeat-config-dir': - ensure => directory, + ensure => $filebeat::directory_ensure, path => $filebeat::config_dir, recurse => $filebeat::purge_conf_dir, purge => $filebeat::purge_conf_dir, + force => true, } } # end Windows diff --git a/manifests/init.pp b/manifests/init.pp index 7a388566..3827783b 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -12,12 +12,12 @@ # }, # } # -# @param major_version [String] The major version of filebeat to install. Should be either undef, 1, or -# 5. (default 5 if 1 not already installed) # @param package_ensure [String] The ensure parameter for the filebeat package (default: present) # @param manage_repo [Boolean] Whether or not the upstream (elastic) repo should be configured or not (default: true) +# @param major_version [Enum] The major version of Filebeat to be installed. # @param service_ensure [String] The ensure parameter on the filebeat service (default: running) # @param service_enable [String] The enable parameter on the filebeat service (default: true) +# @param repo_priority [Integer] Repository priority. yum and apt supported (default: undef) # @param spool_size [Integer] How large the spool should grow before being flushed to the network (default: 2048) # @param idle_timeout [String] How often the spooler should be flushed even if spool size isn't reached (default: 5s) # @param publish_async [Boolean] If set to true filebeat will publish while preparing the next batch of lines to send (defualt: false) @@ -33,110 +33,101 @@ # @param download_url [String] The URL of the zip file that should be downloaded to install filebeat (windows only) # @param install_dir [String] Where filebeat should be installed (windows only) # @param tmp_dir [String] Where filebeat should be temporarily downloaded to so it can be installed (windows only) -# @param use_generic_template [Boolean] Use a more generic version of the configuration template. The generic template is more -# future proof (if types are correct), but looks very different than the example file (default: false) # @param shutdown_timeout [String] How long filebeat waits on shutdown for the publisher to finish sending events # @param beat_name [String] The name of the beat shipper (default: hostname) # @param tags [Array] A list of tags that will be included with each published transaction # @param queue_size [String] The internal queue size for events in the pipeline -# @param max_procs [Number] The maximum number of CPUs that can be simultaneously used +# @param max_procs [Integer] The maximum number of CPUs that can be simultaneously used # @param fields [Hash] Optional fields that should be added to each event output # @param fields_under_root [Boolean] If set to true, custom fields are stored in the top level instead of under fields +# @param processors [Hash] Processors that will be added. Commonly used to create processors using hiera. # @param prospectors [Hash] Prospectors that will be created. Commonly used to create prospectors using hiera # @param prospectors_merge [Boolean] Whether $prospectors should merge all hiera sources, or use simple automatic parameter lookup +# proxy_address [String] Proxy server to use for downloading files class filebeat ( - $major_version = undef, - $package_ensure = $filebeat::params::package_ensure, - $manage_repo = $filebeat::params::manage_repo, - $service_ensure = $filebeat::params::service_ensure, - $service_enable = $filebeat::params::service_enable, - $service_provider = $filebeat::params::service_provider, - $spool_size = $filebeat::params::spool_size, - $idle_timeout = $filebeat::params::idle_timeout, - $publish_async = $filebeat::params::publish_async, - $registry_file = $filebeat::params::registry_file, - $config_file = $filebeat::params::config_file, - $config_dir_mode = $filebeat::params::config_dir_mode, - $config_dir = $filebeat::params::config_dir, - $config_file_mode = $filebeat::params::config_file_mode, - $purge_conf_dir = $filebeat::params::purge_conf_dir, - $outputs = $filebeat::params::outputs, - $shipper = $filebeat::params::shipper, - $logging = $filebeat::params::logging, - $run_options = $filebeat::params::run_options, - $conf_template = undef, - $download_url = $filebeat::params::download_url, - $install_dir = $filebeat::params::install_dir, - $tmp_dir = $filebeat::params::tmp_dir, - #### v5 only #### - $use_generic_template = $filebeat::params::use_generic_template, - $shutdown_timeout = $filebeat::params::shutdown_timeout, - $beat_name = $filebeat::params::beat_name, - $tags = $filebeat::params::tags, - $queue_size = $filebeat::params::queue_size, - $max_procs = $filebeat::params::max_procs, - $fields = $filebeat::params::fields, - $fields_under_root = $filebeat::params::fields_under_root, - #### End v5 onlly #### - $prospectors = {}, - $prospectors_merge = false, + String $package_ensure = $filebeat::params::package_ensure, + Boolean $manage_repo = $filebeat::params::manage_repo, + Enum['5','6'] $major_version = $filebeat::params::major_version, + Variant[Boolean, Enum['stopped', 'running']] $service_ensure = $filebeat::params::service_ensure, + Boolean $service_enable = $filebeat::params::service_enable, + Optional[String] $service_provider = $filebeat::params::service_provider, + Optional[Integer] $repo_priority = undef, + Integer $spool_size = $filebeat::params::spool_size, + String $idle_timeout = $filebeat::params::idle_timeout, + Boolean $publish_async = $filebeat::params::publish_async, + String $registry_file = $filebeat::params::registry_file, + String $config_file = $filebeat::params::config_file, + Optional[String] $config_file_owner = $filebeat::params::config_file_owner, + Optional[String] $config_file_group = $filebeat::params::config_file_group, + String[4,4] $config_dir_mode = $filebeat::params::config_dir_mode, + String $config_dir = $filebeat::params::config_dir, + String[4,4] $config_file_mode = $filebeat::params::config_file_mode, + Optional[String] $config_dir_owner = $filebeat::params::config_dir_owner, + Optional[String] $config_dir_group = $filebeat::params::config_dir_group, + Boolean $purge_conf_dir = $filebeat::params::purge_conf_dir, + Hash $outputs = $filebeat::params::outputs, + Hash $shipper = $filebeat::params::shipper, + Hash $logging = $filebeat::params::logging, + Hash $run_options = $filebeat::params::run_options, + String $conf_template = $filebeat::params::conf_template, + Optional[Pattern[/^(http(?:s)?\:\/\/[a-zA-Z0-9]+(?:(?:\.|\-)[a-zA-Z0-9]+)+(?:\:\d+)?(?:\/[\w\-\.]+)*(?:\/?|\/\w+\.[a-zA-Z]{2,4}(?:\?[\w]+\=[\w\-]+)?)?(?:\&[\w]+\=[\w\-]+)*)$/]] $download_url = undef, # lint:ignore:140chars + Optional[String] $install_dir = $filebeat::params::install_dir, + String $tmp_dir = $filebeat::params::tmp_dir, + Integer $shutdown_timeout = $filebeat::params::shutdown_timeout, + String $beat_name = $filebeat::params::beat_name, + Array $tags = $filebeat::params::tags, + Integer $queue_size = $filebeat::params::queue_size, + Optional[Integer] $max_procs = $filebeat::params::max_procs, + Hash $fields = $filebeat::params::fields, + Boolean $fields_under_root = $filebeat::params::fields_under_root, + Boolean $disable_config_test = $filebeat::params::disable_config_test, + Hash $processors = {}, + Hash $prospectors = {}, + Optional[Pattern[/^(http(?:s)?\:\/\/[a-zA-Z0-9]+(?:(?:\.|\-)[a-zA-Z0-9]+)+(?:\:\d+)?(?:\/[\w\-\.]+)*(?:\/?|\/\w+\.[a-zA-Z]{2,4}(?:\?[\w]+\=[\w\-]+)?)?(?:\&[\w]+\=[\w\-]+)*)$/]] $proxy_address = undef # lint:ignore:140chars ) inherits filebeat::params { - $kernel_fail_message = "${::kernel} is not supported by filebeat." + include ::stdlib - validate_bool($manage_repo, $prospectors_merge) - - if $major_version == undef and $::filebeat_version == undef { - $real_version = '5' - } elsif $major_version == undef and versioncmp($::filebeat_version, '5.0.0') >= 0 { - $real_version = '5' - } elsif $major_version == undef and versioncmp($::filebeat_version, '5.0.0') < 0 { - $real_version = '1' - } else { - $real_version = $major_version - } - - if $conf_template != undef { - $real_conf_template = $conf_template - } elsif $real_version == '1' { - if versioncmp('1.9.1', $::rubyversion) > 0 { - $real_conf_template = "${module_name}/filebeat.yml.ruby18.erb" - } else { - $real_conf_template = "${module_name}/filebeat.yml.erb" - } - } elsif $real_version == '5' { - if $use_generic_template { - $real_conf_template = "${module_name}/filebeat.yml.erb" - } else { - $real_conf_template = "${module_name}/filebeat5.yml.erb" - } - } - - - if $prospectors_merge { - $prospectors_final = hiera_hash('filebeat::prospectors', $prospectors) - } else { - $prospectors_final = $prospectors + $real_download_url = $download_url ? { + undef => "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${package_ensure}-windows-${filebeat::params::url_arch}.zip", + default => $download_url, } if $config_file != $filebeat::params::config_file { warning('You\'ve specified a non-standard config_file location - filebeat may fail to start unless you\'re doing something to fix this') } - validate_hash($outputs, $logging, $prospectors_final) - validate_string($idle_timeout, $registry_file, $config_dir, $package_ensure) - - if $package_ensure == '1.0.0-beta4' or $package_ensure == '1.0.0-rc1' { - fail('Filebeat versions 1.0.0-rc1 and before are unsupported because they don\'t parse normal YAML headers') + if $package_ensure == 'absent' { + $alternate_ensure = 'absent' + $real_service_ensure = 'stopped' + $file_ensure = 'absent' + $directory_ensure = 'absent' + } else { + $alternate_ensure = 'present' + $file_ensure = 'file' + $directory_ensure = 'directory' + $real_service_ensure = $service_ensure } - anchor { 'filebeat::begin': } -> - class { 'filebeat::install': } -> - class { 'filebeat::config': } -> - class { 'filebeat::service': } -> - anchor { 'filebeat::end': } + # If we're removing filebeat, do things in a different order to make sure + # we remove as much as possible + if $package_ensure == 'absent' { + anchor { 'filebeat::begin': } + -> class { '::filebeat::config': } + -> class { '::filebeat::install': } + -> class { '::filebeat::service': } + -> anchor { 'filebeat::end': } + } else { + anchor { 'filebeat::begin': } + -> class { '::filebeat::install': } + -> class { '::filebeat::config': } + -> class { '::filebeat::service': } + -> anchor { 'filebeat::end': } + } - if !empty($prospectors_final) { - create_resources('filebeat::prospector', $prospectors_final) + if $package_ensure != 'absent' { + if !empty($prospectors) { + create_resources('filebeat::prospector', $prospectors) + } } } diff --git a/manifests/install.pp b/manifests/install.pp index 1071de51..a36c1792 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -1,19 +1,24 @@ +# filebeat::install +# +# A private class to manage the installation of Filebeat +# +# @summary A private class that manages the install of Filebeat class filebeat::install { anchor { 'filebeat::install::begin': } case $::kernel { 'Linux': { - class{ 'filebeat::install::linux': + class{ '::filebeat::install::linux': notify => Class['filebeat::service'], } Anchor['filebeat::install::begin'] -> Class['filebeat::install::linux'] -> Anchor['filebeat::install::end'] if $::filebeat::manage_repo { - class { 'filebeat::repo': } + class { '::filebeat::repo': } Class['filebeat::repo'] -> Class['filebeat::install::linux'] } } 'Windows': { - class{'filebeat::install::windows': + class{'::filebeat::install::windows': notify => Class['filebeat::service'], } Anchor['filebeat::install::begin'] -> Class['filebeat::install::windows'] -> Anchor['filebeat::install::end'] @@ -24,4 +29,5 @@ } anchor { 'filebeat::install::end': } + } diff --git a/manifests/install/linux.pp b/manifests/install/linux.pp index 1a75123f..bbe4db85 100644 --- a/manifests/install/linux.pp +++ b/manifests/install/linux.pp @@ -1,3 +1,9 @@ +# filebeat::install::linux +# +# Install the linux filebeat package +# +# @summary A simple class to install the filebeat package +# class filebeat::install::linux { package {'filebeat': ensure => $filebeat::package_ensure, diff --git a/manifests/install/windows.pp b/manifests/install/windows.pp index 34cd88cd..7b831455 100644 --- a/manifests/install/windows.pp +++ b/manifests/install/windows.pp @@ -1,39 +1,82 @@ +# filebeat::install::windows +# +# Download and install filebeat on Windows +# +# @summary A private class that installs filebeat on Windows +# class filebeat::install::windows { - $filename = regsubst($filebeat::download_url, '^https.*\/([^\/]+)\.[^.].*', '\1') + # I'd like to use chocolatey to do this install, but the package for chocolatey is + # failing for updates and seems rather unpredictable at the moment. We may revisit + # that in the future as it would greatly simplify this code and basically reduce it to + # one package resource with type => chocolatey.... + + $filename = regsubst($filebeat::real_download_url, '^https?.*\/([^\/]+)\.[^.].*', '\1') $foldername = 'Filebeat' + $zip_file = join([$filebeat::tmp_dir, "${filename}.zip"], '/') + $install_folder = join([$filebeat::install_dir, $foldername], '/') + $version_file = join([$install_folder, $filename], '/') + + Exec { + provider => powershell, + } - file { $filebeat::install_dir: - ensure => directory + if ! defined(File[$filebeat::install_dir]) { + file { $filebeat::install_dir: + ensure => directory, + } } - remote_file {"${filebeat::tmp_dir}/${filename}.zip": - ensure => present, - source => $filebeat::download_url, - verify_peer => false, + # Note: We can use archive for unzip and cleanup, thus removing the following two resources. + # However, this requires 7zip, which archive can install via chocolatey: + # https://github.com/voxpupuli/puppet-archive/blob/master/manifests/init.pp#L31 + # I'm not choosing to impose those dependencies on anyone at this time... + archive { $zip_file: + source => $filebeat::real_download_url, + cleanup => false, + creates => $version_file, + proxy_server => $filebeat::proxy_address, } exec { "unzip ${filename}": - command => "\$sh=New-Object -COM Shell.Application;\$sh.namespace((Convert-Path '${filebeat::install_dir}')).Copyhere(\$sh.namespace((Convert-Path '${filebeat::tmp_dir}/${filename}.zip')).items(), 16)", - creates => "${filebeat::install_dir}/Filebeat", - provider => powershell, - require => [ + command => "\$sh=New-Object -COM Shell.Application;\$sh.namespace((Convert-Path '${filebeat::install_dir}')).Copyhere(\$sh.namespace((Convert-Path '${zip_file}')).items(), 16)", # lint:ignore:140chars + creates => $version_file, + require => [ File[$filebeat::install_dir], - Remote_file["${filebeat::tmp_dir}/${filename}.zip"], + Archive[$zip_file], ], } - exec { 'rename folder': - command => "Rename-Item '${filebeat::install_dir}/${filename}' Filebeat", - creates => "${filebeat::install_dir}/Filebeat", - provider => powershell, - require => Exec["unzip ${filename}"], + # Clean up after ourselves + file { $zip_file: + ensure => absent, + backup => false, + require => Exec["unzip ${filename}"], + } + + # You can't remove the old dir while the service has files locked... + exec { "stop service ${filename}": + command => 'Set-Service -Name filebeat -Status Stopped', + creates => $version_file, + onlyif => 'if(Get-WmiObject -Class Win32_Service -Filter "Name=\'filebeat\'") {exit 0} else {exit 1}', + require => Exec["unzip ${filename}"], + } + + exec { "rename ${filename}": + command => "Remove-Item '${install_folder}' -Recurse -Force -ErrorAction SilentlyContinue;Rename-Item '${filebeat::install_dir}/${filename}' '${install_folder}'", # lint:ignore:140chars + creates => $version_file, + require => Exec["stop service ${filename}"], + } + + exec { "mark ${filename}": + command => "New-Item '${version_file}' -ItemType file", + creates => $version_file, + require => Exec["rename ${filename}"], } exec { "install ${filename}": - cwd => "${filebeat::install_dir}/Filebeat", - command => './install-service-filebeat.ps1', - onlyif => 'if(Get-WmiObject -Class Win32_Service -Filter "Name=\'filebeat\'") { exit 1 } else {exit 0 }', - provider => powershell, - require => Exec['rename folder'], + cwd => $install_folder, + command => './install-service-filebeat.ps1', + refreshonly => true, + subscribe => Exec["mark ${filename}"], } } diff --git a/manifests/params.pp b/manifests/params.pp index 309462e1..f986e5c8 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -1,44 +1,49 @@ +# filebeat::params +# +# Set a number of default parameters +# +# @summary Set a bunch of default parameters class filebeat::params { - $package_ensure = present - $manage_repo = true - $service_ensure = running - $service_enable = true - $spool_size = 2048 - $idle_timeout = '5s' - $publish_async = false - $shutdown_timeout = 0 - $beat_name = $::fqdn - $tags = [] - $queue_size = 1000 - $max_procs = undef - $registry_file = '.filebeat' - $config_dir_mode = '0755' - $config_file_mode = '0644' - $purge_conf_dir = true - $fields = {} - $fields_under_root = false - $outputs = {} - $shipper = {} - $logging = {} - $run_options = {} - $use_generic_template = false + $manage_repo = true + $major_version = '5' + $service_ensure = running + $service_enable = true + $spool_size = 2048 + $idle_timeout = '5s' + $publish_async = false + $shutdown_timeout = 0 + $beat_name = $::fqdn + $tags = [] + $queue_size = 1000 + $max_procs = undef + $config_file_mode = '0644' + $config_dir_mode = '0755' + $purge_conf_dir = true + $fields = {} + $fields_under_root = false + $outputs = {} + $shipper = {} + $logging = {} + $run_options = {} + $kernel_fail_message = "${::kernel} is not supported by filebeat." + $conf_template = "${module_name}/pure_hash.yml.erb" + $disable_config_test = false + + $config_file_owner = 'root' + $config_file_group = 'root' + $config_dir_owner = 'root' + $config_dir_group = 'root' - # These are irrelevant as long as the template is set based on the major_version parameter - # if versioncmp('1.9.1', $::rubyversion) > 0 { - # $conf_template = "${module_name}/filebeat.yml.ruby18.erb" - # } else { - # $conf_template = "${module_name}/filebeat.yml.erb" - # } - # case $::kernel { 'Linux' : { + $package_ensure = present $config_file = '/etc/filebeat/filebeat.yml' $config_dir = '/etc/filebeat/conf.d' + $registry_file = '/var/lib/filebeat/registry' # These parameters are ignored if/until tarball installs are supported in Linux $tmp_dir = '/tmp' $install_dir = undef - $download_url = undef case $::osfamily { 'RedHat': { $service_provider = 'redhat' @@ -47,19 +52,26 @@ $service_provider = undef } } + $url_arch = undef } 'Windows' : { + $package_ensure = '5.6.2' $config_file = 'C:/Program Files/Filebeat/filebeat.yml' $config_dir = 'C:/Program Files/Filebeat/conf.d' - $download_url = 'https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.0.1-windows-x86_64.zip' + $registry_file = 'C:/ProgramData/filebeat/registry' $install_dir = 'C:/Program Files' $tmp_dir = 'C:/Windows/Temp' $service_provider = undef + $url_arch = $::architecture ? { + 'x86' => 'x86', + 'x64' => 'x86_64', + default => fail("${::architecture} is not supported by filebeat."), + } } default : { - fail("${::kernel} is not supported by filebeat.") + fail($kernel_fail_message) } } } diff --git a/manifests/prospector.pp b/manifests/prospector.pp index eea3c981..42058ba2 100644 --- a/manifests/prospector.pp +++ b/manifests/prospector.pp @@ -1,3 +1,11 @@ +# filebeat::prospector +# +# A description of what this defined type does +# +# @summary A short summary of the purpose of this defined type. +# +# @example +# filebeat::prospector { 'namevar': } define filebeat::prospector ( $ensure = present, $paths = [], @@ -15,39 +23,74 @@ $backoff = '1s', $max_backoff = '10s', $backoff_factor = 2, + $close_inactive = '5m', + $close_renamed = false, + $close_removed = true, + $close_eof = false, + $clean_inactive = 0, + $clean_removed = true, + $close_timeout = 0, $force_close_files = false, $include_lines = [], $exclude_lines = [], $max_bytes = '10485760', $multiline = {}, + $json = {}, + $tags = [], + $symlinks = false, + $pipeline = undef, ) { - validate_hash($fields, $multiline) - validate_array($paths, $exclude_files, $include_lines, $exclude_lines) + validate_hash($fields, $multiline, $json) + validate_array($paths, $exclude_files, $include_lines, $exclude_lines, $tags) + validate_bool($tail_files, $close_renamed, $close_removed, $close_eof, $clean_removed, $symlinks) - $prospector_template = $filebeat::real_version ? { - '5' => 'prospector5.yml.erb', - '1' => 'prospector.yml.erb', - } + $prospector_template = 'prospector.yml.erb' case $::kernel { 'Linux' : { - file { "filebeat-${name}": - ensure => $ensure, - path => "${filebeat::config_dir}/${name}.yml", - owner => 'root', - group => 'root', - mode => $::filebeat::config_file_mode, - content => template("${module_name}/${prospector_template}"), - notify => Service['filebeat'], + if !$filebeat::disable_config_test { + file { "filebeat-${name}": + ensure => $ensure, + path => "${filebeat::config_dir}/${name}.yml", + owner => 'root', + group => 'root', + mode => $::filebeat::config_file_mode, + content => template("${module_name}/${prospector_template}"), + validate_cmd => '/usr/share/filebeat/bin/filebeat -N -configtest -c %', + notify => Service['filebeat'], + } + } else { + file { "filebeat-${name}": + ensure => $ensure, + path => "${filebeat::config_dir}/${name}.yml", + owner => 'root', + group => 'root', + mode => $::filebeat::config_file_mode, + content => template("${module_name}/${prospector_template}"), + notify => Service['filebeat'], + } + } } 'Windows' : { - file { "filebeat-${name}": - ensure => $ensure, - path => "${filebeat::config_dir}/${name}.yml", - content => template("${module_name}/${prospector_template}"), - notify => Service['filebeat'], + $filebeat_path = 'c:\Program Files\Filebeat\filebeat.exe' + + if !$filebeat::disable_config_test { + file { "filebeat-${name}": + ensure => $ensure, + path => "${filebeat::config_dir}/${name}.yml", + content => template("${module_name}/${prospector_template}"), + validate_cmd => "\"${filebeat_path}\" -N -configtest -c \"%\"", + notify => Service['filebeat'], + } + } else { + file { "filebeat-${name}": + ensure => $ensure, + path => "${filebeat::config_dir}/${name}.yml", + content => template("${module_name}/${prospector_template}"), + notify => Service['filebeat'], + } } } default : { diff --git a/manifests/repo.pp b/manifests/repo.pp index 18e1d04e..8133466d 100644 --- a/manifests/repo.pp +++ b/manifests/repo.pp @@ -1,13 +1,11 @@ +# filebeat::repo +# +# Manage the repository for Filebeat (Linux only for now) +# +# @summary Manages the yum, apt, and zypp repositories for Filebeat class filebeat::repo { - $debian_repo_url = $filebeat::real_version ? { - '1' => 'http://packages.elastic.co/beats/apt', - '5' => 'https://artifacts.elastic.co/packages/5.x/apt', - } - - $yum_repo_url = $filebeat::real_version ? { - '1' => 'https://packages.elastic.co/beats/yum/el/$basearch', - '5' => 'https://artifacts.elastic.co/packages/5.x/yum', - } + $debian_repo_url = "https://artifacts.elastic.co/packages/${filebeat::major_version}.x/apt" + $yum_repo_url = "https://artifacts.elastic.co/packages/${filebeat::major_version}.x/yum" case $::osfamily { 'Debian': { @@ -17,9 +15,11 @@ if !defined(Apt::Source['beats']){ apt::source { 'beats': + ensure => $::filebeat::alternate_ensure, location => $debian_repo_url, release => 'stable', repos => 'main', + pin => $::filebeat::repo_priority, key => { id => '46095ACC8548582C1A2699A9D27D666CD88E42B4', source => 'https://artifacts.elastic.co/GPG-KEY-elasticsearch', @@ -30,10 +30,12 @@ 'RedHat', 'Linux': { if !defined(Yumrepo['beats']){ yumrepo { 'beats': + ensure => $::filebeat::alternate_ensure, descr => 'elastic beats repo', baseurl => $yum_repo_url, gpgcheck => 1, gpgkey => 'https://artifacts.elastic.co/GPG-KEY-elasticsearch', + priority => $::filebeat::repo_priority, enabled => 1, } } @@ -46,6 +48,7 @@ } if !defined(Zypprepo['beats']){ zypprepo { 'beats': + ensure => $::filebeat::alternate_ensure, baseurl => $yum_repo_url, enabled => 1, autorefresh => 1, @@ -60,4 +63,5 @@ fail($filebeat::kernel_fail_message) } } + } diff --git a/manifests/service.pp b/manifests/service.pp index a466ec87..80afd08f 100644 --- a/manifests/service.pp +++ b/manifests/service.pp @@ -1,6 +1,11 @@ +# filebeat::service +# +# Manage the filebeat service +# +# @summary Manage the filebeat service class filebeat::service { service { 'filebeat': - ensure => $filebeat::service_ensure, + ensure => $filebeat::real_service_ensure, enable => $filebeat::service_enable, provider => $filebeat::service_provider, } diff --git a/metadata.json b/metadata.json index 21b0785f..e364458c 100644 --- a/metadata.json +++ b/metadata.json @@ -1,30 +1,32 @@ { "name": "pcfens-filebeat", - "version": "0.8.6", + "version": "2.3.0", "author": "pcfens", - "license": "Apache-2.0", "summary": "A module to install and manage the filebeat log shipper", + "license": "Apache-2.0", "source": "https://github.com/pcfens/puppet-filebeat", "project_page": "https://github.com/pcfens/puppet-filebeat", "issues_url": "https://github.com/pcfens/puppet-filebeat/issues", "tags": ["filebeat", "logstash", "elasticsearch", "elastic"], - "operatingsystem_support": [ + "dependencies": [ { - "operatingsystem": "RedHat", - "operatingsystemrelease": [ - "5.0", - "6.0", - "7.0" - ] + "name": "puppetlabs/stdlib", + "version_requirement": ">=4.6.0 <5.0.0" }, { - "operatingsystem": "Ubuntu", - "operatingsystemrelease": [ - "12.04", - "14.04", - "16.04" - ] + "name": "puppetlabs/apt", + "version_requirement": ">=2.0.0 <5.0.0" + }, + { + "name": "puppetlabs/powershell", + "version_requirement": ">= 1.0.1 < 3.0.0" }, + { + "name": "puppet/archive", + "version_requirement": ">= 0.5.0 < 3.0.0" + } + ], + "operatingsystem_support": [ { "operatingsystem": "CentOS", "operatingsystemrelease": [ @@ -34,15 +36,16 @@ ] }, { - "operatingsystem": "Fedora", + "operatingsystem": "OracleLinux", "operatingsystemrelease": [ - "21", - "22" + "6", + "7" ] }, { - "operatingsystem": "OracleLinux", + "operatingsystem": "RedHat", "operatingsystemrelease": [ + "5", "6", "7" ] @@ -56,39 +59,34 @@ ] }, { - "operatingsystem": "Windows", + "operatingsystem": "Debian", "operatingsystemrelease": [ - "Server 2012", - "Server 2012 R2" + "8" ] - } - ], - "dependencies": [ - { - "name": "puppetlabs/stdlib", - "version_requirement": ">=4.6.0 <5.0.0" - }, - { - "name": "puppetlabs/apt", - "version_requirement": ">=2.0.0 <3.0.0" }, { - "name": "puppetlabs/powershell", - "version_requirement": ">= 1.0.1 < 2.0.0" + "operatingsystem": "Ubuntu", + "operatingsystemrelease": [ + "12.04", + "14.04", + "16.04" + ] }, { - "name": "lwf/remote_file", - "version_requirement": ">= 0.2.0 < 2.0.0" + "operatingsystem": "windows", + "operatingsystemrelease": [ + "2012", + "2012 R2" + ] } ], "requirements": [ - { - "name": "pe", - "version_requirement": ">= 3.0.0 < 2016.5.0" - }, { "name": "puppet", - "version_requirement": ">= 3.0.0 < 5.0.0" + "version_requirement": ">= 4.0.0 < 6.0.0" } - ] + ], + "pdk-version": "1.1.0", + "template-url": "file:///opt/puppetlabs/pdk/share/cache/pdk-module-template.git", + "template-ref": "heads/master-0-g54e88a4" } diff --git a/spec/acceptance/001_basic_spec.rb b/spec/acceptance/001_basic_spec.rb index ae5a9afc..433cf919 100644 --- a/spec/acceptance/001_basic_spec.rb +++ b/spec/acceptance/001_basic_spec.rb @@ -1,73 +1,92 @@ require 'spec_helper_acceptance' -describe "filebeat class" do +RSpec.shared_examples 'filebeat' do + describe package('filebeat') do + it { is_expected.to be_installed } + end - package_name = 'filebeat' - service_name = 'filebeat' + describe service('filebeat') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end - context 'default parameters' do + describe file('/etc/filebeat/filebeat.yml') do + it { is_expected.to be_file } + it { is_expected.to contain('---') } + it { is_expected.not_to contain('max_procs: !ruby') } + end +end - let(:pp) do - <<-EOS - if $::osfamily == 'Debian' { - include ::apt +describe 'filebeat class' do + let(:pp) do + <<-EOS + if $::osfamily == 'Debian' { + include ::apt - package { 'apt-transport-https': - ensure => present, - } + package { 'apt-transport-https': + ensure => present, } + } - class { 'filebeat': - outputs => { - 'logstash' => { - 'bulk_max_size' => 1024, - 'hosts' => [ - 'localhost:5044', - ], - }, - 'file' => { - 'path' => '/tmp', - 'filename' => 'filebeat', - 'rotate_every_kb' => 10240, - 'number_of_files' => 2, - }, + class { 'filebeat': + major_version => '#{major_version}', + outputs => { + 'logstash' => { + 'bulk_max_size' => 1024, + 'hosts' => [ + 'localhost:5044', + ], }, - shipper => { - refresh_topology_freq => 10, - topology_expire => 15, - queue_size => 1000, + 'file' => { + 'path' => '/tmp', + 'filename' => 'filebeat', + 'rotate_every_kb' => 10240, + 'number_of_files' => 2, }, - logging => { - files => { - rotateeverybytes => 10485760, - keepfiles => 7, - } - }, - prospectors => { - 'system-logs' => { - doc_type => 'system', - paths => [ - '/var/log/dmesg', - ], - fields => { - service => 'system', - file => 'dmesg', - } - } + }, + shipper => { + refresh_topology_freq => 10, + topology_expire => 15, + queue_size => 1000, + }, + logging => { + files => { + rotateeverybytes => 10485760, + keepfiles => 7, + } + }, + prospectors => { + 'system-logs' => { + doc_type => 'system', + paths => [ + '/var/log/dmesg', + ], + fields => { + service => 'system', + file => 'dmesg', + }, + tags => [ + 'tag1', + 'tag2', + 'tag3', + ] } } - EOS - end + } + EOS + end + + context 'with $major_version = 5' do + let(:major_version) { 5 } - it_behaves_like "an idempotent resource" + it_behaves_like 'an idempotent resource' + include_examples 'filebeat' + end - describe service(service_name) do - it { should be_enabled } - it { should be_running } - end + context 'with $major_version = 6' do + let(:major_version) { 6 } - describe package(package_name) do - it { should be_installed } - end + it_behaves_like 'an idempotent resource' + include_examples 'filebeat' end end diff --git a/spec/acceptance/002_generic_template_spec.rb b/spec/acceptance/002_generic_template_spec.rb deleted file mode 100644 index 09ebbdc6..00000000 --- a/spec/acceptance/002_generic_template_spec.rb +++ /dev/null @@ -1,74 +0,0 @@ -require 'spec_helper_acceptance' - -describe "filebeat class" do - - package_name = 'filebeat' - service_name = 'filebeat' - - context 'use generic template' do - - let(:pp) do - <<-EOS - if $::osfamily == 'Debian' { - include ::apt - - package { 'apt-transport-https': - ensure => present, - } - } - - class { 'filebeat': - use_generic_template => true, - outputs => { - 'logstash' => { - 'bulk_max_size' => 1024, - 'hosts' => [ - 'localhost:5044', - ], - }, - 'file' => { - 'path' => '/tmp', - 'filename' => 'filebeat', - 'rotate_every_kb' => 10240, - 'number_of_files' => 2, - }, - }, - shipper => { - refresh_topology_freq => 10, - topology_expire => 15, - queue_size => 1000, - }, - logging => { - files => { - rotateeverybytes => 10485760, - keepfiles => 7, - } - }, - prospectors => { - 'system-logs' => { - doc_type => 'system', - paths => [ - '/var/log/dmesg', - ], - fields => { - service => 'system', - file => 'dmesg', - } - } - } - } - EOS - end - - it_behaves_like "an idempotent resource" - - describe service(service_name) do - it { should be_enabled } - it { should be_running } - end - - describe package(package_name) do - it { should be_installed } - end - end -end diff --git a/spec/acceptance/nodesets/default.yml b/spec/acceptance/nodesets/default.yml index 81bb47fa..4203e73f 100644 --- a/spec/acceptance/nodesets/default.yml +++ b/spec/acceptance/nodesets/default.yml @@ -1,17 +1,15 @@ HOSTS: - ubuntu-1404-x64: + ubuntu-1604-x64: roles: - - master - - database - - dashboard - platform: ubuntu-14.04-amd64 + - agent + platform: ubuntu-16.04-amd64 hypervisor: docker - image: ubuntu:14.04 + image: ubuntu:16.04 docker_preserve_image: true docker_cmd: '["/sbin/init"]' docker_image_commands: # ensure that upstart is booting correctly in the container - - 'rm /usr/sbin/policy-rc.d && rm /sbin/initctl && dpkg-divert --rename --remove /sbin/initctl && apt-get update && apt-get install -y net-tools wget && locale-gen en_US.UTF-8' + - 'rm /usr/sbin/policy-rc.d && rm /sbin/initctl && dpkg-divert --rename --remove /sbin/initctl && apt-get update && apt-get install -y net-tools wget locales && locale-gen en_US.UTF-8' CONFIG: trace_limit: 200 type: foss diff --git a/spec/acceptance/nodesets/ubuntu-16.04.yml b/spec/acceptance/nodesets/ubuntu-16.04.yml deleted file mode 100644 index 59f1d3ed..00000000 --- a/spec/acceptance/nodesets/ubuntu-16.04.yml +++ /dev/null @@ -1,15 +0,0 @@ -HOSTS: - ubuntu-1604-x64: - roles: - - agent - platform: ubuntu-16.04-amd64 - hypervisor: docker - image: ubuntu:16.04 - docker_preserve_image: true - docker_cmd: '["/sbin/init"]' - docker_image_commands: - # ensure that upstart is booting correctly in the container - - 'rm /usr/sbin/policy-rc.d && rm /sbin/initctl && dpkg-divert --rename --remove /sbin/initctl && apt-get update && apt-get install -y net-tools wget && locale-gen en_US.UTF-8' -CONFIG: - trace_limit: 200 - type: foss diff --git a/spec/classes/config_spec.rb b/spec/classes/config_spec.rb new file mode 100644 index 00000000..b89b9468 --- /dev/null +++ b/spec/classes/config_spec.rb @@ -0,0 +1,76 @@ +require 'spec_helper' + +describe 'filebeat::config' do + let :pre_condition do + 'include ::filebeat' + end + + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + let(:pre_condition) { "class { 'filebeat': major_version => '#{major_version}' }" } + + [5, 6].each do |version| + context "with $major_version == #{version}" do + let(:major_version) { version } + + let(:validate_cmd) do + case major_version + when 5 + '/usr/share/filebeat/bin/filebeat -N -configtest -c %' + else + '/usr/share/filebeat/bin/filebeat -c % test config' + end + end + + case os_facts[:kernel] + when 'Linux' + it { is_expected.to compile } + it { + is_expected.to contain_file('filebeat.yml').with( + ensure: 'file', + path: '/etc/filebeat/filebeat.yml', + owner: 'root', + group: 'root', + mode: '0644', + validate_cmd: validate_cmd, + notify: 'Service[filebeat]', + require: 'File[filebeat-config-dir]', + ) + } + + it { + is_expected.to contain_file('filebeat-config-dir').with( + ensure: 'directory', + path: '/etc/filebeat/conf.d', + owner: 'root', + group: 'root', + mode: '0755', + recurse: true, + purge: true, + ) + } + when 'Windows' + it { + is_expected.to contain_file('filebeat.yml').with( + ensure: 'file', + path: 'C:/Program Files/Filebeat/filebeat.yml', + notify: 'Service[filebeat]', + require: 'File[filebeat-config-dir]', + ) + } + + it { + is_expected.to contain_file('filebeat-config-dir').with( + ensure: 'directory', + path: 'C:/Program Files/Filebeat/conf.d', + recurse: true, + purge: true, + ) + } + end + end + end + end + end +end diff --git a/spec/classes/coverage_spec.rb b/spec/classes/coverage_spec.rb new file mode 100644 index 00000000..c083569a --- /dev/null +++ b/spec/classes/coverage_spec.rb @@ -0,0 +1,3 @@ +require 'rspec-puppet' + +at_exit { RSpec::Puppet::Coverage.report! } diff --git a/spec/classes/filebeat_spec.rb b/spec/classes/filebeat_spec.rb index d90f8539..dfecf6c4 100644 --- a/spec/classes/filebeat_spec.rb +++ b/spec/classes/filebeat_spec.rb @@ -1,284 +1,25 @@ require 'spec_helper' -describe 'filebeat', :type => :class do - context 'defaults' do - context 'on a system without filebeat installed' do - describe 'on a Debian system' do - let :facts do - { - :kernel => 'Linux', - :osfamily => 'Debian', - :lsbdistid => 'Ubuntu', - :lsbdistrelease => '16.04', - :rubyversion => '2.3.1', - :puppetversion => Puppet.version, - :filebeat_version => nil - } - end - - it { is_expected.to contain_package('filebeat') } - it { is_expected.to contain_file('filebeat.yml').with( - :path => '/etc/filebeat/filebeat.yml', - :mode => '0644', - )} - it { is_expected.to contain_file('filebeat-config-dir').with( - :ensure => 'directory', - :path => '/etc/filebeat/conf.d', - :mode => '0755', - :recurse => true, - )} - it { is_expected.to contain_service('filebeat').with( - :enable => true, - :ensure => 'running', - :provider => nil, # Provider should use the resource default - )} - it { is_expected.to contain_apt__source('beats').with( - :location => 'https://artifacts.elastic.co/packages/5.x/apt', - :key => { - 'id' => '46095ACC8548582C1A2699A9D27D666CD88E42B4', - 'source' => 'https://artifacts.elastic.co/GPG-KEY-elasticsearch', - } - )} - end - - describe 'on a RHEL system' do - let :facts do - { - :kernel => 'Linux', - :osfamily => 'RedHat', - :rubyversion => '2.3.1', - :puppetversion => Puppet.version, - :filebeat_version => nil - } - end - - it { is_expected.to contain_yumrepo('beats').with( - :baseurl => 'https://artifacts.elastic.co/packages/5.x/yum', - :gpgkey => 'https://artifacts.elastic.co/GPG-KEY-elasticsearch', - ) } - - it { is_expected.to contain_service('filebeat').with( - :enable => true, - :ensure => 'running', - :provider => 'redhat', - )} - - end - - describe 'on a Windows system' do - let :facts do - { - :kernel => 'Windows', - :rubyversion => '2.3.1', - :puppetversion => Puppet.version, - :filebeat_version => nil - } - end - - it { is_expected.to contain_file('filebeat.yml').with( - :path => 'C:/Program Files/Filebeat/filebeat.yml', - )} - it { is_expected.to contain_file('filebeat-config-dir').with( - :ensure => 'directory', - :path => 'C:/Program Files/Filebeat/conf.d', - :recurse => true, - )} - it { is_expected.to contain_service('filebeat').with( - :enable => true, - :ensure => 'running', - :provider => nil, # Provider should use the resource default - )} - end - end - - context 'on a system with 5.x already installed' do - describe 'on a Debian system' do - let :facts do - { - :kernel => 'Linux', - :osfamily => 'Debian', - :lsbdistid => 'Ubuntu', - :lsbdistrelease => '16.04', - :rubyversion => '2.3.1', - :puppetversion => Puppet.version, - :filebeat_version => '5.0.0' - } - end - - it { is_expected.to contain_package('filebeat') } - it { is_expected.to contain_file('filebeat.yml').with( - :path => '/etc/filebeat/filebeat.yml', - :mode => '0644', - )} - it { is_expected.to contain_file('filebeat-config-dir').with( - :ensure => 'directory', - :path => '/etc/filebeat/conf.d', - :mode => '0755', - :recurse => true, - )} - it { is_expected.to contain_service('filebeat').with( - :enable => true, - :ensure => 'running', - :provider => nil, # Provider should use the resource default - )} - it { is_expected.to contain_apt__source('beats').with( - :location => 'https://artifacts.elastic.co/packages/5.x/apt', - :key => { - 'id' => '46095ACC8548582C1A2699A9D27D666CD88E42B4', - 'source' => 'https://artifacts.elastic.co/GPG-KEY-elasticsearch', - } - )} - end - - describe 'on a RHEL system' do - let :facts do - { - :kernel => 'Linux', - :osfamily => 'RedHat', - :rubyversion => '2.3.1', - :puppetversion => Puppet.version, - :filebeat_version => '5.0.0' - } - end - - it { is_expected.to contain_yumrepo('beats').with( - :baseurl => 'https://artifacts.elastic.co/packages/5.x/yum', - :gpgkey => 'https://artifacts.elastic.co/GPG-KEY-elasticsearch', - ) } - - it { is_expected.to contain_service('filebeat').with( - :enable => true, - :ensure => 'running', - :provider => 'redhat', - )} - - end - - describe 'on a Windows system' do - let :facts do - { - :kernel => 'Windows', - :rubyversion => '2.3.1', - :puppetversion => Puppet.version, - :filebeat_version => '5.0.0' - } - end - - it { is_expected.to contain_file('filebeat.yml').with( - :path => 'C:/Program Files/Filebeat/filebeat.yml', - )} - it { is_expected.to contain_file('filebeat-config-dir').with( - :ensure => 'directory', - :path => 'C:/Program Files/Filebeat/conf.d', - :recurse => true, - )} - it { is_expected.to contain_service('filebeat').with( - :enable => true, - :ensure => 'running', - :provider => nil, # Provider should use the resource default - )} - end - end - - context 'on a system with 1.x already installed' do - describe 'on a Debian system' do - let :facts do - { - :kernel => 'Linux', - :osfamily => 'Debian', - :lsbdistid => 'Ubuntu', - :lsbdistrelease => '16.04', - :rubyversion => '2.3.1', - :puppetversion => Puppet.version, - :filebeat_version => '1.3.1' - } - end - - it { is_expected.to contain_package('filebeat') } - it { is_expected.to contain_file('filebeat.yml').with( - :path => '/etc/filebeat/filebeat.yml', - :mode => '0644', - )} - it { is_expected.to contain_file('filebeat-config-dir').with( - :ensure => 'directory', - :path => '/etc/filebeat/conf.d', - :mode => '0755', - :recurse => true, - )} - it { is_expected.to contain_service('filebeat').with( - :enable => true, - :ensure => 'running', - :provider => nil, # Provider should use the resource default - )} - it { is_expected.to contain_apt__source('beats').with( - :location => 'http://packages.elastic.co/beats/apt', - :key => { - 'id' => '46095ACC8548582C1A2699A9D27D666CD88E42B4', - 'source' => 'https://artifacts.elastic.co/GPG-KEY-elasticsearch', - } - )} - - end - - describe 'on a RHEL system' do - let :facts do - { - :kernel => 'Linux', - :osfamily => 'RedHat', - :rubyversion => '2.3.1', - :puppetversion => Puppet.version, - :filebeat_version => '1' - } - end - - it { is_expected.to contain_yumrepo('beats').with( - :baseurl => 'https://packages.elastic.co/beats/yum/el/$basearch', - :gpgkey => 'https://artifacts.elastic.co/GPG-KEY-elasticsearch', - ) } - - it { is_expected.to contain_service('filebeat').with( - :enable => true, - :ensure => 'running', - :provider => 'redhat', - )} - - end +describe 'filebeat' do + let :pre_condition do + 'include ::filebeat' + end - describe 'on a Windows system' do - let :facts do - { - :kernel => 'Windows', - :rubyversion => '2.3.1', - :puppetversion => Puppet.version, - :filebeat_version => '1' - } - end + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } - it { is_expected.to contain_file('filebeat.yml').with( - :path => 'C:/Program Files/Filebeat/filebeat.yml', - )} - it { is_expected.to contain_file('filebeat-config-dir').with( - :ensure => 'directory', - :path => 'C:/Program Files/Filebeat/conf.d', - :recurse => true, - )} - it { is_expected.to contain_service('filebeat').with( - :enable => true, - :ensure => 'running', - :provider => nil, # Provider should use the resource default - )} + if os_facts[:kernel] != 'windows' + it { is_expected.to compile } end - end - end - describe 'on a Solaris system' do - let :facts do - { - :osfamily => 'Solaris', - } - end - context 'it should fail as unsupported' do - it { expect { should raise_error(Puppet::Error) } } + it { is_expected.to contain_class('filebeat') } + it { is_expected.to contain_class('filebeat::params') } + it { is_expected.to contain_anchor('filebeat::begin') } + it { is_expected.to contain_anchor('filebeat::end') } + it { is_expected.to contain_class('filebeat::install') } + it { is_expected.to contain_class('filebeat::config') } + it { is_expected.to contain_class('filebeat::service') } end end end diff --git a/spec/classes/install/linux_spec.rb b/spec/classes/install/linux_spec.rb new file mode 100644 index 00000000..fbf02ba0 --- /dev/null +++ b/spec/classes/install/linux_spec.rb @@ -0,0 +1,21 @@ +require 'spec_helper' + +describe 'filebeat::install::linux' do + let :pre_condition do + 'include ::filebeat' + end + + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + case os_facts[:kernel] + when 'Linux' + it { is_expected.to compile } + it { is_expected.to contain_package('filebeat').with_ensure('present') } + else + it { is_expected.not_to compile } + end + end + end +end diff --git a/spec/classes/install/windows_spec.rb b/spec/classes/install/windows_spec.rb new file mode 100644 index 00000000..8dba7eaf --- /dev/null +++ b/spec/classes/install/windows_spec.rb @@ -0,0 +1,59 @@ +require 'spec_helper' + +describe 'filebeat::install::windows' do + let :pre_condition do + 'include ::filebeat' + end + + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + case os_facts[:kernel] + when 'windows' + # it { is_expected.to compile } + it { is_expected.to contain_file('C:/Program Files').with_ensure('directory') } + it { + is_expected.to contain_archive('C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip').with( + creates: 'C:/Program Files/Filebeat/filebeat-5.6.2-windows-x86_64', + ) + } + it { + is_expected.to contain_exec('install filebeat-5.6.2-windows-x86_64').with( + command: './install-service-filebeat.ps1', + ) + } + it { + is_expected.to contain_exec('unzip filebeat-5.6.2-windows-x86_64').with( + command: '$sh=New-Object -COM Shell.Application;$sh.namespace((Convert-Path \'C:/Program Files\')).'\ + 'Copyhere($sh.namespace((Convert-Path \'C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip\')).items(), 16)', + ) + } + it { + is_expected.to contain_exec('mark filebeat-5.6.2-windows-x86_64').with( + command: 'New-Item \'C:/Program Files/Filebeat/filebeat-5.6.2-windows-x86_64\' -ItemType file', + ) + } + it { + is_expected.to contain_exec('rename filebeat-5.6.2-windows-x86_64').with( + command: 'Remove-Item \'C:/Program Files/Filebeat\' -Recurse -Force -ErrorAction SilentlyContinue;'\ + 'Rename-Item \'C:/Program Files/filebeat-5.6.2-windows-x86_64\' \'C:/Program Files/Filebeat\'', + ) + } + it { + is_expected.to contain_exec('stop service filebeat-5.6.2-windows-x86_64').with( + command: 'Set-Service -Name filebeat -Status Stopped', + ) + } + it { + is_expected.to contain_file('C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip').with( + ensure: 'absent', + ) + } + + else + it { is_expected.not_to compile } + end + end + end +end diff --git a/spec/classes/install_spec.rb b/spec/classes/install_spec.rb new file mode 100644 index 00000000..d426819f --- /dev/null +++ b/spec/classes/install_spec.rb @@ -0,0 +1,31 @@ +require 'spec_helper' + +describe 'filebeat::install' do + let :pre_condition do + 'include ::filebeat' + end + + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + if os_facts[:kernel] != 'windows' + it { is_expected.to compile } + end + + it { is_expected.to contain_anchor('filebeat::install::begin') } + it { is_expected.to contain_anchor('filebeat::install::end') } + + case os_facts[:kernel] + when 'Linux' + it { is_expected.to contain_class('filebeat::install::linux') } + it { is_expected.to contain_class('filebeat::repo') } + it { is_expected.not_to contain_class('filebeat::install::windows') } + + when 'Windows' + it { is_expected.to contain_class('filebeat::install::windows') } + it { is_expected.not_to contain_class('filebeat::install::linux') } + end + end + end +end diff --git a/spec/classes/params_spec.rb b/spec/classes/params_spec.rb new file mode 100644 index 00000000..f509e153 --- /dev/null +++ b/spec/classes/params_spec.rb @@ -0,0 +1,11 @@ +require 'spec_helper' + +describe 'filebeat::params' do + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + it { is_expected.to compile } + end + end +end diff --git a/spec/classes/repo_spec.rb b/spec/classes/repo_spec.rb new file mode 100644 index 00000000..3016b65e --- /dev/null +++ b/spec/classes/repo_spec.rb @@ -0,0 +1,39 @@ +require 'spec_helper' + +describe 'filebeat::repo' do + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + let(:pre_condition) { "class { 'filebeat': major_version => '#{major_version}' }" } + + [5, 6].each do |version| + context "with $major_version == #{version}" do + let(:major_version) { version } + + case os_facts[:kernel] + when 'Linux' + it { is_expected.to compile } + case os_facts[:osfamily] + when 'Debian' + it { + is_expected.to contain_apt__source('beats').with( + ensure: 'present', + location: "https://artifacts.elastic.co/packages/#{major_version}.x/apt", + ) + } + when 'RedHat' + it { + is_expected.to contain_yumrepo('beats').with( + ensure: 'present', + baseurl: "https://artifacts.elastic.co/packages/#{major_version}.x/yum", + ) + } + end + else + it { is_expected.not_to compile } + end + end + end + end + end +end diff --git a/spec/classes/service_spec.rb b/spec/classes/service_spec.rb new file mode 100644 index 00000000..a8292355 --- /dev/null +++ b/spec/classes/service_spec.rb @@ -0,0 +1,24 @@ +require 'spec_helper' + +describe 'filebeat::service' do + let :pre_condition do + 'include ::filebeat' + end + + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + if os_facts[:kernel] != 'windows' + it { is_expected.to compile } + end + + it { + is_expected.to contain_service('filebeat').with( + ensure: 'running', + enable: true, + ) + } + end + end +end diff --git a/spec/default_facts.yml b/spec/default_facts.yml new file mode 100644 index 00000000..3248be5a --- /dev/null +++ b/spec/default_facts.yml @@ -0,0 +1,8 @@ +# Use default_module_facts.yml for module specific facts. +# +# Facts specified here will override the values provided by rspec-puppet-facts. +--- +concat_basedir: "/tmp" +ipaddress: "172.16.254.254" +is_pe: false +macaddress: "AA:AA:AA:AA:AA:AA" diff --git a/spec/defines/prospector_spec.rb b/spec/defines/prospector_spec.rb index ea524366..56429f07 100644 --- a/spec/defines/prospector_spec.rb +++ b/spec/defines/prospector_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe 'filebeat::prospector', :type => :define do +describe 'filebeat::prospector' do let :pre_condition do 'class { "filebeat": outputs => { @@ -12,153 +12,35 @@ }, }' end - let :title do - 'test-logs' - end - - context 'with no parameters' do - it { expect { should raise_error(Puppet::Error) } } - end - context 'On Linux' do - let :facts do { - :kernel => 'Linux', - :osfamily => 'Linux', - :rubyversion => '2.3.1', - :filebeat_version => '1', + let(:title) { 'test-logs' } + let(:params) do + { + 'paths' => [ + '/var/log/auth.log', + '/var/log/syslog', + ], + 'doc_type' => 'syslog-beat', } - end + end - context 'with file blobs set' do - let :params do - { - :paths => [ - '/var/log/apache2/*.log', - ], - :doc_type => 'apache', - } - end + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } - it { is_expected.to contain_file('filebeat-test-logs').with( - :path => '/etc/filebeat/conf.d/test-logs.yml', - :mode => '0644', - :content => 'filebeat: - prospectors: - - paths: - - /var/log/apache2/*.log - encoding: plain - fields_under_root: false - input_type: log - document_type: apache - scan_frequency: 10s - harvester_buffer_size: 16384 - tail_files: false - force_close_files: false - backoff: 1s - max_backoff: 10s - backoff_factor: 2 - max_bytes: 10485760 -', - )} - end - context 'with some java like multiline settings' do - let :params do - { - :paths => [ - '/var/log/java_app/some.log', - ], - :doc_type => 'java_app', - :exclude_lines => [ - '^DEBUG', - ], - :include_lines => [ - '^ERROR', - '^WARN', - ], - :exclude_files => [ - '.gz$', - ], - :multiline => { - 'pattern' => '^\[', - 'negate' => 'true', - 'match' => 'after', - }, - } + if os_facts[:kernel] != 'windows' + it { is_expected.to compile } end - it { is_expected.to contain_file('filebeat-test-logs').with( - :path => '/etc/filebeat/conf.d/test-logs.yml', - :mode => '0644', - :content => 'filebeat: - prospectors: - - paths: - - /var/log/java_app/some.log - exclude_files: - - .gz$ - encoding: plain - fields_under_root: false - input_type: log - document_type: java_app - scan_frequency: 10s - harvester_buffer_size: 16384 - tail_files: false - force_close_files: false - backoff: 1s - max_backoff: 10s - backoff_factor: 2 - max_bytes: 10485760 - multiline: - pattern: \'^\[\' - negate: true - match: after - include_lines: - - \'^ERROR\' - - \'^WARN\' - exclude_lines: - - \'^DEBUG\' -', - )} + it { + is_expected.to contain_file('filebeat-test-logs').with( + notify: 'Service[filebeat]', + ) + } end end - context 'On Windows' do - let :facts do { - :kernel => 'Windows', - :rubyversion => '2.3.1', - :filebeat_version => '1', - } - end - - context 'with file blobs set' do - let :params do - { - :paths => [ - 'C:/Program Files/Apache Software Foundation/Apache2.2/*.log', - ], - :doc_type => 'apache', - } - end - - it { is_expected.to contain_file('filebeat-test-logs').with( - :path => 'C:/Program Files/Filebeat/conf.d/test-logs.yml', - :content => 'filebeat: - prospectors: - - paths: - - C:/Program Files/Apache Software Foundation/Apache2.2/*.log - encoding: plain - fields_under_root: false - input_type: log - document_type: apache - scan_frequency: 10s - harvester_buffer_size: 16384 - tail_files: false - force_close_files: false - backoff: 1s - max_backoff: 10s - backoff_factor: 2 - max_bytes: 10485760 -', - )} - end + context 'with no parameters' do + it { is_expected.to raise_error(Puppet::Error) } end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 44a55384..15266c20 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,17 +1,23 @@ require 'puppetlabs_spec_helper/module_spec_helper' +require 'rspec-puppet-facts' +include RspecPuppetFacts -RSpec.configure do |c| - c.before :each do - # Ensure that we don't accidentally cache facts and environment - # between test cases. - Facter::Util::Loader.any_instance.stubs(:load_all) - Facter.clear - Facter.clear_messages +default_facts = { + puppetversion: Puppet.version, + facterversion: Facter.version, +} + +default_facts_path = File.expand_path(File.join(File.dirname(__FILE__), 'default_facts.yml')) +default_module_facts_path = File.expand_path(File.join(File.dirname(__FILE__), 'default_module_facts.yml')) + +if File.exist?(default_facts_path) && File.readable?(default_facts_path) + default_facts.merge!(YAML.safe_load(File.read(default_facts_path))) +end - Puppet.settings[:strict_variables]=true +if File.exist?(default_module_facts_path) && File.readable?(default_module_facts_path) + default_facts.merge!(YAML.safe_load(File.read(default_module_facts_path))) +end - # Store any environment variables away to be restored later - @old_env = {} - ENV.each_key {|k| @old_env[k] = ENV[k]} - end +RSpec.configure do |c| + c.default_facts = default_facts end diff --git a/spec/spec_helper_acceptance.rb b/spec/spec_helper_acceptance.rb index 76d9e05f..b8bbd361 100644 --- a/spec/spec_helper_acceptance.rb +++ b/spec/spec_helper_acceptance.rb @@ -1,9 +1,13 @@ -require 'beaker-rspec' +require 'beaker-rspec/spec_helper' +require 'beaker-rspec/helpers/serverspec' require 'beaker/puppet_install_helper' +require 'beaker/module_install_helper' run_puppet_install_helper +install_module_on(hosts) +install_module_dependencies_on(hosts) -UNSUPPORTED_PLATFORMS = ['aix', 'Solaris', 'BSD'] +UNSUPPORTED_PLATFORMS = %w[aix Solaris BSD].freeze RSpec.configure do |c| module_root = File.expand_path(File.join(File.dirname(__FILE__), '..')) @@ -13,22 +17,22 @@ # Configure all nodes in nodeset c.before :suite do # Install module - puppet_module_install(:source => module_root, :module_name => 'filebeat') + puppet_module_install(source: module_root, module_name: 'filebeat') hosts.each do |host| - on host, puppet('module','install','puppetlabs-stdlib'), { :acceptable_exit_codes => [0,1] } - on host, puppet('module','install','puppetlabs-apt'), { :acceptable_exit_codes => [0,1] } - on host, puppet('module','install','puppetlabs-powershell'), { :acceptable_exit_codes => [0,1] } - on host, puppet('module','install','lwf-remote_file'), { :acceptable_exit_codes => [0,1] } + on host, puppet('module', 'install', 'puppetlabs-stdlib'), acceptable_exit_codes: [0, 1] + on host, puppet('module', 'install', 'puppetlabs-apt'), acceptable_exit_codes: [0, 1] + on host, puppet('module', 'install', 'puppetlabs-powershell'), acceptable_exit_codes: [0, 1] + on host, puppet('module', 'install', 'lwf-remote_file'), acceptable_exit_codes: [0, 1] end end end -shared_examples "an idempotent resource" do - it 'should apply with no errors' do - apply_manifest(pp, :catch_failures => true) +shared_examples 'an idempotent resource' do + it 'apply with no errors' do + apply_manifest(pp, catch_failures: true) end - it 'should apply a second time without changes', :skip_pup_5016 do - apply_manifest(pp, :catch_changes => true) + it 'apply a second time without changes', :skip_pup_5016 do + apply_manifest(pp, catch_changes: true) end end diff --git a/spec/unit/facter/filebeat_version_spec.rb b/spec/unit/facter/filebeat_version_spec.rb index 48633d34..a9fdc556 100644 --- a/spec/unit/facter/filebeat_version_spec.rb +++ b/spec/unit/facter/filebeat_version_spec.rb @@ -1,18 +1,19 @@ require 'spec_helper' describe 'filebeat_version' do - before do + before :each do + Facter.clear Facter.fact(:kernel).stubs(:value).returns('Linux') end context 'on a Linux host' do - before do + before :each do File.stubs(:executable?) Facter::Util::Resolution.stubs(:exec) - File.expects(:executable?).with('/usr/bin/filebeat').returns true - Facter::Util::Resolution.stubs(:exec).with('/usr/bin/filebeat --version').returns('filebeat version 1.3.1 (amd64)') + File.expects(:executable?).with('/usr/share/filebeat/bin/filebeat').returns true + Facter::Util::Resolution.stubs(:exec).with('/usr/share/filebeat/bin/filebeat --version').returns('filebeat version 5.1.1 (amd64), libbeat 5.1.1') end it 'returns the correct version' do - expect(Facter.fact(:filebeat_version).value).to eq('1.3.1') + expect(Facter.fact(:filebeat_version).value).to eq('5.1.1') end end end diff --git a/templates/filebeat.yml.erb b/templates/filebeat.yml.erb index c19d297c..b063c27d 100644 --- a/templates/filebeat.yml.erb +++ b/templates/filebeat.yml.erb @@ -1,64 +1,623 @@ -<%- filebeat_config = @filebeat_config.delete_if { |key, value| !defined?(value) } -%> -<%- @filebeat_config['filebeat']['spool_size'] = Integer(@filebeat_config['filebeat']['spool_size']) -%> -<%- if @filebeat_config['output']['file'] -%> -<%- if @filebeat_config['output']['file']['rotate_every_kb'] -%> -<%- @filebeat_config['output']['file']['rotate_every_kb'] = Integer(@filebeat_config['output']['file']['rotate_every_kb']) -%> +#========================= Filebeat global options ============================ + +<% unless scope.function_versioncmp([@major_version, '6']) >= 0 -%> +filebeat.spool_size: <%= @filebeat_config['filebeat']['spool_size'] %> +filebeat.publish_async: <%= @filebeat_config['filebeat']['publish_async'] %> +filebeat.idle_timeout: <%= @filebeat_config['filebeat']['idle_timeout'] %> +<% end -%> +filebeat.registry_file: <%= @filebeat_config['filebeat']['registry_file'] %> +filebeat.config_dir: <%= @filebeat_config['filebeat']['config_dir'] %> +filebeat.shutdown_timeout: <%= @filebeat_config['filebeat']['shutdown_timeout'] %> + +#================================ General ===================================== + +name: <%= @filebeat_config['beat_name'] %> +<%- if @filebeat_config['tags'].length > 0 -%> +tags: +<%- @filebeat_config['tags'].each do |tag| -%> + - <%= tag %> <%- end -%> -<%- if @filebeat_config['output']['file']['number_of_files'] -%> -<%- @filebeat_config['output']['file']['number_of_files'] = Integer(@filebeat_config['output']['file']['number_of_files']) -%> <%- end -%> + +<%- if @filebeat_config['fields'].length > 0 -%> +fields: + <%- @filebeat_config['fields'].each_pair do |k, v| -%> + <%= k %>: <%= v %> + <%- end -%> <%- end -%> -<%- if @filebeat_config['output']['elasticsearch'] -%> -<%- if @filebeat_config['output']['elasticsearch']['port'] -%> -<%- @filebeat_config['output']['elasticsearch']['port'] = Integer(@filebeat_config['output']['elasticsearch']['port']) -%> + +fields_under_root: <%= @filebeat_config['fields_under_root'] %> +<% unless scope.function_versioncmp([@major_version, '6']) >= 0 -%> +queue_size: <%= @filebeat_config['queue_size'] %> +<% end -%> + +# The internal queue size for bulk events in the processing pipeline. +# Do not modify this value. +#bulk_queue_size: 0 + +<%- if @filebeat_config['max_procs'] != nil -%> +max_procs: <%= @filebeat_config['max_procs'] %> <%- end -%> + +#================================ Processors ===================================== + +# Processors are used to reduce the number of fields in the exported event or to +# enhance the event with external meta data. This section defines a list of processors +# that are applied one by one and the first one receives the initial event: +# +# event -> filter1 -> event1 -> filter2 ->event2 ... +# +# Supported processors: drop_fields, drop_event, include_fields +# +# For example, you can use the following processors to keep +# the fields that contain CPU load percentages, but remove the fields that +# contain CPU ticks values: +# +#processors: +#- include_fields: +# fields: ["cpu"] +#- drop_fields: +# fields: ["cpu.user", "cpu.system"] +# +# The following example drops the events that have the HTTP response code 200: +# +#processors: +#- drop_event: +# when: +# equals: +# http.code: 200 +# + +#================================ Outputs ===================================== + +<%- if @filebeat_config['output']['elasticsearch'] != nil -%> +#-------------------------- Elasticsearch output ------------------------------ +output.elasticsearch: + <%- if @filebeat_config['output']['elasticsearch']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['elasticsearch']['enabled'] %> + <%- end -%> + hosts: + <%- @filebeat_config['output']['elasticsearch']['hosts'].each do |host| -%> + - <%= host %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['loadbalance'] != nil -%> + loadbalance: <%= @filebeat_config['output']['elasticsearch']['loadbalance'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['compression_level'] != nil -%> + compression_level: <%= @filebeat_config['output']['elasticsearch']['compression_level'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['protocol'] != nil -%> + protocol: "<%= @filebeat_config['output']['elasticsearch']['protocol'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['username'] != nil -%> + username: "<%= @filebeat_config['output']['elasticsearch']['username'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['password'] != nil -%> + password: "<%= @filebeat_config['output']['elasticsearch']['password'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['parameters'] != nil && @filebeat_config['output']['elasticsearch']['parameters'].length > 0 -%> + parameters: + <%- @filebeat_config['output']['elasticsearch']['parameters'].each_pair do |k, v| -%> + <%= k %>: <%= v %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['worker'] != nil -%> + worker: <%= @filebeat_config['output']['elasticsearch']['worker'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['index'] != nil -%> + index: "<%= @filebeat_config['output']['elasticsearch']['index'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['pipeline'] != nil -%> + pipeline: "<%= @filebeat_config['output']['elasticsearch']['pipeline'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['pipelines'] != nil -%> + pipelines: + <%- @filebeat_config['output']['elasticsearch']['pipelines'].each do |pipeline| -%> + <%- if pipeline['name'] != nil -%> + - pipeline: "<%= pipeline['name'] %>" + when.equals: + <%= pipeline['filter'] %>: "<%= pipeline['pattern'] %>" + <%- end -%> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['indices'] != nil -%> + indices: + <%- @filebeat_config['output']['elasticsearch']['indices'].each do |indice| -%> + <%- if indice['name'] != nil -%> + - index: "<%= indice['name'] %>" + <%- if indice['pipeline'] != nil -%> + pipeline: "<%= indice['pipeline'] %>" + <%- end -%> + when.contains: + <%= indice['filter'] %>: "<%= indice['pattern'] %>" + <%- end -%> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['path'] != nil -%> + path: "<%= @filebeat_config['output']['elasticsearch']['path'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['proxy_url'] != nil -%> + proxy_url: "<%= @filebeat_config['output']['elasticsearch']['proxy_url'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['max_retries'] != nil -%> + max_retries: <%= @filebeat_config['output']['elasticsearch']['max_retries'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['bulk_max_size'] != nil -%> + bulk_max_size: <%= @filebeat_config['output']['elasticsearch']['bulk_max_size'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['timeout'] != nil -%> + timeout: <%= @filebeat_config['output']['elasticsearch']['timeout'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['flush_interval'] != nil -%> + flush_interval: <%= @filebeat_config['output']['elasticsearch']['flush_interval'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['template'] != nil -%> + template: + <%- if @filebeat_config['output']['elasticsearch']['template']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['elasticsearch']['template']['enabled'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['template']['name'] != nil -%> + name: "<%= @filebeat_config['output']['elasticsearch']['template']['name'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['template']['path'] != nil -%> + path: "<%= @filebeat_config['output']['elasticsearch']['template']['path'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['template']['overwrite'] != nil -%> + overwrite: <%= @filebeat_config['output']['elasticsearch']['template']['name'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['template']['versions']['2.x'] != nil -%> + <%- if @filebeat_config['output']['elasticsearch']['template']['versions']['2.x']['enabled'] != nil -%> + versions.2x.enabled: <%= @filebeat_config['output']['elasticsearch']['template']['versions']['2.x']['enabled'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['template']['versions']['2.x']['path'] != nil -%> + versions.2x.path: "<%= @filebeat_config['output']['elasticsearch']['template']['versions']['2.x']['path'] %>" + <%- end -%> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl'] != nil -%> + ssl: + <%- if @filebeat_config['output']['elasticsearch']['ssl']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['elasticsearch']['ssl']['enabled'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl']['verification_mode'] != nil -%> + verification_mode: <%= @filebeat_config['output']['elasticsearch']['ssl']['verification_mode'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl']['supported_protocols'] != nil -%> + supported_protocols: + <%- @filebeat_config['output']['elasticsearch']['ssl']['supported_protocols'].each do |protocol| -%> + - <%= protocol %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl']['certificate_authorities'] != nil -%> + certificate_authorities: + <%- @filebeat_config['output']['elasticsearch']['ssl']['certificate_authorities'].each do |ca| -%> + - <%= ca %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl']['certificate'] != nil -%> + certificate: "<%= @filebeat_config['output']['elasticsearch']['ssl']['certificate'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl']['key'] != nil -%> + key: "<%= @filebeat_config['output']['elasticsearch']['ssl']['key'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl']['key_passphrase'] != nil -%> + key_passphrase: '<%= @filebeat_config['output']['elasticsearch']['ssl']['key_passphrase'] %>' + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl']['cipher_suites'] != nil -%> + cipher_suites: + <%- @filebeat_config['output']['elasticsearch']['ssl']['cipher_suites'].each do |cipher_suite| -%> + - <%= cipher_suite %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl']['curve_types'] != nil -%> + curve_types: + <%- @filebeat_config['output']['elasticsearch']['ssl']['curve_types'].each do |curve_type| -%> + - <%= curve_type %> + <%- end -%> + <%- end -%> + <%- end -%> <%- end -%> -<%- if @filebeat_config['output']['logstash'] -%> -<%- if @filebeat_config['output']['logstash']['bulk_max_size'] -%> -<%- @filebeat_config['output']['logstash']['bulk_max_size'] = Integer(@filebeat_config['output']['logstash']['bulk_max_size']) -%> +<%- if @filebeat_config['output']['logstash'] != nil -%> +#----------------------------- Logstash output -------------------------------- +output.logstash: + <%- if @filebeat_config['output']['logstash']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['logstash']['enabled'] %> + <%- end -%> + hosts: + <%- @filebeat_config['output']['logstash']['hosts'].each do |host| -%> + - <%= host %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl_certificate'] != nil -%> + ssl.certificate: <%= @filebeat_config['output']['logstash']['ssl_certificate'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl_key'] != nil -%> + ssl.key: <%= @filebeat_config['output']['logstash']['ssl_key'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['worker'] != nil -%> + worker: <%= @filebeat_config['output']['logstash']['worker'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['compression_level'] != nil -%> + compression_level: <%= @filebeat_config['output']['logstash']['compression_level'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['loadbalance'] != nil -%> + loadbalance: <%= @filebeat_config['output']['logstash']['loadbalance'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['pipelining'] != nil -%> + pipelining: <%= @filebeat_config['output']['logstash']['pipelining'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['index'] != nil -%> + index: <%= @filebeat_config['output']['logstash']['index'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['proxy_url'] != nil -%> + proxy_url: <%= @filebeat_config['output']['logstash']['proxy_url'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['proxy_use_local_resolver'] != nil -%> + proxy_use_local_resolver: <%= @filebeat_config['output']['logstash']['proxy_use_local_resolver'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl'] != nil -%> + ssl: + <%- if @filebeat_config['output']['logstash']['ssl']['enabled'] == false -%> + enabled: false + <%- else -%> + enabled: true + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl']['verification_mode'] != nil -%> + verification_mode: <%= @filebeat_config['output']['logstash']['ssl']['verification_mode'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl']['certificate_authorities'] != nil -%> + certificate_authorities: + <%- @filebeat_config['output']['logstash']['ssl']['certificate_authorities'].each do |ca| -%> + - <%= ca %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl']['certificate'] != nil -%> + certificate: "<%= @filebeat_config['output']['logstash']['ssl']['certificate'] %>" + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl']['key'] != nil -%> + key: "<%= @filebeat_config['output']['logstash']['ssl']['key'] %>" + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl']['key_passphrase'] != nil -%> + key_passphrase: '<%= @filebeat_config['output']['logstash']['ssl']['key_passphrase'] %>' + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl']['cipher_suites'] != nil -%> + cipher_suites: + <%- @filebeat_config['output']['logstash']['ssl']['cipher_suites'].each do |cipher_suite| -%> + - <%= cipher_suite %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl']['curve_types'] != nil -%> + curve_types: + <%- @filebeat_config['output']['logstash']['ssl']['curve_types'].each do |curve_type| -%> + - <%= curve_type %> + <%- end -%> + <%- end -%> + <%- end -%> <%- end -%> -<%- if @filebeat_config['output']['logstash']['port'] -%> -<%- @filebeat_config['output']['logstash']['port'] = Integer(@filebeat_config['output']['logstash']['port']) -%> +<%- if @filebeat_config['output']['kafka'] != nil -%> +#------------------------------- Kafka output --------------------------------- +output.kafka: + <%- if @filebeat_config['output']['kafka']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['kafka']['enabled'] %> + <%- end -%> + hosts: + <%- @filebeat_config['output']['kafka']['hosts'].each do |host| -%> + - <%= host %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['topic'] != nil -%> + topic: <%= @filebeat_config['output']['kafka']['topic'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['round_robin'] != nil -%> + round_robin: + group_events: <%= @filebeat_config['output']['kafka']['round_robin']['group_events'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['key'] != nil -%> + key: '<%= @filebeat_config['output']['kafka']['key'] %>' + <%- end -%> + <%- if @filebeat_config['output']['kafka']['codec.format'] != nil -%> + codec.format: + <%- if @filebeat_config['output']['kafka']['codec.format']['string'] != nil -%> + string: <%= @filebeat_config['output']['kafka']['codec.format']['string'] %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['partition'] != nil and @filebeat_config['output']['kafka']['partition']['hash'] != nil -%> + partition.hash: + <%- if @filebeat_config['output']['kafka']['partition']['hash']['reachable_only'] != nil -%> + reachable_only: <%= @filebeat_config['output']['kafka']['partition']['hash']['reachable_only'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['partition']['hash']['random'] != nil -%> + random: <%= @filebeat_config['output']['kafka']['partition']['hash']['random'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['partition']['hash']['hash'] != nil -%> + hash: + <%- @filebeat_config['output']['kafka']['partition']['hash']['hash'].each do |value| -%> + - <%= value %> + <%- end -%> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['username'] != nil -%> + username: '<%= @filebeat_config['output']['kafka']['username'] %>' + <%- end -%> + <%- if @filebeat_config['output']['kafka']['password'] != nil -%> + password: '<%= @filebeat_config['output']['kafka']['password'] %>' + <%- end -%> + <%- if @filebeat_config['output']['kafka']['version'] != nil -%> + version: <%= @filebeat_config['output']['kafka']['version'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['metadata'] != nil -%> + metadata: + <%- if @filebeat_config['output']['kafka']['retry']['max'] != nil -%> + retry.max: 3 + <%- end -%> + <%- if @filebeat_config['output']['kafka']['retry']['backoff'] != nil -%> + retry.backoff: 3 + <%- end -%> + <%- if @filebeat_config['output']['kafka']['retry']['frequency'] != nil -%> + retry.frequency: 3 + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['worker'] != nil -%> + worker: <%= @filebeat_config['output']['kafka']['worker'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['max_retries'] != nil -%> + max_retries: <%= @filebeat_config['output']['kafka']['max_retries'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['bulk_max_size'] != nil -%> + bulk_max_size: <%= @filebeat_config['output']['kafka']['bulk_max_size'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['timeout'] != nil -%> + timeout: <%= @filebeat_config['output']['kafka']['timeout'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['broker_timeout'] != nil -%> + broker_timeout: <%= @filebeat_config['output']['kafka']['broker_timeout'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['channel_buffer_size'] != nil -%> + channel_buffer_size: <%= @filebeat_config['output']['kafka']['channel_buffer_size'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['keep_alive'] != nil -%> + keep_alive: <%= @filebeat_config['output']['kafka']['keep_alive'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['compression'] != nil -%> + compression: <%= @filebeat_config['output']['kafka']['compression'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['max_message_bytes'] != nil -%> + max_message_bytes: <%= @filebeat_config['output']['kafka']['max_message_bytes'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['required_acks'] != nil -%> + required_acks: <%= @filebeat_config['output']['kafka']['required_acks'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['flush_interval'] != nil -%> + flush_interval: <%= @filebeat_config['output']['kafka']['flush_interval'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['client_id'] != nil -%> + client_id: <%= @filebeat_config['output']['kafka']['client_id'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl'] != nil -%> + ssl: + <%- if @filebeat_config['output']['kafka']['ssl']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['kafka']['ssl']['enabled'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl']['verification_mode'] != nil -%> + verification_mode: <%= @filebeat_config['output']['kafka']['ssl']['verification_mode'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl']['supported_protocols'] != nil -%> + supported_protocols: + <%- @filebeat_config['output']['kafka']['ssl']['supported_protocols'].each do |protocol| -%> + - <%= protocol %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl']['certificate_authorities'] != nil -%> + certificate_authorities: + <%- @filebeat_config['output']['kafka']['ssl']['certificate_authorities'].each do |ca| -%> + - <%= ca %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl']['certificate'] != nil -%> + certificate: "<%= @filebeat_config['output']['kafka']['ssl']['certificate'] %>" + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl']['key'] != nil -%> + key: "<%= @filebeat_config['output']['kafka']['ssl']['key'] %>" + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl']['key_passphrase'] != nil -%> + key_passphrase: '<%= @filebeat_config['output']['kafka']['ssl']['key_passphrase'] %>' + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl']['cipher_suites'] != nil -%> + cipher_suites: + <%- @filebeat_config['output']['kafka']['ssl']['cipher_suites'].each do |cipher_suite| -%> + - <%= cipher_suite %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl']['curve_types'] != nil -%> + curve_types: + <%- @filebeat_config['output']['kafka']['ssl']['curve_types'].each do |curve_type| -%> + - <%= curve_type %> + <%- end -%> + <%- end -%> + <%- end -%> <%- end -%> -<%- if @filebeat_config['output']['logstash']['timeout'] -%> -<%- @filebeat_config['output']['logstash']['timeout'] = Integer(@filebeat_config['output']['logstash']['timeout']) -%> +<%- if @filebeat_config['output']['redis'] != nil -%> +#------------------------------- Redis output --------------------------------- +output.redis: + <%- if @filebeat_config['output']['redis']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['redis']['enabled'] %> + <%- end -%> + hosts: + <%- @filebeat_config['output']['redis']['hosts'].each do |host| -%> + - <%= host %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['port'] != nil -%> + port: <%= @filebeat_config['output']['redis']['port'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['key'] != nil -%> + key: <%= @filebeat_config['output']['redis']['key'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['password'] != nil -%> + password: <%= @filebeat_config['output']['redis']['password'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['db'] != nil -%> + db: <%= @filebeat_config['output']['redis']['db'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['datetype'] != nil -%> + datetype: <%= @filebeat_config['output']['redis']['datetype'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['worker'] != nil -%> + worker: <%= @filebeat_config['output']['redis']['worker'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['loadbalance'] != nil -%> + loadbalance: <%= @filebeat_config['output']['redis']['loadbalance'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['timeout'] != nil -%> + timeout: <%= @filebeat_config['output']['redis']['timeout'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['max_retries'] != nil -%> + max_retries: <%= @filebeat_config['output']['redis']['max_retries'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['bulk_max_size'] != nil -%> + bulk_max_size: <%= @filebeat_config['output']['redis']['bulk_max_size'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['proxy_url'] != nil -%> + proxy_url: <%= @filebeat_config['output']['redis']['proxy_url'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['proxy_use_local_resolver'] != nil -%> + proxy_use_local_resolver: <%= @filebeat_config['output']['redis']['proxy_use_local_resolver'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl'] != nil -%> + ssl: + <%- if @filebeat_config['output']['redis']['ssl']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['logstash']['ssl']['enabled'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl']['verification_mode'] != nil -%> + verification_mode: <%= @filebeat_config['output']['redis']['ssl']['verification_mode'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl']['supported_protocols'] != nil -%> + supported_protocols: + <%- @filebeat_config['output']['redis']['ssl']['supported_protocols'].each do |protocol| -%> + - <%= protocol %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl']['certificate_authorities'] != nil -%> + certificate_authorities: + <%- @filebeat_config['output']['redis']['ssl']['certificate_authorities'].each do |ca| -%> + - <%= ca %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl']['certificate'] != nil -%> + certificate: "<%= @filebeat_config['output']['redis']['ssl']['certificate'] %>" + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl']['key'] != nil -%> + key: "<%= @filebeat_config['output']['redis']['ssl']['key'] %>" + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl']['key_passphrase'] != nil -%> + key_passphrase: '<%= @filebeat_config['output']['redis']['ssl']['key_passphrase'] %>' + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl']['cipher_suites'] != nil -%> + cipher_suites: + <%- @filebeat_config['output']['redis']['ssl']['cipher_suites'].each do |cipher_suite| -%> + - <%= cipher_suite %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl']['curve_types'] != nil -%> + curve_types: + <%- @filebeat_config['output']['redis']['ssl']['curve_types'].each do |curve_type| -%> + - <%= curve_type %> + <%- end -%> + <%- end -%> + <%- end -%> <%- end -%> +<%- if @filebeat_config['output']['file'] != nil -%> +#------------------------------- File output ---------------------------------- +output.file: + <%- if @filebeat_config['output']['file']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['file']['enabled'] %> + <%- end -%> + <%- if @filebeat_config['output']['file']['path'] != nil -%> + path: "<%= @filebeat_config['output']['file']['path'] %>" + <%- end -%> + <%- if @filebeat_config['output']['file']['filename'] != nil -%> + filename: <%= @filebeat_config['output']['file']['filename'] %> + <%- end -%> + <%- if @filebeat_config['output']['file']['rotate_every_kb'] != nil -%> + rotate_every_kb: <%= @filebeat_config['output']['file']['rotate_every_kb'] %> + <%- end -%> + <%- if @filebeat_config['output']['file']['number_of_files'] != nil -%> + number_of_files: <%= @filebeat_config['output']['file']['number_of_files'] %> + <%- end -%> <%- end -%> -<%- if @filebeat_config['output']['redis'] -%> -<%- if @filebeat_config['output']['redis']['port'] -%> -<%- @filebeat_config['output']['redis']['port'] = Integer(@filebeat_config['output']['redis']['port']) -%> +<%- if @filebeat_config['output']['console'] != nil -%> +#-------------------------- Console output ------------------------------------ +output.console: + <%- if @filebeat_config['output']['console']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['console']['enabled'] %> + <%- end -%> + <%- if @filebeat_config['output']['console']['pretty'] != nil -%> + pretty: <%= @filebeat_config['output']['console']['pretty'] %> + <%- end -%> + <%- if @filebeat_config['output']['console']['bulk_max_size'] != nil -%> + bulk_max_size: <%= @filebeat_config['output']['console']['bulk_max_size'] %> + <%- end -%> <%- end -%> -<%- if @filebeat_config['output']['redis']['db'] -%> -<%- @filebeat_config['output']['redis']['db'] = Integer(@filebeat_config['output']['redis']['db']) -%> + +#================================= Paths ====================================== + +# The home path for the filebeat installation. This is the default base path +# for all other path settings and for miscellaneous files that come with the +# distribution (for example, the sample dashboards). +# If not set by a CLI flag or in the configuration file, the default for the +# home path is the location of the binary. +#path.home: + +# The configuration path for the filebeat installation. This is the default +# base path for configuration files, including the main YAML configuration file +# and the Elasticsearch template file. If not set by a CLI flag or in the +# configuration file, the default for the configuration path is the home path. +#path.config: <%= @filebeat_config['filebeat']['config_dir'] %> + +# The data path for the filebeat installation. This is the default base path +# for all the files in which filebeat needs to store its data. If not set by a +# CLI flag or in the configuration file, the default for the data path is a data +# subdirectory inside the home path. +#path.data: ${path.home}/data + +# The logs path for a filebeat installation. This is the default location for +# the Beat's log files. If not set by a CLI flag or in the configuration file, +# the default for the logs path is a logs subdirectory inside the home path. +#path.logs: ${path.home}/logs + +#================================ Logging ===================================== +<%- if @filebeat_config['logging']['level'] != nil -%> +logging.level: <%= @filebeat_config['logging']['level'] %> <%- end -%> -<%- if @filebeat_config['output']['redis']['db_topology'] -%> -<%- @filebeat_config['output']['redis']['db_topology'] = Integer(@filebeat_config['output']['redis']['db_topology']) -%> +<%- if @filebeat_config['logging']['selectors'] != nil && @filebeat_config['logging']['selectors'].length > 0 -%> +logging.selectors: +<%- @filebeat_config['logging']['selectors'].each do |selector| -%> + - <%= selector %> <%- end -%> -<%- if @filebeat_config['output']['redis']['timeout'] -%> -<%- @filebeat_config['output']['redis']['timeout'] = Integer(@filebeat_config['output']['redis']['timeout']) -%> <%- end -%> -<%- if @filebeat_config['output']['redis']['reconnect_interval'] -%> -<%- @filebeat_config['output']['redis']['reconnect_interval'] = Integer(@filebeat_config['output']['redis']['reconnect_interval']) -%> +<%- if @filebeat_config['logging']['to_syslog'] != nil -%> +logging.to_syslog: <%= @filebeat_config['logging']['to_syslog'] %> <%- end -%> +<%- if @filebeat_config['logging']['metrics'] != nil -%> +<%- if @filebeat_config['logging']['metrics']['enabled'] != nil -%> +logging.metrics.enabled: <%= @filebeat_config['logging']['metrics']['enabled'] %> <%- end -%> -<%- if @filebeat_config['shipper'] -%> -<%- if @filebeat_config['shipper']['refresh_topology_freq'] -%> -<%- @filebeat_config['shipper']['refresh_topology_freq'] = Integer(@filebeat_config['shipper']['refresh_topology_freq']) -%> +<%- if @filebeat_config['logging']['metrics']['period'] != nil -%> +logging.metrics.period: <%= @filebeat_config['logging']['metrics']['period'] %> <%- end -%> -<%- if @filebeat_config['shipper']['topology_expire'] -%> -<%- @filebeat_config['shipper']['topology_expire'] = Integer(@filebeat_config['shipper']['topology_expire']) -%> <%- end -%> -<%- if @filebeat_config['shipper']['queue_size'] -%> -<%- @filebeat_config['shipper']['queue_size'] = Integer(@filebeat_config['shipper']['queue_size']) -%> +<%- if @filebeat_config['logging']['to_files'] != nil -%> +logging.to_files: <%= @filebeat_config['logging']['to_files'] %> <%- end -%> +<%- if @filebeat_config['logging']['files'] != nil -%> +logging.files: + <%- if @filebeat_config['logging']['files']['path'] != nil -%> + path: <%= @filebeat_config['logging']['files']['path'] %> + <%- end -%> + <%- if @filebeat_config['logging']['files']['name'] != nil -%> + name: <%= @filebeat_config['logging']['files']['name'] %> + <%- end -%> + <%- if @filebeat_config['logging']['files']['rotateeverybytes'] != nil -%> + rotateeverybytes: <%= @filebeat_config['logging']['files']['rotateeverybytes'] %> + <%- end -%> + <%- if @filebeat_config['logging']['files']['keepfiles'] != nil -%> + keepfiles: <%= @filebeat_config['logging']['files']['keepfiles'] %> + <%- end -%> <%- end -%> -<%- if @filebeat_config['logging']['files'] -%> -<%- if @filebeat_config['logging']['files']['rotateeverybytes'] -%> -<%- @filebeat_config['logging']['files']['rotateeverybytes'] = Integer(@filebeat_config['logging']['files']['rotateeverybytes']) -%> -<%- end -%> -<%- if @filebeat_config['logging']['files']['keepfiles'] -%> -<%- @filebeat_config['logging']['files']['keepfiles'] = Integer(@filebeat_config['logging']['files']['keepfiles']) -%> -<%- end -%> -<%- end -%> -### Filebeat configuration managed by Puppet ### -<%= @filebeat_config.to_yaml() %> diff --git a/templates/filebeat.yml.ruby18.erb b/templates/filebeat.yml.ruby18.erb deleted file mode 100644 index 9a8ad8b2..00000000 --- a/templates/filebeat.yml.ruby18.erb +++ /dev/null @@ -1,285 +0,0 @@ -### Filebeat configuration managed by Puppet (Ruby 1.8 version) ### ---- - filebeat: - spool_size: <%= @filebeat_config['filebeat']['spool_size'] %> - publish_async: <%= @filebeat_config['filebeat']['publish_async'] %> - idle_timeout: <%= @filebeat_config['filebeat']['idle_timeout'] %> - registry_file: <%= @filebeat_config['filebeat']['registry_file'] %> - config_dir: <%= @filebeat_config['filebeat']['config_dir'] %> - - output: - <%- if @filebeat_config['output']['elasticsearch'] != nil -%> - elasticsearch: - hosts: - <%- @filebeat_config['output']['elasticsearch']['hosts'].each do |host| -%> - - <%= host %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['port'] != nil -%> - port: <%= @filebeat_config['output']['elasticsearch']['port'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['protocol'] != nil -%> - protocol: "<%= @filebeat_config['output']['elasticsearch']['protocol'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['username'] != nil -%> - username: "<%= @filebeat_config['output']['elasticsearch']['username'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['password'] != nil -%> - password: "<%= @filebeat_config['output']['elasticsearch']['password'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['worker'] != nil -%> - worker: <%= @filebeat_config['output']['elasticsearch']['worker'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['index'] != nil -%> - index: "<%= @filebeat_config['output']['elasticsearch']['index'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['path'] != nil -%> - path: "<%= @filebeat_config['output']['elasticsearch']['path'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['proxy_url'] != nil -%> - proxy_url: "<%= @filebeat_config['output']['elasticsearch']['proxy_url'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['max_retries'] != nil -%> - max_retries: <%= @filebeat_config['output']['elasticsearch']['max_retries'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['bulk_max_size'] != nil -%> - bulk_max_size: <%= @filebeat_config['output']['elasticsearch']['bulk_max_size'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['timeout'] != nil -%> - timeout: <%= @filebeat_config['output']['elasticsearch']['timeout'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['flush_interval'] != nil -%> - flush_interval: <%= @filebeat_config['output']['elasticsearch']['flush_interval'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['save_topology'] != nil -%> - save_topology: <%= @filebeat_config['output']['elasticsearch']['save_topology'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['topology_expire'] != nil -%> - topology_expire: <%= @filebeat_config['output']['elasticsearch']['topology_expire'] %> - <%- end -%> - - <%- if @filebeat_config['output']['elasticsearch']['template'] != nil -%> - template: - <%- if @filebeat_config['output']['elasticsearch']['template']['name'] != nil -%> - name: "<%= @filebeat_config['output']['elasticsearch']['name'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['template']['path'] != nil -%> - path: "<%= @filebeat_config['output']['elasticsearch']['path'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['template']['overwrite'] != nil -%> - overwrite: <%= @filebeat_config['output']['elasticsearch']['name'] %> - <%- end -%> - <%- end -%> - - <%- if @filebeat_config['output']['elasticsearch']['tls'] != nil -%> - tls: - <%- if @filebeat_config['output']['elasticsearch']['tls']['certificate_authorities'] != nil -%> - certificate_authorities: - <%- @filebeat_config['output']['elasticsearch']['tls']['certificate_authorities'].each do |ca| -%> - - <%= ca %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['tls']['certificate'] != nil -%> - certificate: "<%= @filebeat_config['output']['elasticsearch']['tls']['certificate'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['tls']['certificate_key'] != nil -%> - certificate_key: "<%= @filebeat_config['output']['elasticsearch']['tls']['certificate_key'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['tls']['insecure'] != nil -%> - insecure: <%= @filebeat_config['output']['elasticsearch']['tls']['insecure'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['tls']['cipher_suites'] != nil -%> - cipher_suites: - <%- @filebeat_config['output']['elasticsearch']['tls']['cipher_suites'].each do |cipher_suite| -%> - - <%= cipher_suite %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['tls']['curve_types'] != nil -%> - curve_types: - <%- @filebeat_config['output']['elasticsearch']['tls']['curve_types'].each do |curve_type| -%> - - <%= curve_type %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['tls']['min_version'] != nil -%> - min_version: <%= @filebeat_config['output']['elasticsearch']['tls']['min_version'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['tls']['max_version'] != nil -%> - max_version: <%= @filebeat_config['output']['elasticsearch']['tls']['max_version'] %> - <%- end -%> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['logstash'] != nil -%> - logstash: - hosts: - <%- @filebeat_config['output']['logstash']['hosts'].each do |host| -%> - - <%= host %> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['port'] != nil -%> - port: <%= @filebeat_config['output']['logstash']['port'] %> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['worker'] != nil -%> - worker: <%= @filebeat_config['output']['logstash']['worker'] %> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['compression_level'] != nil -%> - compression_level: <%= @filebeat_config['output']['logstash']['compression_level'] %> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['loadbalance'] != nil -%> - loadbalance: <%= @filebeat_config['output']['logstash']['loadbalance'] %> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['index'] != nil -%> - index: <%= @filebeat_config['output']['logstash']['index'] %> - <%- end -%> - - <%- if @filebeat_config['output']['logstash']['tls'] != nil -%> - tls: - <%- if @filebeat_config['output']['logstash']['tls']['certificate_authorities'] != nil -%> - certificate_authorities: - <%- @filebeat_config['output']['logstash']['tls']['certificate_authorities'].each do |ca| -%> - - <%= ca %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['tls']['certificate'] != nil -%> - certificate: "<%= @filebeat_config['output']['logstash']['tls']['certificate'] %>" - <%- end -%> - <%- if @filebeat_config['output']['logstash']['tls']['certificate_key'] != nil -%> - certificate_key: "<%= @filebeat_config['output']['logstash']['tls']['certificate_key'] %>" - <%- end -%> - <%- if @filebeat_config['output']['logstash']['tls']['insecure'] != nil -%> - insecure: <%= @filebeat_config['output']['logstash']['tls']['insecure'] %> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['tls']['cipher_suites'] != nil -%> - cipher_suites: - <%- @filebeat_config['output']['logstash']['tls']['cipher_suites'].each do |cipher_suite| -%> - - <%= cipher_suite %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['tls']['curve_types'] != nil -%> - curve_types: - <%- @filebeat_config['output']['logstash']['tls']['curve_types'].each do |curve_type| -%> - - <%= curve_type %> - <%- end -%> - <%- end -%> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['redis'] != nil -%> - redis: - <%- if @filebeat_config['output']['redis']['host'] != nil -%> - host: "<%= @filebeat_config['output']['redis']['host'] %>" - <%- end -%> - <%- if @filebeat_config['output']['redis']['port'] != nil -%> - port: <%= @filebeat_config['output']['redis']['port'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['save_topology'] != nil -%> - save_topology: <%= @filebeat_config['output']['redis']['save_topology'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['index'] != nil -%> - index: "<%= @filebeat_config['output']['redis']['index'] %>" - <%- end -%> - <%- if @filebeat_config['output']['redis']['db'] != nil -%> - db: <%= @filebeat_config['output']['redis']['db'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['db_topology'] != nil -%> - db_topology: <%= @filebeat_config['output']['redis']['db_topology'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['password'] != nil -%> - password: "<%= @filebeat_config['output']['redis']['password'] %>" - <%- end -%> - <%- if @filebeat_config['output']['redis']['timeout'] != nil -%> - timeout: <%= @filebeat_config['output']['redis']['timeout'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['reconnect_interval'] != nil -%> - reconnect_interval: <%= @filebeat_config['output']['redis']['reconnect_interval'] %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['file'] != nil -%> - file: - <%- if @filebeat_config['output']['file']['path'] != nil -%> - path: "<%= @filebeat_config['output']['file']['path'] %>" - <%- end -%> - <%- if @filebeat_config['output']['file']['filename'] != nil -%> - filename: <%= @filebeat_config['output']['file']['filename'] %> - <%- end -%> - <%- if @filebeat_config['output']['file']['rotate_every_kb'] != nil -%> - rotate_every_kb: <%= @filebeat_config['output']['file']['rotate_every_kb'] %> - <%- end -%> - <%- if @filebeat_config['output']['file']['number_of_files'] != nil -%> - number_of_files: <%= @filebeat_config['output']['file']['number_of_files'] %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['console'] != nil -%> - console: - <%- if @filebeat_config['output']['console']['pretty'] != nil -%> - pretty: <%= @filebeat_config['output']['console']['pretty'] %> - <%- end -%> - <%- end -%> - - <%- unless @filebeat_config['shipper'].empty? -%> - shipper: - <%- if @filebeat_config['shipper']['name'] != nil -%> - name: <%= @filebeat_config['shipper']['name'] %> - <%- end -%> - <%- if @filebeat_config['shipper']['tags'] != nil -%> - tags: - <%- @filebeat_config['shipper']['tags'].each do |tag| -%> - - <%= tag %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['shipper']['ignore_outgoing'] != nil -%> - ignore_outgoing: <%= @filebeat_config['shipper']['ignore_outgoing'] %> - <%- end -%> - <%- if @filebeat_config['shipper']['refresh_topology_freq'] != nil -%> - refresh_topology_freq: <%= @filebeat_config['shipper']['refresh_topology_freq'] %> - <%- end -%> - <%- if @filebeat_config['shipper']['topology_expire'] != nil -%> - topology_expire: <%= @filebeat_config['shipper']['topology_expire'] %> - <%- end -%> - <%- if @filebeat_config['shipper']['queue_size'] != nil -%> - queue_size: <%= @filebeat_config['shipper']['queue_size'] %> - <%- end -%> - <%- end -%> - - <%- unless @filebeat_config['logging'].empty? -%> - logging: - <%- if @filebeat_config['logging']['to_syslog'] != nil -%> - to_syslog: <%= @filebeat_config['logging']['to_syslog'] %> - <%- end -%> - <%- if @filebeat_config['logging']['to_files'] != nil -%> - to_files: <%= @filebeat_config['logging']['to_files'] %> - <%- end -%> - - <%- if @filebeat_config['logging']['files'] != nil -%> - files: - <%- if @filebeat_config['logging']['files']['path'] != nil -%> - path: <%= @filebeat_config['logging']['files']['path'] %> - <%- end -%> - <%- if @filebeat_config['logging']['files']['name'] != nil -%> - name: <%= @filebeat_config['logging']['files']['name'] %> - <%- end -%> - <%- if @filebeat_config['logging']['files']['rotateeverybytes'] != nil -%> - rotateeverybytes: <%= @filebeat_config['logging']['files']['rotateeverybytes'] %> - <%- end -%> - <%- if @filebeat_config['logging']['files']['keepfiles'] != nil -%> - keepfiles: <%= @filebeat_config['logging']['files']['keepfiles'] %> - <%- end -%> - <%- end -%> - - <%- if @filebeat_config['logging']['selectors'] != nil -%> - selectors: - <%- @filebeat_config['logging']['selectors'].each do |selector| -%> - - <%= selector %> - <%- end -%> - <%- end -%> - - <%- if @filebeat_config['logging']['level'] != nil -%> - level: <%= @filebeat_config['logging']['level'] %> - <%- end -%> - <%- end -%> - - <%- unless @filebeat_config['runoptions'].empty? -%> - runoptions: - <%- if @filebeat_config['runoptions']['uid'] != nil -%> - uid=<%= @filebeat_config['runoptions']['uid'] %> - <%- end -%> - <%- if @filebeat_config['runoptions']['gid'] != nil -%> - gid=<%= @filebeat_config['runoptions']['gid'] %> - <%- end -%> - <%- end -%> diff --git a/templates/filebeat5.yml.erb b/templates/filebeat5.yml.erb deleted file mode 100644 index 13d12184..00000000 --- a/templates/filebeat5.yml.erb +++ /dev/null @@ -1,561 +0,0 @@ -#========================= Filebeat global options ============================ - -filebeat.spool_size: <%= @filebeat_config['filebeat']['spool_size'] %> -filebeat.publish_async: <%= @filebeat_config['filebeat']['publish_async'] %> -filebeat.idle_timeout: <%= @filebeat_config['filebeat']['idle_timeout'] %> -filebeat.registry_file: <%= @filebeat_config['filebeat']['registry_file'] %> -filebeat.config_dir: <%= @filebeat_config['filebeat']['config_dir'] %> -filebeat.shutdown_timeout: <%= @filebeat_config['filebeat']['shutdown_timeout'] %> - -#================================ General ===================================== - -name: <%= @filebeat_config['beat_name'] %> -<%- if @filebeat_config['tags'].length > 0 -%> -tags: -<%- @filebeat_config['tags'].each do |tag| -%> - - <%= tag %> -<%- end -%> -<%- end -%> - -<%- if @filebeat_config['fields'].length > 0 -%> -fields: - <%- @filebeat_config['fields'].each_pair do |k, v| -%> - <%= k %>: <%= v %> - <%- end -%> -<%- end -%> - -fields_under_root: <%= @filebeat_config['fields_under_root'] %> -queue_size: <%= @filebeat_config['queue_size'] %> - -# The internal queue size for bulk events in the processing pipeline. -# Do not modify this value. -#bulk_queue_size: 0 - -<%- if @filebeat_config['max_procs'] != :undef -%> -max_procs: <%= @filebeat_config['max_procs'] %> -<%- end -%> - -#================================ Processors ===================================== - -# Processors are used to reduce the number of fields in the exported event or to -# enhance the event with external meta data. This section defines a list of processors -# that are applied one by one and the first one receives the initial event: -# -# event -> filter1 -> event1 -> filter2 ->event2 ... -# -# Supported processors: drop_fields, drop_event, include_fields -# -# For example, you can use the following processors to keep -# the fields that contain CPU load percentages, but remove the fields that -# contain CPU ticks values: -# -#processors: -#- include_fields: -# fields: ["cpu"] -#- drop_fields: -# fields: ["cpu.user", "cpu.system"] -# -# The following example drops the events that have the HTTP response code 200: -# -#processors: -#- drop_event: -# when: -# equals: -# http.code: 200 -# - -#================================ Outputs ===================================== - -<%- if @filebeat_config['output']['elasticsearch'] != nil -%> -#-------------------------- Elasticsearch output ------------------------------ -output.elasticsearch: - <%- if @filebeat_config['output']['elasticsearch']['enabled'] != nil -%> - enabled: <%= @filebeat_config['output']['elasticsearch']['enabled'] %> - <%- end -%> - hosts: - <%- @filebeat_config['output']['elasticsearch']['hosts'].each do |host| -%> - - <%= host %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['compression_level'] != nil -%> - compression_level: <%= @filebeat_config['output']['elasticsearch']['compression_level'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['protocol'] != nil -%> - protocol: "<%= @filebeat_config['output']['elasticsearch']['protocol'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['username'] != nil -%> - username: "<%= @filebeat_config['output']['elasticsearch']['username'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['password'] != nil -%> - password: "<%= @filebeat_config['output']['elasticsearch']['password'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['parameters'] != nil && @filebeat_config['output']['elasticsearch']['parameters'].length > 0 -%> - parameters: - <%- @filebeat_config['output']['elasticsearch']['parameters'].each_pair do |k, v| -%> - <%= k %>: <%= v %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['worker'] != nil -%> - worker: <%= @filebeat_config['output']['elasticsearch']['worker'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['index'] != nil -%> - index: "<%= @filebeat_config['output']['elasticsearch']['index'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['pipeline'] != nil -%> - pipeline: "<%= @filebeat_config['output']['elasticsearch']['pipeline'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['path'] != nil -%> - path: "<%= @filebeat_config['output']['elasticsearch']['path'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['proxy_url'] != nil -%> - proxy_url: "<%= @filebeat_config['output']['elasticsearch']['proxy_url'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['max_retries'] != nil -%> - max_retries: <%= @filebeat_config['output']['elasticsearch']['max_retries'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['bulk_max_size'] != nil -%> - bulk_max_size: <%= @filebeat_config['output']['elasticsearch']['bulk_max_size'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['timeout'] != nil -%> - timeout: <%= @filebeat_config['output']['elasticsearch']['timeout'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['flush_interval'] != nil -%> - flush_interval: <%= @filebeat_config['output']['elasticsearch']['flush_interval'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['template'] != nil -%> - template: - <%- if @filebeat_config['output']['elasticsearch']['template']['enabled'] != nil -%> - enabled: <%= @filebeat_config['output']['elasticsearch']['template']['enabled'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['template']['name'] != nil -%> - name: "<%= @filebeat_config['output']['elasticsearch']['template']['name'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['template']['path'] != nil -%> - path: "<%= @filebeat_config['output']['elasticsearch']['template']['path'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['template']['overwrite'] != nil -%> - overwrite: <%= @filebeat_config['output']['elasticsearch']['template']['name'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['template']['versions']['2.x'] != nil -%> - <%- if @filebeat_config['output']['elasticsearch']['template']['versions']['2.x']['enabled'] != nil -%> - versions.2x.enabled: <%= @filebeat_config['output']['elasticsearch']['template']['versions']['2.x']['enabled'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['template']['versions']['2.x']['path'] != nil -%> - versions.2x.path: "<%= @filebeat_config['output']['elasticsearch']['template']['versions']['2.x']['path'] %>" - <%- end -%> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['ssl'] != nil -%> - ssl: - <%- if @filebeat_config['output']['elasticsearch']['ssl']['enabled'] != nil -%> - enabled: <%= @filebeat_config['output']['elasticsearch']['ssl']['enabled'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['ssl']['verification_mode'] != nil -%> - verification_mode: <%= @filebeat_config['output']['elasticsearch']['ssl']['verification_mode'] %> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['ssl']['supported_protocols'] != nil -%> - supported_protocols: - <%- @filebeat_config['output']['elasticsearch']['ssl']['supported_protocols'].each do |protocol| -%> - - <%= protocol %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['ssl']['certificate_authorities'] != nil -%> - certificate_authorities: - <%- @filebeat_config['output']['elasticsearch']['ssl']['certificate_authorities'].each do |ca| -%> - - <%= ca %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['ssl']['certificate'] != nil -%> - certificate: "<%= @filebeat_config['output']['elasticsearch']['ssl']['certificate'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['ssl']['key'] != nil -%> - key: "<%= @filebeat_config['output']['elasticsearch']['ssl']['key'] %>" - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['ssl']['key_passphrase'] != nil -%> - key_passphrase: '<%= @filebeat_config['output']['elasticsearch']['ssl']['key_passphrase'] %>' - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['ssl']['cipher_suites'] != nil -%> - cipher_suites: - <%- @filebeat_config['output']['elasticsearch']['ssl']['cipher_suites'].each do |cipher_suite| -%> - - <%= cipher_suite %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['elasticsearch']['ssl']['curve_types'] != nil -%> - curve_types: - <%- @filebeat_config['output']['elasticsearch']['ssl']['curve_types'].each do |curve_type| -%> - - <%= curve_type %> - <%- end -%> - <%- end -%> - <%- end -%> -<%- end -%> -<%- if @filebeat_config['output']['logstash'] != nil -%> -#----------------------------- Logstash output -------------------------------- -output.logstash: - <%- if @filebeat_config['output']['logstash']['enabled'] != nil -%> - enabled: <%= @filebeat_config['output']['logstash']['enabled'] %> - <%- end -%> - hosts: - <%- @filebeat_config['output']['logstash']['hosts'].each do |host| -%> - - <%= host %> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['worker'] != nil -%> - worker: <%= @filebeat_config['output']['logstash']['worker'] %> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['compression_level'] != nil -%> - compression_level: <%= @filebeat_config['output']['logstash']['compression_level'] %> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['loadbalance'] != nil -%> - loadbalance: <%= @filebeat_config['output']['logstash']['loadbalance'] %> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['pipelining'] != nil -%> - pipelining: <%= @filebeat_config['output']['logstash']['pipelining'] %> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['index'] != nil -%> - index: <%= @filebeat_config['output']['logstash']['index'] %> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['proxy_url'] != nil -%> - proxy_url: <%= @filebeat_config['output']['logstash']['proxy_url'] %> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['proxy_use_local_resolver'] != nil -%> - proxy_use_local_resolver: <%= @filebeat_config['output']['logstash']['proxy_use_local_resolver'] %> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['ssl'] != nil -%> - ssl: - <%- if @filebeat_config['output']['logstash']['ssl']['enabled'] == false -%> - enabled: false - <%- else -%> - enabled: true - <%- end -%> - <%- if @filebeat_config['output']['logstash']['ssl']['verification_mode'] != nil -%> - verification_mode: <%= @filebeat_config['output']['logstash']['ssl']['verification_mode'] %> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['ssl']['certificate_authorities'] != nil -%> - certificate_authorities: - <%- @filebeat_config['output']['logstash']['ssl']['certificate_authorities'].each do |ca| -%> - - <%= ca %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['ssl']['certificate'] != nil -%> - certificate: "<%= @filebeat_config['output']['logstash']['ssl']['certificate'] %>" - <%- end -%> - <%- if @filebeat_config['output']['logstash']['ssl']['key'] != nil -%> - key: "<%= @filebeat_config['output']['logstash']['ssl']['key'] %>" - <%- end -%> - <%- if @filebeat_config['output']['logstash']['ssl']['key_passphrase'] != nil -%> - key_passphrase: '<%= @filebeat_config['output']['logstash']['ssl']['key_passphrase'] %>' - <%- end -%> - <%- if @filebeat_config['output']['logstash']['ssl']['cipher_suites'] != nil -%> - cipher_suites: - <%- @filebeat_config['output']['logstash']['ssl']['cipher_suites'].each do |cipher_suite| -%> - - <%= cipher_suite %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['logstash']['ssl']['curve_types'] != nil -%> - curve_types: - <%- @filebeat_config['output']['logstash']['ssl']['curve_types'].each do |curve_type| -%> - - <%= curve_type %> - <%- end -%> - <%- end -%> - <%- end -%> -<%- end -%> -<%- if @filebeat_config['output']['kafka'] != nil -%> -#------------------------------- Kafka output --------------------------------- -output.kafka: - <%- if @filebeat_config['output']['kafka']['enabled'] != nil -%> - enabled: <%= @filebeat_config['output']['kafka']['enabled'] %> - <%- end -%> - hosts: - <%- @filebeat_config['output']['kafka']['hosts'].each do |host| -%> - - <%= host %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['topic'] != nil -%> - topic: <%= @filebeat_config['output']['kafka']['topic'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['key'] != nil -%> - key: '<%= @filebeat_config['output']['kafka']['key'] %>' - <%- end -%> - <%- if @filebeat_config['output']['kafka']['partition'] != nil and @filebeat_config['output']['kafka']['partition']['hash'] != nil -%> - partition.hash: - <%- if @filebeat_config['output']['kafka']['partition']['hash']['reachable_only'] != nil -%> - reachable_only: <%= @filebeat_config['output']['kafka']['partition']['hash']['reachable_only'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['partition']['hash']['hash'] != nil -%> - hash: - <%- @filebeat_config['output']['kafka']['partition']['hash']['hash'].each do |value| -%> - - <%= value %> - <%- end -%> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['username'] != nil -%> - username: '<%= @filebeat_config['output']['kafka']['username'] %>' - <%- end -%> - <%- if @filebeat_config['output']['kafka']['password'] != nil -%> - password: '<%= @filebeat_config['output']['kafka']['password'] %>' - <%- end -%> - <%- if @filebeat_config['output']['kafka']['version'] != nil -%> - version: <%= @filebeat_config['output']['kafka']['version'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['metadata'] != nil -%> - metadata: - <%- if @filebeat_config['output']['kafka']['retry']['max'] != nil -%> - retry.max: 3 - <%- end -%> - <%- if @filebeat_config['output']['kafka']['retry']['backoff'] != nil -%> - retry.backoff: 3 - <%- end -%> - <%- if @filebeat_config['output']['kafka']['retry']['frequency'] != nil -%> - retry.frequency: 3 - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['worker'] != nil -%> - worker: <%= @filebeat_config['output']['kafka']['worker'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['max_retries'] != nil -%> - max_retries: <%= @filebeat_config['output']['kafka']['max_retries'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['bulk_max_size'] != nil -%> - bulk_max_size: <%= @filebeat_config['output']['kafka']['bulk_max_size'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['timeout'] != nil -%> - timeout: <%= @filebeat_config['output']['kafka']['timeout'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['broker_timeout'] != nil -%> - broker_timeout: <%= @filebeat_config['output']['kafka']['broker_timeout'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['channel_buffer_size'] != nil -%> - channel_buffer_size: <%= @filebeat_config['output']['kafka']['channel_buffer_size'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['keep_alive'] != nil -%> - keep_alive: <%= @filebeat_config['output']['kafka']['keep_alive'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['compression'] != nil -%> - compression: <%= @filebeat_config['output']['kafka']['compression'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['max_message_bytes'] != nil -%> - max_message_bytes: <%= @filebeat_config['output']['kafka']['max_message_bytes'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['required_acks'] != nil -%> - required_acks: <%= @filebeat_config['output']['kafka']['required_acks'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['flush_interval'] != nil -%> - flush_interval: <%= @filebeat_config['output']['kafka']['flush_interval'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['client_id'] != nil -%> - client_id: <%= @filebeat_config['output']['kafka']['client_id'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['ssl'] != nil -%> - ssl: - <%- if @filebeat_config['output']['kafka']['ssl']['enabled'] != nil -%> - enabled: <%= @filebeat_config['output']['kafka']['ssl']['enabled'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['ssl']['verification_mode'] != nil -%> - verification_mode: <%= @filebeat_config['output']['kafka']['ssl']['verification_mode'] %> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['ssl']['supported_protocols'] != nil -%> - supported_protocols: - <%- @filebeat_config['output']['kafka']['ssl']['supported_protocols'].each do |protocol| -%> - - <%= protocol %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['ssl']['certificate_authorities'] != nil -%> - certificate_authorities: - <%- @filebeat_config['output']['kafka']['ssl']['certificate_authorities'].each do |ca| -%> - - <%= ca %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['ssl']['certificate'] != nil -%> - certificate: "<%= @filebeat_config['output']['kafka']['ssl']['certificate'] %>" - <%- end -%> - <%- if @filebeat_config['output']['kafka']['ssl']['key'] != nil -%> - key: "<%= @filebeat_config['output']['kafka']['ssl']['key'] %>" - <%- end -%> - <%- if @filebeat_config['output']['kafka']['ssl']['key_passphrase'] != nil -%> - key_passphrase: '<%= @filebeat_config['output']['kafka']['ssl']['key_passphrase'] %>' - <%- end -%> - <%- if @filebeat_config['output']['kafka']['ssl']['cipher_suites'] != nil -%> - cipher_suites: - <%- @filebeat_config['output']['kafka']['ssl']['cipher_suites'].each do |cipher_suite| -%> - - <%= cipher_suite %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['kafka']['ssl']['curve_types'] != nil -%> - curve_types: - <%- @filebeat_config['output']['kafka']['ssl']['curve_types'].each do |curve_type| -%> - - <%= curve_type %> - <%- end -%> - <%- end -%> - <%- end -%> -<%- end -%> -<%- if @filebeat_config['output']['redis'] != nil -%> -#------------------------------- Redis output --------------------------------- -output.redis: - <%- if @filebeat_config['output']['redis']['enabled'] != nil -%> - enabled: <%= @filebeat_config['output']['redis']['enabled'] %> - <%- end -%> - hosts: - <%- @filebeat_config['output']['redis']['hosts'].each do |host| -%> - - <%= host %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['port'] != nil -%> - port: <%= @filebeat_config['output']['redis']['port'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['key'] != nil -%> - key: <%= @filebeat_config['output']['redis']['key'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['password'] != nil -%> - password: <%= @filebeat_config['output']['redis']['password'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['db'] != nil -%> - db: <%= @filebeat_config['output']['redis']['db'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['datetype'] != nil -%> - datetype: <%= @filebeat_config['output']['redis']['datetype'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['worker'] != nil -%> - worker: <%= @filebeat_config['output']['redis']['worker'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['loadbalance'] != nil -%> - loadbalance: <%= @filebeat_config['output']['redis']['loadbalance'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['timeout'] != nil -%> - timeout: <%= @filebeat_config['output']['redis']['timeout'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['max_retries'] != nil -%> - max_retries: <%= @filebeat_config['output']['redis']['max_retries'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['bulk_max_size'] != nil -%> - bulk_max_size: <%= @filebeat_config['output']['redis']['bulk_max_size'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['proxy_url'] != nil -%> - proxy_url: <%= @filebeat_config['output']['redis']['proxy_url'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['proxy_use_local_resolver'] != nil -%> - proxy_use_local_resolver: <%= @filebeat_config['output']['redis']['proxy_use_local_resolver'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['ssl'] != nil -%> - ssl: - <%- if @filebeat_config['output']['redis']['ssl']['enabled'] != nil -%> - enabled: <%= @filebeat_config['output']['logstash']['ssl']['enabled'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['ssl']['verification_mode'] != nil -%> - verification_mode: <%= @filebeat_config['output']['redis']['ssl']['verification_mode'] %> - <%- end -%> - <%- if @filebeat_config['output']['redis']['ssl']['supported_protocols'] != nil -%> - supported_protocols: - <%- @filebeat_config['output']['redis']['ssl']['supported_protocols'].each do |protocol| -%> - - <%= protocol %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['redis']['ssl']['certificate_authorities'] != nil -%> - certificate_authorities: - <%- @filebeat_config['output']['redis']['ssl']['certificate_authorities'].each do |ca| -%> - - <%= ca %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['redis']['ssl']['certificate'] != nil -%> - certificate: "<%= @filebeat_config['output']['redis']['ssl']['certificate'] %>" - <%- end -%> - <%- if @filebeat_config['output']['redis']['ssl']['key'] != nil -%> - key: "<%= @filebeat_config['output']['redis']['ssl']['key'] %>" - <%- end -%> - <%- if @filebeat_config['output']['redis']['ssl']['key_passphrase'] != nil -%> - key_passphrase: '<%= @filebeat_config['output']['redis']['ssl']['key_passphrase'] %>' - <%- end -%> - <%- if @filebeat_config['output']['redis']['ssl']['cipher_suites'] != nil -%> - cipher_suites: - <%- @filebeat_config['output']['redis']['ssl']['cipher_suites'].each do |cipher_suite| -%> - - <%= cipher_suite %> - <%- end -%> - <%- end -%> - <%- if @filebeat_config['output']['redis']['ssl']['curve_types'] != nil -%> - curve_types: - <%- @filebeat_config['output']['redis']['ssl']['curve_types'].each do |curve_type| -%> - - <%= curve_type %> - <%- end -%> - <%- end -%> - <%- end -%> -<%- end -%> -<%- if @filebeat_config['output']['file'] != nil -%> -#------------------------------- File output ---------------------------------- -output.file: - <%- if @filebeat_config['output']['file']['enabled'] != nil -%> - enabled: <%= @filebeat_config['output']['file']['enabled'] %> - <%- end -%> - <%- if @filebeat_config['output']['file']['path'] != nil -%> - path: "<%= @filebeat_config['output']['file']['path'] %>" - <%- end -%> - <%- if @filebeat_config['output']['file']['filename'] != nil -%> - filename: <%= @filebeat_config['output']['file']['filename'] %> - <%- end -%> - <%- if @filebeat_config['output']['file']['rotate_every_kb'] != nil -%> - rotate_every_kb: <%= @filebeat_config['output']['file']['rotate_every_kb'] %> - <%- end -%> - <%- if @filebeat_config['output']['file']['number_of_files'] != nil -%> - number_of_files: <%= @filebeat_config['output']['file']['number_of_files'] %> - <%- end -%> -<%- end -%> - -#================================= Paths ====================================== - -# The home path for the filebeat installation. This is the default base path -# for all other path settings and for miscellaneous files that come with the -# distribution (for example, the sample dashboards). -# If not set by a CLI flag or in the configuration file, the default for the -# home path is the location of the binary. -#path.home: - -# The configuration path for the filebeat installation. This is the default -# base path for configuration files, including the main YAML configuration file -# and the Elasticsearch template file. If not set by a CLI flag or in the -# configuration file, the default for the configuration path is the home path. -#path.config: <%= @filebeat_config['filebeat']['config_dir'] %> - -# The data path for the filebeat installation. This is the default base path -# for all the files in which filebeat needs to store its data. If not set by a -# CLI flag or in the configuration file, the default for the data path is a data -# subdirectory inside the home path. -#path.data: ${path.home}/data - -# The logs path for a filebeat installation. This is the default location for -# the Beat's log files. If not set by a CLI flag or in the configuration file, -# the default for the logs path is a logs subdirectory inside the home path. -#path.logs: ${path.home}/logs - -#================================ Logging ===================================== -<%- if @filebeat_config['logging']['level'] != nil -%> -logging.level: <%= @filebeat_config['logging']['level'] %> -<%- end -%> -<%- if @filebeat_config['logging']['selectors'] != nil && @filebeat_config['logging']['selectors'].length > 0 -%> -logging.selectors: -<%- @filebeat_config['logging']['selectors'].each do |selector| -%> - - <%= selector %> -<%- end -%> -<%- end -%> -<%- if @filebeat_config['logging']['to_syslog'] != nil -%> -logging.to_syslog: <%= @filebeat_config['logging']['to_syslog'] %> -<%- end -%> -<%- if @filebeat_config['logging']['metrics'] != nil -%> -<%- if @filebeat_config['logging']['metrics']['enabled'] != nil -%> -logging.metrics.enabled: <%= @filebeat_config['logging']['metrics']['enabled'] %> -<%- end -%> -<%- if @filebeat_config['logging']['metrics']['period'] != nil -%> -logging.metrics.period: <%= @filebeat_config['logging']['metrics']['period'] %> -<%- end -%> -<%- end -%> -<%- if @filebeat_config['logging']['to_files'] != nil -%> -logging.to_files: <%= @filebeat_config['logging']['to_files'] %> -<%- end -%> -<%- if @filebeat_config['logging']['files'] != nil -%> -logging.files: - <%- if @filebeat_config['logging']['files']['path'] != nil -%> - path: <%= @filebeat_config['logging']['files']['path'] %> - <%- end -%> - <%- if @filebeat_config['logging']['files']['name'] != nil -%> - name: <%= @filebeat_config['logging']['files']['name'] %> - <%- end -%> - <%- if @filebeat_config['logging']['files']['rotateeverybytes'] != nil -%> - rotateeverybytes: <%= @filebeat_config['logging']['files']['rotateeverybytes'] %> - <%- end -%> - <%- if @filebeat_config['logging']['files']['keepfiles'] != nil -%> - keepfiles: <%= @filebeat_config['logging']['files']['keepfiles'] %> - <%- end -%> -<%- end -%> diff --git a/templates/prospector.yml.erb b/templates/prospector.yml.erb index c0daaf73..1d1fcfab 100644 --- a/templates/prospector.yml.erb +++ b/templates/prospector.yml.erb @@ -1,8 +1,25 @@ +--- filebeat: prospectors: - - paths: - <%- @paths.each do |log_path| -%> - - <%= log_path %> + - <% if scope.function_versioncmp([scope.lookupvar('filebeat::major_version'), '6']) >= 0 %>type<% else %>input_type<% end %>: <%= @input_type %> + paths: + <%- @paths.each do |log_path| -%> + - <%= log_path %> + <%- end -%> + <%- if @encoding -%> + encoding: <%= @encoding %> + <%- end -%> + <%- if @include_lines.length > 0 -%> + include_lines: + <%- @include_lines.each do |include_line| -%> + - '<%= include_line %>' + <%- end -%> + <%- end -%> + <%- if @exclude_lines.length > 0 -%> + exclude_lines: + <%- @exclude_lines.each do |exclude_line| -%> + - '<%= exclude_line %>' + <%- end -%> <%- end -%> <%- if @exclude_files.length > 0 -%> exclude_files: @@ -10,25 +27,22 @@ filebeat: - <%= exclude_file %> <%- end -%> <%- end -%> - <%- if @encoding -%> - encoding: <%= @encoding %> - <%- end -%> <%- if @fields.length > 0 -%> fields: - <%- @fields.sort_by{|k,v| k}.each do |key, value| -%> - <%= key %>: <%= value %> + <%- @fields.each_pair do |k, v| -%> + <%= k %>: <%= v %> <%- end -%> <%- end -%> fields_under_root: <%= @fields_under_root %> - <%- if @input_type -%> - input_type: <%= @input_type %> + <%- if @tags.length > 0 -%> + tags: + <%- @tags.each do |tag| -%> + - <%= tag %> + <%- end -%> <%- end -%> <%- if @ignore_older -%> ignore_older: <%= @ignore_older %> <%- end -%> - <%- if @close_older -%> - close_older: <%= @close_older %> - <%- end -%> <%- if @doc_type -%> document_type: <%= @doc_type %> <%- end -%> @@ -38,20 +52,54 @@ filebeat: <%- if @harvester_buffer_size -%> harvester_buffer_size: <%= @harvester_buffer_size %> <%- end -%> - tail_files: <%= @tail_files %> - force_close_files: <%= @force_close_files %> - <%- if @backoff -%> - backoff: <%= @backoff %> + <%- if @max_bytes -%> + max_bytes: <%= @max_bytes %> <%- end -%> - <%- if @max_backoff -%> - max_backoff: <%= @max_backoff %> + <%- if @symlinks -%> + symlinks: <%= @symlinks %> <%- end -%> - <%- if @backoff_factor -%> - backoff_factor: <%= @backoff_factor %> + <%- if @close_older -%> + close_older: <%= @close_older %> <%- end -%> - <%- if @max_bytes -%> - max_bytes: <%= @max_bytes %> + <%- if @force_close_files -%> + force_close_files: <%= @force_close_files %> <%- end -%> + <%- if @pipeline -%> + pipeline: <%= @pipeline %> + <%- end -%> + + <%- if @json.length > 0 -%> + ### JSON configuration + json: + # Decode JSON options. Enable this if your logs are structured in JSON. + # JSON key on which to apply the line filtering and multiline settings. This key + # must be top level and its value must be string, otherwise it is ignored. If + # no text key is defined, the line filtering and multiline features cannot be used. + <%- if @json['message_key'] != nil-%> + message_key: '<%= @json['message_key'] %>' + <%- end -%> + + # By default, the decoded JSON is placed under a "json" key in the output document. + # If you enable this setting, the keys are copied top level in the output document. + <%- if @json['keys_under_root'] != nil -%> + keys_under_root: <%= @json['keys_under_root'] %> + <%- end -%> + + # If keys_under_root and this setting are enabled, then the values from the decoded + # JSON object overwrite the fields that Filebeat normally adds (type, source, offset, etc.) + # in case of conflicts. + <%- if @json['overwrite_keys'] != nil -%> + overwrite_keys: <%= @json['overwrite_keys'] %> + <%- end -%> + + # If this setting is enabled, Filebeat adds a "json_error" key in case of JSON + # unmarshaling errors or when a text key is defined in the configuration but cannot + # be used. + <%- if @json['add_error_key'] != nil -%> + add_error_key: <%= @json['add_error_key'] %> + <%- end -%> + <%- end -%> + <%- if @multiline.length > 0 -%> multiline: <%- if @multiline['pattern'] -%> @@ -70,15 +118,64 @@ filebeat: timeout: <%= @multiline['timeout'] %> <%- end -%> <%- end -%> - <%- if @include_lines.length > 0 -%> - include_lines: - <%- @include_lines.each do |include_line| -%> - - '<%= include_line %>' - <%- end -%> + tail_files: <%= @tail_files %> + + # Experimental: If symlinks is enabled, symlinks are opened and harvested. The harvester is openening the + # original for harvesting but will report the symlink name as source. + #symlinks: false + + <%- if @backoff -%> + backoff: <%= @backoff %> <%- end -%> - <%- if @exclude_lines.length > 0 -%> - exclude_lines: - <%- @exclude_lines.each do |exclude_line| -%> - - '<%= exclude_line %>' - <%- end -%> + <%- if @max_backoff -%> + max_backoff: <%= @max_backoff %> + <%- end -%> + <%- if @backoff_factor -%> + backoff_factor: <%= @backoff_factor %> + <%- end -%> + + # Experimental: Max number of harvesters that are started in parallel. + # Default is 0 which means unlimited + #harvester_limit: 0 + + ### Harvester closing options + + # Close inactive closes the file handler after the predefined period. + # The period starts when the last line of the file was, not the file ModTime. + # Time strings like 2h (2 hours), 5m (5 minutes) can be used. + <%- if @close_inactive -%> + close_inactive: <%= @close_inactive %> + <%- end -%> + + # Close renamed closes a file handler when the file is renamed or rotated. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + close_renamed: <%= @close_renamed %> + + # When enabling this option, a file handler is closed immediately in case a file can't be found + # any more. In case the file shows up again later, harvesting will continue at the last known position + # after scan_frequency. + close_removed: <%= @close_removed %> + + # Closes the file handler as soon as the harvesters reaches the end of the file. + # By default this option is disabled. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + close_eof: <%= @close_eof %> + + ### State options + + # Files for the modification data is older then clean_inactive the state from the registry is removed + # By default this is disabled. + <%- if @clean_inactive -%> + clean_inactive: <%= @clean_inactive %> + <%- end -%> + + # Removes the state for file which cannot be found on disk anymore immediately + clean_removed: <%= @clean_removed %> + + # Close timeout closes the harvester after the predefined time. + # This is independent if the harvester did finish reading the file or not. + # By default this option is disabled. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + <%- if @close_timeout -%> + close_timeout: <%= @close_timeout %> <%- end -%> diff --git a/templates/prospector5.yml.erb b/templates/prospector5.yml.erb deleted file mode 100644 index 97a3872e..00000000 --- a/templates/prospector5.yml.erb +++ /dev/null @@ -1,147 +0,0 @@ ---- -filebeat: - prospectors: - - input_type: <%= @input_type %> - paths: - <%- @paths.each do |log_path| -%> - - <%= log_path %> - <%- end -%> - <%- if @encoding -%> - encoding: <%= @encoding %> - <%- end -%> - <%- if @include_lines.length > 0 -%> - include_lines: - <%- @include_lines.each do |include_line| -%> - - '<%= include_line %>' - <%- end -%> - <%- end -%> - <%- if @exclude_lines.length > 0 -%> - exclude_lines: - <%- @exclude_lines.each do |exclude_line| -%> - - '<%= exclude_line %>' - <%- end -%> - <%- end -%> - <%- if @exclude_files.length > 0 -%> - exclude_files: - <%- @exclude_files.each do |exclude_file| -%> - - <%= exclude_file %> - <%- end -%> - <%- end -%> - <%- if @fields.length > 0 -%> - fields: - <%- @fields.each_pair do |k, v| -%> - <%= k %>: <%= v %> - <%- end -%> - <%- end -%> - fields_under_root: <%= @fields_under_root %> - <%- if @ignore_older -%> - ignore_older: <%= @ignore_older %> - <%- end -%> - <%- if @doc_type -%> - document_type: <%= @doc_type %> - <%- end -%> - <%- if @scan_frequency -%> - scan_frequency: <%= @scan_frequency %> - <%- end -%> - <%- if @harvester_buffer_size -%> - harvester_buffer_size: <%= @harvester_buffer_size %> - <%- end -%> - <%- if @max_bytes -%> - max_bytes: <%= @max_bytes %> - <%- end -%> - - ### JSON configuration - - # Decode JSON options. Enable this if your logs are structured in JSON. - # JSON key on which to apply the line filtering and multiline settings. This key - # must be top level and its value must be string, otherwise it is ignored. If - # no text key is defined, the line filtering and multiline features cannot be used. - #json.message_key: - - # By default, the decoded JSON is placed under a "json" key in the output document. - # If you enable this setting, the keys are copied top level in the output document. - #json.keys_under_root: false - - # If keys_under_root and this setting are enabled, then the values from the decoded - # JSON object overwrite the fields that Filebeat normally adds (type, source, offset, etc.) - # in case of conflicts. - #json.overwrite_keys: false - - # If this setting is enabled, Filebeat adds a "json_error" key in case of JSON - # unmarshaling errors or when a text key is defined in the configuration but cannot - # be used. - #json.add_error_key: false - - <%- if @multiline.length > 0 -%> - multiline: - <%- if @multiline['pattern'] -%> - pattern: '<%= @multiline['pattern'] %>' - <%- end -%> - <%- if @multiline['negate'] -%> - negate: <%= @multiline['negate'] %> - <%- end -%> - <%- if @multiline['match'] -%> - match: <%= @multiline['match'] %> - <%- end -%> - <%- if @multiline['max_lines'] -%> - max_lines: <%= @multiline['max_lines'] %> - <%- end -%> - <%- if @multiline['timeout'] -%> - timeout: <%= @multiline['timeout'] %> - <%- end -%> - <%- end -%> - tail_files: <%= @tail_files %> - - # Experimental: If symlinks is enabled, symlinks are opened and harvested. The harvester is openening the - # original for harvesting but will report the symlink name as source. - #symlinks: false - - <%- if @backoff -%> - backoff: <%= @backoff %> - <%- end -%> - <%- if @max_backoff -%> - max_backoff: <%= @max_backoff %> - <%- end -%> - <%- if @backoff_factor -%> - backoff_factor: <%= @backoff_factor %> - <%- end -%> - - # Experimental: Max number of harvesters that are started in parallel. - # Default is 0 which means unlimited - #harvester_limit: 0 - - ### Harvester closing options - - # Close inactive closes the file handler after the predefined period. - # The period starts when the last line of the file was, not the file ModTime. - # Time strings like 2h (2 hours), 5m (5 minutes) can be used. - #close_inactive: 5m - - # Close renamed closes a file handler when the file is renamed or rotated. - # Note: Potential data loss. Make sure to read and understand the docs for this option. - #close_renamed: false - - # When enabling this option, a file handler is closed immediately in case a file can't be found - # any more. In case the file shows up again later, harvesting will continue at the last known position - # after scan_frequency. - #close_removed: true - - # Closes the file handler as soon as the harvesters reaches the end of the file. - # By default this option is disabled. - # Note: Potential data loss. Make sure to read and understand the docs for this option. - #close_eof: false - - ### State options - - # Files for the modification data is older then clean_inactive the state from the registry is removed - # By default this is disabled. - #clean_inactive: 0 - - # Removes the state for file which cannot be found on disk anymore immediately - #clean_removed: true - - # Close timeout closes the harvester after the predefined time. - # This is independent if the harvester did finish reading the file or not. - # By default this option is disabled. - # Note: Potential data loss. Make sure to read and understand the docs for this option. - #close_timeout: 0 diff --git a/templates/pure_hash.yml.erb b/templates/pure_hash.yml.erb new file mode 100644 index 00000000..04e16804 --- /dev/null +++ b/templates/pure_hash.yml.erb @@ -0,0 +1,2 @@ +### Filebeat configuration managed by Puppet ### +<%= @filebeat_config.to_yaml() %>