diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f6216f2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +/.bundle/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ +*.gem diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..b1a6a17 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,12 @@ +language: ruby +rvm: + - 2.3 + - 2.4 + - 2.5 +# deploy: +# provider: rubygems +# gemspec: fluent-plugin-splunk-hec.gemspec +# on: +# tags: true +# api_key: +# secure: blah blah blah diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..a22ee54 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,74 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or +advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at zliang@splunk.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..f7826ef --- /dev/null +++ b/Gemfile @@ -0,0 +1,6 @@ +source "https://rubygems.org" + +git_source(:github) {|repo_name| "https://github.com/#{repo_name}" } + +# Specify your gem's dependencies in fluent-plugin-splunk_hec_output.gemspec +gemspec diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..486e594 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,64 @@ +PATH + remote: . + specs: + fluent-plugin-splunk_hec_output (1.0.0.pre.alpha.1) + fluentd (~> 1.0) + net-http-persistent (~> 3.0) + +GEM + remote: https://rubygems.org/ + specs: + addressable (2.5.2) + public_suffix (>= 2.0.2, < 4.0) + connection_pool (2.2.1) + cool.io (1.5.3) + crack (0.4.3) + safe_yaml (~> 1.0.0) + dig_rb (1.0.1) + fluentd (1.1.0) + cool.io (>= 1.4.5, < 2.0.0) + dig_rb (~> 1.0.0) + http_parser.rb (>= 0.5.1, < 0.7.0) + msgpack (>= 0.7.0, < 2.0.0) + serverengine (>= 2.0.4, < 3.0.0) + sigdump (~> 0.2.2) + strptime (>= 0.2.2, < 1.0.0) + tzinfo (~> 1.0) + tzinfo-data (~> 1.0) + yajl-ruby (~> 1.0) + hashdiff (0.3.7) + http_parser.rb (0.6.0) + minitest (5.11.1) + msgpack (1.2.2) + net-http-persistent (3.0.0) + connection_pool (~> 2.2) + public_suffix (3.0.1) + rake (10.5.0) + safe_yaml (1.0.4) + serverengine (2.0.6) + sigdump (~> 0.2.2) + sigdump (0.2.4) + strptime (0.2.3) + thread_safe (0.3.6) + tzinfo (1.2.4) + thread_safe (~> 0.1) + tzinfo-data (1.2017.3) + tzinfo (>= 1.0.0) + webmock (3.3.0) + addressable (>= 2.3.6) + crack (>= 0.3.2) + hashdiff + yajl-ruby (1.3.1) + +PLATFORMS + ruby + +DEPENDENCIES + bundler (~> 1.16) + fluent-plugin-splunk_hec_output! + minitest (~> 5.0) + rake (~> 10.0) + webmock (~> 3.2) + +BUNDLED WITH + 1.16.0 diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..45dfe34 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [2018] [Gimi Liang @ Splunk Inc.] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 79cb5a3..8dadaa6 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,123 @@ # fluent-plugin-splunk-hec -This is the Fluentd output plugin for sending events to Splunk via HEC. + +[Fluentd](https://fluentd.org/) output plugin to send events to [Splunk](https://www.splunk.com) over the HEC (HTTP Event Collector) API. + +## Installation + +### RubyGems + +``` +$ gem install fluent-plugin-splunk-hec +``` + +### Bundler + +Add following line to your Gemfile: + +```ruby +gem "fluent-plugin-splunk-hec" +``` + +And then execute: + +``` +$ bundle +``` + +## Configuration + +* See also: [Output Plugin Overview](https://docs.fluentd.org/v1.0/articles/output-plugin-overview) + +### protocol (enum) (optional) + +Which protocol to use to call HEC api, "http" or "https", default "https". + +Available values: http, https + +Default value: `https`. + +### hec_host (string) (required) + +The hostname/IP of the Splunk instance which has HTTP input enabled, or a HEC load balancer. + +### hec_port (integer) (optional) + +The port number of the HTTP input, or the HEC load balancer. + +Default value: `8088`. + +### hec_token (string) (required) + +The HEC token. + +### index (string) (optional) + +The Splunk index indexs events, by default it is not set, and will use what is configured in the HTTP input. Liquid template is supported. + +### host (string) (optional) + +Set the host field for events, by default it's the hostname of the machine that runnning fluentd. Liquid template is supported. + +### source (string) (optional) + +The source will be applied to the events, by default it uses the event's tag. Liquid template is supported. + +### sourcetype (string) (optional) + +The sourcetype will be applied to the events, by default it is not set, and leave it to Splunk to figure it out. Liquid template is supported. + +### disable_template (bool) (optional) + +Disable Liquid template support. Once disabled, it cannot use Liquid templates in the `host`, `index`, `source`, `sourcetype` fields. + +### coerce_to_utf8 (bool) (optional) + + + +Default value: `true`. + +### non_utf8_replacement_string (string) (optional) + + + +Default value: ` `. + + +### \ section (optional) (single) + +#### client_cert (string) (optional) + +The path to a file containing a PEM-format CA certificate for this client. + +#### ca_file (string) (optional) + +The path to a file containing a PEM-format CA certificate. + +#### ca_path (string) (optional) + +The path to a directory containing CA certificates in PEM format. + +#### ciphers (array) (optional) + +List of SSl ciphers allowed. + +#### client_pkey (string) (optional) + +The client's SSL private key. + +#### insecure (bool) (optional) + +If `insecure` is set to true, it will not verify the server's certificate. If `ca_file` or `ca_path` is set, `insecure` will be ignored. + + + +### \ section (optional) (single) + +#### @type (string) (required) + + +## Copyright + +* Copyright(c) 2018- Gimi Liang @ Splunk Inc. +* License + * Apache License, Version 2.0 diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..d433a1e --- /dev/null +++ b/Rakefile @@ -0,0 +1,10 @@ +require "bundler/gem_tasks" +require "rake/testtask" + +Rake::TestTask.new(:test) do |t| + t.libs << "test" + t.libs << "lib" + t.test_files = FileList["test/**/*_test.rb"] +end + +task :default => :test diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..3eefcb9 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.0.0 diff --git a/fluent-plugin-splunk-hec.gemspec b/fluent-plugin-splunk-hec.gemspec new file mode 100644 index 0000000..88163ee --- /dev/null +++ b/fluent-plugin-splunk-hec.gemspec @@ -0,0 +1,39 @@ +Gem::Specification.new do |spec| + spec.name = "fluent-plugin-splunk-hec" + spec.version = File.read("VERSION") + spec.authors = ["Zhimin (Gimi) Liang"] + spec.email = ["zliang@splunk.com"] + + spec.summary = %q{A fluentd output plugin writes events to Splunk via HEC.} + spec.description = %q{A fluentd output plugin writes events to Splunk via HEC.} + spec.homepage = "https://github.com/splunk/fluent-plugin-splunk-hec" + spec.license = "Apache-2.0" + + # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host' + # to allow pushing to a single host or delete this section to allow pushing to any host. + if spec.respond_to?(:metadata) + spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'" + else + raise "RubyGems 2.0 or newer is required to protect against " \ + "public gem pushes." + end + + spec.require_paths = ["lib"] + spec.test_files = Dir.glob('test/**/**.rb') + spec.files = %w[ + CODE_OF_CONDUCT.md README.md LICENSE.txt + fluent-plugin-splunk-hec.gemspec + Gemfile Gemfile.lock + Rakefile + ] + Dir.glob('lib/**/**').reject(&File.method(:directory?)) + + spec.required_ruby_version = '>= 2.3.0' + + spec.add_runtime_dependency "fluentd", "~> 1.0" + spec.add_runtime_dependency "net-http-persistent", "~> 3.0" + + spec.add_development_dependency "bundler", "~> 1.16" + spec.add_development_dependency "rake", "~> 10.0" + spec.add_development_dependency "minitest", "~> 5.0" + spec.add_development_dependency "webmock", "~> 3.2" +end diff --git a/lib/fluent/plugin/formatter_nil.rb b/lib/fluent/plugin/formatter_nil.rb new file mode 100644 index 0000000..560a1a0 --- /dev/null +++ b/lib/fluent/plugin/formatter_nil.rb @@ -0,0 +1,28 @@ +# +# Copyright 2018- Zhimin (Gimi) Liang @ Splunk Inc. (https://github.com/Gimi) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "fluent/plugin/formatter" + +module Fluent + module Plugin + class NilFormatter < Fluent::Plugin::Formatter + Fluent::Plugin.register_formatter("nil", self) + + def format(tag, time, record) + '' + end + end + end +end diff --git a/lib/fluent/plugin/out_splunk_hec.rb b/lib/fluent/plugin/out_splunk_hec.rb new file mode 100644 index 0000000..778cc4c --- /dev/null +++ b/lib/fluent/plugin/out_splunk_hec.rb @@ -0,0 +1,246 @@ +require "fluent/plugin/output" + +require 'openssl' +require 'net/http/persistent' + +module Fluent::Plugin + class SplunkHecOutput < Fluent::Plugin::Output + Fluent::Plugin.register_output('splunk_hec', self) + + helpers :formatter + + autoload :VERSION, "fluent/plugin/out_splunk_hec/version" + + desc 'Which protocol to use to call HEC api, "http" or "https", default "https".' + config_param :protocol, :enum, list: %i[http https], default: :https + + desc 'The hostname/IP of the Splunk instance which has HTTP input enabled, or a HEC load balancer.' + config_param :hec_host, :string + + desc 'The port number of the HTTP input, or the HEC load balancer.' + config_param :hec_port, :integer, default: 8088 + + desc 'The HEC token.' + config_param :hec_token, :string + + desc 'SSL configurations.' + config_section :ssl, param_name: 'ssl', required: false, multi: false, init: true do + desc "The path to a file containing a PEM-format CA certificate for this client." + config_param :client_cert, :string, default: nil + + desc 'The path to a file containing a PEM-format CA certificate.' + config_param :ca_file, :string, default: nil + + desc 'The path to a directory containing CA certificates in PEM format.' + config_param :ca_path, :string, default: nil + + desc 'List of SSl ciphers allowed.' + config_param :ciphers, :array, default: nil + + desc "The client's SSL private key." + config_param :client_pkey, :string, default: nil + + desc "If `insecure` is set to true, it will not verify the server's certificate. If `ca_file` or `ca_path` is set, `insecure` will be ignored." + config_param :insecure, :bool, default: false + end + + desc 'The Splunk index indexs events, by default it is not set, and will use what is configured in the HTTP input. Liquid template is supported.' + config_param :index, :string, default: nil + + desc "Set the host field for events, by default it's the hostname of the machine that runnning fluentd. Liquid template is supported." + config_param :host, :string, default: nil + + desc "The source will be applied to the events, by default it uses the event's tag. Liquid template is supported." + config_param :source, :string, default: nil + + desc 'The sourcetype will be applied to the events, by default it is not set, and leave it to Splunk to figure it out. Liquid template is supported.' + config_param :sourcetype, :string, default: nil + + desc 'Disable Liquid template support. Once disabled, it cannot use Liquid templates in the `host`, `index`, `source`, `sourcetype` fields.' + config_param :disable_template, :bool, default: false + + # Whether to allow non-UTF-8 characters in user logs. If set to true, any + # non-UTF-8 character would be replaced by the string specified by + # 'non_utf8_replacement_string'. If set to false, any non-UTF-8 character + # would trigger the plugin to error out. + config_param :coerce_to_utf8, :bool, :default => true + + # If 'coerce_to_utf8' is set to true, any non-UTF-8 character would be + # replaced by the string specified here. + config_param :non_utf8_replacement_string, :string, :default => ' ' + + config_section :format do + # the format section defined in formatter plugin help requires init. + # just defined a useless formatter as a placeholder. + config_param :@type, :string, default: 'nil' + end + + def initialize + super + @default_host = Socket.gethostname + @chunk_queue = SizedQueue.new 1 + end + + def configure(conf) + super + prepare_templates + construct_api + + @formatter = formatter_create + @formatter = nil if @formatter.is_a?(::Fluent::Plugin::NilFormatter) + end + + def start + super + start_worker_threads + end + + def format(tag, time, record) + values = { + 'tag' => tag, + 'record' => record + } + event = @formatter ? @formatter.format(tag, time, record) : record + + { + host: @host ? @host.render(values) : @default_host, + source: @source ? @source.render(values) : tag, + event: convert_to_utf8(event), + time: time.to_i + }.tap { |payload| + payload.update sourcetype: @sourcetype.render(values) if @sourcetype + payload.update index: @index.render(values) if @index + }.to_json + end + + def try_write(chunk) + log.debug { "Received new chunk, size=#{chunk.read.bytesize}" } + @chunk_queue << chunk + end + + def stop + @chunk_queue.close + super + end + + def multi_workers_ready? + true + end + + private + + def prepare_templates + template_fields = %w[@index @host @source @sourcetype] + + if @disable_template + # provides `render` method when template is diabled, so that + # we can handle the fields in the same ways no matter if templating + # is enabled or not. + self_render = Module.new { + def render(*args) self end + } + template_fields.each { |field| + v = instance_variable_get field + v.extend self_render if v + } + else + require 'liquid' + template_fields.each { |field| + v = instance_variable_get field + instance_variable_set field, Liquid::Template.parse(v) if v + } + end + end + + def construct_api + @hec_api = URI("#{@protocol}://#{@hec_host}:#{@hec_port}/services/collector") + rescue + raise Fluent::ConfigError, "hec_host (#{@hec_host}) and/or hec_port (#{@hec_port}) are invalid." + end + + def start_worker_threads + thread_create :"hec_worker_#{@hec_api}" do + http = new_connection + while chunk = get_next_chunk + send_to_hec http, chunk + end + end + end + + def get_next_chunk + @chunk_queue.pop @chunk_queue.closed? + rescue ThreadError # see SizedQueue#pop doc + nil + end + + def new_connection + Net::HTTP::Persistent.new.tap do |c| + c.verify_mode = @ssl.insecure ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER + c.cert = OpenSSL::X509::Certificate.new File.read(@ssl.client_cert) if @ssl.client_cert + c.key = OpenSSL::PKey::RSA.new File.read(@ssl.client_pkey) if @ssl.client_pkey + c.ca_file = @ssl.ca_file + c.ca_path = @ssl.ca_path + c.ciphers = @ssl.ciphers + + c.override_headers['Content-Type'] = 'application/json' + c.override_headers['User-Agent'] = "fluent-plugin-splunk_hec_out/#{VERSION}" + c.override_headers['Authorization'] = "Splunk #{@hec_token}" + end + end + + def send_to_hec(http, chunk) + post = Net::HTTP::Post.new @hec_api.request_uri + post.body = chunk.read + log.debug { "Sending #{post.body.bytesize} bytes to Splunk." } + + log.trace { "POST #{@hec_api} body=#{post.body}" } + response = http.request @hec_api, post + log.debug { "[Response] POST #{@hec_api}: #{response.inspect}" } + + # raise Exception to utilize Fluentd output plugin retry machanism + raise "Server error for POST #{@hec_api}, response: #{response.body}" if response.code.start_with?('5') + + # For both success response (2xx) and client errors (4xx), we will consume the chunk. + # Because there probably a bug in the code if we get 4xx errors, retry won't do any good. + commit_write(chunk.unique_id) + log.error "Failed POST to #{@hec_api}, response: #{response.body}" if not response.code.start_with?('2') + end + + # Encode as UTF-8. If 'coerce_to_utf8' is set to true in the config, any + # non-UTF-8 character would be replaced by the string specified by + # 'non_utf8_replacement_string'. If 'coerce_to_utf8' is set to false, any + # non-UTF-8 character would trigger the plugin to error out. + # Thanks to + # https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud/blob/dbc28575/lib/fluent/plugin/out_google_cloud.rb#L1284 + def convert_to_utf8(input) + if input.is_a?(Hash) + record = {} + input.each do |key, value| + record[convert_to_utf8(key)] = convert_to_utf8(value) + end + + return record + end + return input.map { |value| convert_to_utf8(value) } if input.is_a?(Array) + return input unless input.respond_to?(:encode) + + if @coerce_to_utf8 + input.encode( + 'utf-8', + invalid: :replace, + undef: :replace, + replace: @non_utf8_replacement_string) + else + begin + input.encode('utf-8') + rescue EncodingError + log.error { 'Encountered encoding issues potentially due to non ' \ + 'UTF-8 characters. To allow non-UTF-8 characters and ' \ + 'replace them with spaces, please set "coerce_to_utf8" ' \ + 'to true.' } + raise + end + end + end + end +end diff --git a/lib/fluent/plugin/out_splunk_hec/version.rb b/lib/fluent/plugin/out_splunk_hec/version.rb new file mode 100644 index 0000000..f03c9d8 --- /dev/null +++ b/lib/fluent/plugin/out_splunk_hec/version.rb @@ -0,0 +1 @@ +Fluent::Plugin::SplunkHecOutput::VERSION = File.read(File.expand_path('../../../../VERSION', File.dirname(__FILE__))).chomp.strip diff --git a/test/fluent/plugin/out_splunk_hec_test.rb b/test/fluent/plugin/out_splunk_hec_test.rb new file mode 100644 index 0000000..78869fd --- /dev/null +++ b/test/fluent/plugin/out_splunk_hec_test.rb @@ -0,0 +1,128 @@ +require "test_helper" + +describe Fluent::Plugin::SplunkHecOutput do + include Fluent::Test::Helpers + include PluginTestHelper + + before { Fluent::Test.setup } # setup router and others + + it { expect(::Fluent::Plugin::SplunkHecOutput::VERSION).wont_be_nil } + + describe "hec_host validation" do + describe "invalid host" do + it "should require hec_host" do + expect{ create_output_driver }.must_raise Fluent::ConfigError + end + + it { expect{ create_output_driver('hec_host %bad-host%') }.must_raise Fluent::ConfigError } + end + + describe "good host" do + it { + expect(create_output_driver('hec_host splunk.com').instance.hec_host).must_equal "splunk.com" + } + end + end + + it "should send request to Splunk" do + req = verify_sent_events { |r| + expect(r.body.scan(/test message/).size).must_equal 2 + } + expect(req).must_be_requested times: 1 + end + + describe "source" do + it "should use event tags by default" do + verify_sent_events() { |r| + expect(r.body).must_match(/"source"\s*:\s*"tag.event1"/) + expect(r.body).must_match(/"source"\s*:\s*"tag.event2"/) + } + end + + describe "use liquid templates" do + it "can use tag" do + verify_sent_events(%q) { |r| + expect(r.body).must_match(/"source"\s*:\s*"tag-event1"/) + expect(r.body).must_match(/"source"\s*:\s*"tag-event2"/) + } + end + + it "can use record" do + verify_sent_events('source "{{ record.id }}"') { |r| + expect(r.body).must_match(/"source"\s*:\s*"1st"/) + expect(r.body).must_match(/"source"\s*:\s*"2nd"/) + } + end + end + end + + describe "host" do + it "should use host machine's hostname by default" do + verify_sent_events() { |r| + expect(r.body).must_match(/"host"\s*:\s*"#{Socket.gethostname}"/) + } + end + + it "should understand liquid tempaltes" do + verify_sent_events(%q) { |r| + expect(r.body).must_match(/"host"\s*:\s*"tag-event1"/) + expect(r.body).must_match(/"host"\s*:\s*"tag-event2"/) + } + end + end + + describe "sourcetype" do + it "should not be set by default" do + verify_sent_events() { |r| + expect(r.body).wont_match(/"sourcetype"\s*:\s*"/) + true # `wont_match` returns `false` which will make webmock think it fails + } + end + + it "should understand liquid tempaltes" do + verify_sent_events(%q) { |r| + expect(r.body).must_match(/"sourcetype"\s*:\s*"tag-event1"/) + expect(r.body).must_match(/"sourcetype"\s*:\s*"tag-event2"/) + } + end + end + + it "should be able to disable liquid tempalte" do + verify_sent_events(<<~CONF) { |r| + disable_template true + host "{{ host }}" + source "{{ source }}" + sourcetype "{{ sourcetype }}" + CONF + expect(r.body.scan(/"host"\s*:\s*"{{ host }}"/).size).must_equal 2 + expect(r.body.scan(/"source"\s*:\s*"{{ source }}"/).size).must_equal 2 + expect(r.body.scan(/"sourcetype"\s*:\s*"{{ sourcetype }}"/).size).must_equal 2 + } + end + + it "should support use a formatter" do + verify_sent_events(<<~CONF) { |r| + + @type single_value + message_key message + add_newline false + + CONF + expect(r.body.scan(/"event"\s*:\s*"test message"/).size).must_equal 2 + } + end + + def verify_sent_events(conf = '', &blk) + host = "hec.splunk.com" + d = create_output_driver("hec_host #{host}", conf) + + hec_req = stub_hec_request("https://#{host}:8088").with &blk + + d.run do + d.feed("tag.event1", event_time, {"message" => "test message", "id" => "1st"}) + d.feed("tag.event2", event_time, {"message" => "test message", "id" => "2nd"}) + end + + hec_req + end +end diff --git a/test/lib/webmock/README.md b/test/lib/webmock/README.md new file mode 100644 index 0000000..9b68eed --- /dev/null +++ b/test/lib/webmock/README.md @@ -0,0 +1,3 @@ +There are two reasons why we stub out all these webmock adapter: +* Requiring 'http' (by the http_rb_adapter) will trigger a circle require warning (http/client <-> http/connection) +* We only need mocking the standard library `net/http`, and we don't want to load a bunch of not used libraries. diff --git a/test/lib/webmock/http_lib_adapters/curb_adapter.rb b/test/lib/webmock/http_lib_adapters/curb_adapter.rb new file mode 100644 index 0000000..e69de29 diff --git a/test/lib/webmock/http_lib_adapters/em_http_request_adapter.rb b/test/lib/webmock/http_lib_adapters/em_http_request_adapter.rb new file mode 100644 index 0000000..e69de29 diff --git a/test/lib/webmock/http_lib_adapters/excon_adapter.rb b/test/lib/webmock/http_lib_adapters/excon_adapter.rb new file mode 100644 index 0000000..e69de29 diff --git a/test/lib/webmock/http_lib_adapters/http_rb_adapter.rb b/test/lib/webmock/http_lib_adapters/http_rb_adapter.rb new file mode 100644 index 0000000..e69de29 diff --git a/test/lib/webmock/http_lib_adapters/httpclient_adapter.rb b/test/lib/webmock/http_lib_adapters/httpclient_adapter.rb new file mode 100644 index 0000000..e69de29 diff --git a/test/lib/webmock/http_lib_adapters/manticore_adapter.rb b/test/lib/webmock/http_lib_adapters/manticore_adapter.rb new file mode 100644 index 0000000..e69de29 diff --git a/test/lib/webmock/http_lib_adapters/patron_adapter.rb b/test/lib/webmock/http_lib_adapters/patron_adapter.rb new file mode 100644 index 0000000..e69de29 diff --git a/test/lib/webmock/http_lib_adapters/typhoeus_hydra_adapter.rb b/test/lib/webmock/http_lib_adapters/typhoeus_hydra_adapter.rb new file mode 100644 index 0000000..e69de29 diff --git a/test/test_helper.rb b/test/test_helper.rb new file mode 100644 index 0000000..30d9fed --- /dev/null +++ b/test/test_helper.rb @@ -0,0 +1,38 @@ +$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__) +$LOAD_PATH.unshift File.expand_path("../lib", __FILE__) +require "fluent/plugin/out_splunk_hec" + +require "fluent/test" +require "fluent/test/driver/output" +require "fluent/test/helpers" +require "minitest/autorun" +require "webmock/minitest" + +# make assertions from webmock available in minitest/spec +module Minitest::Expectations + infect_an_assertion :assert_requested, :must_be_requested, :reverse + infect_an_assertion :assert_not_requested, :wont_be_requested, :reverse +end + +TEST_HEC_TOKEN = "some-token".freeze + +module PluginTestHelper + def fluentd_conf_for(*lines) + basic_config = [ + "hec_token #{TEST_HEC_TOKEN}" + ] + (basic_config + lines).join("\n") + end + + def create_output_driver(*configs) + Fluent::Test::Driver::Output.new(Fluent::Plugin::SplunkHecOutput).tap { |d| + d.configure(fluentd_conf_for(*configs)) + } + end + + def stub_hec_request(endpoint) + stub_request(:post, "#{endpoint}/services/collector"). + with(headers: {"Authorization" => "Splunk #{TEST_HEC_TOKEN}", "User-Agent" => "fluent-plugin-splunk_hec_out/#{Fluent::Plugin::SplunkHecOutput::VERSION}"}). + to_return(body: '{"text":"Success","code":0}') + end +end