From a82194fe9bb1e4593f6d3ca21cfab3c730ea0a42 Mon Sep 17 00:00:00 2001 From: sskim Date: Fri, 8 Sep 2017 10:49:25 +0900 Subject: [PATCH 1/3] capture response headers --- fluent-plugin-http-pull.gemspec | 2 +- lib/fluent/plugin/in_http_pull.rb | 14 ++++++++++++++ test/plugin/test_in_http_pull.rb | 32 +++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/fluent-plugin-http-pull.gemspec b/fluent-plugin-http-pull.gemspec index e35a387..0242107 100644 --- a/fluent-plugin-http-pull.gemspec +++ b/fluent-plugin-http-pull.gemspec @@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) Gem::Specification.new do |spec| spec.name = "fluent-plugin-http-pull" - spec.version = "0.4.0" + spec.version = "0.5.0" spec.authors = ["filepang"] spec.email = ["filepang@gmail.com"] diff --git a/lib/fluent/plugin/in_http_pull.rb b/lib/fluent/plugin/in_http_pull.rb index 43ec0b5..67ff1db 100644 --- a/lib/fluent/plugin/in_http_pull.rb +++ b/lib/fluent/plugin/in_http_pull.rb @@ -45,6 +45,11 @@ def initialize desc 'password of basic auth' config_param :password, :string, default: nil + config_section :response_header, param_name: :response_headers, multi: true do + desc 'The name of header to cature from response' + config_param :header, :string + end + def configure(conf) compat_parameters_convert(conf, :parser) super @@ -69,8 +74,17 @@ def on_timer request_options[:password] = @password if @password res = RestClient::Request.execute request_options + record["status"] = res.code record["body"] = res.body + + record["header"] = {} unless @response_headers.empty? + @response_headers.each do |section| + name = section["header"] + symbolize_name = name.downcase.gsub(/-/, '_').to_sym + + record["header"][name] = res.headers[symbolize_name] + end rescue StandardError => err if err.respond_to? :http_code record["status"] = err.http_code || 0 diff --git a/test/plugin/test_in_http_pull.rb b/test/plugin/test_in_http_pull.rb index 6e68a71..e37c853 100644 --- a/test/plugin/test_in_http_pull.rb +++ b/test/plugin/test_in_http_pull.rb @@ -453,6 +453,38 @@ class HttpPullInputTest < Test::Unit::TestCase end end + sub_test_case "capture response header" do + TEST_INTERVAL_3_RES_HEADER_CONFIG = %[ + tag test + url http://127.0.0.1:3939 + + interval 3s + format json + + + header Content-Type + + ] + + test 'interval 3' do + d = create_driver TEST_INTERVAL_3_RES_HEADER_CONFIG + assert_equal("test", d.instance.tag) + assert_equal(3, d.instance.interval) + + d.run(timeout: 8) do + sleep 7 + end + assert_equal(2, d.events.size) + + d.events.each do |tag, time, record| + assert_equal("test", tag) + + assert_equal({"url"=>"http://127.0.0.1:3939","status"=>200,"message"=>{"status"=>"OK"},"header"=>{"Content-Type"=>"application/json"}}, record) + assert(time.is_a?(Fluent::EventTime)) + end + end + end + private def create_driver(conf) From 53a0ce16095466168c9c460111db6d709637d41e Mon Sep 17 00:00:00 2001 From: sskim Date: Fri, 8 Sep 2017 13:04:33 +0900 Subject: [PATCH 2/3] add request support --- lib/fluent/plugin/in_http_pull.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/fluent/plugin/in_http_pull.rb b/lib/fluent/plugin/in_http_pull.rb index 67ff1db..8a81c0a 100644 --- a/lib/fluent/plugin/in_http_pull.rb +++ b/lib/fluent/plugin/in_http_pull.rb @@ -50,11 +50,25 @@ def initialize config_param :header, :string end + config_section :request_header, param_name: :request_headers, multi: true do + desc 'The name of request header' + config_param :header, :string + + desc 'The value of request header' + config_param :value, :string + end + def configure(conf) compat_parameters_convert(conf, :parser) super @parser = parser_create unless @status_only + @_request_headers = @request_headers.map do |section| + header = section["header"] + value = section["value"] + + [header.to_sym, value] + end.to_h end def start @@ -72,6 +86,7 @@ def on_timer request_options[:proxy] = @proxy if @proxy request_options[:user] = @user if @user request_options[:password] = @password if @password + request_options[:headers] = @_request_headers unless @request_headers.empty? res = RestClient::Request.execute request_options From 1a3104a8b00d90508031a355a6a0de951385074f Mon Sep 17 00:00:00 2001 From: sskim Date: Fri, 8 Sep 2017 14:32:40 +0900 Subject: [PATCH 3/3] add req/res header testcase --- test/helper/stub_server.rb | 9 ++++ test/plugin/test_in_http_pull.rb | 73 +++++++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/test/helper/stub_server.rb b/test/helper/stub_server.rb index 56853fa..b0da31a 100644 --- a/test/helper/stub_server.rb +++ b/test/helper/stub_server.rb @@ -11,6 +11,7 @@ def initialize @server.mount_proc '/internal_error', &method(:internal_error) @server.mount_proc '/redirect', &method(:redirect) @server.mount_proc '/protected', &method(:protected) + @server.mount_proc '/custom_header', &method(:custom_header) end def start @@ -81,4 +82,12 @@ def protected(req, res) res['Content-Type'] = 'application/json' res.body = '{ "status": "OK" }' end + + def custom_header(req, res) + res.header["HATSUNE-MIKU"] = req["HATSUNE-MIKU"] if req["HATSUNE-MIKU"] + + res.status = 200 + res['Content-Type'] = 'application/json' + res.body = '{ "status": "OK" }' + end end diff --git a/test/plugin/test_in_http_pull.rb b/test/plugin/test_in_http_pull.rb index e37c853..977b023 100644 --- a/test/plugin/test_in_http_pull.rb +++ b/test/plugin/test_in_http_pull.rb @@ -466,7 +466,23 @@ class HttpPullInputTest < Test::Unit::TestCase ] - test 'interval 3' do + TEST_INTERVAL_3_RES_MULTI_HEADER_CONFIG = %[ + tag test + url http://127.0.0.1:3939 + + interval 3s + format json + + + header Content-Type + + + + header Content-Length + + ] + + test 'interval 3 with single header' do d = create_driver TEST_INTERVAL_3_RES_HEADER_CONFIG assert_equal("test", d.instance.tag) assert_equal(3, d.instance.interval) @@ -483,6 +499,61 @@ class HttpPullInputTest < Test::Unit::TestCase assert(time.is_a?(Fluent::EventTime)) end end + + test 'interval 3 with multiple header' do + d = create_driver TEST_INTERVAL_3_RES_MULTI_HEADER_CONFIG + assert_equal("test", d.instance.tag) + assert_equal(3, d.instance.interval) + + d.run(timeout: 8) do + sleep 7 + end + assert_equal(2, d.events.size) + + d.events.each do |tag, time, record| + assert_equal("test", tag) + + assert_equal({"url"=>"http://127.0.0.1:3939","status"=>200,"message"=>{"status"=>"OK"},"header"=>{"Content-Type"=>"application/json","Content-Length"=>"18"}}, record) + assert(time.is_a?(Fluent::EventTime)) + end + end + end + + sub_test_case "custom request header" do + TEST_INTERVAL_3_CUSTOM_HEADER_CONFIG = %[ + tag test + url http://127.0.0.1:3939/custom_header + + interval 3s + format json + + + header HATSUNE-MIKU + value 3939 + + + + header HATSUNE-MIKU + + ] + + test 'interval 3 with single header' do + d = create_driver TEST_INTERVAL_3_CUSTOM_HEADER_CONFIG + assert_equal("test", d.instance.tag) + assert_equal(3, d.instance.interval) + + d.run(timeout: 8) do + sleep 7 + end + assert_equal(2, d.events.size) + + d.events.each do |tag, time, record| + assert_equal("test", tag) + + assert_equal({"url"=>"http://127.0.0.1:3939/custom_header","status"=>200,"message"=>{"status"=>"OK"},"header"=>{"HATSUNE-MIKU"=>"3939"}}, record) + assert(time.is_a?(Fluent::EventTime)) + end + end end private