From f7173207130b593114fff6f8573a9c0627a56ebc Mon Sep 17 00:00:00 2001 From: David Venable Date: Mon, 1 Apr 2024 14:49:09 -0500 Subject: [PATCH 1/4] Implements AWS SigV4 for the HTTP output plugin. Resolves #4444. Signed-off-by: David Venable --- fluentd.gemspec | 3 ++ lib/fluent/plugin/out_http.rb | 60 ++++++++++++++++++++-- test/plugin/test_out_http.rb | 94 +++++++++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 5 deletions(-) diff --git a/fluentd.gemspec b/fluentd.gemspec index fd460fb240..71f7081aae 100644 --- a/fluentd.gemspec +++ b/fluentd.gemspec @@ -29,6 +29,9 @@ Gem::Specification.new do |gem| gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"]) gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"]) gem.add_runtime_dependency("webrick", ["~> 1.4"]) + gem.add_runtime_dependency("aws-sigv4", ["~> 1.8"]) + gem.add_runtime_dependency("aws-sdk-sts", ["~> 1.11"]) + gem.add_runtime_dependency("rexml", ["~> 3.2"]) # gems that aren't default gems as of Ruby 3.4 gem.add_runtime_dependency("base64", ["~> 0.2"]) diff --git a/lib/fluent/plugin/out_http.rb b/lib/fluent/plugin/out_http.rb index b4c149feb0..02ff027984 100644 --- a/lib/fluent/plugin/out_http.rb +++ b/lib/fluent/plugin/out_http.rb @@ -20,6 +20,8 @@ require 'fluent/tls' require 'fluent/plugin/output' require 'fluent/plugin_helper/socket' +require 'aws-sigv4' +require 'aws-sdk-core' # patch Net::HTTP to support extra_chain_cert which was added in Ruby feature #9758. # see: https://github.com/ruby/ruby/commit/31af0dafba6d3769d2a39617c0dddedb97883712 @@ -87,11 +89,17 @@ class RetryableResponse < StandardError; end config_section :auth, required: false, multi: false do desc 'The method for HTTP authentication' - config_param :method, :enum, list: [:basic], default: :basic + config_param :method, :enum, list: [:basic, :aws_sigv4], default: :basic desc 'The username for basic authentication' config_param :username, :string, default: nil desc 'The password for basic authentication' config_param :password, :string, default: nil, secret: true + desc 'The AWS service to authenticate against' + config_param :aws_service, :string, default: nil + desc 'The AWS region to use when authenticating' + config_param :aws_region, :string, default: nil + desc 'The AWS role ARN to assume when authenticating' + config_param :aws_role_arn, :string, default: nil end def initialize @@ -121,6 +129,30 @@ def configure(conf) end define_singleton_method(:format, method(:format_json_array)) end + + if @auth and @auth.method == :aws_sigv4 + + raise Fluent::ConfigError, "aws_service is required for aws_sigv4 auth" unless @auth.aws_service != nil + raise Fluent::ConfigError, "aws_region is required for aws_sigv4 auth" unless @auth.aws_region != nil + + if @auth.aws_role_arn == nil + aws_credentials = Aws::CredentialProviderChain.new.resolve + else + aws_credentials = Aws::AssumeRoleCredentials.new( + client: Aws::STS::Client.new( + region: @auth.aws_region + ), + role_arn: @auth.aws_role_arn, + role_session_name: "fluentd" + ) + end + + @aws_signer = Aws::Sigv4::Signer.new( + service: @auth.aws_service, + region: @auth.aws_region, + credentials_provider: aws_credentials + ) + end end def multi_workers_ready? @@ -215,7 +247,7 @@ def parse_endpoint(chunk) URI.parse(endpoint) end - def set_headers(req, chunk) + def set_headers(req, uri, chunk) if @headers @headers.each do |k, v| req[k] = v @@ -227,6 +259,7 @@ def set_headers(req, chunk) end end req['Content-Type'] = @content_type + req['Host'] = uri.host end def create_request(chunk, uri) @@ -236,11 +269,28 @@ def create_request(chunk, uri) when :put Net::HTTP::Put.new(uri.request_uri) end + set_headers(req, uri, chunk) + req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read + if @auth - req.basic_auth(@auth.username, @auth.password) + if @auth.method == :basic + req.basic_auth(@auth.username, @auth.password) + elsif @auth.method == :aws_sigv4 + signature = @aws_signer.sign_request( + http_method: req.method, + url: uri.request_uri, + headers: { + 'Content-Type' => @content_type, + 'Host' => uri.host + }, + body: req.body + ) + req.add_field('x-amz-date', signature.headers['x-amz-date']) + req.add_field('x-amz-security-token', signature.headers['x-amz-security-token']) + req.add_field('x-amz-content-sha256', signature.headers['x-amz-content-sha256']) + req.add_field('authorization', signature.headers['authorization']) + end end - set_headers(req, chunk) - req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read req end diff --git a/test/plugin/test_out_http.rb b/test/plugin/test_out_http.rb index b0e1a469a7..efb118d55e 100644 --- a/test/plugin/test_out_http.rb +++ b/test/plugin/test_out_http.rb @@ -7,6 +7,7 @@ require 'net/http' require 'uri' require 'json' +require 'aws-sdk-core' # WEBrick's ProcHandler doesn't handle PUT by default module WEBrick::HTTPServlet @@ -390,6 +391,99 @@ def test_basic_auth_with_invalid_auth end end + + sub_test_case 'aws sigv4 auth' do + setup do + @@fake_aws_credentials = Aws::Credentials.new( + 'fakeaccess', + 'fakesecret', + 'fake session token' + ) + end + + def server_port + 19883 + end + + def test_aws_sigv4_sts_role_arn + stub(Aws::AssumeRoleCredentials).new do |credentials_provider| + stub(credentials_provider).credentials { + @@fake_aws_credentials + } + credentials_provider + end + + d = create_driver(config + %[ + + method aws_sigv4 + aws_service someservice + aws_region my-region-1 + aws_role_arn arn:aws:iam::123456789012:role/MyRole + + ]) + d.run(default_tag: 'test.http') do + test_events.each { |event| + d.feed(event) + } + end + + result = @@result + assert_equal 'POST', result.method + assert_equal 'application/x-ndjson', result.content_type + assert_equal test_events, result.data + assert_not_empty result.headers + assert_equal '127.0.0.1', result.headers['host'] + assert_not_nil result.headers['authorization'] + assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization'] + assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization'] + assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token'] + assert_not_nil result.headers['x-amz-content-sha256'] + assert_not_empty result.headers['x-amz-content-sha256'] + assert_not_nil result.headers['x-amz-security-token'] + assert_not_empty result.headers['x-amz-security-token'] + assert_not_nil result.headers['x-amz-date'] + assert_not_empty result.headers['x-amz-date'] + end + + def test_aws_sigv4_no_role + stub(Aws::CredentialProviderChain).new do |provider_chain| + stub(provider_chain).resolve { + @@fake_aws_credentials + } + provider_chain + end + d = create_driver(config + %[ + + method aws_sigv4 + aws_service someservice + aws_region my-region-1 + + ]) + d.run(default_tag: 'test.http') do + test_events.each { |event| + d.feed(event) + } + end + + result = @@result + assert_equal 'POST', result.method + assert_equal 'application/x-ndjson', result.content_type + assert_equal test_events, result.data + assert_not_empty result.headers + assert_equal '127.0.0.1', result.headers['host'] + assert_not_nil result.headers['authorization'] + assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization'] + assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization'] + assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token'] + assert_not_nil result.headers['x-amz-content-sha256'] + assert_not_empty result.headers['x-amz-content-sha256'] + assert_not_nil result.headers['x-amz-security-token'] + assert_not_empty result.headers['x-amz-security-token'] + assert_not_nil result.headers['x-amz-date'] + assert_not_empty result.headers['x-amz-date'] + end + end + sub_test_case 'HTTPS' do def server_port 19882 From 43753219e215a9e8da53d2fd57000d687dfbc7fe Mon Sep 17 00:00:00 2001 From: David Venable Date: Wed, 3 Apr 2024 10:25:31 -0500 Subject: [PATCH 2/4] Make the AWS dependencies optional requirements and only require them as needed for SigV4 authentication. Signed-off-by: David Venable --- fluentd.gemspec | 6 +++--- lib/fluent/plugin/out_http.rb | 34 ++++++++++++++++++++++------------ 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/fluentd.gemspec b/fluentd.gemspec index 71f7081aae..dc6f6b4e8f 100644 --- a/fluentd.gemspec +++ b/fluentd.gemspec @@ -29,9 +29,6 @@ Gem::Specification.new do |gem| gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"]) gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"]) gem.add_runtime_dependency("webrick", ["~> 1.4"]) - gem.add_runtime_dependency("aws-sigv4", ["~> 1.8"]) - gem.add_runtime_dependency("aws-sdk-sts", ["~> 1.11"]) - gem.add_runtime_dependency("rexml", ["~> 3.2"]) # gems that aren't default gems as of Ruby 3.4 gem.add_runtime_dependency("base64", ["~> 0.2"]) @@ -59,4 +56,7 @@ Gem::Specification.new do |gem| gem.add_development_dependency("oj", [">= 2.14", "< 4"]) gem.add_development_dependency("async", "~> 1.23") gem.add_development_dependency("async-http", ">= 0.50.0") + gem.add_development_dependency("aws-sigv4", ["~> 1.8"]) + gem.add_development_dependency("aws-sdk-core", ["~> 3.191"]) + gem.add_development_dependency("rexml", ["~> 3.2"]) end diff --git a/lib/fluent/plugin/out_http.rb b/lib/fluent/plugin/out_http.rb index 02ff027984..db6ba8158d 100644 --- a/lib/fluent/plugin/out_http.rb +++ b/lib/fluent/plugin/out_http.rb @@ -20,8 +20,6 @@ require 'fluent/tls' require 'fluent/plugin/output' require 'fluent/plugin_helper/socket' -require 'aws-sigv4' -require 'aws-sdk-core' # patch Net::HTTP to support extra_chain_cert which was added in Ruby feature #9758. # see: https://github.com/ruby/ruby/commit/31af0dafba6d3769d2a39617c0dddedb97883712 @@ -131,6 +129,12 @@ def configure(conf) end if @auth and @auth.method == :aws_sigv4 + begin + require 'aws-sigv4' + require 'aws-sdk-core' + rescue LoadError + raise Fluent::ConfigError, "The aws-sigv4 and aws-sdk-core gems are required for aws_sigv4 auth. Run: 'gem install aws-sigv4 -v 1.8.0' and 'gem install aws-sdk-core -v 3.191'." + end raise Fluent::ConfigError, "aws_service is required for aws_sigv4 auth" unless @auth.aws_service != nil raise Fluent::ConfigError, "aws_region is required for aws_sigv4 auth" unless @auth.aws_region != nil @@ -262,16 +266,7 @@ def set_headers(req, uri, chunk) req['Host'] = uri.host end - def create_request(chunk, uri) - req = case @http_method - when :post - Net::HTTP::Post.new(uri.request_uri) - when :put - Net::HTTP::Put.new(uri.request_uri) - end - set_headers(req, uri, chunk) - req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read - + def set_auth(req, uri) if @auth if @auth.method == :basic req.basic_auth(@auth.username, @auth.password) @@ -291,9 +286,24 @@ def create_request(chunk, uri) req.add_field('authorization', signature.headers['authorization']) end end + end + + def create_request(chunk, uri) + req = case @http_method + when :post + Net::HTTP::Post.new(uri.request_uri) + when :put + Net::HTTP::Put.new(uri.request_uri) + end + set_headers(req, uri, chunk) + req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read + + # At least one authentication method requires the body and other headers, so the order of this call matters + set_auth(req, uri) req end + def send_request(uri, req) res = if @proxy_uri Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt) { |http| From e4c2044b3d42afbd1d9962f297e142ba2ad2f55d Mon Sep 17 00:00:00 2001 From: David Venable Date: Wed, 3 Apr 2024 10:28:11 -0500 Subject: [PATCH 3/4] Remove the Host header from HTTP output requests. Signed-off-by: David Venable --- lib/fluent/plugin/out_http.rb | 1 - test/plugin/test_out_http.rb | 2 -- 2 files changed, 3 deletions(-) diff --git a/lib/fluent/plugin/out_http.rb b/lib/fluent/plugin/out_http.rb index db6ba8158d..ec295db7b5 100644 --- a/lib/fluent/plugin/out_http.rb +++ b/lib/fluent/plugin/out_http.rb @@ -263,7 +263,6 @@ def set_headers(req, uri, chunk) end end req['Content-Type'] = @content_type - req['Host'] = uri.host end def set_auth(req, uri) diff --git a/test/plugin/test_out_http.rb b/test/plugin/test_out_http.rb index efb118d55e..04c80137b3 100644 --- a/test/plugin/test_out_http.rb +++ b/test/plugin/test_out_http.rb @@ -432,7 +432,6 @@ def test_aws_sigv4_sts_role_arn assert_equal 'application/x-ndjson', result.content_type assert_equal test_events, result.data assert_not_empty result.headers - assert_equal '127.0.0.1', result.headers['host'] assert_not_nil result.headers['authorization'] assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization'] assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization'] @@ -470,7 +469,6 @@ def test_aws_sigv4_no_role assert_equal 'application/x-ndjson', result.content_type assert_equal test_events, result.data assert_not_empty result.headers - assert_equal '127.0.0.1', result.headers['host'] assert_not_nil result.headers['authorization'] assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization'] assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization'] From c26d8e77b0ed23582100f7c2b7bd0e62a8ae47e1 Mon Sep 17 00:00:00 2001 From: David Venable Date: Fri, 5 Apr 2024 08:53:12 -0500 Subject: [PATCH 4/4] Addressed PR comments - update the gem installation to allow for more flexible versions and use a return to invert a conditional. Signed-off-by: David Venable --- lib/fluent/plugin/out_http.rb | 38 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/lib/fluent/plugin/out_http.rb b/lib/fluent/plugin/out_http.rb index ec295db7b5..55887065a7 100644 --- a/lib/fluent/plugin/out_http.rb +++ b/lib/fluent/plugin/out_http.rb @@ -133,7 +133,7 @@ def configure(conf) require 'aws-sigv4' require 'aws-sdk-core' rescue LoadError - raise Fluent::ConfigError, "The aws-sigv4 and aws-sdk-core gems are required for aws_sigv4 auth. Run: 'gem install aws-sigv4 -v 1.8.0' and 'gem install aws-sdk-core -v 3.191'." + raise Fluent::ConfigError, "The aws-sdk-core and aws-sigv4 gems are required for aws_sigv4 auth. Run: gem install aws-sdk-core -v '~> 3.191'" end raise Fluent::ConfigError, "aws_service is required for aws_sigv4 auth" unless @auth.aws_service != nil @@ -266,24 +266,24 @@ def set_headers(req, uri, chunk) end def set_auth(req, uri) - if @auth - if @auth.method == :basic - req.basic_auth(@auth.username, @auth.password) - elsif @auth.method == :aws_sigv4 - signature = @aws_signer.sign_request( - http_method: req.method, - url: uri.request_uri, - headers: { - 'Content-Type' => @content_type, - 'Host' => uri.host - }, - body: req.body - ) - req.add_field('x-amz-date', signature.headers['x-amz-date']) - req.add_field('x-amz-security-token', signature.headers['x-amz-security-token']) - req.add_field('x-amz-content-sha256', signature.headers['x-amz-content-sha256']) - req.add_field('authorization', signature.headers['authorization']) - end + return unless @auth + + if @auth.method == :basic + req.basic_auth(@auth.username, @auth.password) + elsif @auth.method == :aws_sigv4 + signature = @aws_signer.sign_request( + http_method: req.method, + url: uri.request_uri, + headers: { + 'Content-Type' => @content_type, + 'Host' => uri.host + }, + body: req.body + ) + req.add_field('x-amz-date', signature.headers['x-amz-date']) + req.add_field('x-amz-security-token', signature.headers['x-amz-security-token']) + req.add_field('x-amz-content-sha256', signature.headers['x-amz-content-sha256']) + req.add_field('authorization', signature.headers['authorization']) end end