From ab6f3640c5d1035ba9c483d6513d7f8ff755a07f Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Sun, 6 Jan 2019 21:44:08 +0900 Subject: [PATCH 1/8] Drop support for Ruby 2.2 which has reached its EOL --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d199ca840e..ef480d6d5f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,7 +53,6 @@ matrix: - rvm: 2.3.7 env: RSPEC_TASK=spec:features DATABASE_ADAPTER=postgresql DATABASE_USERNAME=postgres rvm: -- 2.2.10 - 2.3.7 - 2.4.4 - 2.5.1 From 9fbeaa085c5d4f603c4c482369523aabe6076a85 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Sun, 6 Jan 2019 21:47:09 +0900 Subject: [PATCH 2/8] Bundler 2.0.1 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 59f24091c2..213fea68a8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -781,4 +781,4 @@ RUBY VERSION ruby 2.5.1p57 BUNDLED WITH - 1.16.3 + 2.0.1 From 55ca37c157a21d92e299df7ecaa04bc75f564c5e Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Sun, 6 Jan 2019 21:47:12 +0900 Subject: [PATCH 3/8] Use `--no-document` instead of obsolete `--no-ri --no-rdoc` --- doc/manual/installation.md | 2 +- docker/scripts/prepare | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/installation.md b/doc/manual/installation.md index 199288450f..c9fdb864f7 100644 --- a/doc/manual/installation.md +++ b/doc/manual/installation.md @@ -79,7 +79,7 @@ Download Ruby and compile it: Install the bundler and foreman gems: - sudo gem install rake bundler foreman --no-ri --no-rdoc + sudo gem install rake bundler foreman --no-document ## 3. System Users diff --git a/docker/scripts/prepare b/docker/scripts/prepare index 31ce504cb2..2f842ff4a6 100755 --- a/docker/scripts/prepare +++ b/docker/scripts/prepare @@ -33,7 +33,7 @@ $minimal_apt_get_install build-essential checkinstall git-core \ ruby2.5 ruby2.5-dev locale-gen en_US.UTF-8 update-locale LANG=en_US.UTF-8 LC_CTYPE=en_US.UTF-8 -gem install --no-ri --no-rdoc bundler +gem install --no-document bundler apt-get purge -y python3* rsyslog rsync manpages apt-get -y clean From a128863e566938737d5b9c4fc58baa546180c54a Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Sun, 6 Jan 2019 21:49:39 +0900 Subject: [PATCH 4/8] Add Ruby 2.6.0 to CI with allow_failures specified --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index ef480d6d5f..49f3466593 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,10 @@ matrix: env: DATABASE_ADAPTER=mysql2 DOCKER_IMAGE=huginn/huginn-single-process DOCKERFILE=docker/single-process/Dockerfile - rvm: 2.4.4 env: DATABASE_ADAPTER=mysql2 DOCKER_IMAGE=huginn/huginn DOCKERFILE=docker/multi-process/Dockerfile + - rvm: 2.6.0 + env: RSPEC_TASK=spec:features DATABASE_ADAPTER=mysql2 + - rvm: 2.6.0 + env: RSPEC_TASK=spec:features DATABASE_ADAPTER=postgresql DATABASE_USERNAME=postgres - rvm: 2.5.1 env: RSPEC_TASK=spec:features DATABASE_ADAPTER=mysql2 - rvm: 2.5.1 @@ -52,10 +56,13 @@ matrix: env: RSPEC_TASK=spec:features DATABASE_ADAPTER=mysql2 - rvm: 2.3.7 env: RSPEC_TASK=spec:features DATABASE_ADAPTER=postgresql DATABASE_USERNAME=postgres + allow_failures: + - rvm: 2.6.0 rvm: - 2.3.7 - 2.4.4 - 2.5.1 +- 2.6.0 cache: bundler bundler_args: --without development production script: From 6182c4c867bf3963f5165ddd705790abe67560d2 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Sun, 6 Jan 2019 22:49:19 +0900 Subject: [PATCH 5/8] Update ffi to 1.9.25 to silence the security alert It seems CVE-2018-1000201 is only applicable to Windows which we don't support, but it's certainly nice to update a gem whenever we can. --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 213fea68a8..f8b6a462f0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -253,7 +253,7 @@ GEM faraday_middleware (>= 0.9) loofah (>= 2.0) sax-machine (>= 1.0) - ffi (1.9.18) + ffi (1.9.25) font-awesome-sass (4.7.0) sass (>= 3.2) forecast_io (2.0.1) From 11ae9f420d04ff012d47643555e737f8ceb4ae91 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Sun, 6 Jan 2019 18:10:42 +0900 Subject: [PATCH 6/8] Factor out EventHeadersConcern from PostAgent for use in other agents --- app/concerns/event_headers_concern.rb | 61 +++++++++++++++++++++++++++ app/models/agents/post_agent.rb | 39 +++++------------ spec/models/agents/post_agent_spec.rb | 15 +++++-- 3 files changed, 82 insertions(+), 33 deletions(-) create mode 100644 app/concerns/event_headers_concern.rb diff --git a/app/concerns/event_headers_concern.rb b/app/concerns/event_headers_concern.rb new file mode 100644 index 0000000000..78e61ec26f --- /dev/null +++ b/app/concerns/event_headers_concern.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module EventHeadersConcern + private + + def validate_event_headers_options! + event_headers_payload({}) + rescue ArgumentError => e + errors.add(:base, e.message) + rescue Liquid::Error => e + errors.add(:base, "has an error with Liquid templating: #{e.message}") + end + + def event_headers_normalizer + case interpolated['event_headers_style'] + when nil, '', 'capitalized' + ->name { name.gsub(/[^-]+/, &:capitalize) } + when 'downcased' + :downcase.to_proc + when 'snakecased', nil + ->name { name.tr('A-Z-', 'a-z_') } + when 'raw' + :itself.to_proc + else + raise ArgumentError, "if provided, event_headers_style must be 'capitalized', 'downcased', 'snakecased' or 'raw'" + end + end + + def event_headers_key + case key = interpolated['event_headers_key'] + when nil, String + key.presence + else + raise ArgumentError, "if provided, event_headers_key must be a string" + end + end + + def event_headers_payload(headers) + key = event_headers_key or return {} + + normalize = event_headers_normalizer + + hash = headers.transform_keys(&normalize) + + names = + case event_headers = interpolated['event_headers'] + when Array + event_headers.map(&:to_s) + when String + event_headers.split(',') + when nil + nil + else + raise ArgumentError, "if provided, event_headers must be an array of strings or a comma separated string" + end + + { + key => names ? hash.slice(*names.map(&normalize)) : hash + } + end +end diff --git a/app/models/agents/post_agent.rb b/app/models/agents/post_agent.rb index 60009fbfe4..662b24a915 100644 --- a/app/models/agents/post_agent.rb +++ b/app/models/agents/post_agent.rb @@ -1,5 +1,6 @@ module Agents class PostAgent < Agent + include EventHeadersConcern include WebRequestConcern include FileHandling @@ -33,6 +34,8 @@ class PostAgent < Agent If `output_mode` is set to `merge`, the emitted Event will be merged into the original contents of the received Event. + Set `event_headers` to a list of header names, either in an array of string or in a comma-separated string, to include only some of the header values. + Set `event_headers_style` to one of the following values to normalize the keys of "headers" for downstream agents' convenience: * `capitalized` (default) - Header names are capitalized; e.g. "Content-Type" @@ -125,11 +128,7 @@ def validate_options errors.add(:base, "if provided, emit_events must be true or false") end - begin - normalize_response_headers({}) - rescue ArgumentError => e - errors.add(:base, e.message) - end + validate_event_headers_options! unless %w[post get put delete patch].include?(method) errors.add(:base, "method must be 'post', 'get', 'put', 'delete', or 'patch'") @@ -169,29 +168,6 @@ def check private - def normalize_response_headers(headers) - case interpolated['event_headers_style'] - when nil, '', 'capitalized' - normalize = ->name { - name.gsub(/(?:\A|(?<=-))([[:alpha:]])|([[:alpha:]]+)/) { - $1 ? $1.upcase : $2.downcase - } - } - when 'downcased' - normalize = :downcase.to_proc - when 'snakecased', nil - normalize = ->name { name.tr('A-Z-', 'a-z_') } - when 'raw' - normalize = ->name { name } # :itself.to_proc in Ruby >= 2.2 - else - raise ArgumentError, "if provided, event_headers_style must be 'capitalized', 'downcased', 'snakecased' or 'raw'" - end - - headers.each_with_object({}) { |(key, value), hash| - hash[normalize[key]] = value - } - end - def handle(data, event = Event.new, headers) url = interpolated(event.payload)[:post_url] @@ -234,10 +210,15 @@ def handle(data, event = Event.new, headers) new_event = interpolated['output_mode'].to_s == 'merge' ? event.payload.dup : {} create_event payload: new_event.merge( body: response.body, - headers: normalize_response_headers(response.headers), status: response.status + ).merge( + event_headers_payload(response.headers) ) end end + + def event_headers_key + super || 'headers' + end end end diff --git a/spec/models/agents/post_agent_spec.rb b/spec/models/agents/post_agent_spec.rb index 60f6b10e57..00a2524cb3 100644 --- a/spec/models/agents/post_agent_spec.rb +++ b/spec/models/agents/post_agent_spec.rb @@ -52,7 +52,7 @@ raise "unexpected Content-Type: #{content_type}" end end - { status: 200, body: "a webpage!", headers: { 'Content-type' => 'text/html' } } + { status: 200, body: "a webpage!", headers: { 'Content-type' => 'text/html', 'X-Foo-Bar' => 'baz' } } } end @@ -272,24 +272,31 @@ it "emits the response headers capitalized by default" do @checker.check - expect(@checker.events.last.payload['headers']).to eq({ 'Content-Type' => 'text/html' }) + expect(@checker.events.last.payload['headers']).to eq({ 'Content-Type' => 'text/html', 'X-Foo-Bar' => 'baz' }) end it "emits the response headers capitalized" do @checker.options['event_headers_style'] = 'capitalized' @checker.check - expect(@checker.events.last.payload['headers']).to eq({ 'Content-Type' => 'text/html' }) + expect(@checker.events.last.payload['headers']).to eq({ 'Content-Type' => 'text/html', 'X-Foo-Bar' => 'baz' }) end it "emits the response headers downcased" do @checker.options['event_headers_style'] = 'downcased' @checker.check - expect(@checker.events.last.payload['headers']).to eq({ 'content-type' => 'text/html' }) + expect(@checker.events.last.payload['headers']).to eq({ 'content-type' => 'text/html', 'x-foo-bar' => 'baz' }) end it "emits the response headers snakecased" do @checker.options['event_headers_style'] = 'snakecased' @checker.check + expect(@checker.events.last.payload['headers']).to eq({ 'content_type' => 'text/html', 'x_foo_bar' => 'baz' }) + end + + it "emits the response headers only including those specified by event_headers" do + @checker.options['event_headers_style'] = 'snakecased' + @checker.options['event_headers'] = 'content-type' + @checker.check expect(@checker.events.last.payload['headers']).to eq({ 'content_type' => 'text/html' }) end From b427488ce0003676aadfa53a5380adaf9b2bc390 Mon Sep 17 00:00:00 2001 From: Justin Hammond Date: Mon, 10 Dec 2018 11:27:10 +0800 Subject: [PATCH 7/8] add HTTP headers to payload of Webhook Agent --- app/models/agents/webhook_agent.rb | 16 +- spec/models/agents/webhook_agent_spec.rb | 352 ++++++++++++++++++++--- 2 files changed, 327 insertions(+), 41 deletions(-) diff --git a/app/models/agents/webhook_agent.rb b/app/models/agents/webhook_agent.rb index e7298503b2..111b4ae01c 100644 --- a/app/models/agents/webhook_agent.rb +++ b/app/models/agents/webhook_agent.rb @@ -22,6 +22,8 @@ class WebhookAgent < Agent * `payload_path` - JSONPath of the attribute in the POST body to be used as the Event payload. Set to `.` to return the entire message. If `payload_path` points to an array, Events will be created for each element. + * `headers` - Comma-separated list of HTTP headers your agent will include in the payload. + * `header_key` - The key to use to store all the headers recieved * `verbs` - Comma-separated list of http verbs your agent will accept. For example, "post,get" will enable POST and GET requests. Defaults to "post". @@ -43,11 +45,17 @@ class WebhookAgent < Agent def default_options { "secret" => "supersecretstring", "expected_receive_period_in_days" => 1, - "payload_path" => "some_key" + "payload_path" => "some_key", + "headers" => "", + "header_key" => "X-HTTP-HEADERS" } end - def receive_web_request(params, method, format) + def receive_web_request(request) + params = request.params.except(:action, :controller, :agent_id, :user_id, :format) + method = request.method_symbol.to_s + headers = request.headers.select {|k,v| k.to_s[/^HTTP_/]}.to_h + # check the secret secret = params.delete('secret') return ["Not Authorized", 401] unless secret == interpolated['secret'] @@ -86,6 +94,10 @@ def receive_web_request(params, method, format) end [payload_for(params)].flatten.each do |payload| + if interpolated['header_key'].present? + acceptedheaders = interpolated['headers'].split(/,/).map { |x| x.strip } + payload[interpolated['header_key']] = headers.slice(*acceptedheaders) + end create_event(payload: payload) end diff --git a/spec/models/agents/webhook_agent_spec.rb b/spec/models/agents/webhook_agent_spec.rb index 377b0ce9d9..5e0cf9a8df 100644 --- a/spec/models/agents/webhook_agent_spec.rb +++ b/spec/models/agents/webhook_agent_spec.rb @@ -3,7 +3,7 @@ describe Agents::WebhookAgent do let(:agent) do _agent = Agents::WebhookAgent.new(:name => 'webhook', - :options => { 'secret' => 'foobar', 'payload_path' => 'some_key' }) + :options => { 'secret' => 'foobar', 'payload_path' => 'some_key', 'headers' => 'HTTP_ACCEPT, HTTP_X_HELLO_WORLD', 'header_key' => 'X-HTTP-HEADERS' }) _agent.user = users(:bob) _agent.save! _agent @@ -12,82 +12,143 @@ describe 'receive_web_request' do it 'should create event if secret matches' do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(1) expect(out).to eq(['Event Created', 201]) - expect(Event.last.payload).to eq(payload) + expect(Event.last.payload).to eq( {"people"=>[{"name"=>"bob"}, {"name"=>"jon"}], "X-HTTP-HEADERS"=>{"HTTP_ACCEPT"=>"application/xml", "HTTP_X_HELLO_WORLD"=>"Hello Huginn"}}) end it 'should be able to create multiple events when given an array' do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) out = nil agent.options['payload_path'] = 'some_key.people' expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(2) expect(out).to eq(['Event Created', 201]) - expect(Event.last.payload).to eq({ 'name' => 'jon' }) + expect(Event.last.payload).to eq({"name"=>"jon", "X-HTTP-HEADERS"=>{"HTTP_ACCEPT"=>"application/xml", "HTTP_X_HELLO_WORLD"=>"Hello Huginn"}}) end it 'should not create event if secrets do not match' do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "bazbat", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'bazbat', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(0) expect(out).to eq(['Not Authorized', 401]) end it 'should respond with customized response message if configured with `response` option' do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + agent.options['response'] = 'That Worked' - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) expect(out).to eq(['That Worked', 201]) # Empty string is a valid response agent.options['response'] = '' - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) expect(out).to eq(['', 201]) end it 'should respond with interpolated response message if configured with `response` option' do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + agent.options['response'] = '{{some_key.people[1].name}}' - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) expect(out).to eq(['jon', 201]) end it 'should respond with custom response header if configured with `response_headers` option' do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) agent.options['response_headers'] = {"X-My-Custom-Header" => 'hello'} - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) expect(out).to eq(['Event Created', 201, "text/plain", {"X-My-Custom-Header" => 'hello'}]) end it 'should respond with `Event Created` if the response option is nil or missing' do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + agent.options['response'] = nil - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) expect(out).to eq(['Event Created', 201]) agent.options.delete('response') - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) expect(out).to eq(['Event Created', 201]) end it 'should respond with customized response code if configured with `code` option' do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + agent.options['code'] = '200' - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) expect(out).to eq(['Event Created', 200]) end it 'should respond with `201` if the code option is empty, nil or missing' do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + agent.options['code'] = '' - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) expect(out).to eq(['Event Created', 201]) - + agent.options['code'] = nil - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) expect(out).to eq(['Event Created', 201]) agent.options.delete('code') - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) expect(out).to eq(['Event Created', 201]) end @@ -96,17 +157,31 @@ context "default settings" do it "should not accept GET" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "GET", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "get", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(0) expect(out).to eq(['Please use POST requests only', 401]) end it "should accept POST" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(1) expect(out).to eq(['Event Created', 201]) end @@ -118,25 +193,46 @@ before { agent.options['verbs'] = 'get,post' } it "should accept GET" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "GET", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "get", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(1) expect(out).to eq(['Event Created', 201]) end it "should accept POST" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(1) expect(out).to eq(['Event Created', 201]) end it "should not accept PUT" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "PUT", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "put", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(0) expect(out).to eq(['Please use GET/POST requests only', 401]) end @@ -148,17 +244,31 @@ before { agent.options['verbs'] = 'get' } it "should accept GET" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "GET", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "get", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(1) expect(out).to eq(['Event Created', 201]) end it "should not accept POST" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(0) expect(out).to eq(['Please use GET requests only', 401]) end @@ -170,17 +280,31 @@ before { agent.options['verbs'] = 'post' } it "should not accept GET" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "GET", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "get", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(0) expect(out).to eq(['Please use POST requests only', 401]) end it "should accept POST" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(1) expect(out).to eq(['Event Created', 201]) end @@ -192,25 +316,46 @@ before { agent.options['verbs'] = 'put' } it "should accept PUT" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "PUT", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "put", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(1) expect(out).to eq(['Event Created', 201]) end it "should not accept GET" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "GET", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "get", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(0) expect(out).to eq(['Please use PUT requests only', 401]) end it "should not accept POST" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(0) expect(out).to eq(['Please use PUT requests only', 401]) end @@ -222,33 +367,61 @@ before { agent.options['verbs'] = ',, PUT,POST, gEt , ,' } it "should accept PUT" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "PUT", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "put", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(1) expect(out).to eq(['Event Created', 201]) end it "should accept GET" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "GET", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "get", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(1) expect(out).to eq(['Event Created', 201]) end it "should accept POST" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(1) expect(out).to eq(['Event Created', 201]) end it "should not accept DELETE" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "DELETE", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + out = nil expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "delete", "text/html") + out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(0) expect(out).to eq(['Please use PUT/POST/GET requests only', 401]) end @@ -257,6 +430,13 @@ context "with reCAPTCHA" do it "should not check a reCAPTCHA response unless recaptcha_secret is set" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + checked = false out = nil @@ -266,7 +446,7 @@ } expect { - out= agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out= agent.receive_web_request(webpayload) }.not_to change { checked } expect(out).to eq(["Event Created", 201]) @@ -275,6 +455,13 @@ it "should reject a request if recaptcha_secret is set but g-recaptcha-response is not given" do agent.options['recaptcha_secret'] = 'supersupersecret' + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + checked = false out = nil @@ -284,7 +471,7 @@ } expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload }, "post", "text/html") + out = agent.receive_web_request(webpayload) }.not_to change { checked } expect(out).to eq(["Not Authorized", 401]) @@ -292,6 +479,12 @@ it "should reject a request if recaptcha_secret is set and g-recaptcha-response given is not verified" do agent.options['recaptcha_secret'] = 'supersupersecret' + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload, 'g-recaptcha-response' => 'somevalue'}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) checked = false out = nil @@ -300,9 +493,8 @@ checked = true { status: 200, body: '{"success":false}' } } - - expect { - out = agent.receive_web_request({ 'secret' => 'foobar', 'some_key' => payload, 'g-recaptcha-response' => 'somevalue' }, "post", "text/html") + expect { + out = agent.receive_web_request(webpayload) }.to change { checked } expect(out).to eq(["Not Authorized", 401]) @@ -311,6 +503,12 @@ it "should accept a request if recaptcha_secret is set and g-recaptcha-response given is verified" do agent.options['payload_path'] = '.' agent.options['recaptcha_secret'] = 'supersupersecret' + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'g-recaptcha-response' => 'somevalue'}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) checked = false out = nil @@ -321,13 +519,89 @@ } expect { - out = agent.receive_web_request(payload.merge({ 'secret' => 'foobar', 'g-recaptcha-response' => 'somevalue' }), "post", "text/html") + out = agent.receive_web_request(webpayload) }.to change { checked } expect(out).to eq(["Event Created", 201]) expect(Event.last.payload).to eq(payload) end end + end + context "with Headers" do + it "should not pass any headers if Header_Key is not set" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + + agent.options['header_key'] = '' + + out = nil + + expect { + out = agent.receive_web_request(webpayload) + }.to change { Event.count }.by(1) + expect(out).to eq(['Event Created', 201]) + expect(Event.last.payload).to eq(payload) + end + it "should pass selected headers specified in Header_Key" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + + agent.options['headers'] = 'HTTP_X_HELLO_WORLD' + + out = nil + + expect { + out= agent.receive_web_request(webpayload) + }.to change { Event.count }.by(1) + expect(out).to eq(['Event Created', 201]) + expect(Event.last.payload).to eq({"people"=>[{"name"=>"bob"}, {"name"=>"jon"}], "X-HTTP-HEADERS"=>{"HTTP_X_HELLO_WORLD"=>"Hello Huginn"}}) + end + + it "should pass empty header_key if none of the headers exist" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + + agent.options['headers'] = 'HTTP_X_HELLO_WORLD1' + + out = nil + + expect { + out= agent.receive_web_request(webpayload) + }.to change { Event.count }.by(1) + expect(out).to eq(['Event Created', 201]) + expect(Event.last.payload).to eq({"people"=>[{"name"=>"bob"}, {"name"=>"jon"}], "X-HTTP-HEADERS"=>{}}) + end + + it "should pass empty header_key if headers is empty" do + webpayload = ActionDispatch::Request.new({ + 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), + 'REQUEST_METHOD' => "POST", + 'HTTP_ACCEPT' => 'application/xml', + 'HTTP_X_HELLO_WORLD' => "Hello Huginn" + }) + + agent.options['headers'] = '' + + out = nil + + expect { + out= agent.receive_web_request(webpayload) + }.to change { Event.count }.by(1) + expect(out).to eq(['Event Created', 201]) + expect(Event.last.payload).to eq({"people"=>[{"name"=>"bob"}, {"name"=>"jon"}], "X-HTTP-HEADERS"=>{}}) + end end From a2942b66f72e1a87af9b328f6419f722cfa685d8 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Sun, 6 Jan 2019 20:06:41 +0900 Subject: [PATCH 8/8] Use EventHeadersConcern --- app/models/agents/webhook_agent.rb | 26 ++++++++++------- spec/models/agents/webhook_agent_spec.rb | 37 +++++++++++++++--------- 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/app/models/agents/webhook_agent.rb b/app/models/agents/webhook_agent.rb index 111b4ae01c..55d1e76fee 100644 --- a/app/models/agents/webhook_agent.rb +++ b/app/models/agents/webhook_agent.rb @@ -1,6 +1,7 @@ module Agents class WebhookAgent < Agent - include WebRequestConcern + include EventHeadersConcern + include WebRequestConcern # to make reCAPTCHA verification requests cannot_be_scheduled! cannot_receive_events! @@ -22,8 +23,8 @@ class WebhookAgent < Agent * `payload_path` - JSONPath of the attribute in the POST body to be used as the Event payload. Set to `.` to return the entire message. If `payload_path` points to an array, Events will be created for each element. - * `headers` - Comma-separated list of HTTP headers your agent will include in the payload. - * `header_key` - The key to use to store all the headers recieved + * `event_headers` - Comma-separated list of HTTP headers your agent will include in the payload. + * `event_headers_key` - The key to use to store all the headers received * `verbs` - Comma-separated list of http verbs your agent will accept. For example, "post,get" will enable POST and GET requests. Defaults to "post". @@ -46,15 +47,20 @@ def default_options { "secret" => "supersecretstring", "expected_receive_period_in_days" => 1, "payload_path" => "some_key", - "headers" => "", - "header_key" => "X-HTTP-HEADERS" + "event_headers" => "", + "event_headers_key" => "headers" } end def receive_web_request(request) params = request.params.except(:action, :controller, :agent_id, :user_id, :format) method = request.method_symbol.to_s - headers = request.headers.select {|k,v| k.to_s[/^HTTP_/]}.to_h + headers = request.headers.each_with_object({}) { |(name, value), hash| + case name + when /\AHTTP_([A-Z0-9_]+)\z/ + hash[$1.tr('_', '-').gsub(/[^-]+/, &:capitalize)] = value + end + } # check the secret secret = params.delete('secret') @@ -94,11 +100,7 @@ def receive_web_request(request) end [payload_for(params)].flatten.each do |payload| - if interpolated['header_key'].present? - acceptedheaders = interpolated['headers'].split(/,/).map { |x| x.strip } - payload[interpolated['header_key']] = headers.slice(*acceptedheaders) - end - create_event(payload: payload) + create_event(payload: payload.merge(event_headers_payload(headers))) end if interpolated['response_headers'].presence @@ -124,6 +126,8 @@ def validate_options if options['code'].to_s.in?(['301', '302']) && !options['response'].present? errors.add(:base, "Must specify a url for request redirect") end + + validate_event_headers_options! end def payload_for(params) diff --git a/spec/models/agents/webhook_agent_spec.rb b/spec/models/agents/webhook_agent_spec.rb index 5e0cf9a8df..527bfe64b9 100644 --- a/spec/models/agents/webhook_agent_spec.rb +++ b/spec/models/agents/webhook_agent_spec.rb @@ -3,7 +3,7 @@ describe Agents::WebhookAgent do let(:agent) do _agent = Agents::WebhookAgent.new(:name => 'webhook', - :options => { 'secret' => 'foobar', 'payload_path' => 'some_key', 'headers' => 'HTTP_ACCEPT, HTTP_X_HELLO_WORLD', 'header_key' => 'X-HTTP-HEADERS' }) + :options => { 'secret' => 'foobar', 'payload_path' => 'some_key' }) _agent.user = users(:bob) _agent.save! _agent @@ -20,11 +20,13 @@ }) out = nil + agent.options['event_headers'] = 'Accept,X-Hello-World' + agent.options['event_headers_key'] = 'X-HTTP-HEADERS' expect { out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(1) expect(out).to eq(['Event Created', 201]) - expect(Event.last.payload).to eq( {"people"=>[{"name"=>"bob"}, {"name"=>"jon"}], "X-HTTP-HEADERS"=>{"HTTP_ACCEPT"=>"application/xml", "HTTP_X_HELLO_WORLD"=>"Hello Huginn"}}) + expect(Event.last.payload).to eq( {"people"=>[{"name"=>"bob"}, {"name"=>"jon"}], "X-HTTP-HEADERS"=>{"Accept"=>"application/xml", "X-Hello-World"=>"Hello Huginn"}}) end it 'should be able to create multiple events when given an array' do @@ -36,11 +38,13 @@ }) out = nil agent.options['payload_path'] = 'some_key.people' + agent.options['event_headers'] = 'Accept,X-Hello-World' + agent.options['event_headers_key'] = 'X-HTTP-HEADERS' expect { out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(2) expect(out).to eq(['Event Created', 201]) - expect(Event.last.payload).to eq({"name"=>"jon", "X-HTTP-HEADERS"=>{"HTTP_ACCEPT"=>"application/xml", "HTTP_X_HELLO_WORLD"=>"Hello Huginn"}}) + expect(Event.last.payload).to eq({"name"=>"jon", "X-HTTP-HEADERS"=>{"Accept"=>"application/xml", "X-Hello-World"=>"Hello Huginn"}}) end it 'should not create event if secrets do not match' do @@ -52,6 +56,8 @@ }) out = nil + agent.options['event_headers'] = 'Accept,X-Hello-World' + agent.options['event_headers_key'] = 'X-HTTP-HEADERS' expect { out = agent.receive_web_request(webpayload) }.to change { Event.count }.by(0) @@ -527,8 +533,8 @@ end end end - context "with Headers" do - it "should not pass any headers if Header_Key is not set" do + context "with headers" do + it "should not pass any headers if event_headers_key is not set" do webpayload = ActionDispatch::Request.new({ 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), 'REQUEST_METHOD' => "POST", @@ -536,7 +542,8 @@ 'HTTP_X_HELLO_WORLD' => "Hello Huginn" }) - agent.options['header_key'] = '' + agent.options['event_headers'] = 'Accept,X-Hello-World' + agent.options['event_headers_key'] = '' out = nil @@ -546,7 +553,8 @@ expect(out).to eq(['Event Created', 201]) expect(Event.last.payload).to eq(payload) end - it "should pass selected headers specified in Header_Key" do + + it "should pass selected headers specified in event_headers_key" do webpayload = ActionDispatch::Request.new({ 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), 'REQUEST_METHOD' => "POST", @@ -554,7 +562,8 @@ 'HTTP_X_HELLO_WORLD' => "Hello Huginn" }) - agent.options['headers'] = 'HTTP_X_HELLO_WORLD' + agent.options['event_headers'] = 'X-Hello-World' + agent.options['event_headers_key'] = 'X-HTTP-HEADERS' out = nil @@ -562,10 +571,10 @@ out= agent.receive_web_request(webpayload) }.to change { Event.count }.by(1) expect(out).to eq(['Event Created', 201]) - expect(Event.last.payload).to eq({"people"=>[{"name"=>"bob"}, {"name"=>"jon"}], "X-HTTP-HEADERS"=>{"HTTP_X_HELLO_WORLD"=>"Hello Huginn"}}) + expect(Event.last.payload).to eq({"people"=>[{"name"=>"bob"}, {"name"=>"jon"}], "X-HTTP-HEADERS"=>{"X-Hello-World"=>"Hello Huginn"}}) end - it "should pass empty header_key if none of the headers exist" do + it "should pass empty event_headers_key if none of the headers exist" do webpayload = ActionDispatch::Request.new({ 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), 'REQUEST_METHOD' => "POST", @@ -573,7 +582,8 @@ 'HTTP_X_HELLO_WORLD' => "Hello Huginn" }) - agent.options['headers'] = 'HTTP_X_HELLO_WORLD1' + agent.options['event_headers'] = 'x-hello-world1' + agent.options['event_headers_key'] = 'X-HTTP-HEADERS' out = nil @@ -584,7 +594,7 @@ expect(Event.last.payload).to eq({"people"=>[{"name"=>"bob"}, {"name"=>"jon"}], "X-HTTP-HEADERS"=>{}}) end - it "should pass empty header_key if headers is empty" do + it "should pass empty event_headers_key if event_headers is empty" do webpayload = ActionDispatch::Request.new({ 'action_dispatch.request.request_parameters' => payload.merge({"secret" => "foobar", 'some_key' => payload}), 'REQUEST_METHOD' => "POST", @@ -592,7 +602,8 @@ 'HTTP_X_HELLO_WORLD' => "Hello Huginn" }) - agent.options['headers'] = '' + agent.options['event_headers'] = '' + agent.options['event_headers_key'] = 'X-HTTP-HEADERS' out = nil