Merge a2942b6 into b513fca

.travis.yml

@@ -40,6 +40,10 @@ matrix:
       env: DATABASE_ADAPTER=mysql2 DOCKER_IMAGE=huginn/huginn-single-process DOCKERFILE=docker/single-process/Dockerfile
     - rvm: 2.4.4
       env: DATABASE_ADAPTER=mysql2 DOCKER_IMAGE=huginn/huginn DOCKERFILE=docker/multi-process/Dockerfile
+    - rvm: 2.6.0
+      env: RSPEC_TASK=spec:features DATABASE_ADAPTER=mysql2
+    - rvm: 2.6.0
+      env: RSPEC_TASK=spec:features DATABASE_ADAPTER=postgresql DATABASE_USERNAME=postgres
     - rvm: 2.5.1
       env: RSPEC_TASK=spec:features DATABASE_ADAPTER=mysql2
     - rvm: 2.5.1
@@ -52,11 +56,13 @@ matrix:
       env: RSPEC_TASK=spec:features DATABASE_ADAPTER=mysql2
     - rvm: 2.3.7
       env: RSPEC_TASK=spec:features DATABASE_ADAPTER=postgresql DATABASE_USERNAME=postgres
+  allow_failures:
+    - rvm: 2.6.0
 rvm:
-- 2.2.10
 - 2.3.7
 - 2.4.4
 - 2.5.1
+- 2.6.0
 cache: bundler
 bundler_args: --without development production
 script:

Gemfile.lock

@@ -253,7 +253,7 @@ GEM
       faraday_middleware (>= 0.9)
       loofah (>= 2.0)
       sax-machine (>= 1.0)
-    ffi (1.9.18)
+    ffi (1.9.25)
     font-awesome-sass (4.7.0)
       sass (>= 3.2)
     forecast_io (2.0.1)
@@ -781,4 +781,4 @@ RUBY VERSION
    ruby 2.5.1p57
 
 BUNDLED WITH
-   1.16.3
+   2.0.1

app/concerns/event_headers_concern.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module EventHeadersConcern
+  private
+
+  def validate_event_headers_options!
+    event_headers_payload({})
+  rescue ArgumentError => e
+    errors.add(:base, e.message)
+  rescue Liquid::Error => e
+    errors.add(:base, "has an error with Liquid templating: #{e.message}")
+  end
+
+  def event_headers_normalizer
+    case interpolated['event_headers_style']
+    when nil, '', 'capitalized'
+      ->name { name.gsub(/[^-]+/, &:capitalize) }
+    when 'downcased'
+      :downcase.to_proc
+    when 'snakecased', nil
+      ->name { name.tr('A-Z-', 'a-z_') }
+    when 'raw'
+      :itself.to_proc
+    else
+      raise ArgumentError, "if provided, event_headers_style must be 'capitalized', 'downcased', 'snakecased' or 'raw'"
+    end
+  end
+
+  def event_headers_key
+    case key = interpolated['event_headers_key']
+    when nil, String
+      key.presence
+    else
+      raise ArgumentError, "if provided, event_headers_key must be a string"
+    end
+  end
+
+  def event_headers_payload(headers)
+    key = event_headers_key or return {}
+
+    normalize = event_headers_normalizer
+
+    hash = headers.transform_keys(&normalize)
+
+    names =
+      case event_headers = interpolated['event_headers']
+      when Array
+        event_headers.map(&:to_s)
+      when String
+        event_headers.split(',')
+      when nil
+        nil
+      else
+        raise ArgumentError, "if provided, event_headers must be an array of strings or a comma separated string"
+      end
+
+    {
+      key => names ? hash.slice(*names.map(&normalize)) : hash
+    }
+  end
+end

app/models/agents/post_agent.rb

@@ -1,5 +1,6 @@
 module Agents
   class PostAgent < Agent
+    include EventHeadersConcern
     include WebRequestConcern
     include FileHandling
 
@@ -33,6 +34,8 @@ class PostAgent < Agent
 
         If `output_mode` is set to `merge`, the emitted Event will be merged into the original contents of the received Event.
 
+        Set `event_headers` to a list of header names, either in an array of string or in a comma-separated string, to include only some of the header values.
+
         Set `event_headers_style` to one of the following values to normalize the keys of "headers" for downstream agents' convenience:
 
           * `capitalized` (default) - Header names are capitalized; e.g. "Content-Type"
@@ -125,11 +128,7 @@ def validate_options
         errors.add(:base, "if provided, emit_events must be true or false")
       end
 
-      begin
-        normalize_response_headers({})
-      rescue ArgumentError => e
-        errors.add(:base, e.message)
-      end
+      validate_event_headers_options!
 
       unless %w[post get put delete patch].include?(method)
         errors.add(:base, "method must be 'post', 'get', 'put', 'delete', or 'patch'")
@@ -169,29 +168,6 @@ def check
 
     private
 
-    def normalize_response_headers(headers)
-      case interpolated['event_headers_style']
-      when nil, '', 'capitalized'
-        normalize = ->name {
-          name.gsub(/(?:\A|(?<=-))([[:alpha:]])|([[:alpha:]]+)/) {
-            $1 ? $1.upcase : $2.downcase
-          }
-        }
-      when 'downcased'
-        normalize = :downcase.to_proc
-      when 'snakecased', nil
-        normalize = ->name { name.tr('A-Z-', 'a-z_') }
-      when 'raw'
-        normalize = ->name { name }  # :itself.to_proc in Ruby >= 2.2
-      else
-        raise ArgumentError, "if provided, event_headers_style must be 'capitalized', 'downcased', 'snakecased' or 'raw'"
-      end
-
-      headers.each_with_object({}) { |(key, value), hash|
-        hash[normalize[key]] = value
-      }
-    end
-
     def handle(data, event = Event.new, headers)
       url = interpolated(event.payload)[:post_url]
 
@@ -234,10 +210,15 @@ def handle(data, event = Event.new, headers)
         new_event = interpolated['output_mode'].to_s == 'merge' ? event.payload.dup : {}
         create_event payload: new_event.merge(
           body: response.body,
-          headers: normalize_response_headers(response.headers),
           status: response.status
+        ).merge(
+          event_headers_payload(response.headers)
         )
       end
     end
+
+    def event_headers_key
+      super || 'headers'
+    end
   end
 end

app/models/agents/webhook_agent.rb

@@ -1,6 +1,7 @@
 module Agents
   class WebhookAgent < Agent
-    include WebRequestConcern
+    include EventHeadersConcern
+    include WebRequestConcern  # to make reCAPTCHA verification requests
 
     cannot_be_scheduled!
     cannot_receive_events!
@@ -22,6 +23,8 @@ class WebhookAgent < Agent
         * `payload_path` - JSONPath of the attribute in the POST body to be
           used as the Event payload.  Set to `.` to return the entire message.
           If `payload_path` points to an array, Events will be created for each element.
+        * `event_headers` - Comma-separated list of HTTP headers your agent will include in the payload.
+        * `event_headers_key` - The key to use to store all the headers received
         * `verbs` - Comma-separated list of http verbs your agent will accept.
           For example, "post,get" will enable POST and GET requests. Defaults
           to "post".
@@ -43,11 +46,22 @@ class WebhookAgent < Agent
     def default_options
       { "secret" => "supersecretstring",
         "expected_receive_period_in_days" => 1,
-        "payload_path" => "some_key"
+        "payload_path" => "some_key",
+        "event_headers" => "",
+        "event_headers_key" => "headers"
       }
     end
 
-    def receive_web_request(params, method, format)
+    def receive_web_request(request)
+      params = request.params.except(:action, :controller, :agent_id, :user_id, :format)
+      method = request.method_symbol.to_s
+      headers = request.headers.each_with_object({}) { |(name, value), hash|
+        case name
+        when /\AHTTP_([A-Z0-9_]+)\z/
+          hash[$1.tr('_', '-').gsub(/[^-]+/, &:capitalize)] = value
+        end
+      }
+
       # check the secret
       secret = params.delete('secret')
       return ["Not Authorized", 401] unless secret == interpolated['secret']
@@ -86,7 +100,7 @@ def receive_web_request(params, method, format)
       end
 
       [payload_for(params)].flatten.each do |payload|
-        create_event(payload: payload)
+        create_event(payload: payload.merge(event_headers_payload(headers)))
       end
 
       if interpolated['response_headers'].presence
@@ -112,6 +126,8 @@ def validate_options
       if options['code'].to_s.in?(['301', '302']) && !options['response'].present?
         errors.add(:base, "Must specify a url for request redirect")
       end
+
+      validate_event_headers_options!
     end
 
     def payload_for(params)

doc/manual/installation.md

@@ -79,7 +79,7 @@ Download Ruby and compile it:
 
 Install the bundler and foreman gems:
 
-    sudo gem install rake bundler foreman --no-ri --no-rdoc
+    sudo gem install rake bundler foreman --no-document
 
 ## 3. System Users
 

docker/scripts/prepare

@@ -33,7 +33,7 @@ $minimal_apt_get_install build-essential checkinstall git-core \
   ruby2.5 ruby2.5-dev
 locale-gen en_US.UTF-8
 update-locale LANG=en_US.UTF-8 LC_CTYPE=en_US.UTF-8
-gem install --no-ri --no-rdoc bundler
+gem install --no-document bundler
 
 apt-get purge -y python3* rsyslog rsync manpages
 apt-get -y clean

spec/models/agents/post_agent_spec.rb

@@ -52,7 +52,7 @@
           raise "unexpected Content-Type: #{content_type}"
         end
       end
-      { status: 200, body: "<html>a webpage!</html>", headers: { 'Content-type' => 'text/html' } }
+      { status: 200, body: "<html>a webpage!</html>", headers: { 'Content-type' => 'text/html', 'X-Foo-Bar' => 'baz' } }
     }
   end
 
@@ -272,24 +272,31 @@
 
         it "emits the response headers capitalized by default" do
           @checker.check
-          expect(@checker.events.last.payload['headers']).to eq({ 'Content-Type' => 'text/html' })
+          expect(@checker.events.last.payload['headers']).to eq({ 'Content-Type' => 'text/html', 'X-Foo-Bar' => 'baz' })
         end
 
         it "emits the response headers capitalized" do
           @checker.options['event_headers_style'] = 'capitalized'
           @checker.check
-          expect(@checker.events.last.payload['headers']).to eq({ 'Content-Type' => 'text/html' })
+          expect(@checker.events.last.payload['headers']).to eq({ 'Content-Type' => 'text/html', 'X-Foo-Bar' => 'baz' })
         end
 
         it "emits the response headers downcased" do
           @checker.options['event_headers_style'] = 'downcased'
           @checker.check
-          expect(@checker.events.last.payload['headers']).to eq({ 'content-type' => 'text/html' })
+          expect(@checker.events.last.payload['headers']).to eq({ 'content-type' => 'text/html', 'x-foo-bar' => 'baz' })
         end
 
         it "emits the response headers snakecased" do
           @checker.options['event_headers_style'] = 'snakecased'
           @checker.check
+          expect(@checker.events.last.payload['headers']).to eq({ 'content_type' => 'text/html', 'x_foo_bar' => 'baz' })
+        end
+
+        it "emits the response headers only including those specified by event_headers" do
+          @checker.options['event_headers_style'] = 'snakecased'
+          @checker.options['event_headers'] = 'content-type'
+          @checker.check
           expect(@checker.events.last.payload['headers']).to eq({ 'content_type' => 'text/html' })
         end