Merge pull request #1690 from kreuzwerker/post-agent-file-upload

Add multipart file upload to PostAgent

app/concerns/file_handling.rb

@@ -5,13 +5,21 @@ def get_file_pointer(file)
     { file_pointer: { file: file, agent_id: id } }
   end
 
+  def has_file_pointer?(event)
+    event.payload['file_pointer'] &&
+      event.payload['file_pointer']['file'] &&
+      event.payload['file_pointer']['agent_id']
+  end
+
   def get_io(event)
-    return nil unless event.payload['file_pointer'] &&
-                      event.payload['file_pointer']['file'] &&
-                      event.payload['file_pointer']['agent_id']
+    return nil unless has_file_pointer?(event)
     event.user.agents.find(event.payload['file_pointer']['agent_id']).get_io(event.payload['file_pointer']['file'])
   end
 
+  def get_upload_io(event)
+    Faraday::UploadIO.new(get_io(event), MIME::Types.type_for(File.basename(event.payload['file_pointer']['file'])).first.try(:content_type))
+  end
+
   def emitting_file_handling_agent_description
     @emitting_file_handling_agent_description ||=
       "This agent only emits a 'file pointer', not the data inside the files, the following agents can consume the created events: `#{receiving_file_handling_agents.join('`, `')}`. Read more about the concept in the [wiki](https://github.com/cantino/huginn/wiki/How-Huginn-works-with-files)."

app/concerns/web_request_concern.rb

@@ -113,6 +113,7 @@ def faraday
       unless boolify(interpolated['disable_redirect_follow'])
         builder.use FaradayMiddleware::FollowRedirects
       end
+      builder.request :multipart
       builder.request :url_encoded
 
       if boolify(interpolated['disable_url_encoding'])

app/models/agents/post_agent.rb

@@ -1,45 +1,54 @@
 module Agents
   class PostAgent < Agent
     include WebRequestConcern
+    include FileHandling
+
+    consumes_file_pointer!
 
     MIME_RE = /\A\w+\/.+\z/
 
     can_dry_run!
     no_bulk_receive!
     default_schedule "never"
 
-    description <<-MD
-      A Post Agent receives events from other agents (or runs periodically), merges those events with the [Liquid-interpolated](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) contents of `payload`, and sends the results as POST (or GET) requests to a specified url.  To skip merging in the incoming event, but still send the interpolated payload, set `no_merge` to `true`.
+    description do
+      <<-MD
+        A Post Agent receives events from other agents (or runs periodically), merges those events with the [Liquid-interpolated](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) contents of `payload`, and sends the results as POST (or GET) requests to a specified url.  To skip merging in the incoming event, but still send the interpolated payload, set `no_merge` to `true`.
 
-      The `post_url` field must specify where you would like to send requests. Please include the URI scheme (`http` or `https`).
+        The `post_url` field must specify where you would like to send requests. Please include the URI scheme (`http` or `https`).
 
-      The `method` used can be any of `get`, `post`, `put`, `patch`, and `delete`.
+        The `method` used can be any of `get`, `post`, `put`, `patch`, and `delete`.
 
-      By default, non-GETs will be sent with form encoding (`application/x-www-form-urlencoded`).
+        By default, non-GETs will be sent with form encoding (`application/x-www-form-urlencoded`).
 
-      Change `content_type` to `json` to send JSON instead.
+        Change `content_type` to `json` to send JSON instead.
 
-      Change `content_type` to `xml` to send XML, where the name of the root element may be specified using `xml_root`, defaulting to `post`.
+        Change `content_type` to `xml` to send XML, where the name of the root element may be specified using `xml_root`, defaulting to `post`.
 
-      When `content_type` contains a [MIME](https://en.wikipedia.org/wiki/Media_type) type, and `payload` is a string, its interpolated value will be sent as a string in the HTTP request's body and the request's `Content-Type` HTTP header will be set to `content_type`. When `payload` is a string `no_merge` has to be set to `true`.
+        When `content_type` contains a [MIME](https://en.wikipedia.org/wiki/Media_type) type, and `payload` is a string, its interpolated value will be sent as a string in the HTTP request's body and the request's `Content-Type` HTTP header will be set to `content_type`. When `payload` is a string `no_merge` has to be set to `true`.
 
-      If `emit_events` is set to `true`, the server response will be emitted as an Event and can be fed to a WebsiteAgent for parsing (using its `data_from_event` and `type` options). No data processing
-      will be attempted by this Agent, so the Event's "body" value will always be raw text.
-      The Event will also have a "headers" hash and a "status" integer value.
-      Set `event_headers_style` to one of the following values to normalize the keys of "headers" for downstream agents' convenience:
+        If `emit_events` is set to `true`, the server response will be emitted as an Event and can be fed to a WebsiteAgent for parsing (using its `data_from_event` and `type` options). No data processing
+        will be attempted by this Agent, so the Event's "body" value will always be raw text.
+        The Event will also have a "headers" hash and a "status" integer value.
+        Set `event_headers_style` to one of the following values to normalize the keys of "headers" for downstream agents' convenience:
 
-        * `capitalized` (default) - Header names are capitalized; e.g. "Content-Type"
-        * `downcased` - Header names are downcased; e.g. "content-type"
-        * `snakecased` - Header names are snakecased; e.g. "content_type"
-        * `raw` - Backward compatibility option to leave them unmodified from what the underlying HTTP library returns.
+          * `capitalized` (default) - Header names are capitalized; e.g. "Content-Type"
+          * `downcased` - Header names are downcased; e.g. "content-type"
+          * `snakecased` - Header names are snakecased; e.g. "content_type"
+          * `raw` - Backward compatibility option to leave them unmodified from what the underlying HTTP library returns.
 
-      Other Options:
+        Other Options:
 
-        * `headers` - When present, it should be a hash of headers to send with the request.
-        * `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`.
-        * `disable_ssl_verification` - Set to `true` to disable ssl verification.
-        * `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}").
-    MD
+          * `headers` - When present, it should be a hash of headers to send with the request.
+          * `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`.
+          * `disable_ssl_verification` - Set to `true` to disable ssl verification.
+          * `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}").
+
+        #{receiving_file_handling_agent_description}
+
+        When receiving a `file_pointer` the request will be sent with multipart encoding (`multipart/form-data`) and `content_type` is ignored. `upload_key` can be used to specify the parameter in which the file will be sent, it defaults to `file`.
+      MD
+    end
 
     event_description <<-MD
       Events look like this:
@@ -125,9 +134,9 @@ def receive(incoming_events)
         interpolate_with(event) do
           outgoing = interpolated['payload'].presence || {}
           if boolify(interpolated['no_merge'])
-            handle outgoing, event.payload, headers(interpolated[:headers])
+            handle outgoing, event, headers(interpolated[:headers])
           else
-            handle outgoing.merge(event.payload), event.payload, headers(interpolated[:headers])
+            handle outgoing.merge(event.payload), event, headers(interpolated[:headers])
           end
         end
       end
@@ -162,22 +171,30 @@ def normalize_response_headers(headers)
       }
     end
 
-    def handle(data, payload = {}, headers)
-      url = interpolated(payload)[:post_url]
+    def handle(data, event = Event.new, headers)
+      url = interpolated(event.payload)[:post_url]
 
       case method
       when 'get', 'delete'
         params, body = data, nil
       when 'post', 'put', 'patch'
         params = nil
 
-        case (content_type = interpolated(payload)['content_type'])
+        content_type =
+          if has_file_pointer?(event)
+            data[interpolated(event.payload)['upload_key'].presence || 'file'] = get_upload_io(event)
+            nil
+          else
+            interpolated(event.payload)['content_type']
+          end
+
+        case content_type
         when 'json'
           headers['Content-Type'] = 'application/json; charset=utf-8'
           body = data.to_json
         when 'xml'
           headers['Content-Type'] = 'text/xml; charset=utf-8'
-          body = data.to_xml(root: (interpolated(payload)[:xml_root] || 'post'))
+          body = data.to_xml(root: (interpolated(event.payload)[:xml_root] || 'post'))
         when MIME_RE
           headers['Content-Type'] = content_type
           body = data.to_s

spec/models/agents/post_agent_spec.rb

@@ -57,6 +57,11 @@
   end
 
   it_behaves_like WebRequestConcern
+  it_behaves_like 'FileHandlingConsumer'
+
+  it 'renders the description markdown without errors' do
+    expect { @checker.description }.not_to raise_error
+  end
 
   describe "making requests" do
     it "can make requests of each type" do
@@ -149,6 +154,19 @@
       headers = @sent_requests[:post].first.headers
       expect(headers["Foo"]).to eq("a_variable")
     end
+
+    it 'makes a multipart request when receiving a file_pointer' do
+      WebMock.reset!
+      stub_request(:post, "http://www.example.com/").
+        with(:body => "-------------RubyMultipartPost\r\nContent-Disposition: form-data; name=\"default\"\r\n\r\nvalue\r\n-------------RubyMultipartPost\r\nContent-Disposition: form-data; name=\"file\"; filename=\"local.path\"\r\nContent-Length: 8\r\nContent-Type: \r\nContent-Transfer-Encoding: binary\r\n\r\ntestdata\r\n-------------RubyMultipartPost--\r\n\r\n",
+             :headers => {'Accept-Encoding'=>'gzip,deflate', 'Content-Length'=>'307', 'Content-Type'=>'multipart/form-data; boundary=-----------RubyMultipartPost', 'User-Agent'=>'Huginn - https://github.com/cantino/huginn'}).
+        to_return(:status => 200, :body => "", :headers => {})
+      event = Event.new(payload: {file_pointer: {agent_id: 111, file: 'test'}})
+      io_mock = mock()
+      mock(@checker).get_io(event) { StringIO.new("testdata") }
+      @checker.options['no_merge'] = true
+      @checker.receive([event])
+    end
   end
 
   describe "#check" do

spec/support/shared_examples/file_handling_consumer.rb

@@ -1,6 +1,8 @@
 require 'rails_helper'
 
 shared_examples_for 'FileHandlingConsumer' do
+  let(:event) { Event.new(user: @checker.user, payload: {'file_pointer' => {'file' => 'text.txt', 'agent_id' => @checker.id}}) }
+
   it 'returns a file pointer' do
     expect(@checker.get_file_pointer('testfile')).to eq(file_pointer: { file: "testfile", agent_id: @checker.id})
   end
@@ -9,8 +11,26 @@
     @checker2 = @checker.dup
     @checker2.user = users(:bob)
     @checker2.save!
-    expect(@checker2.user.id).not_to eq(@checker.user.id)
-    event = Event.new(user: @checker.user, payload: {'file_pointer' => {'file' => 'test', 'agent_id' => @checker2.id}})
+    event.payload['file_pointer']['agent_id'] = @checker2.id
     expect { @checker.get_io(event) }.to raise_error(ActiveRecord::RecordNotFound)
   end
-end
+
+  context '#has_file_pointer?' do
+    it 'returns true if the event contains a file pointer' do
+      expect(@checker.has_file_pointer?(event)).to be_truthy
+    end
+
+    it 'returns false if the event does not contain a file pointer' do
+      expect(@checker.has_file_pointer?(Event.new)).to be_falsy
+    end
+  end
+
+  it '#get_upload_io returns a Faraday::UploadIO instance' do
+    io_mock = mock()
+    mock(@checker).get_io(event) { StringIO.new("testdata") }
+
+    upload_io = @checker.get_upload_io(event)
+    expect(upload_io).to be_a(Faraday::UploadIO)
+    expect(upload_io.content_type).to eq('text/plain')
+  end
+end