Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Post agent can parse a JSON response by enabling the parse_body option #3287

Merged
merged 2 commits into from Jul 2, 2023
Merged

Post agent can parse a JSON response by enabling the parse_body option #3287

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c823d11
PostAgent can parse a JSON response by enabling the `parse_body` option
knu Jun 27, 2023
a62fe20
Reformat the code with rubocop
knu Jun 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
8 changes: 8 additions & 0 deletions app/concerns/web_request_concern.rb
View file
Open in desktop
Expand Up @@ -106,6 +106,10 @@ def default_encoding
Encoding::UTF_8
end

def parse_body?
false
end

def faraday
faraday_options = {
ssl: {
Expand All @@ -114,6 +118,10 @@ def faraday
}

@faraday ||= Faraday.new(faraday_options) { |builder|
if parse_body?
builder.response :json
end

builder.response :character_encoding,
force_encoding: interpolated['force_encoding'].presence,
default_encoding:,
Expand Down
12 changes: 10 additions & 2 deletions app/models/agents/post_agent.rb
View file
Open in desktop
Expand Up @@ -28,8 +28,7 @@ class PostAgent < Agent

When `content_type` contains a [MIME](https://en.wikipedia.org/wiki/Media_type) type, and `payload` is a string, its interpolated value will be sent as a string in the HTTP request's body and the request's `Content-Type` HTTP header will be set to `content_type`. When `payload` is a string `no_merge` has to be set to `true`.

If `emit_events` is set to `true`, the server response will be emitted as an Event and can be fed to a WebsiteAgent for parsing (using its `data_from_event` and `type` options). No data processing
will be attempted by this Agent, so the Event's "body" value will always be raw text.
If `emit_events` is set to `true`, the server response will be emitted as an Event. The "body" value of the Event is the response body. If the `parse_body` option is set to `true` and the content type of the response is JSON, it is parsed to a JSON object. Otherwise it is raw text. A raw HTML/XML text can be fed to a WebsiteAgent for parsing (using its `data_from_event` and `type` options).
The Event will also have a "headers" hash and a "status" integer value.

If `output_mode` is set to `merge`, the emitted Event will be merged into the original contents of the received Event.
Expand Down Expand Up @@ -83,6 +82,7 @@ def default_options
},
'headers' => {},
'emit_events' => 'false',
'parse_body' => 'true',
'no_merge' => 'true',
'output_mode' => 'clean'
}
Expand Down Expand Up @@ -144,13 +144,21 @@ def validate_options
errors.add(:base, "if provided, output_mode must be 'clean' or 'merge'")
end

if options['parse_body'].present? && /\A(?:true|false)\z|\{/.match?(options['parse_body'].to_s)
errors.add(:base, "if provided, parse_body must be 'true' or 'false'")
end

unless headers.is_a?(Hash)
errors.add(:base, "if provided, headers must be a hash")
end

validate_web_request_options!
end

def parse_body?
boolify(interpolated['parse_body'])
end

def receive(incoming_events)
incoming_events.each do |event|
interpolate_with(event) do
Expand Down
68 changes: 53 additions & 15 deletions spec/models/agents/post_agent_spec.rb
View file
Open in desktop
Expand Up @@ -2,6 +2,17 @@
require 'ostruct'

describe Agents::PostAgent do
let(:mocked_response) do
{
status: 200,
body: "<html>a webpage!</html>",
headers: {
'Content-type' => 'text/html',
'X-Foo-Bar' => 'baz',
}
}
end

before do
@valid_options = {
'post_url' => "http://www.example.com",
Expand Down Expand Up @@ -52,7 +63,7 @@
raise "unexpected Content-Type: #{content_type}"
end
end
{ status: 200, body: "<html>a webpage!</html>", headers: { 'Content-type' => 'text/html', 'X-Foo-Bar' => 'baz' } }
mocked_response
}
end

Expand Down Expand Up @@ -89,7 +100,7 @@
expect {
@checker.receive([@event, event1])
}.to change { @sent_requests[:post].length }.by(2)
}.not_to change { @sent_requests[:get].length }
}.not_to(change { @sent_requests[:get].length })

expect(@sent_requests[:post][0].data).to eq(@event.payload.merge('default' => 'value').to_query)
expect(@sent_requests[:post][1].data).to eq(event1.payload.to_query)
Expand All @@ -102,7 +113,7 @@
expect {
@checker.receive([@event])
}.to change { @sent_requests[:get].length }.by(1)
}.not_to change { @sent_requests[:post].length }
}.not_to(change { @sent_requests[:post].length })

expect(@sent_requests[:get][0].data).to eq(@event.payload.merge('default' => 'value').to_query)
end
Expand Down Expand Up @@ -157,17 +168,17 @@

it 'makes a multipart request when receiving a file_pointer' do
WebMock.reset!
stub_request(:post, "http://www.example.com/").
with(headers: {
'Accept-Encoding' => 'gzip,deflate',
'Content-Type' => /\Amultipart\/form-data; boundary=/,
'User-Agent' => 'Huginn - https://github.com/huginn/huginn'
stub_request(:post, "http://www.example.com/")
.with(headers: {
'Accept-Encoding' => 'gzip,deflate',
'Content-Type' => /\Amultipart\/form-data; boundary=/,
'User-Agent' => 'Huginn - https://github.com/huginn/huginn'
}) { |request|
qboundary = Regexp.quote(request.headers['Content-Type'][/ boundary=(.+)/, 1])
/\A--#{qboundary}\r\nContent-Disposition: form-data; name="default"\r\n\r\nvalue\r\n--#{qboundary}\r\nContent-Disposition: form-data; name="file"; filename="local.path"\r\nContent-Length: 8\r\nContent-Type: \r\nContent-Transfer-Encoding: binary\r\n\r\ntestdata\r\n--#{qboundary}--\r\n\z/ === request.body
}.to_return(status: 200, body: "", headers: {})
event = Event.new(payload: {file_pointer: {agent_id: 111, file: 'test'}})
io_mock = double()
event = Event.new(payload: { file_pointer: { agent_id: 111, file: 'test' } })
io_mock = double
expect(@checker).to receive(:get_io).with(event) { StringIO.new("testdata") }
@checker.options['no_merge'] = true
@checker.receive([event])
Expand Down Expand Up @@ -198,7 +209,7 @@
@checker.check
}.to change { @sent_requests[:post].length }.by(1)

expect(@sent_requests[:post][0].data.keys).to eq([ 'post' ])
expect(@sent_requests[:post][0].data.keys).to eq(['post'])
expect(@sent_requests[:post][0].data['post']).to eq(@checker.options['payload'])
end

Expand All @@ -209,7 +220,7 @@
@checker.check
}.to change { @sent_requests[:post].length }.by(1)

expect(@sent_requests[:post][0].data.keys).to eq([ 'foobar' ])
expect(@sent_requests[:post][0].data.keys).to eq(['foobar'])
expect(@sent_requests[:post][0].data['foobar']).to eq(@checker.options['payload'])
end

Expand All @@ -219,7 +230,7 @@
expect {
@checker.check
}.to change { @sent_requests[:get].length }.by(1)
}.not_to change { @sent_requests[:post].length }
}.not_to(change { @sent_requests[:post].length })

expect(@sent_requests[:get][0].data).to eq(@checker.options['payload'].to_query)
end
Expand Down Expand Up @@ -248,7 +259,7 @@
it "does not emit events" do
expect {
@checker.check
}.not_to change { @checker.events.count }
}.not_to(change { @checker.events.count })
end
end

Expand All @@ -270,6 +281,33 @@
expect(@checker.events.last.payload['body']).to eq '<html>a webpage!</html>'
end

context "and the response is in JSON" do
let(:json_data) {
{ "foo" => 123, "bar" => 456 }
}
let(:mocked_response) do
{
status: 200,
body: json_data.to_json,
headers: {
'Content-type' => 'application/json',
'X-Foo-Bar' => 'baz',
}
}
end

it "emits the unparsed JSON body" do
@checker.check
expect(@checker.events.last.payload['body']).to eq json_data.to_json
end

it "emits the parsed JSON body when parse_body is true" do
@checker.options['parse_body'] = 'true'
@checker.check
expect(@checker.events.last.payload['body']).to eq json_data
end
end

it "emits the response headers capitalized by default" do
@checker.check
expect(@checker.events.last.payload['headers']).to eq({ 'Content-Type' => 'text/html', 'X-Foo-Bar' => 'baz' })
Expand Down Expand Up @@ -437,7 +475,7 @@
end

it "requires headers to be a hash, if present" do
@checker.options['headers'] = [1,2,3]
@checker.options['headers'] = [1, 2, 3]
expect(@checker).not_to be_valid

@checker.options['headers'] = "hello world"
Expand Down