Skip to content

Commit

Permalink
allow separate events for each answer in the HumanTaskAgent
Browse files Browse the repository at this point in the history
  • Loading branch information
cantino committed Sep 1, 2014
1 parent 3afd215 commit 2c820e8
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 23 deletions.
18 changes: 15 additions & 3 deletions app/models/agents/human_task_agent.rb
Expand Up @@ -62,6 +62,8 @@ class HumanTaskAgent < Agent
which contain `key` and `text`. For _free\\_text_, the special configuration options are all optional, and are
`default`, `min_length`, and `max_length`.
By default, all answers are emitted in a single event. If you'd like separate events for each answer, set `separate_answers` to `true`.
# Combining answers
There are a couple of ways to combine HITs that have multiple `assignments`, all of which involve setting `combination_mode` at the top level.
Expand Down Expand Up @@ -105,7 +107,7 @@ class HumanTaskAgent < Agent
}
}
Resulting events will have the original `answers`, as well as the `poll` results, and a field called `best_answer` that contains the best answer as determined by the poll.
Resulting events will have the original `answers`, as well as the `poll` results, and a field called `best_answer` that contains the best answer as determined by the poll. (Note that `separate_answers` won't work when doing a poll.)
# Other settings
Expand Down Expand Up @@ -351,8 +353,18 @@ def review_hits

log "Poll HIT created with ID #{poll_hit.id} and URL #{poll_hit.url}. Original HIT: #{hit_id}", :inbound_event => inbound_event
else
event = create_event :payload => payload
log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => inbound_event
if options[:separate_answers]
payload['answers'].each.with_index do |answer, index|
sub_payload = payload.dup
sub_payload.delete('answers')
sub_payload['answer'] = answer
event = create_event :payload => sub_payload
log "Event emitted with answer ##{index}", :outbound_event => event, :inbound_event => inbound_event
end
else
event = create_event :payload => payload
log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => inbound_event
end
end
end

Expand Down
63 changes: 43 additions & 20 deletions spec/models/agents/human_task_agent_spec.rb
Expand Up @@ -345,30 +345,53 @@ def approve!
@checker.memory['hits'].should == { "JH3132836336DHG" => { 'event_id' => @event.id } }
end

it "should create events when all assignments are ready" do
@checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
assignments = [
FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>""}),
FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
]
hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
hit.should_not be_disposed
mock(RTurk::Hit).new("JH3132836336DHG") { hit }
context "emitting events" do
before do
@checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
@assignments = [
FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>""}),
FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
]
@hit = FakeHit.new(:max_assignments => 2, :assignments => @assignments)
@hit.should_not be_disposed
mock(RTurk::Hit).new("JH3132836336DHG") { @hit }
end

lambda {
@checker.send :review_hits
}.should change { Event.count }.by(1)
it "should create events when all assignments are ready" do
lambda {
@checker.send :review_hits
}.should change { Event.count }.by(1)

assignments.all? {|a| a.approved == true }.should be_truthy
hit.should be_disposed
@assignments.all? {|a| a.approved == true }.should be_truthy
@hit.should be_disposed

@checker.events.last.payload['answers'].should == [
{'sentiment' => "neutral", 'feedback' => ""},
{'sentiment' => "happy", 'feedback' => "Take 2"}
]
@checker.events.last.payload['answers'].should == [
{'sentiment' => "neutral", 'feedback' => ""},
{'sentiment' => "happy", 'feedback' => "Take 2"}
]

@checker.memory['hits'].should == {}
end

it "should emit separate answers when options[:separate_answers] is true" do
@checker.options[:separate_answers] = true

@checker.memory['hits'].should == {}
lambda {
@checker.send :review_hits
}.should change { Event.count }.by(2)

@assignments.all? {|a| a.approved == true }.should be_truthy
@hit.should be_disposed

event1, event2 = @checker.events.last(2)
event1.payload.should_not have_key('answers')
event2.payload.should_not have_key('answers')
event1.payload['answer'].should == { 'sentiment' => "happy", 'feedback' => "Take 2" }
event2.payload['answer'].should == { 'sentiment' => "neutral", 'feedback' => "" }

@checker.memory['hits'].should == {}
end
end

describe "taking majority votes" do
Expand Down

0 comments on commit 2c820e8

Please sign in to comment.