From 370297ced3923937385d45dcf8c478fcad653583 Mon Sep 17 00:00:00 2001 From: Glenn 'devalias' Grant Date: Mon, 2 Jun 2014 20:22:41 +1000 Subject: [PATCH 1/5] Solves #357 #359 --- app/models/agents/twitter_user_agent.rb | 50 ++++++++++++++++++------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/app/models/agents/twitter_user_agent.rb b/app/models/agents/twitter_user_agent.rb index 79f87326dc..8761a9ccdc 100644 --- a/app/models/agents/twitter_user_agent.rb +++ b/app/models/agents/twitter_user_agent.rb @@ -17,11 +17,15 @@ class TwitterUserAgent < Agent You must also provide the `username` of the Twitter user to monitor. + Set `include_retweets` to `false` to not include retweets (default: `true`) + Set `expected_update_period_in_days` to the maximum amount of time that you'd expect to pass between Events being created by this Agent. + + Set `starting_at` to the date/time (eg. `Mon Jun 02 00:38:12 +0000 2014`) you want to start receiving tweets from (default: agent's `created_at`) MD event_description <<-MD - Events are the raw JSON provided by the Twitter API. Should look something like: + Events are the raw JSON provided by the [Twitter API](https://dev.twitter.com/docs/api/1.1/get/statuses/user_timeline). Should look something like: { ... every Tweet field, including ... @@ -46,38 +50,58 @@ class TwitterUserAgent < Agent default_schedule "every_1h" + def working? + event_created_within?(options['expected_update_period_in_days']) && !recent_error_logs? + end + + def default_options + { + 'username' => 'tectonic', + 'include_retweets' => 'true', + 'expected_update_period_in_days' => '2' + } + end + def validate_options unless options['username'].present? && options['expected_update_period_in_days'].present? errors.add(:base, "username and expected_update_period_in_days are required") - end + end end - def working? - event_created_within?(options['expected_update_period_in_days']) && !recent_error_logs? + def starting_at + if options[:starting_at].present? + Time.parse(options[:starting_at]) + else + created_at + end end - def default_options - { - 'username' => "tectonic", - 'expected_update_period_in_days' => "2" - } + def include_retweets? + if options[:include_retweets].present? + !!options[:include_retweets] + else + true + end end def check since_id = memory['since_id'] || nil - opts = {:count => 200, :include_rts => true, :exclude_replies => false, :include_entities => true, :contributor_details => true} + opts = {:count => 200, :include_rts => include_retweets?, :exclude_replies => false, :include_entities => true, :contributor_details => true} opts.merge! :since_id => since_id unless since_id.nil? + # http://rdoc.info/gems/twitter/Twitter/REST/Timelines#user_timeline-instance_method tweets = twitter.user_timeline(options['username'], opts) tweets.each do |tweet| - memory['since_id'] = tweet.id if !memory['since_id'] || (tweet.id > memory['since_id']) + if tweet.created_at >= starting_at + memory['since_id'] = tweet.id if !memory['since_id'] || (tweet.id > memory['since_id']) - create_event :payload => tweet.attrs + create_event :payload => tweet.attrs + end end save! end end -end \ No newline at end of file +end From 2eee88e03371c076f27c8e02559de1d71ae76eb5 Mon Sep 17 00:00:00 2001 From: Glenn 'devalias' Grant Date: Mon, 2 Jun 2014 20:40:09 +1000 Subject: [PATCH 2/5] Fix spec --- spec/models/agents/twitter_user_agent_spec.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spec/models/agents/twitter_user_agent_spec.rb b/spec/models/agents/twitter_user_agent_spec.rb index 8739e8535f..f7c10afd51 100644 --- a/spec/models/agents/twitter_user_agent_spec.rb +++ b/spec/models/agents/twitter_user_agent_spec.rb @@ -4,10 +4,11 @@ before do # intercept the twitter API request for @tectonic's user profile stub_request(:any, /tectonic/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/user_tweets.json")), :status => 200) - + @opts = { :username => "tectonic", :expected_update_period_in_days => "2", + :starting_at => "Jan 01 00:00:01 +0000 2014", :consumer_key => "---", :consumer_secret => "---", :oauth_token => "---", @@ -25,4 +26,4 @@ end end -end \ No newline at end of file +end From fe75d6be0acba1531758d301e6f7e49c53f3ab79 Mon Sep 17 00:00:00 2001 From: Glenn 'devalias' Grant Date: Tue, 3 Jun 2014 20:27:37 +1000 Subject: [PATCH 3/5] Bump up error checking and add new spec --- app/models/agents/twitter_user_agent.rb | 17 +++++++++++------ spec/models/agents/twitter_user_agent_spec.rb | 14 +++++++++++++- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/app/models/agents/twitter_user_agent.rb b/app/models/agents/twitter_user_agent.rb index 8761a9ccdc..4affba02d7 100644 --- a/app/models/agents/twitter_user_agent.rb +++ b/app/models/agents/twitter_user_agent.rb @@ -67,22 +67,27 @@ def validate_options options['expected_update_period_in_days'].present? errors.add(:base, "username and expected_update_period_in_days are required") end + + if options[:include_retweets].present? && + !(!!options[:include_retweets] === true || !!options[:include_retweets] === false) + errors.add(:base, "include_retweets must be a boolean (true/false)") + end + + if options[:starting_at].present? + Time.parse(options[:starting_at]) rescue errors.add(:base, "Error parsing starting_at") + end end def starting_at if options[:starting_at].present? - Time.parse(options[:starting_at]) + Time.parse(options[:starting_at]) rescue created_at else created_at end end def include_retweets? - if options[:include_retweets].present? - !!options[:include_retweets] - else - true - end + options[:include_retweets] != false end def check diff --git a/spec/models/agents/twitter_user_agent_spec.rb b/spec/models/agents/twitter_user_agent_spec.rb index f7c10afd51..dba9c7ba29 100644 --- a/spec/models/agents/twitter_user_agent_spec.rb +++ b/spec/models/agents/twitter_user_agent_spec.rb @@ -8,7 +8,7 @@ @opts = { :username => "tectonic", :expected_update_period_in_days => "2", - :starting_at => "Jan 01 00:00:01 +0000 2014", + :starting_at => "Jan 01 00:00:01 +0000 2000", :consumer_key => "---", :consumer_secret => "---", :oauth_token => "---", @@ -26,4 +26,16 @@ end end + describe "#check with starting_at=future date" do + it "should check for changes starting_at a future date, thus not find any" do + opts = @opts.merge({ :starting_at => "Jan 01 00:00:01 +0000 2999", }) + + checker = Agents::TwitterUserAgent.new(:name => "tectonic", :options => opts) + checker.user = users(:bob) + checker.save! + + lambda { checker.check }.should change { Event.count }.by(0) + end + end + end From 4b48a6e578cb520f37cff82efd574cbde73744d1 Mon Sep 17 00:00:00 2001 From: Glenn 'devalias' Grant Date: Tue, 3 Jun 2014 21:44:05 +1000 Subject: [PATCH 4/5] Validate a little nicer --- app/models/agents/twitter_user_agent.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/app/models/agents/twitter_user_agent.rb b/app/models/agents/twitter_user_agent.rb index 4affba02d7..86175e30a4 100644 --- a/app/models/agents/twitter_user_agent.rb +++ b/app/models/agents/twitter_user_agent.rb @@ -63,10 +63,8 @@ def default_options end def validate_options - unless options['username'].present? && - options['expected_update_period_in_days'].present? - errors.add(:base, "username and expected_update_period_in_days are required") - end + errors.add(:base, "username is required") unless options['username'].present? + errors.add(:base, "expected_update_period_in_days is required") unless options['expected_update_period_in_days'].present? if options[:include_retweets].present? && !(!!options[:include_retweets] === true || !!options[:include_retweets] === false) From 1bf0e3bc628d188a7fd75cf264d9605a745ebfef Mon Sep 17 00:00:00 2001 From: Glenn 'devalias' Grant Date: Thu, 5 Jun 2014 15:18:25 +1000 Subject: [PATCH 5/5] Fixups based on @cantino 's comments --- app/models/agents/twitter_user_agent.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/models/agents/twitter_user_agent.rb b/app/models/agents/twitter_user_agent.rb index 86175e30a4..520cefe339 100644 --- a/app/models/agents/twitter_user_agent.rb +++ b/app/models/agents/twitter_user_agent.rb @@ -66,9 +66,8 @@ def validate_options errors.add(:base, "username is required") unless options['username'].present? errors.add(:base, "expected_update_period_in_days is required") unless options['expected_update_period_in_days'].present? - if options[:include_retweets].present? && - !(!!options[:include_retweets] === true || !!options[:include_retweets] === false) - errors.add(:base, "include_retweets must be a boolean (true/false)") + if options[:include_retweets].present? && !%w[true false].include?(options[:include_retweets]) + errors.add(:base, "include_retweets must be a boolean value string (true/false)") end if options[:starting_at].present? @@ -85,7 +84,7 @@ def starting_at end def include_retweets? - options[:include_retweets] != false + options[:include_retweets] != "false" end def check