Merge branch 'master' of git://github.com/technoweenie/islostonyet.com

README

@@ -3,4 +3,4 @@ Is LOST on yet?
 
 Step 1: Setup config/lost.rb.  Use lost.default.rb as a guide.  Be sure to setup your database and twitter account.
 Step 2: Create your database with `rake lost:schema`
-Step 3: Setup cron jobs for `rake lost:process_replies` and `rake lost:process_updates`.
+Step 3: Setup cron jobs for `rake lost:process_updates lost:process_search lost:process_replies`.

Rakefile

@@ -25,6 +25,11 @@ namespace :lost do
     IsLOSTOnYet::Post.process_updates
   end
 
+  desc "Process all search results from Twitter"
+  task :process_search => :init do
+    IsLOSTOnYet::Post.process_search
+  end
+
   desc "Process all replies to the existing Twitter user"
   task :process_replies => :init do
     IsLOSTOnYet::Post.process_replies

config/lost.default.rb

@@ -10,6 +10,27 @@
   IsLOSTOnYet.time_zone        = "Eastern Time (US & Canada)"
   IsLOSTOnYet.twitter_login    = ''
   IsLOSTOnYet.twitter_password = ''
+
+  # see http://search.twitter.com/advanced
+  # Probably only want :main_keywords and :secondary_keywords, but go nuts if you like
+  #
+  # :main_keywords and :secondary_keywords are used for basic scoring of search results to determine
+  # if they are really about the tv show.  :main_keywords build the the :containing query part of the search.  
+  # When parsing search results, a score is kept for the occurence of main and secondary words.  Ideally we want
+  # AT LEAST 1 main word, and a total of 1 main + 1 secondary (2 points).  A hashtag counts as 2 points though.
+  IsLOSTOnYet.twitter_search_options = {
+    :main_keywords => %w(lost #lost kate sayid), # these are joined with an OR to make the same :containing query below
+    :secondary_keywords => %w(tv season)
+    :from => 'technoweenie',
+    :to   => 'technoweenie',
+    :referencing => 'technoweenie',
+    :containing  => "lost OR kate OR sayid",
+    :hashed      => "lost",
+    :lang        => "en",
+    :per_page    => 50,
+    :since       => '13423423', # unneeded, this wonderful site will fill this in for you!
+    :geocode     => [@long, @lat, @range]
+  }
 end
 
 IsLOSTOnYet.load_episodes :lost

islostonyet.rb

@@ -35,9 +35,8 @@ def render_sass(content, options = {})
 
 get '/' do
   @tags    = IsLOSTOnYet::Tag.list
-  @posts   = IsLOSTOnYet::Post.find_replies
-  @updates = IsLOSTOnYet::Post.find_updates
-  @users   = users_for @posts + @updates
+  @posts   = IsLOSTOnYet::Post.list
+  @users   = users_for @posts
   @body_class = "latest"
   haml :index
 end
@@ -50,22 +49,22 @@ def render_sass(content, options = {})
 end
 
 get '/updates.atom' do
-  @posts = IsLOSTOnYet::Post.find_replies
+  @posts = IsLOSTOnYet::Post.list
   @users = users_for @posts
   builder :updates
 end
 
 get '/tags' do
   @tags  = IsLOSTOnYet::Tag.list
-  @posts = IsLOSTOnYet::Post.find_replies
+  @posts = IsLOSTOnYet::Post.list
   @users = users_for @posts
   @body_class = "tags"
   haml :tags
 end
 
 get '/episodes' do
-  @tags  = IsLOSTOnYet::Tag.list
-  @episodes = IsLOSTOnYet.episodes
+  @tags       = IsLOSTOnYet::Tag.list
+  @episodes   = IsLOSTOnYet.episodes
   @body_class = "episodes"
   haml :episodes
 end

lib/is_lost_on_yet.rb

@@ -26,6 +26,7 @@ class << self
     attr_writer   :twitter_user
     attr_accessor :twitter_login
     attr_accessor :twitter_password
+    attr_accessor :twitter_search_options
     attr_accessor :time_zone
     attr_accessor :show_title
     attr_accessor :show_abbrev
@@ -47,6 +48,7 @@ def init
     end
   end
 
+  self.twitter_search_options = {}
   self.counter_url = "http://errcount.com/ctr/750.js"
   self.show_title  = "LOST"
   self.show_abbrev = "LOST"

lib/is_lost_on_yet/post.rb

@@ -1,20 +1,46 @@
+# used to transform twitter search results into duck typable twitter objects
+class Faux
+  class User < Struct.new(:id, :screen_name, :profile_image_url)
+  end
+
+  class Post < Struct.new(:id, :text, :user, :created_at)
+    def to_search_result
+      {'id' => id, 'text' => text, 'from_user' => user.screen_name, 'from_user_id' => user.id, 'created_at' => created_at, 'profile_image_url' => user.profile_image_url}
+    end
+  end
+end
+
 module IsLOSTOnYet
   class Post < Sequel.Model(:posts)
     many_to_one :user, :class => "IsLOSTOnYet::User"
 
-    def self.find_updates(page = 1)
-      filtered_for_updates.where(:visible => true).paginate(page, 30).to_a
-    end
-
-    def self.find_replies(page = 1)
-      filtered_for_replies.where(:visible => true).paginate(page, 30).to_a
+    def self.list(page = 1)
+      filter_and_order(:visible => true).paginate(page, 30).to_a
     end
 
     def self.find_by_tags(tags, page = 1)
       return [] if tags.empty?
-      filtered_for_replies.
+      filter_and_order(:visible => true).
         where([Array.new(tags.size, "tag LIKE ?") * " AND ", *tags.map { |t| "%[#{t}]%" }]).
-        where(:visible => true).paginate(page, 30).to_a
+        paginate(page, 30).to_a
+    end
+
+    def self.process_search
+      search = Twitter::Search.new
+      if post = latest_search
+        search.since(post.external_id)
+      end
+      if keywords = IsLOSTOnYet.twitter_search_options[:main_keywords]
+        search.contains keywords.join(" OR ")
+      end
+      IsLOSTOnYet.twitter_search_options.each do |key, args|
+        next if key == :main_keywords || key == :secondary_keywords
+        args = [args]; args.flatten!
+        search.send(key, *args)
+      end
+      process_search_results(search) do |user, post|
+        !post.reply_to_bot? && user.external_id != IsLOSTOnYet.twitter_user.external_id && post.valid_search_result?
+      end
     end
 
     def self.process_updates
@@ -48,7 +74,11 @@ def self.latest_update
     end
 
     def self.latest_reply
-      filtered_for_replies.select(:external_id).first
+      filtered_for_replies.where("body LIKE ?", "@#{IsLOSTOnYet.twitter_login}%").select(:external_id).first
+    end
+
+    def self.latest_search
+      filtered_for_replies.where("body NOT LIKE ?", "@#{IsLOSTOnYet.twitter_login}%").select(:external_id).first
     end
 
     def formatted_body
@@ -65,6 +95,10 @@ def reply?
       body.strip =~ /^@/
     end
 
+    def reply_to_bot?
+      body.strip =~ /^@#{IsLOSTOnYet.twitter_login}/i
+    end
+
     # A tweet from a user asking the twitter bot if the show is on
     def inquiry?
       body.strip =~ /^@#{IsLOSTOnYet.twitter_login}\s*\?$/i
@@ -95,6 +129,16 @@ def save_hash_tags
       save
     end
 
+    def valid_search_result?
+      if IsLOSTOnYet.twitter_search_options[:main_keywords].nil? then return true ; end
+      score          = 0
+      downcased_body = body.downcase
+      score += score_from downcased_body, IsLOSTOnYet.twitter_search_options[:main_keywords]
+      if score.zero? then return false ; end
+      score += score_from downcased_body, IsLOSTOnYet.twitter_search_options[:secondary_keywords]
+      score > 1
+    end
+
   protected
     def self.filtered_for_updates
       user_id = IsLOSTOnYet.twitter_user ? IsLOSTOnYet.twitter_user.id : 0
@@ -110,11 +154,19 @@ def self.filter_and_order(*args)
       where(*args).order(:created_at.desc)
     end
 
+    def self.process_search_results(search, &block)
+      posts = []
+      search.fetch['results'].each do |hash|
+        user   = Faux::User.new(hash['from_user_id'], hash['from_user'], hash['profile_image_url'])
+        posts << Faux::Post.new(hash['id'], hash['text'], user, hash['created_at'])
+      end
+      process_tweets(posts, &block)
+    end
+
     def self.process_tweets(tweets, &block)
       return nil if tweets.empty?
       users = {}
       posts = []
-      tweets.reverse!
       tweets.each do |s|
         users[s.user.id.to_i] = {:login => s.user.screen_name, :avatar_url => s.user.profile_image_url}
         posts << {:body => s.text, :user_id => s.user.id, :created_at => Time.parse(s.created_at).utc, :external_id => s.id}
@@ -149,6 +201,7 @@ def self.process_user(users, user)
     end
 
     def self.process_posts(posts, users, &block)
+      posts.reverse!
       posts.each do |attributes|
         user = users[attributes.delete(:user_id).to_i]
         post = Post.new(:user_id => user.id)
@@ -160,5 +213,16 @@ def self.process_posts(posts, users, &block)
         post.save_hash_tags
       end
     end
+
+    def score_from(downcased_body, words)
+      return 0 if words.nil?
+      score = 0
+      words = words.dup
+      words.each do |key|
+        this_score = key =~ /^#/ ? 2 : 1 # hash keywords worth 2 points
+        score += this_score if downcased_body =~ %r{(^|\s|\W)#{key}($|\s|\W)}
+      end
+      score
+    end
   end
 end