store properties, HNBot with whenever, mark time

config/schedule.rb

@@ -19,6 +19,15 @@
 
 # Learn more: http://github.com/javan/whenever
 
-every 30.minutes do 
-  runner "HackerNews.refresh"
-end
+every 2.minutes do 
+  runner "HNBot.fetch_comments"
+end
+
+every 10.minutes do 
+  runner "HNBot.fetch_postings"
+end
+
+every 20.minutes do 
+  runner "HNBot.post_activity"
+end
+

lib/comment_list.rb

@@ -61,8 +61,7 @@ def process_page
         count += 1 
       end
     end
-    # since we only fetch one page at a time, wait here
-    sleep wait_interval
+    # since we only fetch one page at a time, don't wait 
     count
   end
 

lib/crawler.rb

@@ -18,6 +18,14 @@ def wait_interval
     @min_wait + rand*@var_wait + rand*@var_wait
   end
 
+  def self.decent_interval
+    23 + 42*rand
+  end
+
+  def self.polite?(time)
+    (Time.now - time)/1.second > decent_interval
+  end
+
   # Fetch URI, deals with redirects
   def self.fetch(link, redir_limit = 5)
     raise "Too many HTTP redirects." if redir_limit == 0

lib/hnbot.rb

@@ -1,9 +1,7 @@
-require 'watchbot'
-require 'daemons'
 
 # HNBot lets you track comments made by people you follow via mavenn, 
 #
-class HNBot < Watchbot
+class HNBot 
   BASE_URL = 'http://news.ycombinator.com'
   NEW_LNKS = "#{BASE_URL}/newest"
   NEW_CMTS = "#{BASE_URL}/newcomments"
@@ -13,12 +11,9 @@ class HNBot < Watchbot
   # Politeness restricts us to /, /newest, /newcomments and /item with a
   # wait_interval between 30sec to 1min.
   #
-  # Daemonized fetch of /newcomments storing Comments and stub Postings
-  # Fetches full Postings info less often, say every 10 min.
-
-  def initialize
-    self.load    
-  end
+  # Fetch of /newcomments storing Comments and stub Postings
+  # Fetches full Postings for stub postings
+  # Posts activity to mavenn
 
 	def self.stats
 		{ 
@@ -30,11 +25,22 @@ def self.stats
 
   # Fetch newest comments (first page only)
   def self.fetch_comments
-    CommentList.new(NEW_CMTS).crawl
+    last_fetch = Setting.getval(:method) || Time.now - 1.day
+    sleep 42*rand                     # create some variability
+    Setting.setval(:method, Time.now)
+    CommentList.new(NEW_CMTS).crawl   # only gets one page
   end
 
   # Fetch posts on which watchlist avatars have commented. 
 	def self.fetch_postings
+    last_fetch = Setting.getval(:method) || Time.now - 1.day
+    fetching = Setting.getval(:fetch_postings_underway)
+    return false if fetching
+
+    Setting.setval(:fetch_postings_underway, true)
+    Setting.setval(:method, Time.now)
+    sleep 42*rand
+
     link = Link.new(BASE_URL)
     Posting.unfetched.each do |posting|
       begin
@@ -45,12 +51,16 @@ def self.fetch_postings
         # soldier on 
       end
     end
+    tm = (Time.now - tm)/1.minute
+    Setting.setval("fetch_postings_minutes", tm)
+    Setting.setval(:fetch_postings_underway, false)
 	end
 
   # Post latest activity to mavenn via API
-  def post_activity
-    before = self.last_post
-    self.set(:last_post => Time.now)
+  def self.post_activity
+    before = Setting.getval(:method) || Time.now - 1.day
+    Setting.setval(:method, Time.now)
+
     Stream.all.each do |stream|
       activity = stream.tuples(:since => before)
       json = activity.to_json
@@ -61,10 +71,11 @@ def post_activity
 
       req = Net::HTTP::Post.new(uri.request_uri)
       req.set_form_data(:activity => activity)
+
       req.basic_auth(SiteConfig.apid, SiteConfig.token)
       rsp = http.request(req)
 
-      sleep 2
+      sleep 30
     end
   end
 

lib/setting.rb

@@ -0,0 +1,46 @@
+# A property-value store 
+class Setting 
+  include MongoMapper::Document
+
+  key :name, String, :required => true
+  key :value, String
+  key :ptyp, String
+
+  def self.setval(name, value) 
+    name = (caller[1] =~ /`([^']*)'/ and $1) if name == :method
+    kv = {:value => value.to_s, :ptyp => value.class.to_s}
+    prop = Setting.first_or_create({:name => name})
+    prop.set(kv)
+  end
+
+  def self.getval(name)
+    name = (caller[1] =~ /`([^']*)'/ and $1) if name == :method
+    setting = Setting.first(:name => name) or return nil
+    val = case setting.ptyp
+    when "Fixnum"; setting.value.to_i
+    when "Time"; Time.parse(setting.value)
+    when "Float"; setting.value.to_f
+    when "Boolean"; (setting.value == "false") ? false : true
+    else; setting.value
+    end
+  end
+
+  def self.remove(name)
+    name = (caller[1] =~ /`([^']*)'/ and $1) if name == :method
+    if prop = Setting.first(:name => name)
+      prop.destroy
+    end
+    prop.value
+  end
+end
+
+module Kernel
+private
+  def this_method
+    caller[0] =~ /`([^']*)'/ and $1
+  end
+  def calling_method
+    caller[1] =~ /`([^']*)'/ and $1
+  end
+end
+