First pass, twitter + hackernews

.gitignore

@@ -1,2 +1,3 @@
 .DS_Store
-tmp
+tmp
+db/*

Rakefile

@@ -17,26 +17,28 @@ namespace :test do
   end
 end
 
+namespace :mongodb do
+  desc "Start MongoDB for development"
+  task :start do
+    mkdir_p "db"
+    system "mongod --dbpath db/"
+  end
+end
+
 namespace :app do
-  task :setup do
-    require "lib/burndown"
-    Burndown.new(File.dirname(__FILE__) + "/config/config.yml")
+  task :environment do
+    require "lib/watchtower"
+    Watchtower.new(File.dirname(__FILE__) + "/config/config.yml")
   end
-  
-  task :update_milestones => :setup do
-    Burndown::Milestone.sync_with_lighthouse
+
+  task :poll => :environment do
+    Watchtower::Beam.poll_all
   end
-end
 
-task :environment do
-  require "lib/burndown"
-  Burndown.new(File.dirname(__FILE__) + "/config/config.yml")
+  desc "Start Haystack for development"
+  task :start do
+    system "shotgun config.ru"
+  end
 end
 
-task :cron => :environment do
-  if Time.now.hour == 01
-    puts "Updating milestones..."
-    Burndown::Milestone.sync_with_lighthouse
-    puts "done."
-  end
-end
+multitask :start => [ 'mongodb:start', 'app:start' ]

config.ru

@@ -1,4 +1,7 @@
 #!/usr/bin/env ruby
+
+use Rack::ShowExceptions
+
 require 'lib/watchtower'
 
 # Load configuration and initialize Watchtower

lib/watchtower.rb

@@ -5,10 +5,19 @@
 
 require 'mongo'
 require 'mustache/sinatra'
+require 'nokogiri'
+require 'open-uri'
+require 'digest/sha1'
 
+require 'watchtower/beam'
+require 'watchtower/beam/hacker_news'
+require 'watchtower/beam/twitter'
 require 'watchtower/helpers'
 require 'watchtower/app'
 
+MONGO = Mongo::Connection.new.db("watchtower-#{Watchtower::App.environment}")
+BEAMS = MONGO.collection('beams')
+
 module Watchtower
   def self.new(config=nil)
     if config.is_a?(String) && File.file?(config)

lib/watchtower/app.rb

@@ -5,9 +5,9 @@ class App < Sinatra::Default
     enable :sessions
 
     include Watchtower
-    include Watchtower::Helpers
 
     register Mustache::Sinatra
+    helpers Helpers
 
     dir = File.dirname(File.dirname(__FILE__) + "/../../..")
 
@@ -24,5 +24,10 @@ class App < Sinatra::Default
       show :index
     end
 
+    get '/poll' do
+      @results = Beam.poll_all
+      show :poll, :layout => false
+    end
+
   end
 end

lib/watchtower/beam.rb

@@ -0,0 +1,36 @@
+module Watchtower
+  module Beam
+    def self.poll(beam)
+      results = []
+      beam.new.results.each do |result|
+        existing = BEAMS.find('unique' => result['unique']).first
+        if !existing
+          self.save(result)
+          results << result
+        end
+      end
+      results
+    end
+
+    def self.poll_all
+      self.poll(Beam::HackerNews)
+      self.poll(Beam::Twitter)
+    end
+
+    def self.save(object)
+      BEAMS.insert(object.merge({ 'created_at' => Time.now}))
+    end
+
+    # Returns all Needles. Can be passed an options hash
+    #   :sort  - An array of [ field, order ] pairs
+    #   :limit - How many results to return
+    def self.all(options = {})
+      default_options = {
+        :sort  => [['created_at', 'descending']],
+        :limit => 100
+      }
+
+      BEAMS.find({}, default_options.merge(options))
+    end
+  end
+end

lib/watchtower/beam/hacker_news.rb

@@ -0,0 +1,52 @@
+module Watchtower
+  module Beam
+    class HackerNews
+      def initialize
+        @stories_document = Nokogiri::HTML(open('http://news.ycombinator.com'))
+
+        @results = self.find_github_stories
+      end
+
+      def find_github_stories
+        stories = []
+        @stories_document.css('tr').each do |row|
+          link = row.css('td.title a').first
+          if link && (link['href'].downcase =~ (/github/) || link.content.downcase =~ (/github/))
+            author, author_url, comments, hn_url, points = "?"
+
+            meta_row = row.next
+            meta_row.css('td.subtext span').each do |span|
+              points = span.content.match(/(.*?) points/)[1] if span.content =~ /(.*?) point/
+            end
+            meta_row.css('td.subtext a').each do |inner_link|
+              if inner_link['href'] =~ /^user(.*)/
+                author = inner_link.content
+                author_url = 'http://news.ycombinator.com' + inner_link['href']
+              end
+              if inner_link.content.downcase =~ /(.*?) comment/
+                comments = inner_link.content.downcase.match(/(.*?) comment/)[1]
+                hn_url = 'http://news.ycombinator.com' + inner_link['href']
+              end
+            end
+
+            stories << {
+              'unique' => Digest::SHA1.hexdigest(link['href']),
+              'service' => 'Hacker News',
+              'body' => link.content,
+              'story_url' => link['href'],
+              'author' => author,
+              'author_url' => author_url,
+              'comments' => comments,
+              'points' => points
+            }
+          end
+        end
+        stories
+      end
+
+      def results
+        @results || []
+      end
+    end
+  end
+end

lib/watchtower/beam/twitter.rb

@@ -0,0 +1,34 @@
+module Watchtower
+  module Beam
+    class Twitter
+      def initialize
+        @tweets_document = Nokogiri::HTML(open('http://search.twitter.com/search.atom?q=github'))
+
+        @results = self.find_github_tweets
+      end
+
+      def find_github_tweets
+        tweets = []
+        @tweets_document.css('entry').each do |entry|
+          body = entry.css('title').first.content
+          author_url = entry.css('author uri').first.content
+          author = author_url.match(/twitter.com\/(.*?)$/)[1]
+          tweet_url = entry.css('link').first['href']
+          tweets << {
+            'unique' => Digest::SHA1.hexdigest(tweet_url),
+            'service' => 'Twitter',
+            'body' => body,
+            'author' => '@' + author,
+            'author_url' => author_url,
+            'tweet_url' => tweet_url
+          }
+        end
+        tweets
+      end
+
+      def results
+        @results || []
+      end
+    end
+  end
+end

lib/watchtower/helpers.rb

@@ -5,7 +5,7 @@ module Helpers
 
     def show(template, options = {})
       @title = options[:title]
-      mustache template
+      mustache template, options
     end
 
     def link_to(title, url)
@@ -17,5 +17,58 @@ def page_info(options={})
       @breadcrumb = options[:breadcrumb]
       @page_id = options[:id]
     end
+
+    # Taken from rails
+    AUTO_LINK_RE = %r{
+                ( https?:// | www\. )
+                [^\s<]+
+              }x unless const_defined?(:AUTO_LINK_RE)
+    BRACKETS = { ']' => '[', ')' => '(', '}' => '{' }
+    def auto_link(text, html_options = {})
+      text.gsub(AUTO_LINK_RE) do
+        href = $&
+        punctuation = ''
+        left, right = $`, $'
+        # detect already linked URLs and URLs in the middle of a tag
+        if left =~ /<[^>]+$/ && right =~ /^[^>]*>/
+          # do not change string; URL is alreay linked
+          href
+        else
+          # don't include trailing punctuation character as part of the URL
+          if href.sub!(/[^\w\/-]$/, '') and punctuation = $& and opening = BRACKETS[punctuation]
+            if href.scan(opening).size > href.scan(punctuation).size
+              href << punctuation
+              punctuation = ''
+            end
+          end
+
+          link_text = block_given?? yield(href) : href
+          href = 'http://' + href unless href.index('http') == 0
+
+          "<a href=\"#{href}\">#{h(link_text)}</a>" + punctuation
+        end
+      end
+    end
+
+    def format_beams(beams)
+      beams.collect do |beam|
+        actions = []
+        case beam['service']
+        when 'Twitter'
+          actions << {'url' => beam['tweet_url'], 'name' => 'View Tweet'}
+        when 'Hacker News'
+          actions << {'url' => beam['story_url'], 'name' => 'View Story'}
+        end
+        {
+          'body' => auto_link(beam['body']),
+          'author_url' => beam['author_url'],
+          'author' => beam['author'],
+          'service_short' => (beam['service'] || "unknown").downcase.gsub(/\s/, ''),
+          'created_at' => beam['created_at'] || Time.now.utc,
+          'actions' => actions
+        }
+      end
+    end
+
   end
 end

public/javascripts/application.js

@@ -26,6 +26,9 @@
       this.beams = getBeams()
       setupClickHandlers()
       this.initBeam('all')
+
+      setInterval(Watchtower.App.poll, 60*1000)
+      Watchtower.App.poll()
     },
 
     initBeam: function(service){
@@ -35,6 +38,18 @@
 
       this.currentBeam = this.beams[service]
       this.currentBeam.navElement.addClass('selected')
+    },
+
+    poll: function(){
+      $('p.status').show()
+      $.ajax({
+        type: 'GET',
+        url: '/poll',
+        success: function(html){
+          $('ul.events').prepend(html)
+          $('p.status').hide()
+        }
+      })
     }
   }
 

public/stylesheets/base.css

@@ -60,6 +60,7 @@ h1{
 ------------------------------------------------------------------------------*/
 
 html, body{
+  overflow:auto;
   height:100%;
 }
 
@@ -78,6 +79,12 @@ h2#logo{
   color:#bbb;
 }
 
+#sidebar p.status{
+  margin:15px;
+  color:#999;
+  font-size:11px;
+}
+
 ul#nav{
   margin:15px 0;
   border-top:1px solid #000;
@@ -176,7 +183,7 @@ ul.events .icon{
   height:16px;
   background:0 0 no-repeat;
 }
-ul.events li.tweet .icon{ background-image:url(/images/icons/twitter.png); }
+ul.events li.twitter .icon{ background-image:url(/images/icons/twitter.png); }
 ul.events li.hackernews .icon{ background-image:url(/images/icons/hackernews.png); }
 
 ul.events .meta ul.actions{

templates/index.mustache

@@ -0,0 +1,20 @@
+<h1>All Events</h1>
+
+<ul class="events">
+  {{# beams }}
+    <li class="event {{ service_short }}">
+      <div class="inner">
+        <div class="meta">
+          <span class="icon"></span>
+          <h3><a href="{{ author_url }}">{{ author }}</a> <span class="sep">&mdash;</span> {{ created_at }}</h3>
+          <ul class="actions">
+            {{# actions }}
+            <li><a href="{{ url }}">{{ name }}</a></li>
+            {{/ actions }}
+          </ul>
+        </div>
+        <p>{{{ body }}}</p>
+      </div>
+    </li><!-- /.event -->
+  {{/ beams }}
+</ul>