Tag items that are on the front page every minute

app/assets/templates/api.html.haml

@@ -59,6 +59,7 @@
             %li <code>pollopt</code>
             %li <code>show_hn</code>
             %li <code>ask_hn</code>
+            %li <code>front_page</code>
             %li <code>author_:USERNAME</code>
             %li <code>story_:ID</code>
         %td
@@ -95,6 +96,8 @@
       %dd <code>#{api_v1_search_url(host: 'hn.algolia.com')}?query=bar&tags=comment</code>
       %dt All URLs matching <code>bar</code>
       %dd <code>#{api_v1_search_url(host: 'hn.algolia.com')}?query=bar&restrictSearchableAttributes=url</code>
+      %dt All stories that are on the front/home page right now
+      %dd <code>#{api_v1_search_url(host: 'hn.algolia.com')}?tags=front_page</code>
       %dt Last stories
       %dd <code>#{api_v1_search_by_date_url(host: 'hn.algolia.com')}?tags=story</code>
       %dt Last stories OR polls

app/controllers/home_controller.rb

@@ -18,9 +18,7 @@ def popular
   end
 
   def front_page
-    @stories = SimpleRSS.parse(Net::HTTP.get(URI.parse('https://news.ycombinator.com/rss'))).items.map do |item|
-      Item.find(item[:comments].split('=').last) rescue nil
-    end.compact
+    @stories = Item.where(front_page: true).all
     @updated_at = DateTime.now
     @title = "HN's home page"
     feed

app/models/item.rb

@@ -38,6 +38,7 @@ class Item < ActiveRecord::Base
           t << 'ask_hn'
         end
       end
+      t << 'front_page' if front_page
       t
     end
     queryType 'prefixLast'

app/workers/hacker_news_realtime_crawler.rb

@@ -58,6 +58,16 @@ def indexing_check
     RestClient.post(ENV['HN_STATUS_API'], status: status, name: 'Indexing') rescue nil # not fatal
   end
 
+  def self.refresh_home_page!
+    old_front_pages = Item.where(front_page: true).select(:id).map(&:id)
+    Item.where(id: old_front_pages).update_all front_page: false
+
+    new_front_pages = SimpleRSS.parse(Net::HTTP.get(URI.parse('https://news.ycombinator.com/rss'))).items.map { |item| item[:comments].split('=').last.to_i }
+    Item.where(id: new_front_pages).update_all front_page: true
+
+    Item.where(id: (old_front_pages + new_front_pages)).reindex!
+  end
+
   private
 
   def refresh(data = {})

config/schedule.rb

@@ -5,6 +5,7 @@
 
 every 1.minute, roles: [:cron] do
   runner "HackerNewsRealtimeCrawler.new.sanity_check(100)"
+  runner "HackerNewsRealtimeCrawler.refresh_home_page!"
   runner "User.where('updated_at >= ?', 5.minute.ago).reindex!"
   runner "Item.where('updated_at >= ?', 5.minute.ago).reindex!"
 end

db/migrate/20150307103111_add_front_page_flag.rb

@@ -0,0 +1,6 @@
+class AddFrontPageFlag < ActiveRecord::Migration
+  def change
+    add_column :items, :front_page, :boolean
+    add_index :items, :front_page
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20141009182754) do
+ActiveRecord::Schema.define(version: 20150307103111) do
 
   create_table "delayed_jobs", force: true do |t|
     t.integer  "priority",   default: 0, null: false
@@ -42,11 +42,13 @@
     t.integer  "parent_id"
     t.integer  "story_id"
     t.datetime "updated_at"
+    t.boolean  "front_page"
   end
 
   add_index "items", ["author"], name: "index_items_on_author"
   add_index "items", ["dead"], name: "index_items_on_dead"
   add_index "items", ["deleted"], name: "index_items_on_deleted"
+  add_index "items", ["front_page"], name: "index_items_on_front_page"
   add_index "items", ["item_type_cd"], name: "index_items_on_item_type_cd"
   add_index "items", ["parent_id"], name: "index_items_on_parent_id"
   add_index "items", ["story_id"], name: "index_items_on_story_id"