Better handling of gist timestamps

app/assets/stylesheets/site.sass

@@ -3,6 +3,11 @@
   width: 95%
   text-align: right
 
+  .last-edited
+    margin-right: 15px
+    color: #ccc
+    font-size: 12px
+
 #footer
   position: absolute
   bottom: 0px

app/controllers/gists_controller.rb

@@ -9,7 +9,7 @@ class GistsController < ApplicationController
   # end
 
   def search
-    if(!cache? || stale?(etag: search_etag, last_modified: current_user.last_gh_fetch))
+    if(!cache? || stale?(etag: search_etag, last_modified: current_user.updated_at))
       @results = Gist.search(current_user, normalized_query)
       if(feeling_lucky_directive? && lucky_result = @results.first)
         redirect_to lucky_result.url

app/models/gist.rb

@@ -5,10 +5,12 @@ class Gist < ActiveRecord::Base
   attr_accessible :gh_id, :user_id, :owner_gh_id, :owner_gh_username, :owner_gh_avatar_url, :description, :url, :git_pull_url, :git_push_url, :public,
     :comment_count, :gh_created_at, :gh_updated_at, :starred, :owned
 
-  belongs_to :user
+  belongs_to :user, :touch => true
   has_many :files, :class_name => 'GistFile', :dependent => :delete_all
 
   scope :with_ids, lambda { |ids| where(ids.any? ? ["id in (?)", ids] : "1 = 0") }
+  scope :starred, where(starred: true)
+  scope :not_starred, where(["starred = ? OR starred IS NULL", false])
 
   index_name ELASTICSEARCH_INDEX_NAME
 
@@ -94,7 +96,7 @@ def reindex(gists = scoped)
     end
 
     def search_cache_key(user, q)
-      "#{CACHE_ACTIVE ? CACHE_VERSION : "no-cache"}-user_id:#{user.id}-updated_at:#{user.last_gh_fetch ? user.last_gh_fetch.to_i : "never"}-#{q}"
+      "#{CACHE_ACTIVE ? CACHE_VERSION : "no-cache"}-user_id:#{user.id}-updated_at:#{user.updated_at}-#{q}"
     end
 
     def with_cache(key)

app/models/gist_fetcher.rb

@@ -3,10 +3,8 @@ class GistFetcher
   class << self
 
     def fetch
-      period = ENV['FETCH_INTERVAL_MINS'] ? ENV['FETCH_INTERVAL_MINS'].to_i : 1440
-      since = period.minutes.ago
-      log({ns: self, fn: __method__}, since: since) do
-        User.last_fetched_before(since).active_auth.pluck(:id).each do |user_id|
+      log({ns: self, fn: __method__}) do
+        User.active_auth.pluck(:id).each do |user_id|
           QC.enqueue("GistFetcher.fetch_gists", user_id)
           QC.enqueue("GistFetcher.fetch_starred_gists", user_id)
         end
@@ -17,35 +15,31 @@ def fetch
     def fetch_gists(user_id)
 
       user = User.find(user_id)
+      since = user.last_gist_updated_at(user.gists.not_starred)
 
       gh_client(user) do |gh|
-        log({ns: self, fn: __method__, measure: true}, user) do
-          gh.gists(nil, since: (user.last_gh_fetch ? user.last_gh_fetch.iso8601.to_s : nil)).each do |gh_gist|
+        log({ns: self, fn: __method__, measure: true, since: since}, user) do
+          gh.gists(nil, since: since).each do |gh_gist|
             Gist.import(user_id, gh_gist)
-            QC.enqueue("GistFetcher.fetch_gist_files", user_id, gh_gist.id)
+            GistFetcher.fetch_gist_files(user_id, gh_gist.id)
           end
         end
-        QC.enqueue("User.refresh_index", user_id)
       end
-
-      user.fetched! # If gist imports fail, this could cause gaps in updated gists...
     end
 
     def fetch_starred_gists(user_id)
 
       user = User.find(user_id)
+      since = user.last_gist_updated_at(user.gists.starred)
 
       gh_client(user) do |gh|
-        log({ns: self, fn: __method__, measure: true}, user) do
-          gh.starred_gists(since: (user.last_gh_fetch ? user.last_gh_fetch.iso8601.to_s : nil)).each do |gh_gist|
+        log({ns: self, fn: __method__, measure: true, since: since}, user) do
+          gh.starred_gists(since: since).each do |gh_gist|
             Gist.import(user_id, gh_gist, starred: true)
-            QC.enqueue("GistFetcher.fetch_gist_files", user_id, gh_gist.id)
+            GistFetcher.fetch_gist_files(user_id, gh_gist.id)
           end
         end
-        QC.enqueue("User.refresh_index", user_id)
       end
-
-      user.fetched! # If gist imports fail, this could cause gaps in updated gists...
     end
 
     def fetch_gist_files(user_id, gh_gist_id)

app/models/user.rb

@@ -5,7 +5,6 @@ class User < ActiveRecord::Base
   has_many :gists, :dependent => :destroy
   has_many :files, :through => :gists
 
-  scope :last_fetched_before, lambda { |since| where(["last_gh_fetch < ? OR last_gh_fetch IS NULL", since])}
   scope :active_auth, where(gh_auth_active: true)
 
   class << self
@@ -28,27 +27,29 @@ def authenticate(auth)
       end
     end
 
+    def refresh_indexes
+      User.active_auth.pluck(:id).each do |user_id|
+        refresh_index(user_id)
+      end
+    end
+
     def refresh_index(user_id)
       user = User.find(user_id)
       log({ns: self, fn: __method__}, user) do
         Gist.reindex(user.gists)
       end
     end
+  end
 
-    def fetched!(user_id)
-      user = User.find(user_id)
-      user.fetched!
-    end
+  def last_gist_updated_at(gist_scope = gists)
+    gist = gist_scope.order("gh_updated_at DESC").first
+    gist ? gist.gh_updated_at : nil
   end
 
   def invalidate_auth!
     update_attribute(:gh_auth_active, false)
   end
 
-  def fetched!
-    update_attribute(:last_gh_fetch, Time.now)
-  end
-
   def gists_count
     @gists_count ||= gists.count
   end
@@ -57,10 +58,6 @@ def files_count
     @files_count ||= files.count
   end
 
-  def fetched?
-    !last_gh_fetch.nil?
-  end
-
   def to_log
     { user: gh_username, user_id: id, user_email: gh_email }
   end

app/views/layouts/application.html.haml

@@ -25,7 +25,9 @@
 
   %body.launch
     #profile
-      = link_to("logout", logout_path) if user_logged_in?
+      - if user_logged_in?
+        %span.last-edited= "Most recent gist edited on #{current_user.last_gist_updated_at.to_formatted_s(:short)}"
+        %span.logout= link_to("logout", logout_path) 
     #header
       .container
         = yield

db/migrate/20121124154836_remove_last_gh_fetch.rb

@@ -0,0 +1,12 @@
+class RemoveLastGhFetch < ActiveRecord::Migration
+
+  def up
+    remove_column :users, :last_gh_fetch
+  end
+
+  def down
+    change_table :users do |t|
+      t.timestamp :last_gh_fetch
+    end
+  end
+end

lib/tasks/fetch.rake

@@ -1,7 +1,8 @@
 namespace :fetch do
 
   task :periodic => :environment do
-    QC.enqueue("GistFetcher.fetch")
+    GistFetcher.fetch
+    QC.enqueue("User.refresh_indexes")    
   end
 
 end

lib/tasks/gisted.rake

@@ -3,4 +3,13 @@ namespace :gisted do
   task :work => :environment do
     GistedWorker.new.start
   end
+
+  desc "Delete all gist data, and do a fresh fetch and reindex of all user data"
+  task :rebuild => :environment do
+    Gist.destroy_all
+    Gist.tire.index.delete
+    Gist.tire.index.create
+    GistFetcher.fetch
+    QC.enqueue("User.refresh_indexes")
+  end
 end

lib/tasks/search.rake

@@ -2,14 +2,14 @@ namespace :search do
 
   # Reindex all gists
   task :reindex => :environment do
-    Gist.reindex
+    User.refresh_indexes
   end
 
   # Delete existing gist index, create new one and reindex all gists
   task :rebuild => :environment do
     Gist.tire.index.delete
     Gist.tire.index.create
-    Gist.reindex
+    User.refresh_indexes
   end
 
 end