From 6f53dd50b91b62a2501ee33e9b0b76721dff7faa Mon Sep 17 00:00:00 2001 From: Ryan Daigle Date: Sat, 24 Nov 2012 11:07:31 -0500 Subject: [PATCH] Better handling of gist timestamps --- app/assets/stylesheets/site.sass | 5 ++++ app/controllers/gists_controller.rb | 2 +- app/models/gist.rb | 6 +++-- app/models/gist_fetcher.rb | 26 +++++++------------ app/models/user.rb | 23 +++++++--------- app/views/layouts/application.html.haml | 4 ++- .../20121124154836_remove_last_gh_fetch.rb | 12 +++++++++ lib/tasks/fetch.rake | 3 ++- lib/tasks/gisted.rake | 9 +++++++ lib/tasks/search.rake | 4 +-- 10 files changed, 58 insertions(+), 36 deletions(-) create mode 100644 db/migrate/20121124154836_remove_last_gh_fetch.rb diff --git a/app/assets/stylesheets/site.sass b/app/assets/stylesheets/site.sass index c1a803e..261f9ed 100644 --- a/app/assets/stylesheets/site.sass +++ b/app/assets/stylesheets/site.sass @@ -3,6 +3,11 @@ width: 95% text-align: right + .last-edited + margin-right: 15px + color: #ccc + font-size: 12px + #footer position: absolute bottom: 0px diff --git a/app/controllers/gists_controller.rb b/app/controllers/gists_controller.rb index e946861..1fa2262 100644 --- a/app/controllers/gists_controller.rb +++ b/app/controllers/gists_controller.rb @@ -9,7 +9,7 @@ class GistsController < ApplicationController # end def search - if(!cache? || stale?(etag: search_etag, last_modified: current_user.last_gh_fetch)) + if(!cache? || stale?(etag: search_etag, last_modified: current_user.updated_at)) @results = Gist.search(current_user, normalized_query) if(feeling_lucky_directive? && lucky_result = @results.first) redirect_to lucky_result.url diff --git a/app/models/gist.rb b/app/models/gist.rb index 38ba012..fc6f0c2 100644 --- a/app/models/gist.rb +++ b/app/models/gist.rb @@ -5,10 +5,12 @@ class Gist < ActiveRecord::Base attr_accessible :gh_id, :user_id, :owner_gh_id, :owner_gh_username, :owner_gh_avatar_url, :description, :url, :git_pull_url, :git_push_url, :public, :comment_count, :gh_created_at, :gh_updated_at, :starred, :owned - belongs_to :user + belongs_to :user, :touch => true has_many :files, :class_name => 'GistFile', :dependent => :delete_all scope :with_ids, lambda { |ids| where(ids.any? ? ["id in (?)", ids] : "1 = 0") } + scope :starred, where(starred: true) + scope :not_starred, where(["starred = ? OR starred IS NULL", false]) index_name ELASTICSEARCH_INDEX_NAME @@ -94,7 +96,7 @@ def reindex(gists = scoped) end def search_cache_key(user, q) - "#{CACHE_ACTIVE ? CACHE_VERSION : "no-cache"}-user_id:#{user.id}-updated_at:#{user.last_gh_fetch ? user.last_gh_fetch.to_i : "never"}-#{q}" + "#{CACHE_ACTIVE ? CACHE_VERSION : "no-cache"}-user_id:#{user.id}-updated_at:#{user.updated_at}-#{q}" end def with_cache(key) diff --git a/app/models/gist_fetcher.rb b/app/models/gist_fetcher.rb index c027770..5459a21 100644 --- a/app/models/gist_fetcher.rb +++ b/app/models/gist_fetcher.rb @@ -3,10 +3,8 @@ class GistFetcher class << self def fetch - period = ENV['FETCH_INTERVAL_MINS'] ? ENV['FETCH_INTERVAL_MINS'].to_i : 1440 - since = period.minutes.ago - log({ns: self, fn: __method__}, since: since) do - User.last_fetched_before(since).active_auth.pluck(:id).each do |user_id| + log({ns: self, fn: __method__}) do + User.active_auth.pluck(:id).each do |user_id| QC.enqueue("GistFetcher.fetch_gists", user_id) QC.enqueue("GistFetcher.fetch_starred_gists", user_id) end @@ -17,35 +15,31 @@ def fetch def fetch_gists(user_id) user = User.find(user_id) + since = user.last_gist_updated_at(user.gists.not_starred) gh_client(user) do |gh| - log({ns: self, fn: __method__, measure: true}, user) do - gh.gists(nil, since: (user.last_gh_fetch ? user.last_gh_fetch.iso8601.to_s : nil)).each do |gh_gist| + log({ns: self, fn: __method__, measure: true, since: since}, user) do + gh.gists(nil, since: since).each do |gh_gist| Gist.import(user_id, gh_gist) - QC.enqueue("GistFetcher.fetch_gist_files", user_id, gh_gist.id) + GistFetcher.fetch_gist_files(user_id, gh_gist.id) end end - QC.enqueue("User.refresh_index", user_id) end - - user.fetched! # If gist imports fail, this could cause gaps in updated gists... end def fetch_starred_gists(user_id) user = User.find(user_id) + since = user.last_gist_updated_at(user.gists.starred) gh_client(user) do |gh| - log({ns: self, fn: __method__, measure: true}, user) do - gh.starred_gists(since: (user.last_gh_fetch ? user.last_gh_fetch.iso8601.to_s : nil)).each do |gh_gist| + log({ns: self, fn: __method__, measure: true, since: since}, user) do + gh.starred_gists(since: since).each do |gh_gist| Gist.import(user_id, gh_gist, starred: true) - QC.enqueue("GistFetcher.fetch_gist_files", user_id, gh_gist.id) + GistFetcher.fetch_gist_files(user_id, gh_gist.id) end end - QC.enqueue("User.refresh_index", user_id) end - - user.fetched! # If gist imports fail, this could cause gaps in updated gists... end def fetch_gist_files(user_id, gh_gist_id) diff --git a/app/models/user.rb b/app/models/user.rb index 2a34422..97cde26 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -5,7 +5,6 @@ class User < ActiveRecord::Base has_many :gists, :dependent => :destroy has_many :files, :through => :gists - scope :last_fetched_before, lambda { |since| where(["last_gh_fetch < ? OR last_gh_fetch IS NULL", since])} scope :active_auth, where(gh_auth_active: true) class << self @@ -28,27 +27,29 @@ def authenticate(auth) end end + def refresh_indexes + User.active_auth.pluck(:id).each do |user_id| + refresh_index(user_id) + end + end + def refresh_index(user_id) user = User.find(user_id) log({ns: self, fn: __method__}, user) do Gist.reindex(user.gists) end end + end - def fetched!(user_id) - user = User.find(user_id) - user.fetched! - end + def last_gist_updated_at(gist_scope = gists) + gist = gist_scope.order("gh_updated_at DESC").first + gist ? gist.gh_updated_at : nil end def invalidate_auth! update_attribute(:gh_auth_active, false) end - def fetched! - update_attribute(:last_gh_fetch, Time.now) - end - def gists_count @gists_count ||= gists.count end @@ -57,10 +58,6 @@ def files_count @files_count ||= files.count end - def fetched? - !last_gh_fetch.nil? - end - def to_log { user: gh_username, user_id: id, user_email: gh_email } end diff --git a/app/views/layouts/application.html.haml b/app/views/layouts/application.html.haml index 6a7ec7f..363efcb 100644 --- a/app/views/layouts/application.html.haml +++ b/app/views/layouts/application.html.haml @@ -25,7 +25,9 @@ %body.launch #profile - = link_to("logout", logout_path) if user_logged_in? + - if user_logged_in? + %span.last-edited= "Most recent gist edited on #{current_user.last_gist_updated_at.to_formatted_s(:short)}" + %span.logout= link_to("logout", logout_path) #header .container = yield diff --git a/db/migrate/20121124154836_remove_last_gh_fetch.rb b/db/migrate/20121124154836_remove_last_gh_fetch.rb new file mode 100644 index 0000000..ba980c2 --- /dev/null +++ b/db/migrate/20121124154836_remove_last_gh_fetch.rb @@ -0,0 +1,12 @@ +class RemoveLastGhFetch < ActiveRecord::Migration + + def up + remove_column :users, :last_gh_fetch + end + + def down + change_table :users do |t| + t.timestamp :last_gh_fetch + end + end +end diff --git a/lib/tasks/fetch.rake b/lib/tasks/fetch.rake index 1ca2fec..0b93e31 100644 --- a/lib/tasks/fetch.rake +++ b/lib/tasks/fetch.rake @@ -1,7 +1,8 @@ namespace :fetch do task :periodic => :environment do - QC.enqueue("GistFetcher.fetch") + GistFetcher.fetch + QC.enqueue("User.refresh_indexes") end end \ No newline at end of file diff --git a/lib/tasks/gisted.rake b/lib/tasks/gisted.rake index 0da417b..96476fb 100644 --- a/lib/tasks/gisted.rake +++ b/lib/tasks/gisted.rake @@ -3,4 +3,13 @@ namespace :gisted do task :work => :environment do GistedWorker.new.start end + + desc "Delete all gist data, and do a fresh fetch and reindex of all user data" + task :rebuild => :environment do + Gist.destroy_all + Gist.tire.index.delete + Gist.tire.index.create + GistFetcher.fetch + QC.enqueue("User.refresh_indexes") + end end \ No newline at end of file diff --git a/lib/tasks/search.rake b/lib/tasks/search.rake index 3edc308..50c6a93 100644 --- a/lib/tasks/search.rake +++ b/lib/tasks/search.rake @@ -2,14 +2,14 @@ namespace :search do # Reindex all gists task :reindex => :environment do - Gist.reindex + User.refresh_indexes end # Delete existing gist index, create new one and reindex all gists task :rebuild => :environment do Gist.tire.index.delete Gist.tire.index.create - Gist.reindex + User.refresh_indexes end end \ No newline at end of file