Merge pull request #1014 from shalott/works_index_cache_key

Works index cache key fix

Gemfile

@@ -88,6 +88,7 @@ group :test do
   # cuke fast
   gem 'fakeweb'
   gem 'vcr'
+  gem 'delorean'
 end
 
 # Deploy with Capistrano

Gemfile.lock

@@ -78,6 +78,7 @@ GEM
       selenium-webdriver (>= 0.0.3)
     childprocess (0.1.3)
       ffi (~> 0.6.3)
+    chronic (0.8.0)
     cocaine (0.2.0)
     columnize (0.3.1)
     configuration (1.1.0)
@@ -92,6 +93,8 @@ GEM
       cucumber (>= 0.8.0)
     culerity (0.2.12)
     database_cleaner (0.6.0.rc.3)
+    delorean (2.0.0)
+      chronic
     diff-lcs (1.1.2)
     erubis (2.6.6)
       abstract (>= 1.0.0)
@@ -275,6 +278,7 @@ DEPENDENCIES
   cucumber (>= 0.9.1)
   cucumber-rails
   database_cleaner (>= 0.6.0.rc.3)
+  delorean
   escape_utils
   factory_girl
   fakeweb

app/controllers/works_controller.rb

@@ -123,7 +123,7 @@ def index
           (params[:page].blank? || params[:page].to_i <= ArchiveConfig.PAGES_TO_CACHE)
           # we only cache some first initial number of pages since those are biggest bang for 
           # the buck -- users don't often go past them
-          @works = Rails.cache.fetch(index_cache_key) do
+          @works = Rails.cache.fetch(@owner.works_index_cache_key) do
             results = @search.search_results
             # calling this here to avoid frozen object errors
             results.items
@@ -944,22 +944,4 @@ def index_page_title
     end
   end
 
-  # This will change (and thereby automatically invalidates the cache) any time 
-  # one of the @owner's works is created, updated, deleted, or orphaned. 
-  # * The most-recent-updated-at date will capture any work being created or updated
-  # * The count of works will capture an older work being deleted or orphaned
-  # * Can't keep both the same if one of those things has changed!
-  # * To deal with wrangling changes making the filters stale, works are "touched" when they are 
-  #   reindexed for those changes, in the RedisSearchIndexQueue, which will change the updated_at 
-  #   dates. 
-  def index_cache_key
-    cache_key = "works_index_for_"
-    cache_key << (@owner.is_a?(Tag) ? 'tag' : @owner.class.to_s.underscore)
-    cache_key << @owner.id.to_s
-    cache_key << "_"
-    cache_key << @owner.works.count.to_s
-    cache_key << "_"
-    cache_key << @owner.works.order("updated_at DESC").limit(1).value_of(:updated_at).first.to_s.underscore
-  end
-
 end

app/models/collection.rb

@@ -260,9 +260,15 @@ def all_posting_participants
   def all_participants
     (self.participants + (self.parent ? self.parent.participants : [])).uniq
   end
-
+
+  def all_items
+    CollectionItem.where(:collection_id => ([self.id] + self.children.value_of(:id)))
+  end
+
   def all_approved_works
-    (self.approved_works + (self.children ? self.children.collect(&:approved_works).flatten : [])).uniq
+    work_ids = all_items.where(:item_type => "Work", :user_approval_status => CollectionItem::APPROVED, 
+      :collection_approval_status => CollectionItem::APPROVED).value_of(:item_id)
+    Work.where(:id => work_ids, :posted => true)
   end
 
   def all_approved_works_count
@@ -450,4 +456,12 @@ def clear_icon
     self.icon = nil if delete_icon? && !icon.dirty?
   end
 
+  include WorksOwner  
+  # Used in works_controller to determine whether to expire the cache for this tag's works index page
+  def works_index_cache_key
+    index_works = self.children.present? ? self.all_approved_works : self.approved_works
+    super(index_works)
+  end
+
+
 end

app/models/pseud.rb

@@ -2,6 +2,7 @@ class Pseud < ActiveRecord::Base
 
   include Tire::Model::Search
   include Tire::Model::Callbacks
+  include WorksOwner
 
   attr_protected :description_sanitizer_version
 

app/models/tag.rb

@@ -529,6 +529,14 @@ def can_change_type?
   end
 
   #### FILTERING ####
+
+  include WorksOwner  
+  # Used in works_controller to determine whether to expire the cache for this tag's works index page
+  def works_index_cache_key
+    index_works = self.canonical? ? self.filtered_works : self.works
+    super(index_works.where(:posted => true))
+  end
+
 
   # Usage is either:
   # reindex_taggables

app/models/user.rb

@@ -1,5 +1,7 @@
 class User < ActiveRecord::Base
 
+  include WorksOwner
+
 #### used to be in acts_as_authentable
 ## used in app/views/users/new.html.erb
 ## TODO move to ArchiveConfig

config/config.yml

@@ -318,3 +318,38 @@ ACTION_RENAME: 7
 ACTION_PASSWORD_RESET: 8
 ACTION_NEW_EMAIL: 9
 
+
+##### For dumping the seed database and NOT clearing emails on stage
+
+# users who have webdavs or asked to be added
+# or who have problematic works which need testing
+DUMP_SEEDS: [
+          "Amancham", "Anneli", "astolat", "Atalan", "awils1", "aworldinside", 
+          "bingeling", "Cesy", "cesytest", "Celandine", "Chandri", "eel", 
+          "elz", "erda", "Enigel", "hele", "Hope", "Jaetion", "jennyst", 
+          "justira", "jetta_e_rus", "Lal", "lim", "Lisztful", "mumble", 
+          "open_doors", "Rebecca", "RKlyne", "Rustler", "Sidra", 
+          "staranise", "Stowaway", "testy", "Tel", "tomatopudding", 
+          "velocitygrass", "xparrot", "zelempa", "zoemathemata", 
+          "Zooey_Glass", "zlabya", "zz9pzza"
+]
+
+# seeds who want their email address preserved for testing
+DUMP_EMAIL: [ 
+          "admin-amelia", "admin-elz", "admin-emilie", "admin-franny",
+          "admin-kielix", "admin-shalott", "admin-sidra",
+          "Amancham", "astolat", "aworldinside", "bingeling", "cesy", 
+          "cesytest", "elz", "Enigel", "hele", "Lal", "mumble", "open_doors",
+          "Sidra", "testy", "velocitygrass", "xparrot", "Zooey_Glass" , 
+          "zz9pzza"
+]
+
+# a bunch of bigger collections (>250 works)
+# probably would be scooped up anyway, but just in case
+DUMP_COLLECTION_SEEDS: [
+          "yuletide2009", "ladieschoice", "yuletidemadness2009", "female_focus",
+          "crossgen_slash", "PornBattleIX", "Remix2010", "yuletide2010", "fic_promptly",
+          "chromatic_yuletide_2010", "Fandom_Stocking", "yuletidemadness2010", "PornBattleXI",
+          "Remix2011", "PornBattleXII"
+]
+

config/deploy.rb

@@ -84,54 +84,89 @@
 
 # our tasks which are staging specific
 namespace :stage_only do
+
+  # Use git to pull down the latest version of the master branch
   task :git_in_home do
     run "git pull origin master"
     run "bundle install --quiet"
-#    don't update config files in home. they may have been customized
-#    run "ln -nfs -t config/ #{deploy_to}/shared/config/*"
   end
+
+  # Update the public/ folder from the current release to point to shared/static
+  # folders that we want to carry over
   task :update_public do
     run "ln -nfs -t #{release_path}/public/ #{deploy_to}/shared/downloads"
     run "ln -nfs -t #{release_path}/public/ #{deploy_to}/shared/static"
     run "ln -nfs -t #{release_path}/public/stylesheets/ #{deploy_to}/shared/skins"
   end
+
   task :update_configs do
     run "ln -nfs -t #{release_path}/config/ #{deploy_to}/shared/config/*"
   end
+
+  # Reset the entire database from the latest backup from production -- takes a LONG TIME
   task :reset_db do
     run "/static/bin/reset_database.sh"
   end
+
+  # Get rid of subscriptions so we don't spam people
+  task :clear_subscriptions do
+    run "cd #{release_path}; bundle exec rake deploy:clear_subscriptions RAILS_ENV=production"
+  end
+
+  # Redact emails so we don't spam people
+  task :clear_emails do
+    run "cd #{release_path}; bundle exec rake deploy:clear_emails RAILS_ENV=production"
+  end
+
   task :notify_testers do
     system "echo 'testarchive deployed' | mail -s 'testarchive deployed' #{mail_to}"
   end
 end
 
 # our tasks which are production specific
 namespace :production_only do
+  # Use git to pull down the deploy branch
   task :git_in_home, :roles => [:backend, :search] do
     run "git pull origin deploy"
     run "bundle install --quiet"
-#    don't update config files in home. they may have been customized
-#    run "ln -nfs -t config/ /static/config/*"
   end
+
+  # copy over the unicorn configs to the config folder
+  task :get_local_configs, :roles => [:app] do
+    run "cp /root/unicorn* #{release_path}/config/"
+  end  
+
+  # create symlinks from the new public/ folder in the current release to the
+  # carried-over folders for the downloads, skins, other static files
   task :update_public, :roles => [:web, :backend] do
     run "ln -nfs -t #{release_path}/public/ /static/downloads"
     run "ln -nfs -t #{release_path}/public/ /static/static"
     run "ln -nfs -t #{release_path}/public/stylesheets/ /static/skins"
     run "cp #{release_path}/public/robots.public.txt #{release_path}/public/robots.txt"
   end
+
+  # TEMPORARY FIX: there are too many files in the tags feed folder for 
+  # an ext2 filesystem, ack. They have temporarily been moved to a different
+  # filesys. 
+  task :update_tag_feeds, :roles => [:web] do
+    run "ln -nfs -t #{release_path}/public/tags /mnt1"
+  end
+
   task :update_configs, :roles => [:app, :backend] do
     run "ln -nfs -t #{release_path}/config/ /static/config/*"
   end
+
+  # Back up the production database 
   task :backup_db, {:roles => :search} do
     run "/static/bin/backup_database.sh &"
   end
+
+  # Update the crontabs on various machines
   task :update_cron_email, {:roles => :backend} do
     run "whenever --update-crontab production -f config/schedule_production.rb"
   end
-  task :update_cron_reindex, {:roles => :search} do
-    run "whenever --update-crontab search -f config/schedule_search.rb"
-  end
+
+  # Send out notification 
   task :notify_testers do
     system "echo 'archive deployed' | mail -s 'archive deployed' #{mail_to}"
   end

config/deploy/production.rb

@@ -1,15 +1,18 @@
 # otw1 runs sphinx off a slave database
-server "otw1.ao3.org", :search
+# server "otw1.ao3.org", :search # otw1 is gone
+
 # otw2 runs redis, resque and memcache.
 # it also runs the database migrations and can be used to get a console
 server "otw2.ao3.org", :backend, :db, :primary => true
+
 # otw3 and otw4 are the main web/app combos
 server "otw3.ao3.org", :web, :app
 server "otw4.ao3.org", :web, :app
+
 # otw5 is the actual db server and doesn't need anything from capistrano
 
-before "deploy:update_code", "production_only:git_in_home"
-after "deploy:update_code", "production_only:update_public", "production_only:update_configs"
+before "deploy:update_code", "production_only:git_in_home", "production_only:get_local_configs"
+after "deploy:update_code", "production_only:update_public", "production_only:update_tag_feeds", "production_only:update_configs"
 
 before "deploy:migrate", "production_only:backup_db"
 after "deploy:restart", "production_only:update_cron_email", "production_only:update_cron_reindex"

config/deploy/staging.rb

@@ -5,4 +5,8 @@
 
 before "deploy:migrate", "stage_only:reset_db"
 after "deploy:migrate", "extras:reload_site_skins"
+after "deploy:migrate", "stage_only:clear_subscriptions", "stage_only:clear_emails"
 after "deploy:restart", "stage_only:notify_testers"
+# try restarting resque one extra time to see if this does the trick?
+after "deploy:restart", "extras:restart_delayed_jobs"
+

lib/tasks/after_tasks.rake

@@ -330,39 +330,39 @@ namespace :After do
 #     work.save
 #   end
 # end
-
+# 
+# desc "Set stat counts for works"
+# task(:set_work_stats => :environment) do
+#   Work.find_each do |work|
+#     puts work.id
+#     work.update_stat_counter
+#   end
+# end
+# 
+# desc "Set anon/unrevealed status for works"
+# task(:set_anon_unrevealed => :environment) do
+#   CollectionItem.where("(anonymous = 1 OR unrevealed = 1) AND item_type = 'Work'").each do |collection_item|
+#     puts collection_item.id
+#     work = collection_item.item
+#     if work.present?
+#       work.update_attributes(
+#         in_anon_collection: collection_item.anonymous, 
+#         in_unrevealed_collection: collection_item.unrevealed
+#       )
+#     end
+#   end
+# end
+# 
+# desc "Add filters to external works"
+# task(:external_work_filters => :environment) do
+#   ExternalWork.find_each do |ew|
+#     puts ew.id
+#     ew.check_filter_taggings
+#   end
+# end
 
   #### Add your new tasks here
 
-  desc "Set stat counts for works"
-  task(:set_work_stats => :environment) do
-    Work.find_each do |work|
-      puts work.id
-      work.update_stat_counter
-    end
-  end
-
-  desc "Set anon/unrevealed status for works"
-  task(:set_anon_unrevealed => :environment) do
-    CollectionItem.where("(anonymous = 1 OR unrevealed = 1) AND item_type = 'Work'").each do |collection_item|
-      puts collection_item.id
-      work = collection_item.item
-      if work.present?
-        work.update_attributes(
-          in_anon_collection: collection_item.anonymous, 
-          in_unrevealed_collection: collection_item.unrevealed
-        )
-      end
-    end
-  end
-
-  desc "Add filters to external works"
-  task(:external_work_filters => :environment) do
-    ExternalWork.find_each do |ew|
-      puts ew.id
-      ew.check_filter_taggings
-    end
-  end
 
 end # this is the end that you have to put new tasks above
 
@@ -372,9 +372,6 @@ end # this is the end that you have to put new tasks above
 # Remove tasks from the list once they've been run on the deployed site
 # NOTE: 
 desc "Run all current migrate tasks"
-#task :After => ['After:fix_default_pseuds', 'After:remove_owner_kudos']
-#task :After => ['autocomplete:reload_data']
-#task :After => ['After:set_complete_status', 'After:invite_external_authors']
-# task :After => ['After:convert_tag_sets', 'autocomplete:reload_tagset_data', 'skins:disable_all', 'skins:unapprove_all', 'skins:load_site_skins', 'After:convert_existing_skins', 
-#                 'skins:load_user_skins', 'After:remove_old_epubs']
+# task :After => ['After:convert_tag_sets', 'autocomplete:reload_tagset_data', 'skins:disable_all', 'skins:unapprove_all',
+# 'skins:load_site_skins', 'After:convert_existing_skins', 'skins:load_user_skins', 'After:remove_old_epubs']
 task :After => []