Skip to content
Browse files

Rename hit counters to stat counters, add migrations for stats, work …

…collection info, and upcoming filter deploy
  • Loading branch information...
1 parent 2cda100 commit 16b668b751b1a6fd03ddf319ae13756d20ccf4f9 @elzj elzj committed Sep 16, 2012
View
71 app/models/stat_counter.rb
@@ -0,0 +1,71 @@
+class StatCounter < ActiveRecord::Base
+ include LogfileReader
+ include WorkStats
+
+ belongs_to :work
+
+ after_create :init_redis
+ def init_redis
+ $redis.set(redis_stat_key(:hit_count), 0)
+ $redis.set(redis_stat_key(:download_count), 0)
+ end
+
+ ###############################################
+ ##### MOVING DATA INTO THE DATABASE
+ ###############################################
+
+ # Persist the hit counts to database
+ def self.hits_to_database
+ # persist from redis to db
+ work_ids = $redis.smembers(WORKS_TO_UPDATE_KEY).map{|id| id.to_i}
+ found_works = []
+
+ StatCounter.find_each(:conditions => ["work_id IN (?)", work_ids]) do |stat_counter|
+ redis_hits = get_stat(:hit_count, stat_counter.work_id)
+ if redis_hits > stat_counter.hit_count
+ stat_counter.update_attribute(:hit_count, redis_hits)
+ else
+ Rails.logger.debug "The redis hit count for work id: #{stat_counter.work_id} was #{redis_hits}. redis has been updated with the database value of #{stat_counter.hit_count}"
+ set_stat(:hit_count, stat_counter.work_id, stat_counter.hit_count)
+ end
+ $redis.srem(WORKS_TO_UPDATE_KEY, stat_counter.work_id)
+ found_works << stat_counter.work_id
+ end
+
+ # Create hit counters for works that don't have them yet
+ (work_ids - found_works).each do |work_id|
+ stat_counter = StatCounter.create(:work_id => work_id, :hit_count => get_stat(:hit_count, work_id))
+ $redis.srem(WORKS_TO_UPDATE_KEY, work_id)
+ end
+ end
+
+
+ # Move download counts and referers from logs to database
+ def self.logs_to_database
+ start_date = AdminSetting.stats_updated_at
+
+ # downloads since the start date
+ stats = get_work_statistic_from_logs(:download_count, start_date)
+ stats.each_pair do |work_id, new_count|
+ # add the count to the hit counter
+ hc = StatCounter.find_or_initialize_by_work_id(:work_id => work_id)
+ hc.download_count += new_count || 0
+ hc.save
+ # update redis to current value
+ set_stat(:download_count, work_id, hc.download_count)
+ end
+
+ # referers
+ stats = get_work_statistic_from_logs(:links, start_date)
+ stats.each_pair do |work_id, referers|
+ # group referers by url and pass count
+ # Let WorkLink determine whether url valid, etc
+ referers.group_by {|referer| referer}.each_pair {|referer, referers| WorkLink.create_or_increment(work_id, referer, referers.count)}
+ end
+
+ AdminSetting.set_stats_updated_at(Time.now)
+ end
+
+
+
+end
View
12 app/models/work.rb
@@ -82,10 +82,10 @@ def work_skin_allowed
# statistics
has_many :work_links, :dependent => :destroy
- has_one :hit_counter, :dependent => :destroy
- after_create :create_hit_counter
- def create_hit_counter
- counter = self.build_hit_counter
+ has_one :stat_counter, :dependent => :destroy
+ after_create :create_stat_counter
+ def create_stat_counter
+ counter = self.build_stat_counter
counter.save
end
@@ -1042,7 +1042,7 @@ def self.find_with_options(options = {})
@works = @works.unrestricted
end
if options[:sort_column] == "hit_count"
- @works = @works.select("works.*, hit_counters.hit_count AS hit_count").joins(:hit_counter)
+ @works = @works.select("works.*, stat_counters.hit_count AS hit_count").joins(:stat_counter)
end
@works = @works.order(sort_by).posted.unhidden
@@ -1098,7 +1098,7 @@ def <=>(another_work)
# indexes chapters.content, :as => 'content'
# attributes
- has hit_counter.hit_count, :as => 'hit_count'
+ has stat_counter.hit_count, :as => 'hit_count'
has word_count, revised_at
has posted, restricted, hidden_by_admin
has complete
View
2 config/schedule.rb
@@ -61,7 +61,7 @@
# Move hit counts from redis to database
every 10.minutes do
- rake "statistics:update_hit_counters"
+ rake "statistics:update_stat_counters"
end
# Move readings from redis to database
View
11 db/migrate/20120809161528_add_collection_info_to_works.rb
@@ -0,0 +1,11 @@
+class AddCollectionInfoToWorks < ActiveRecord::Migration
+ def self.up
+ add_column :works, :in_anon_collection, :boolean, :default => false, :null => false
+ add_column :works, :in_unrevealed_collection, :boolean, :default => false, :null => false
+ end
+
+ def self.down
+ remove_column :works, :in_unrevealed_collection
+ remove_column :works, :in_anon_collection
+ end
+end
View
15 db/migrate/20120809164434_change_hit_counters_to_stat_counters.rb
@@ -0,0 +1,15 @@
+class ChangeHitCountersToStatCounters < ActiveRecord::Migration
+ def self.up
+ rename_table :hit_counters, :stat_counters
+ add_column :stat_counters, :comments_count, :integer, :default => 0, :null => false
+ add_column :stat_counters, :kudos_count, :integer, :default => 0, :null => false
+ add_column :stat_counters, :bookmarks_count, :integer, :default => 0, :null => false
+ end
+
+ def self.down
+ remove_column :stat_counters, :bookmarks_count
+ remove_column :stat_counters, :kudos_count
+ remove_column :stat_counters, :comments_count
+ rename_table :stat_counters, :hit_counters
+ end
+end
View
16 db/migrate/20120825165632_create_searches.rb
@@ -0,0 +1,16 @@
+class CreateSearches < ActiveRecord::Migration
+ def self.up
+ create_table :searches do |t|
+ t.references :user
+ t.string :name
+ t.text :options
+ t.string :type
+
+ t.timestamps
+ end
+ end
+
+ def self.down
+ drop_table :searches
+ end
+end
View
9 db/migrate/20120901113344_add_filter_control_to_admin_settings.rb
@@ -0,0 +1,9 @@
+class AddFilterControlToAdminSettings < ActiveRecord::Migration
+ def self.up
+ add_column :admin_settings, :disable_filtering, :boolean, :null => false, :default => false
+ end
+
+ def self.down
+ remove_column :admin_settings, :disable_filtering
+ end
+end
View
15 features/fixtures/stat_counters.yml
@@ -0,0 +1,15 @@
+first:
+ work_id: 1
+ hit_count: 1001
+second:
+ work_id: 2
+ hit_count: 10
+third:
+ work_id: 3
+ hit_count: 10000
+fourth:
+ work_id: 4
+ hit_count: 0
+fifth:
+ work_id: 5
+ hit_count: 2
View
2 lib/tasks/after_tasks.rake
@@ -128,7 +128,7 @@ namespace :After do
# desc "Move hit counts to their own table"
# task(:move_hit_counts => :environment) do
# Work.find_each do |work|
-# counter = work.build_hit_counter(:hit_count => work.hit_count_old, :last_visitor => work.last_visitor_old)
+# counter = work.build_stat_counter(:hit_count => work.hit_count_old, :last_visitor => work.last_visitor_old)
# counter.save
# end
# end
View
2 lib/tasks/database_seed.rake
@@ -8,5 +8,5 @@ namespace :db do
end
desc "Reset and then seed the development database with test data from the fixtures"
- task :otwseed => [:environment, :development_environment_only, :reset, 'fixtures:load', 'work:missing_hit_counters', 'Tag:reset_filters', 'Tag:reset_filter_counts']
+ task :otwseed => [:environment, :development_environment_only, :reset, 'fixtures:load', 'work:missing_stat_counters', 'Tag:reset_filters', 'Tag:reset_filter_counts']
end
View
8 lib/tasks/statistics_tasks.rake
@@ -1,13 +1,13 @@
namespace :statistics do
desc "update database hit counters from redis"
- task(:update_hit_counters => :environment) do
- HitCounter.hits_to_database
+ task(:update_stat_counters => :environment) do
+ StatCounter.hits_to_database
end
desc "update database statistics from nginx logfiles"
task(:update_from_logfiles => :environment) do
- HitCounter.logs_to_database
+ StatCounter.logs_to_database
end
desc "update database hit counts from squid cache logfiles"
@@ -26,7 +26,7 @@ namespace :statistics do
work_hits.keys.each do |work_id|
begin
work = Work.where(:id => work_id).first
- next unless work && work.hit_counter
+ next unless work && work.stat_counter
work.add_to_hit_count(work_hits[work_id])
rescue
end
View
6 lib/tasks/work_tasks.rake
@@ -6,11 +6,11 @@ namespace :work do
end
desc "create missing hit counters"
- task(:missing_hit_counters => :environment) do
+ task(:missing_stat_counters => :environment) do
Work.find_each do |work|
- counter = work.hit_counter
+ counter = work.stat_counter
unless counter
- counter = HitCounter.create(:work=>work, :hit_count => 1)
+ counter = StatCounter.create(:work=>work, :hit_count => 1)
end
end
end
View
4 lib/work_stats.rb
@@ -1,4 +1,4 @@
-# This module is included by both the work and hit_counter models so they use the
+# This module is included by both the work and stat_counter models so they use the
# same redis keys and can both access the data in redis
module WorkStats
@@ -30,7 +30,7 @@ def get_stat(statistic, work_id)
end
def get_database_stat(statistic, work_id)
- HitCounter.where(:work_id => work_id).value_of(statistic).first || 0
+ StatCounter.where(:work_id => work_id).value_of(statistic).first || 0
end
end # ClassMethods
View
2 script/seed_dump.rb
@@ -414,7 +414,7 @@ def work_associations(items)
work.creatorships.each {|x| write_model(x)}
work.pseuds.each {|p| PSEUDS[p.id] = p }
write_model(work.language) if work.language
- write_model(work.hit_counter)
+ write_model(work.stat_counter)
work.gifts.each {|x| write_model(x)}
work.gifts.each {|g| PSEUDS[g.pseud.id] = g.pseud if g.pseud}
work.serial_works.each {|x| write_model(x)}
View
14 spec/models/stat_counter_spec.rb
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+
+describe StatCounter do
+
+ before(:each) do
+ @work = Factory.create(:work)
+ end
+
+ it "should be created for a new work" do
+ @work.stat_counter.nil?.should_not be_true
+ end
+
+end
View
49 test/fixtures/stat_counters.yml
@@ -0,0 +1,49 @@
+stat_counter_00049:
+ work: work_00049
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00063:
+ work: work_00063
+ hit_count: 3
+ last_visitor: 70.222.162.200
+stat_counter_00067:
+ work: work_00067
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00070:
+ work: work_00070
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00090:
+ work: work_00090
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00091:
+ work: work_00091
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00098:
+ work: work_00098
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00101:
+ work: work_00101
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00102:
+ work: work_00102
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00103:
+ work: work_00103
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00104:
+ work: work_00104
+ hit_count: 2
+ last_visitor: 89.136.154.183
+stat_counter_00105:
+ work: work_00105
+ hit_count: 1
+ last_visitor: 87.194.8.88
+

0 comments on commit 16b668b

Please sign in to comment.
Something went wrong with that request. Please try again.