Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Rename hit counters to stat counters, add migrations for stats, work …

…collection info, and upcoming filter deploy
  • Loading branch information...
commit 16b668b751b1a6fd03ddf319ae13756d20ccf4f9 1 parent 2cda100
@elzj elzj authored
View
71 app/models/stat_counter.rb
@@ -0,0 +1,71 @@
+class StatCounter < ActiveRecord::Base
+ include LogfileReader
+ include WorkStats
+
+ belongs_to :work
+
+ after_create :init_redis
+ def init_redis
+ $redis.set(redis_stat_key(:hit_count), 0)
+ $redis.set(redis_stat_key(:download_count), 0)
+ end
+
+ ###############################################
+ ##### MOVING DATA INTO THE DATABASE
+ ###############################################
+
+ # Persist the hit counts to database
+ def self.hits_to_database
+ # persist from redis to db
+ work_ids = $redis.smembers(WORKS_TO_UPDATE_KEY).map{|id| id.to_i}
+ found_works = []
+
+ StatCounter.find_each(:conditions => ["work_id IN (?)", work_ids]) do |stat_counter|
+ redis_hits = get_stat(:hit_count, stat_counter.work_id)
+ if redis_hits > stat_counter.hit_count
+ stat_counter.update_attribute(:hit_count, redis_hits)
+ else
+ Rails.logger.debug "The redis hit count for work id: #{stat_counter.work_id} was #{redis_hits}. redis has been updated with the database value of #{stat_counter.hit_count}"
+ set_stat(:hit_count, stat_counter.work_id, stat_counter.hit_count)
+ end
+ $redis.srem(WORKS_TO_UPDATE_KEY, stat_counter.work_id)
+ found_works << stat_counter.work_id
+ end
+
+ # Create hit counters for works that don't have them yet
+ (work_ids - found_works).each do |work_id|
+ stat_counter = StatCounter.create(:work_id => work_id, :hit_count => get_stat(:hit_count, work_id))
+ $redis.srem(WORKS_TO_UPDATE_KEY, work_id)
+ end
+ end
+
+
+ # Move download counts and referers from logs to database
+ def self.logs_to_database
+ start_date = AdminSetting.stats_updated_at
+
+ # downloads since the start date
+ stats = get_work_statistic_from_logs(:download_count, start_date)
+ stats.each_pair do |work_id, new_count|
+ # add the count to the hit counter
+ hc = StatCounter.find_or_initialize_by_work_id(:work_id => work_id)
+ hc.download_count += new_count || 0
+ hc.save
+ # update redis to current value
+ set_stat(:download_count, work_id, hc.download_count)
+ end
+
+ # referers
+ stats = get_work_statistic_from_logs(:links, start_date)
+ stats.each_pair do |work_id, referers|
+ # group referers by url and pass count
+ # Let WorkLink determine whether url valid, etc
+ referers.group_by {|referer| referer}.each_pair {|referer, referers| WorkLink.create_or_increment(work_id, referer, referers.count)}
+ end
+
+ AdminSetting.set_stats_updated_at(Time.now)
+ end
+
+
+
+end
View
12 app/models/work.rb
@@ -82,10 +82,10 @@ def work_skin_allowed
# statistics
has_many :work_links, :dependent => :destroy
- has_one :hit_counter, :dependent => :destroy
- after_create :create_hit_counter
- def create_hit_counter
- counter = self.build_hit_counter
+ has_one :stat_counter, :dependent => :destroy
+ after_create :create_stat_counter
+ def create_stat_counter
+ counter = self.build_stat_counter
counter.save
end
@@ -1042,7 +1042,7 @@ def self.find_with_options(options = {})
@works = @works.unrestricted
end
if options[:sort_column] == "hit_count"
- @works = @works.select("works.*, hit_counters.hit_count AS hit_count").joins(:hit_counter)
+ @works = @works.select("works.*, stat_counters.hit_count AS hit_count").joins(:stat_counter)
end
@works = @works.order(sort_by).posted.unhidden
@@ -1098,7 +1098,7 @@ def <=>(another_work)
# indexes chapters.content, :as => 'content'
# attributes
- has hit_counter.hit_count, :as => 'hit_count'
+ has stat_counter.hit_count, :as => 'hit_count'
has word_count, revised_at
has posted, restricted, hidden_by_admin
has complete
View
2  config/schedule.rb
@@ -61,7 +61,7 @@
# Move hit counts from redis to database
every 10.minutes do
- rake "statistics:update_hit_counters"
+ rake "statistics:update_stat_counters"
end
# Move readings from redis to database
View
11 db/migrate/20120809161528_add_collection_info_to_works.rb
@@ -0,0 +1,11 @@
+class AddCollectionInfoToWorks < ActiveRecord::Migration
+ def self.up
+ add_column :works, :in_anon_collection, :boolean, :default => false, :null => false
+ add_column :works, :in_unrevealed_collection, :boolean, :default => false, :null => false
+ end
+
+ def self.down
+ remove_column :works, :in_unrevealed_collection
+ remove_column :works, :in_anon_collection
+ end
+end
View
15 db/migrate/20120809164434_change_hit_counters_to_stat_counters.rb
@@ -0,0 +1,15 @@
+class ChangeHitCountersToStatCounters < ActiveRecord::Migration
+ def self.up
+ rename_table :hit_counters, :stat_counters
+ add_column :stat_counters, :comments_count, :integer, :default => 0, :null => false
+ add_column :stat_counters, :kudos_count, :integer, :default => 0, :null => false
+ add_column :stat_counters, :bookmarks_count, :integer, :default => 0, :null => false
+ end
+
+ def self.down
+ remove_column :stat_counters, :bookmarks_count
+ remove_column :stat_counters, :kudos_count
+ remove_column :stat_counters, :comments_count
+ rename_table :stat_counters, :hit_counters
+ end
+end
View
16 db/migrate/20120825165632_create_searches.rb
@@ -0,0 +1,16 @@
+class CreateSearches < ActiveRecord::Migration
+ def self.up
+ create_table :searches do |t|
+ t.references :user
+ t.string :name
+ t.text :options
+ t.string :type
+
+ t.timestamps
+ end
+ end
+
+ def self.down
+ drop_table :searches
+ end
+end
View
9 db/migrate/20120901113344_add_filter_control_to_admin_settings.rb
@@ -0,0 +1,9 @@
+class AddFilterControlToAdminSettings < ActiveRecord::Migration
+ def self.up
+ add_column :admin_settings, :disable_filtering, :boolean, :null => false, :default => false
+ end
+
+ def self.down
+ remove_column :admin_settings, :disable_filtering
+ end
+end
View
15 features/fixtures/stat_counters.yml
@@ -0,0 +1,15 @@
+first:
+ work_id: 1
+ hit_count: 1001
+second:
+ work_id: 2
+ hit_count: 10
+third:
+ work_id: 3
+ hit_count: 10000
+fourth:
+ work_id: 4
+ hit_count: 0
+fifth:
+ work_id: 5
+ hit_count: 2
View
2  lib/tasks/after_tasks.rake
@@ -128,7 +128,7 @@ namespace :After do
# desc "Move hit counts to their own table"
# task(:move_hit_counts => :environment) do
# Work.find_each do |work|
-# counter = work.build_hit_counter(:hit_count => work.hit_count_old, :last_visitor => work.last_visitor_old)
+# counter = work.build_stat_counter(:hit_count => work.hit_count_old, :last_visitor => work.last_visitor_old)
# counter.save
# end
# end
View
2  lib/tasks/database_seed.rake
@@ -8,5 +8,5 @@ namespace :db do
end
desc "Reset and then seed the development database with test data from the fixtures"
- task :otwseed => [:environment, :development_environment_only, :reset, 'fixtures:load', 'work:missing_hit_counters', 'Tag:reset_filters', 'Tag:reset_filter_counts']
+ task :otwseed => [:environment, :development_environment_only, :reset, 'fixtures:load', 'work:missing_stat_counters', 'Tag:reset_filters', 'Tag:reset_filter_counts']
end
View
8 lib/tasks/statistics_tasks.rake
@@ -1,13 +1,13 @@
namespace :statistics do
desc "update database hit counters from redis"
- task(:update_hit_counters => :environment) do
- HitCounter.hits_to_database
+ task(:update_stat_counters => :environment) do
+ StatCounter.hits_to_database
end
desc "update database statistics from nginx logfiles"
task(:update_from_logfiles => :environment) do
- HitCounter.logs_to_database
+ StatCounter.logs_to_database
end
desc "update database hit counts from squid cache logfiles"
@@ -26,7 +26,7 @@ namespace :statistics do
work_hits.keys.each do |work_id|
begin
work = Work.where(:id => work_id).first
- next unless work && work.hit_counter
+ next unless work && work.stat_counter
work.add_to_hit_count(work_hits[work_id])
rescue
end
View
6 lib/tasks/work_tasks.rake
@@ -6,11 +6,11 @@ namespace :work do
end
desc "create missing hit counters"
- task(:missing_hit_counters => :environment) do
+ task(:missing_stat_counters => :environment) do
Work.find_each do |work|
- counter = work.hit_counter
+ counter = work.stat_counter
unless counter
- counter = HitCounter.create(:work=>work, :hit_count => 1)
+ counter = StatCounter.create(:work=>work, :hit_count => 1)
end
end
end
View
4 lib/work_stats.rb
@@ -1,4 +1,4 @@
-# This module is included by both the work and hit_counter models so they use the
+# This module is included by both the work and stat_counter models so they use the
# same redis keys and can both access the data in redis
module WorkStats
@@ -30,7 +30,7 @@ def get_stat(statistic, work_id)
end
def get_database_stat(statistic, work_id)
- HitCounter.where(:work_id => work_id).value_of(statistic).first || 0
+ StatCounter.where(:work_id => work_id).value_of(statistic).first || 0
end
end # ClassMethods
View
2  script/seed_dump.rb
@@ -414,7 +414,7 @@ def work_associations(items)
work.creatorships.each {|x| write_model(x)}
work.pseuds.each {|p| PSEUDS[p.id] = p }
write_model(work.language) if work.language
- write_model(work.hit_counter)
+ write_model(work.stat_counter)
work.gifts.each {|x| write_model(x)}
work.gifts.each {|g| PSEUDS[g.pseud.id] = g.pseud if g.pseud}
work.serial_works.each {|x| write_model(x)}
View
14 spec/models/stat_counter_spec.rb
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+
+describe StatCounter do
+
+ before(:each) do
+ @work = Factory.create(:work)
+ end
+
+ it "should be created for a new work" do
+ @work.stat_counter.nil?.should_not be_true
+ end
+
+end
View
49 test/fixtures/stat_counters.yml
@@ -0,0 +1,49 @@
+stat_counter_00049:
+ work: work_00049
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00063:
+ work: work_00063
+ hit_count: 3
+ last_visitor: 70.222.162.200
+stat_counter_00067:
+ work: work_00067
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00070:
+ work: work_00070
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00090:
+ work: work_00090
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00091:
+ work: work_00091
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00098:
+ work: work_00098
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00101:
+ work: work_00101
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00102:
+ work: work_00102
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00103:
+ work: work_00103
+ hit_count: 1
+ last_visitor: 87.194.8.88
+stat_counter_00104:
+ work: work_00104
+ hit_count: 2
+ last_visitor: 89.136.154.183
+stat_counter_00105:
+ work: work_00105
+ hit_count: 1
+ last_visitor: 87.194.8.88
+
Please sign in to comment.
Something went wrong with that request. Please try again.