Browse files

Merge branch 'downloads'

  • Loading branch information...
2 parents d0ac696 + c9eaab5 commit 71235f11b68fb895d6b8d90cb9a703c15ff3434f @evanphx evanphx committed Sep 15, 2012
View
88 app/models/download.rb
@@ -54,22 +54,48 @@ def self.most_downloaded_all_time(n=5)
def self.counts_by_day_for_versions(versions, days)
dates = (days.days.ago.to_date...Time.zone.today).map(&:to_s)
- versions.inject({}) do |downloads, version|
- $redis.hmget(self.history_key(version), *dates).each_with_index do |count, idx|
- downloads["#{version.id}-#{dates[idx]}"] = count.to_i
+ downloads = {}
+ versions.each do |version|
+ key = history_key(version)
+
+ $redis.hmget(key, *dates).zip(dates).each do |count, date|
+ if count
+ count = count.to_i
+ else
+ vh = VersionHistory.where(:version_id => version.id,
+ :day => date).first
+
+ count = vh ? vh.count : 0
+ end
+
+ downloads["#{version.id}-#{date}"] = count
end
downloads["#{version.id}-#{Time.zone.today}"] = self.today(version)
- downloads
end
+
+ downloads
end
def self.counts_by_day_for_version_in_date_range(version, start, stop)
downloads = ActiveSupport::OrderedHash.new
dates = (start..stop).map(&:to_s)
- $redis.hmget(self.history_key(version), *dates).each_with_index do |count, idx|
- downloads["#{dates[idx]}"] = count.to_i
+ $redis.hmget(history_key(version), *dates).zip(dates).each do |count, date|
+ if count
+ count = count.to_i
+ else
+ vh = VersionHistory.where(:version_id => version.id,
+ :day => date).first
+
+ if vh
+ count = vh.count
+ else
+ count = 0
+ end
+ end
+
+ downloads[date] = count
end
if stop == Time.zone.today
@@ -80,6 +106,52 @@ def self.counts_by_day_for_version_in_date_range(version, start, stop)
downloads
end
+ def self.copy_to_sql(version, date)
+ count = $redis.hget(history_key(version), date)
+
+ if vh = VersionHistory.for(version, date)
+ vh.count = count
+ vh.save
+ else
+ VersionHistory.make(version, date, count)
+ end
+ end
+
+ def self.migrate_to_sql(version)
+ key = history_key version
+
+ dates = $redis.hkeys(key)
+
+ back = 1.days.ago.to_date
+
+ dates.delete_if { |e| Date.parse(e) >= back }
+
+ dates.each do |d|
+ copy_to_sql version, d
+ $redis.hdel key, d
+ end
+
+ dates
+ end
+
+ def self.migrate_all_to_sql
+ i = 0
+ count = 0
+ versions = Version.all
+ total = versions.size
+
+
+ versions.each do |ver|
+ i += 1
+ yield total, i, ver if block_given?
+
+ dates = migrate_to_sql ver
+ count += 1 unless dates.empty?
+ end
+
+ count
+ end
+
def self.counts_by_day_for_version(version)
counts_by_day_for_version_in_date_range(version, Time.zone.today - 89.days, Time.zone.today)
end
@@ -90,6 +162,8 @@ def self.key(what)
version_key(what.full_name)
when Rubygem
rubygem_key(what.name)
+ else
+ raise TypeError, "Unknown type for key - #{what.class}"
end
end
@@ -99,6 +173,8 @@ def self.history_key(what)
version_history_key(what.full_name)
when Rubygem
rubygem_history_key(what.name)
+ else
+ raise TypeError, "Unknown type for history_key - #{what.class}"
end
end
View
14 app/models/version_history.rb
@@ -0,0 +1,14 @@
+class VersionHistory < ActiveRecord::Base
+ attr_accessible :count, :day, :version_id
+ belongs_to :version
+
+ def self.for(version, day)
+ VersionHistory.where(:version_id => version.id, :day => day).first
+ end
+
+ def self.make(version, day, count)
+ VersionHistory.create(:version_id => version.id,
+ :day => day.to_s,
+ :count => count.to_i).save
+ end
+end
View
11 db/migrate/20120915055838_create_version_histories.rb
@@ -0,0 +1,11 @@
+class CreateVersionHistories < ActiveRecord::Migration
+ def change
+ create_table :version_histories do |t|
+ t.integer :version_id
+ t.date :day
+ t.integer :count
+
+ t.timestamps
+ end
+ end
+end
View
9 db/migrate/20120915212528_add_version_history_indexes.rb
@@ -0,0 +1,9 @@
+class AddVersionHistoryIndexes < ActiveRecord::Migration
+ def up
+ add_index :version_histories, [:version_id, :day], :unique => true
+ end
+
+ def down
+ remove_index :version_histories, :column => [:version_id, :day]
+ end
+end
View
12 db/schema.rb
@@ -11,7 +11,7 @@
#
# It's strongly recommended to check this file into your version control system.
-ActiveRecord::Schema.define(:version => 20120118194729) do
+ActiveRecord::Schema.define(:version => 20120915212528) do
create_table "announcements", :force => true do |t|
t.text "body"
@@ -119,6 +119,16 @@
add_index "users", ["remember_token"], :name => "index_users_on_remember_token"
add_index "users", ["token"], :name => "index_users_on_token"
+ create_table "version_histories", :force => true do |t|
+ t.integer "version_id"
+ t.date "day"
+ t.integer "count"
+ t.datetime "created_at", :null => false
+ t.datetime "updated_at", :null => false
+ end
+
+ add_index "version_histories", ["version_id", "day"], :name => "index_version_histories_on_version_id_and_day", :unique => true
+
create_table "versions", :force => true do |t|
t.text "authors"
t.text "description"
View
7 lib/tasks/gemcutter.rake
@@ -58,4 +58,11 @@ namespace :gemcutter do
end
end
end
+
+ desc "Move all but the last 2 days of version history to SQL"
+ task :migrate_history => :environment do
+ Download.migrate_all_to_sql do |t,c,v|
+ puts "#{c} of #{t}: #{v.full_name}"
+ end
+ end
end
View
5 test/factories.rb
@@ -83,6 +83,11 @@
rubygem
end
+ factory :version_history do
+ day { Time.zone.today.to_s }
+ count 1
+ end
+
sequence :url do |n|
"http://example#{n}.com"
end
View
108 test/unit/download_test.rb
@@ -132,6 +132,60 @@ class DownloadTest < ActiveSupport::TestCase
assert_equal downloads, Download.counts_by_day_for_versions([@version_1, @version_2, @version_3], 2)
end
+ should "find counts per day for versions when in DB also" do
+ @rubygem_1 = create(:rubygem)
+ @version_1 = create(:version, :rubygem => @rubygem_1)
+ @version_2 = create(:version, :rubygem => @rubygem_1)
+
+ @rubygem_2 = create(:rubygem)
+ @version_3 = create(:version, :rubygem => @rubygem_2)
+
+ @rubygem_3 = create(:rubygem)
+ @version_4 = create(:version, :rubygem => @rubygem_3)
+
+ Timecop.freeze(1.day.ago) do
+ create :version_history, :version => @version_1, :count => 5
+ create :version_history, :version => @version_2
+ create :version_history, :version => @version_3
+ end
+
+ Download.incr(@rubygem_2, @version_3.full_name)
+ Download.incr(@rubygem_1, @version_1.full_name)
+ Download.incr(@rubygem_2, @version_3.full_name)
+ Download.incr(@rubygem_2, @version_3.full_name)
+ Download.incr(@rubygem_1, @version_2.full_name)
+
+ downloads = {
+ "#{@version_1.id}-#{2.days.ago.to_date}" => 0, "#{@version_1.id}-#{Date.yesterday}" => 5, "#{@version_1.id}-#{Time.zone.today}" => 1,
+ "#{@version_2.id}-#{2.days.ago.to_date}" => 0, "#{@version_2.id}-#{Date.yesterday}" => 1, "#{@version_2.id}-#{Time.zone.today}" => 1,
+ "#{@version_3.id}-#{2.days.ago.to_date}" => 0, "#{@version_3.id}-#{Date.yesterday}" => 1, "#{@version_3.id}-#{Time.zone.today}" => 3 }
+
+ assert_equal downloads.size, 9
+ assert_equal downloads, Download.counts_by_day_for_versions([@version_1, @version_2, @version_3], 2)
+ end
+
+ should "find counts per day for versions in range" do
+ @rubygem_1 = create(:rubygem)
+ @version_1 = create(:version, :rubygem => @rubygem_1)
+
+ Timecop.freeze(1.day.ago) do
+ create :version_history, :version => @version_1, :count => 5
+ end
+
+ Download.incr(@rubygem_1, @version_1.full_name)
+
+ start = 2.days.ago.to_date.to_s
+ fin = Time.zone.today.to_s
+
+ downloads = ActiveSupport::OrderedHash.new.tap do |d|
+ d[start] = 0
+ d["#{Date.yesterday}"] = 5
+ d[fin] = 1
+ end
+
+ assert_equal downloads, Download.counts_by_day_for_version_in_date_range(@version_1, start, fin)
+ end
+
should "find download count by gem name" do
rubygem = create(:rubygem)
version1 = create(:version, :rubygem => rubygem)
@@ -165,4 +219,58 @@ class DownloadTest < ActiveSupport::TestCase
Download.cleanup_today_keys
assert_equal 0, Download.today_keys.size
end
+
+ should "copy data from redis into SQL" do
+ rubygem = create(:rubygem)
+ version = create(:version, :rubygem => rubygem)
+
+ Download.incr rubygem.name, version.full_name
+
+ date = Time.zone.today.to_s
+
+ assert_equal nil, VersionHistory.for(version, date)
+
+ Download.copy_to_sql version, date
+
+ assert_equal 1, VersionHistory.for(version, date).count
+
+ Download.incr rubygem.name, version.full_name
+
+ Download.copy_to_sql version, date
+
+ assert_equal 2, VersionHistory.for(version, date).count
+ end
+
+ should "copy all be the last 2 days into SQL" do
+ rubygem = create(:rubygem)
+ version = create(:version, :rubygem => rubygem)
+
+ 10.times do |n|
+ Timecop.freeze(n.days.ago) do
+ 3.times { Download.incr(rubygem.name, version.full_name) }
+ end
+ end
+
+ Download.migrate_to_sql version
+
+ assert_equal [1.day.ago.to_date.to_s, Time.zone.today.to_s].sort,
+ $redis.hkeys(Download.history_key(version)).sort
+ end
+
+ should "migrate all keys in redis" do
+ rubygem = create(:rubygem)
+ version = create(:version, :rubygem => rubygem)
+
+ 10.times do |n|
+ Timecop.freeze(n.days.ago) do
+ 3.times { Download.incr(rubygem.name, version.full_name) }
+ end
+ end
+
+ assert_equal 1, Download.migrate_all_to_sql
+
+ assert_equal [1.day.ago.to_date.to_s, Time.zone.today.to_s].sort,
+ $redis.hkeys(Download.history_key(version)).sort
+
+ end
end

0 comments on commit 71235f1

Please sign in to comment.