Permalink
Browse files

Add code to migrate data from redis to SQL

  • Loading branch information...
1 parent 1312f80 commit 63a39a9d5ccc99d21c05bdb9c576fcb58f7b96c2 @evanphx evanphx committed Sep 16, 2012
Showing with 102 additions and 0 deletions.
  1. +38 −0 app/models/download.rb
  2. +10 −0 app/models/version_history.rb
  3. +54 −0 test/unit/download_test.rb
View
@@ -106,6 +106,44 @@ def self.counts_by_day_for_version_in_date_range(version, start, stop)
downloads
end
+ def self.copy_to_sql(version, date)
+ count = $redis.hget(history_key(version), date)
+
+ if vh = VersionHistory.for(version, date)
+ vh.count = count
+ vh.save
+ else
+ VersionHistory.make(version, date, count)
+ end
+ end
+
+ def self.migrate_to_sql(version)
+ key = history_key version
+
+ dates = $redis.hkeys(key)
+
+ back = 1.days.ago.to_date
+
+ dates.delete_if { |e| Date.parse(e) >= back }
+
+ dates.each do |d|
+ copy_to_sql version, d
+ $redis.hdel key, d
+ end
+
+ dates
+ end
+
+ def self.migrate_all_to_sql
+ count = 0
+ Version.all.each do |ver|
+ dates = migrate_to_sql ver
+ count += 1 unless dates.empty?
+ end
+
+ count
+ end
+
def self.counts_by_day_for_version(version)
counts_by_day_for_version_in_date_range(version, Time.zone.today - 89.days, Time.zone.today)
end
@@ -1,4 +1,14 @@
class VersionHistory < ActiveRecord::Base
attr_accessible :count, :day, :version_id
belongs_to :version
+
+ def self.for(version, day)
+ VersionHistory.where(:version_id => version.id, :day => day).first
+ end
+
+ def self.make(version, day, count)
+ VersionHistory.create(:version_id => version.id,
+ :day => day.to_s,
+ :count => count.to_i).save
+ end
end
View
@@ -219,4 +219,58 @@ class DownloadTest < ActiveSupport::TestCase
Download.cleanup_today_keys
assert_equal 0, Download.today_keys.size
end
+
+ should "copy data from redis into SQL" do
+ rubygem = create(:rubygem)
+ version = create(:version, :rubygem => rubygem)
+
+ Download.incr rubygem.name, version.full_name
+
+ date = Time.zone.today.to_s
+
+ assert_equal nil, VersionHistory.for(version, date)
+
+ Download.copy_to_sql version, date
+
+ assert_equal 1, VersionHistory.for(version, date).count
+
+ Download.incr rubygem.name, version.full_name
+
+ Download.copy_to_sql version, date
+
+ assert_equal 2, VersionHistory.for(version, date).count
+ end
+
+ should "copy all be the last 2 days into SQL" do
+ rubygem = create(:rubygem)
+ version = create(:version, :rubygem => rubygem)
+
+ 10.times do |n|
+ Timecop.freeze(n.days.ago) do
+ 3.times { Download.incr(rubygem.name, version.full_name) }
+ end
+ end
+
+ Download.migrate_to_sql version
+
+ assert_equal [1.day.ago.to_date.to_s, Time.zone.today.to_s].sort,
+ $redis.hkeys(Download.history_key(version)).sort
+ end
+
+ should "migrate all keys in redis" do
+ rubygem = create(:rubygem)
+ version = create(:version, :rubygem => rubygem)
+
+ 10.times do |n|
+ Timecop.freeze(n.days.ago) do
+ 3.times { Download.incr(rubygem.name, version.full_name) }
+ end
+ end
+
+ assert_equal 1, Download.migrate_all_to_sql
+
+ assert_equal [1.day.ago.to_date.to_s, Time.zone.today.to_s].sort,
+ $redis.hkeys(Download.history_key(version)).sort
+
+ end
end

0 comments on commit 63a39a9

Please sign in to comment.