Skip to content
This repository
Browse code

Add code to migrate data from redis to SQL

  • Loading branch information...
commit 63a39a9d5ccc99d21c05bdb9c576fcb58f7b96c2 1 parent 1312f80
Evan Phoenix authored
38  app/models/download.rb
@@ -106,6 +106,44 @@ def self.counts_by_day_for_version_in_date_range(version, start, stop)
106 106
     downloads
107 107
   end
108 108
 
  109
+  def self.copy_to_sql(version, date)
  110
+    count = $redis.hget(history_key(version), date)
  111
+
  112
+    if vh = VersionHistory.for(version, date)
  113
+      vh.count = count
  114
+      vh.save
  115
+    else
  116
+      VersionHistory.make(version, date, count)
  117
+    end
  118
+  end
  119
+
  120
+  def self.migrate_to_sql(version)
  121
+    key = history_key version
  122
+
  123
+    dates = $redis.hkeys(key)
  124
+
  125
+    back = 1.days.ago.to_date
  126
+
  127
+    dates.delete_if { |e| Date.parse(e) >= back }
  128
+
  129
+    dates.each do |d|
  130
+      copy_to_sql version, d
  131
+      $redis.hdel key, d
  132
+    end
  133
+
  134
+    dates
  135
+  end
  136
+
  137
+  def self.migrate_all_to_sql
  138
+    count = 0
  139
+    Version.all.each do |ver|
  140
+      dates = migrate_to_sql ver
  141
+      count += 1 unless dates.empty?
  142
+    end
  143
+
  144
+    count
  145
+  end
  146
+
109 147
   def self.counts_by_day_for_version(version)
110 148
     counts_by_day_for_version_in_date_range(version, Time.zone.today - 89.days, Time.zone.today)
111 149
   end
10  app/models/version_history.rb
... ...
@@ -1,4 +1,14 @@
1 1
 class VersionHistory < ActiveRecord::Base
2 2
   attr_accessible :count, :day, :version_id
3 3
   belongs_to :version
  4
+
  5
+  def self.for(version, day)
  6
+    VersionHistory.where(:version_id => version.id, :day => day).first
  7
+  end
  8
+
  9
+  def self.make(version, day, count)
  10
+    VersionHistory.create(:version_id => version.id,
  11
+                          :day => day.to_s,
  12
+                          :count => count.to_i).save
  13
+  end
4 14
 end
54  test/unit/download_test.rb
@@ -219,4 +219,58 @@ class DownloadTest < ActiveSupport::TestCase
219 219
     Download.cleanup_today_keys
220 220
     assert_equal 0, Download.today_keys.size
221 221
   end
  222
+
  223
+  should "copy data from redis into SQL" do
  224
+    rubygem = create(:rubygem)
  225
+    version = create(:version, :rubygem => rubygem)
  226
+
  227
+    Download.incr rubygem.name, version.full_name
  228
+
  229
+    date = Time.zone.today.to_s
  230
+
  231
+    assert_equal nil, VersionHistory.for(version, date)
  232
+
  233
+    Download.copy_to_sql version, date
  234
+
  235
+    assert_equal 1, VersionHistory.for(version, date).count
  236
+
  237
+    Download.incr rubygem.name, version.full_name
  238
+
  239
+    Download.copy_to_sql version, date
  240
+
  241
+    assert_equal 2, VersionHistory.for(version, date).count
  242
+  end
  243
+
  244
+  should "copy all be the last 2 days into SQL" do
  245
+    rubygem = create(:rubygem)
  246
+    version = create(:version, :rubygem => rubygem)
  247
+
  248
+    10.times do |n|
  249
+      Timecop.freeze(n.days.ago) do
  250
+        3.times { Download.incr(rubygem.name, version.full_name) }
  251
+      end
  252
+    end
  253
+
  254
+    Download.migrate_to_sql version
  255
+
  256
+    assert_equal [1.day.ago.to_date.to_s, Time.zone.today.to_s].sort,
  257
+                 $redis.hkeys(Download.history_key(version)).sort
  258
+  end
  259
+
  260
+  should "migrate all keys in redis" do
  261
+    rubygem = create(:rubygem)
  262
+    version = create(:version, :rubygem => rubygem)
  263
+
  264
+    10.times do |n|
  265
+      Timecop.freeze(n.days.ago) do
  266
+        3.times { Download.incr(rubygem.name, version.full_name) }
  267
+      end
  268
+    end
  269
+
  270
+    assert_equal 1, Download.migrate_all_to_sql
  271
+
  272
+    assert_equal [1.day.ago.to_date.to_s, Time.zone.today.to_s].sort,
  273
+                 $redis.hkeys(Download.history_key(version)).sort
  274
+
  275
+  end
222 276
 end

0 notes on commit 63a39a9

Please sign in to comment.
Something went wrong with that request. Please try again.