Permalink
Browse files

Move resolver queries into redis

Signed-off-by: Nick Quaranto <nick@quaran.to>
  • Loading branch information...
1 parent 948d4d5 commit a316d254c64c0e03d35d08639b2bad4537858b85 Matt Mongeau committed with qrush Aug 17, 2010
@@ -32,7 +32,7 @@ def yank
render :json => "The version #{params[:version]} has already been yanked.", :status => :unprocessable_entity
end
end
-
+
def unyank
if !@version.indexed?
@version.unyank!
@@ -41,7 +41,7 @@ def unyank
render :json => "The version #{params[:version]} is already indexed.", :status => :unprocessable_entity
end
end
-
+
private
def validate_gem_and_version
if !@rubygem.hosted?
@@ -66,7 +66,7 @@ def serve_via_cf
end
get "/quick/Marshal.4.8/*.gemspec.rz" do
- if Version.here?(full_name)
+ if Version.rubygem_name_for(full_name)
content_type('application/x-deflate')
serve_via_s3
else
@@ -78,7 +78,7 @@ def serve_via_cf
if Rails.env.maintenance?
serve_via_cf
else
- if name = Version.here?(full_name)
+ if name = Version.rubygem_name_for(full_name)
Download.incr(name, full_name)
serve_via_cf
View
@@ -5,6 +5,8 @@ class Dependency < ActiveRecord::Base
before_validation :use_gem_dependency,
:use_existing_rubygem,
:parse_gem_dependency
+ after_create :push_on_to_list
+
validates_presence_of :requirements
validates_inclusion_of :scope, :in => %w( development runtime )
@@ -36,16 +38,26 @@ def to_a
[name, requirements]
end
+ def to_s
+ "#{name} #{requirements}"
+ end
+
+ def self.runtime_key(full_name)
+ "rd:#{full_name}"
+ end
+
# rails,rack,bundler
def self.for(gem_list)
gem_list.split(',').map do |rubygem_name|
- rubygem = Rubygem.find_by_name(rubygem_name)
- rubygem.versions.includes(:dependencies).map do |version|
+ versions = $redis.lrange(Rubygem.versions_key(rubygem_name), 0, -1)
+ versions.map do |version|
+ info = $redis.hgetall(Version.info_key(version))
+ deps = $redis.lrange(Dependency.runtime_key(version), 0, -1)
{
- :name => rubygem.name,
- :number => version.number,
- :platform => version.platform,
- :dependencies => version.dependencies.runtime.map(&:to_a)
+ :name => info["name"],
+ :number => info["number"],
+ :platform => info["platform"],
+ :dependencies => deps.map { |dep| dep.split(" ", 2) }
}
end
end.flatten
@@ -73,4 +85,8 @@ def parse_gem_dependency
self.requirements = gem_dependency.requirements_list.join(', ')
self.scope = gem_dependency.type.to_s
end
+
+ def push_on_to_list
+ $redis.lpush(Dependency.runtime_key(self.version.full_name), self.to_s)
+ end
end
View
@@ -200,4 +200,8 @@ def find_or_initialize_version_from_spec(spec)
version.rubygem = self
version
end
+
+ def self.versions_key(name)
+ "r:#{name}"
+ end
end
View
@@ -27,9 +27,9 @@ class Version < ActiveRecord::Base
scope :indexed, { :conditions => { :indexed => true }}
before_save :update_prerelease
- after_save :reorder_versions
- after_save :full_nameify!
after_validation :join_authors
+ after_create :full_nameify!
+ after_save :reorder_versions
validates_format_of :number, :with => /\A#{Gem::Version::VERSION_PATTERN}\z/
validate :platform_and_number_are_unique, :on => :create
@@ -84,8 +84,12 @@ def self.platforms
select('platform').map(&:platform).uniq
end
- def self.here?(full_name)
- $redis.get("versions:#{full_name}")
+ def self.rubygem_name_for(full_name)
+ $redis.hget(info_key(full_name), :name)
+ end
+
+ def self.info_key(full_name)
+ "v:#{full_name}"
end
def platformed?
@@ -104,7 +108,7 @@ def reorder_versions
def yank!
update_attributes!(:indexed => false)
end
-
+
def unyank!
update_attributes!(:indexed => true)
end
@@ -153,11 +157,17 @@ def full_nameify!
self.full_name = "#{rubygem.name}-#{number}"
self.full_name << "-#{platform}" if platformed?
- $redis.set("versions:#{full_name}", rubygem.name)
-
Version.update_all({:full_name => full_name}, {:id => id})
+
+ $redis.hmset(Version.info_key(full_name),
+ :name, rubygem.name,
+ :number, number,
+ :platform, platform)
+
+ $redis.lpush(Rubygem.versions_key(rubygem.name), full_name)
end
+
def slug
full_name.gsub(/^#{rubygem.name}-/, '')
end
@@ -0,0 +1,32 @@
+class AddVersionHashAndDepsListToRedis < ActiveRecord::Migration
+ def self.up
+ $redis.keys('versions:*').each do |key|
+ $redis.del(key)
+ end
+
+ count = Version.count
+ progress = 0
+ Version.indexed.with_deps.find_each do |version|
+ puts "#{progress += 1}/#{count}"
+ next if version.rubygem.blank?
+
+ $redis.hmset(Version.info_key(version.full_name),
+ :name, version.rubygem.name,
+ :number, version.number,
+ :platform, version.platform)
+
+ runtime_key = Dependency.runtime_key(version.full_name)
+ version.dependencies.each do |dependency|
+ $redis.lpush runtime_key, dependency if dependency.scope == "runtime"
+ end
+
+ $redis.lpush Rubygem.versions_key(version.rubygem.name), version.full_name
+ end
+ end
+
+ def self.down
+ [$redis.keys('v:*') + $redis.keys('rd:*') + $redis.keys('dd:*')].flatten.each do |key|
+ $redis.del(key)
+ end
+ end
+end
View
@@ -112,9 +112,9 @@
t.boolean "email_reset"
end
+ add_index "users", ["confirmation_token", "id"], :name => "index_users_on_id_and_confirmation_token"
add_index "users", ["email"], :name => "index_users_on_email"
add_index "users", ["handle"], :name => "index_users_on_handle"
- add_index "users", ["id", "confirmation_token"], :name => "index_users_on_id_and_confirmation_token"
add_index "users", ["id", "token"], :name => "index_users_on_id_and_token"
add_index "users", ["remember_token"], :name => "index_users_on_remember_token"
add_index "users", ["token"], :name => "index_users_on_token"
@@ -6,7 +6,8 @@ class DependencyTest < ActiveSupport::TestCase
context "with dependency" do
setup do
- @dependency = Factory.build(:dependency)
+ @version = Factory(:version)
+ @dependency = Factory.build(:dependency, :version => @version)
end
should "be valid with factory" do
@@ -31,6 +32,12 @@ class DependencyTest < ActiveSupport::TestCase
assert_equal @dependency.rubygem.name, xml.at_css("name").content
assert_equal @dependency.requirements, xml.at_css("requirements").content
end
+
+ should "be pushed onto a redis list if a runtime dependency" do
+ @dependency.save
+
+ assert_equal "#{@dependency.name} #{@dependency.requirements}", $redis.lindex(Dependency.runtime_key(@version.full_name), 0)
+ end
end
context "with a Gem::Dependency" do
@@ -39,7 +46,7 @@ class DependencyTest < ActiveSupport::TestCase
@rubygem = Factory(:rubygem)
@requirements = ['>= 0.0.0']
@gem_dependency = Gem::Dependency.new(@rubygem.name, @requirements)
- @dependency = Dependency.create!(:gem_dependency => @gem_dependency)
+ @dependency = Factory(:dependency, :rubygem => @rubygem, :gem_dependency => @gem_dependency)
end
should "create a Dependency referring to the existing Rubygem" do
@@ -53,7 +60,7 @@ class DependencyTest < ActiveSupport::TestCase
@rubygem = Factory(:rubygem)
@requirements = ['>= 0.0.0', '< 1.0.0']
@gem_dependency = Gem::Dependency.new(@rubygem.name, @requirements)
- @dependency = Dependency.create!(:gem_dependency => @gem_dependency)
+ @dependency = Factory(:dependency, :rubygem => @rubygem, :gem_dependency => @gem_dependency)
end
should "create a Dependency referring to the existing Rubygem" do
View
@@ -47,8 +47,15 @@ class VersionTest < ActiveSupport::TestCase
assert_equal @version.number, @version.slug
end
- should "map full_name to name in Redis" do
- assert_equal @version.rubygem.name, $redis.get("versions:#{@version.full_name}")
+ should "save info into redis" do
+ info = $redis.hgetall(Version.info_key(@version.full_name))
+ assert_equal @version.rubygem.name, info["name"]
+ assert_equal @version.number, info["number"]
+ assert_equal @version.platform, info["platform"]
+ end
+
+ should "add version onto redis versions list" do
+ assert_equal @version.full_name, $redis.lindex(Rubygem.versions_key(@version.rubygem.name), 0)
end
should "raise an ActiveRecord::RecordNotFound if an invalid slug is given" do
@@ -58,13 +65,13 @@ class VersionTest < ActiveSupport::TestCase
end
%w[x86_64-linux java mswin x86-mswin32-60].each do |platform|
- should "be able to deal with platform of #{platform}" do
- @version.update_attributes(:platform => platform)
- slug = "#{@version.number}-#{platform}"
+ should "be able to find with platform of #{platform}" do
+ version = Factory(:version, :platform => platform)
+ slug = "#{version.number}-#{platform}"
- assert @version.reload.platformed?
- assert_equal @version.reload, Version.find_from_slug!(@version.reload.rubygem_id, slug)
- assert_equal slug, @version.reload.slug
+ assert version.platformed?
+ assert_equal version.reload, Version.find_from_slug!(version.rubygem_id, slug)
+ assert_equal slug, version.slug
end
end

0 comments on commit a316d25

Please sign in to comment.