Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
move indexer to its own class, kill vault
  • Loading branch information
qrush committed Nov 25, 2011
1 parent edf573b commit 6a48bcb
Show file tree
Hide file tree
Showing 17 changed files with 141 additions and 205 deletions.
102 changes: 102 additions & 0 deletions app/jobs/indexer.rb
@@ -0,0 +1,102 @@
class Indexer
def perform
log "Updating the index"
update_index
log "Finished updating the index"
end

def write_gem(body, spec)
gem_file = directory.files.create(
:body => body.string,
:key => "gems/#{spec.original_name}.gem",
:public => true
)

self.class.indexer.abbreviate spec
self.class.indexer.sanitize spec

gem_spec = directory.files.create(
:body => Gem.deflate(Marshal.dump(spec)),
:key => "quick/Marshal.4.8/#{spec.original_name}.gemspec.rz",
:public => true
)
end

def directory
fog.directories.get($rubygems_config[:s3_bucket])
end

private

def fog
$fog || Fog::Storage.new(
:provider => 'Local',
:local_root => Pusher.server_path('gems')
)
end

def stringify(value)
final = StringIO.new
gzip = Zlib::GzipWriter.new(final)
gzip.write(Marshal.dump(value))
gzip.close

final.string
end

def upload(key, value)
file = directory.files.create(
:body => stringify(value),
:key => key,
:public => true
)
end

def update_index
upload("specs.4.8.gz", specs_index)
log "Uploaded all specs index"
upload("latest_specs.4.8.gz", latest_index)
log "Uploaded latest specs index"
upload("prerelease_specs.4.8.gz", prerelease_index)
log "Uploaded prerelease specs index"
end

def minimize_specs(data)
names = Hash.new { |h,k| h[k] = k }
versions = Hash.new { |h,k| h[k] = Gem::Version.new(k) }
platforms = Hash.new { |h,k| h[k] = k }

data.each do |row|
row[0] = names[row[0]]
row[1] = versions[row[1].strip]
row[2] = platforms[row[2]]
end

data
end

def specs_index
minimize_specs Version.rows_for_index
end

def latest_index
minimize_specs Version.rows_for_latest_index
end

def prerelease_index
minimize_specs Version.rows_for_prerelease_index
end

def log(message)
Rails.logger.info "[GEMCUTTER:#{Time.now}] #{message}"
end

def self.indexer
@indexer ||=
begin
indexer = Gem::Indexer.new(Pusher.server_path, :build_legacy => false)
def indexer.say(message) end
indexer
end
end
end
4 changes: 2 additions & 2 deletions app/middleware/hostess.rb
Expand Up @@ -18,13 +18,13 @@ def serve

def serve_via_s3
serve do
redirect Vault.s3_url_for(request.path_info)
redirect "http://#{$rubygems_config[:s3_domain]}#{request.path_info}"
end
end

def serve_via_cf
serve do
redirect Vault.cf_url_for(request.path_info)
redirect "http://#{$rubygems_config[:cf_domain]}#{request.path_info}"
end
end

Expand Down
99 changes: 23 additions & 76 deletions app/models/pusher.rb
@@ -1,11 +1,10 @@
class Pusher
include Vault

attr_reader :user, :spec, :message, :code, :rubygem, :body, :version, :version_id

def initialize(user, body, host_with_port=nil)
@user = user
@body = StringIO.new(body.read)
@indexer = Indexer.new
@host_with_port = host_with_port
end

Expand All @@ -21,34 +20,14 @@ def authorize

def save
if update
write_gem
@indexer.write_gem @body, @spec
after_write
notify("Successfully registered gem: #{version.to_title}", 200)
else
notify("There was a problem saving your gem: #{rubygem.all_errors(version)}", 403)
end
end

def after_write
@version_id = version.id
Delayed::Job.enqueue self, :priority => PRIORITIES[:push]
enqueue_web_hook_jobs
end

def notify(message, code)
@message = message
@code = code
false
end

def update
rubygem.update_attributes_from_gem_specification!(version, spec)
rubygem.create_ownership(user) unless version.new_record?
true
rescue ActiveRecord::RecordInvalid, ActiveRecord::Rollback
false
end

def pull_spec
Gem::Package.open body, "r", nil do |pkg|
@spec = pkg.metadata
Expand Down Expand Up @@ -77,55 +56,36 @@ def find
end
end

def self.server_path(*more)
File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'server', *more))
end

# Overridden so we don't get megabytes of the raw data printing out
def inspect
attrs = [:@rubygem, :@user, :@message, :@code].map { |attr| "#{attr}=#{instance_variable_get(attr) || 'nil'}" }
"<Gemcutter #{attrs.join(' ')}>"
end
private

def minimize_specs(data)
names = Hash.new { |h,k| h[k] = k }
versions = Hash.new { |h,k| h[k] = Gem::Version.new(k) }
platforms = Hash.new { |h,k| h[k] = k }

data.each do |row|
row[0] = names[row[0]]
row[1] = versions[row[1].strip]
row[2] = platforms[row[2]]
end

data
end

def specs_index
minimize_specs Version.rows_for_index
def after_write
@version_id = version.id
Delayed::Job.enqueue Indexer.new, :priority => PRIORITIES[:push]
enqueue_web_hook_jobs
end

def latest_index
minimize_specs Version.rows_for_latest_index
def notify(message, code)
@message = message
@code = code
false
end

def prerelease_index
minimize_specs Version.rows_for_prerelease_index
def update
rubygem.update_attributes_from_gem_specification!(version, spec)
rubygem.create_ownership(user) unless version.new_record?
true
rescue ActiveRecord::RecordInvalid, ActiveRecord::Rollback
false
end

def perform
log "Updating the index"
update_index
log "Finished updating the index"
def self.server_path(*more)
File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'server', *more))
end

def update_index
upload("specs.4.8.gz", specs_index)
log "Uploaded all specs index"
upload("latest_specs.4.8.gz", latest_index)
log "Uploaded latest specs index"
upload("prerelease_specs.4.8.gz", prerelease_index)
log "Uploaded prerelease specs index"
# Overridden so we don't get megabytes of the raw data printing out
def inspect
attrs = [:@rubygem, :@user, :@message, :@code].map { |attr| "#{attr}=#{instance_variable_get(attr) || 'nil'}" }
"<Gemcutter #{attrs.join(' ')}>"
end

def enqueue_web_hook_jobs
Expand All @@ -134,17 +94,4 @@ def enqueue_web_hook_jobs
job.fire(@host_with_port, rubygem, version)
end
end

def self.indexer
@indexer ||=
begin
indexer = Gem::Indexer.new(server_path, :build_legacy => false)
def indexer.say(message) end
indexer
end
end

def log(message)
Rails.logger.info "[GEMCUTTER:#{Time.now}] #{message}"
end
end
54 changes: 0 additions & 54 deletions app/models/vault.rb

This file was deleted.

4 changes: 2 additions & 2 deletions config/application.rb
Expand Up @@ -34,11 +34,11 @@ class Application < Rails::Application

config.after_initialize do
Hostess.local = $rubygems_config[:local_storage]

YAML::ENGINE.yamler = "syck"
end

config.plugins = [:dynamic_form]
config.plugins << :heroku_asset_cacher if $rubygems_config[:asset_cacher]

config.autoload_paths << "./app/jobs"
end
end

0 comments on commit 6a48bcb

Please sign in to comment.