Permalink
Browse files

Initial import, missing coverage and docs after big refactor

  • Loading branch information...
0 parents commit 64e7b62c55d4a1d62e84fabec8b8ea303eaf123b @raggi raggi committed Aug 5, 2010
@@ -0,0 +1,7 @@
+# -*- ruby -*-
+
+require 'autotest/restart'
+
+Autotest.add_hook :initialize do |at|
+ at.testlib = 'minitest/unit' if at.respond_to? :testlib=
+end
@@ -0,0 +1,2 @@
+pkg
+doc
@@ -0,0 +1,5 @@
+=== 1.0.0 / 2010-07-07
+
+* 1 major enhancement
+
+ * Birthday!
@@ -0,0 +1,10 @@
+.autotest
+CHANGELOG.rdoc
+Manifest.txt
+README.rdoc
+Rakefile
+lib/rubygems/mirror.rb
+lib/rubygems/mirror/command.rb
+lib/rubygems_plugin.rb
+test/rubygems/mirror/test_command.rb
+test/test_gem_mirror.rb
@@ -0,0 +1,53 @@
+= rubygems-mirror
+
+* {Website}[http://rubygems.org/]
+* {Documentation}[http://rubygems.rubyforge.org/rubygems-mirror/README_rdoc.html]
+* {Wiki}[http://wiki.github.com/rubygems/rubygems-mirror/]
+* {Source Code}[http://github.com/rubygems/rubygems-mirror/]
+* {Issues}[http://github.com/rubygems/rubygems-mirror/issues]
+* {Rubyforge}[http://rubyforge.org/projects/rubygems]
+
+== DESCRIPTION:
+
+FIX (describe your package)
+
+== FEATURES/PROBLEMS:
+
+* FIX (list of features or problems)
+
+== SYNOPSIS:
+
+ FIX (code sample of usage)
+
+== REQUIREMENTS:
+
+* FIX (list of requirements)
+
+== INSTALL:
+
+* gem install rubygems-mirror
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2010 James Tucker, The RubyGems Team
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,32 @@
+#!/usr/bin/env rake
+
+require 'rubygems'
+require 'hoe'
+Hoe.plugin :doofus, :git, :gemcutter
+
+Hoe.spec 'rubygems-mirror' do
+ developer('James Tucker', 'raggi@rubyforge.org')
+
+ extra_dev_deps << %w[hoe-doofus >=1.0.0]
+ extra_dev_deps << %w[hoe-git >=1.3.0]
+ extra_dev_deps << %w[hoe-gemcutter >=1.0.0]
+ extra_dev_deps << %w[builder >=2.1.2]
+ extra_deps << %w[net-http-persistent >=1.2.5]
+
+ self.extra_rdoc_files = FileList["**/*.rdoc"]
+ self.history_file = "CHANGELOG.rdoc"
+ self.readme_file = "README.rdoc"
+ self.rubyforge_name = 'rubygems'
+ self.testlib = :minitest
+end
+
+namespace :mirror do
+ desc "Run the Gem::Mirror::Command"
+ task :update do
+ $:.unshift 'lib'
+ require 'rubygems/mirror/command'
+
+ mirror = Gem::Mirror::Command.new
+ mirror.execute
+ end
+end
@@ -0,0 +1,86 @@
+require 'rubygems'
+require 'fileutils'
+
+class Gem::Mirror
+ autoload :Fetcher, 'rubygems/mirror/fetcher'
+ autoload :Pool, 'rubygems/mirror/pool'
+
+ SPECS_FILE = "specs.#{Gem.marshal_version}"
+ SPECS_FILE_Z = "specs.#{Gem.marshal_version}.gz"
+
+ DEFAULT_URI = 'http://production.cf.rubygems.org/'
+ DEFAULT_TO = File.join(Gem.user_home, '.gem', 'mirror')
+
+ RUBY = 'ruby'
+
+ def initialize(from = DEFAULT_URI, to = DEFAULT_TO, parallelism = 10)
+ @from, @to = from, to
+ @fetcher = Fetcher.new
+ @pool = Pool.new(parallelism)
+ end
+
+ def from(*args)
+ File.join(@from, *args)
+ end
+
+ def to(*args)
+ File.join(@to, *args)
+ end
+
+ def update_specs
+ specz = to(SPECS_FILE_Z)
+ @fetcher.fetch(from(SPECS_FILE_Z), specz)
+ open(to(SPECS_FILE), 'wb') { |f| f << Gem.gunzip(File.read(specz)) }
+ end
+
+ def gems
+ update_specs unless File.exists?(to(SPECS_FILE))
+
+ gems = Marshal.load(File.read(to(SPECS_FILE)))
+ gems.map! do |name, ver, plat|
+ # If the platform is ruby, it is not in the gem name
+ "#{name}-#{ver}#{"-#{plat}" unless plat == RUBY}.gem"
+ end
+ gems
+ end
+
+ def existing_gems
+ Dir[to('gems', '*.gem')].entries.map { |f| File.basename(f) }
+ end
+
+ def gems_to_fetch
+ gems - existing_gems
+ end
+
+ def gems_to_delete
+ existing_gems - gems
+ end
+
+ def update_gems
+ gems_to_fetch.each do |g|
+ @pool.job do
+ @fetcher.fetch(from('gems', g), to('gems', g))
+ yield
+ end
+ end
+
+ @pool.run_til_done
+ end
+
+ def delete_gems
+ gems_to_delete.each do |g|
+ @pool.job do
+ File.delete(to('gems', g))
+ yield
+ end
+ end
+
+ @pool.run_til_done
+ end
+
+ def update
+ update_specs
+ update_gems
+ cleanup_gems
+ end
+end
@@ -0,0 +1,63 @@
+require 'rubygems/mirror'
+require 'rubygems/command'
+require 'yaml'
+
+class Gem::Mirror::Command < Gem::Command
+
+ def initialize
+ super 'mirror', 'Mirror a gem repository'
+ end
+
+ def description # :nodoc:
+ <<-EOF
+The mirror command uses the ~/.gemmirrorrc config file to mirror remote gem
+repositories to a local path. The config file is a YAML document that looks
+like this:
+
+ ---
+ mirrors:
+ - from: http://gems.example.com # source repository URI
+ to: /path/to/mirror # destination directory
+
+Multiple sources and destinations may be specified.
+ EOF
+ end
+
+ def execute
+ config_file = File.join Gem.user_home, '.gemmirrorrc'
+
+ raise "Config file #{config_file} not found" unless File.exist? config_file
+
+ mirrors = YAML.load_file config_file
+
+ raise "Invalid config file #{config_file}" unless mirrors.respond_to? :each
+
+ mirrors.each do |mir|
+ raise "mirror missing 'from' field" unless mir.has_key? 'from'
+ raise "mirror missing 'to' field" unless mir.has_key? 'to'
+
+ get_from = mir['from']
+ save_to = File.expand_path mir['to']
+
+ raise "Directory not found: #{save_to}" unless File.exist? save_to
+ raise "Not a directory: #{save_to}" unless File.directory? save_to
+
+ mirror = Gem::Mirror.new(get_from, save_to)
+
+ say "Fetching: #{mirror.from(Gem::Mirror::SPECS_FILE_Z)}"
+ mirror.update_specs
+
+ say "Total gems: #{mirror.gems.size}"
+
+ progress = ui.progress_reporter mirror.gems_to_fetch.size,
+ "Fetching #{mirror.gems_to_fetch.size} gems"
+
+ mirror.update_gems { progress.updated true }
+
+ progress = ui.progress_reporter mirror.gems_to_delete.size,
+ "Deleting #{mirror.gems_to_delete.size} gems"
+
+ mirror.delete_gems { progress.updated true }
+ end
+ end
+end
@@ -0,0 +1,56 @@
+require 'net/http'
+require 'net/http/persistent'
+
+class Gem::Mirror::Fetcher
+ # TODO beef
+ class Error < StandardError; end
+
+ def initialize
+ @http = Net::HTTP::Persistent.new
+ end
+
+ # Fetch a source path under the base uri, and put it in the same or given
+ # destination path under the base path.
+ def fetch(uri, path)
+ modified_time = File.exists?(path) && File.stat(path).mtime.rfc822
+
+ req = Net::HTTP::Get.new uri
+ req.add_field 'If-Modified-Since', modified_time if modified_time
+
+ @http.request URI(uri), req do |resp|
+ return handle_response(resp, path)
+ end
+ end
+
+ # Handle an http response, follow redirects, etc. returns true if a file was
+ # downloaded, false if a 304. Raise Error on unknown responses.
+ def handle_response(resp, path)
+ case resp.code.to_i
+ when 304
+ when 302
+ fetch resp['location'], path
+ when 200
+ write_file(resp, path)
+ when 403
+ warn "403 on #{File.basename(path)}"
+ else
+ raise Error, "unexpected response #{resp.inspect}"
+ end
+ # TODO rescue http errors and reraise cleanly
+ end
+
+ # Efficiently writes an http response object to a particular path. If there
+ # is an error, it will remove the target file.
+ def write_file(resp, path)
+ FileUtils.mkdir_p File.dirname(path)
+ File.open(path, 'wb') do |output|
+ resp.read_body { |chunk| output << chunk }
+ end
+ true
+ ensure
+ # cleanup incomplete files, rescue perm errors etc, they're being
+ # raised already.
+ File.delete(path) rescue nil if $!
+ end
+
+end
@@ -0,0 +1,20 @@
+class Gem::Mirror::Pool
+ def initialize(size)
+ @size = size
+ @queue = Queue.new
+ end
+
+ def job(&blk)
+ @queue << blk
+ end
+
+ def run_til_done
+ threads = Array.new(@size) do
+ Thread.new { @queue.pop.call while true }
@deepak

deepak Jan 14, 2011

why do we need 'while true'
if i remove this then the program goes in a loops and i have to manually do a ctrl+c

@raggi

raggi Jan 15, 2011

Collaborator

um, i don't think you understand the purpose of this code.

+ end
+ until @queue.empty? && @queue.num_waiting == @size
@deepak

deepak Jan 14, 2011

i checked the Queue#num_waiting source, and Thread.current is pushed into waiting only when Queue#@que is empty
i tried in https://gist.github.com/779503 (line 35) and always push the same number of items (line 34). when will pop be empty? the last element?

+ threads.each { |t| t.join(0.1) }
@deepak

deepak Jan 14, 2011

why have a timeout on join and then kill it afterwards

@raggi

raggi Jan 15, 2011

Collaborator

as above, you don't understand this code. each of your questions might be valid except that when combined with the other lines the code makes sense.

the intent here (and it works) is to keep the threads running until the queue is empty, then kill them all.

joining onto the threads regularly will make any errors in the threads bubble into the main thread.

+ end
+ threads.each { |t| t.kill }
+ end
+end
Oops, something went wrong.

0 comments on commit 64e7b62

Please sign in to comment.