Initial import, missing coverage and docs after big refactor

.autotest

@@ -0,0 +1,7 @@
+# -*- ruby -*-
+
+require 'autotest/restart'
+
+Autotest.add_hook :initialize do |at|
+  at.testlib = 'minitest/unit' if at.respond_to? :testlib=
+end

.gitignore

@@ -0,0 +1,2 @@
+pkg
+doc

CHANGELOG.rdoc

@@ -0,0 +1,5 @@
+=== 1.0.0 / 2010-07-07
+
+* 1 major enhancement
+
+  * Birthday!

Manifest.txt

@@ -0,0 +1,10 @@
+.autotest
+CHANGELOG.rdoc
+Manifest.txt
+README.rdoc
+Rakefile
+lib/rubygems/mirror.rb
+lib/rubygems/mirror/command.rb
+lib/rubygems_plugin.rb
+test/rubygems/mirror/test_command.rb
+test/test_gem_mirror.rb

README.rdoc

@@ -0,0 +1,53 @@
+= rubygems-mirror
+
+* {Website}[http://rubygems.org/]
+* {Documentation}[http://rubygems.rubyforge.org/rubygems-mirror/README_rdoc.html]
+* {Wiki}[http://wiki.github.com/rubygems/rubygems-mirror/]
+* {Source Code}[http://github.com/rubygems/rubygems-mirror/]
+* {Issues}[http://github.com/rubygems/rubygems-mirror/issues]
+* {Rubyforge}[http://rubyforge.org/projects/rubygems]
+
+== DESCRIPTION:
+
+FIX (describe your package)
+
+== FEATURES/PROBLEMS:
+
+* FIX (list of features or problems)
+
+== SYNOPSIS:
+
+  FIX (code sample of usage)
+
+== REQUIREMENTS:
+
+* FIX (list of requirements)
+
+== INSTALL:
+
+* gem install rubygems-mirror
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2010 James Tucker, The RubyGems Team
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Rakefile

@@ -0,0 +1,32 @@
+#!/usr/bin/env rake
+
+require 'rubygems'
+require 'hoe'
+Hoe.plugin :doofus, :git, :gemcutter
+
+Hoe.spec 'rubygems-mirror' do
+  developer('James Tucker', 'raggi@rubyforge.org')
+
+  extra_dev_deps << %w[hoe-doofus >=1.0.0]
+  extra_dev_deps << %w[hoe-git >=1.3.0]
+  extra_dev_deps << %w[hoe-gemcutter >=1.0.0]
+  extra_dev_deps << %w[builder >=2.1.2]
+  extra_deps << %w[net-http-persistent >=1.2.5]
+
+  self.extra_rdoc_files = FileList["**/*.rdoc"]
+  self.history_file     = "CHANGELOG.rdoc"
+  self.readme_file      = "README.rdoc"
+  self.rubyforge_name   = 'rubygems'
+  self.testlib          = :minitest
+end
+
+namespace :mirror do
+  desc "Run the Gem::Mirror::Command"
+  task :update do
+    $:.unshift 'lib'
+    require 'rubygems/mirror/command'
+
+    mirror = Gem::Mirror::Command.new
+    mirror.execute
+  end
+end

lib/rubygems/mirror.rb

@@ -0,0 +1,86 @@
+require 'rubygems'
+require 'fileutils'
+
+class Gem::Mirror
+  autoload :Fetcher, 'rubygems/mirror/fetcher'
+  autoload :Pool, 'rubygems/mirror/pool'
+
+  SPECS_FILE = "specs.#{Gem.marshal_version}"
+  SPECS_FILE_Z = "specs.#{Gem.marshal_version}.gz"
+
+  DEFAULT_URI = 'http://production.cf.rubygems.org/'
+  DEFAULT_TO = File.join(Gem.user_home, '.gem', 'mirror')
+
+  RUBY = 'ruby'
+
+  def initialize(from = DEFAULT_URI, to = DEFAULT_TO, parallelism = 10)
+    @from, @to = from, to
+    @fetcher = Fetcher.new
+    @pool = Pool.new(parallelism)
+  end
+
+  def from(*args)
+    File.join(@from, *args)
+  end
+
+  def to(*args)
+    File.join(@to, *args)
+  end
+
+  def update_specs
+    specz = to(SPECS_FILE_Z)
+    @fetcher.fetch(from(SPECS_FILE_Z), specz)
+    open(to(SPECS_FILE), 'wb') { |f| f << Gem.gunzip(File.read(specz)) }
+  end
+
+  def gems
+    update_specs unless File.exists?(to(SPECS_FILE))
+
+    gems = Marshal.load(File.read(to(SPECS_FILE)))
+    gems.map! do |name, ver, plat|
+      # If the platform is ruby, it is not in the gem name
+      "#{name}-#{ver}#{"-#{plat}" unless plat == RUBY}.gem"
+    end
+    gems
+  end
+
+  def existing_gems
+    Dir[to('gems', '*.gem')].entries.map { |f| File.basename(f) }
+  end
+
+  def gems_to_fetch
+    gems - existing_gems
+  end
+
+  def gems_to_delete
+    existing_gems - gems
+  end
+
+  def update_gems
+    gems_to_fetch.each do |g|
+      @pool.job do
+        @fetcher.fetch(from('gems', g), to('gems', g))
+        yield
+      end
+    end
+
+    @pool.run_til_done
+  end
+
+  def delete_gems
+    gems_to_delete.each do |g|
+      @pool.job do
+        File.delete(to('gems', g))
+        yield
+      end
+    end
+
+    @pool.run_til_done
+  end
+
+  def update
+    update_specs
+    update_gems
+    cleanup_gems
+  end
+end

lib/rubygems/mirror/command.rb

@@ -0,0 +1,63 @@
+require 'rubygems/mirror'
+require 'rubygems/command'
+require 'yaml'
+
+class Gem::Mirror::Command < Gem::Command
+
+  def initialize
+    super 'mirror', 'Mirror a gem repository'
+  end
+
+  def description # :nodoc:
+    <<-EOF
+The mirror command uses the ~/.gemmirrorrc config file to mirror remote gem
+repositories to a local path. The config file is a YAML document that looks
+like this:
+
+  ---
+  mirrors:
+    - from: http://gems.example.com # source repository URI
+      to: /path/to/mirror           # destination directory
+
+Multiple sources and destinations may be specified.
+    EOF
+  end
+
+  def execute
+    config_file = File.join Gem.user_home, '.gemmirrorrc'
+
+    raise "Config file #{config_file} not found" unless File.exist? config_file
+
+    mirrors = YAML.load_file config_file
+
+    raise "Invalid config file #{config_file}" unless mirrors.respond_to? :each
+
+    mirrors.each do |mir|
+      raise "mirror missing 'from' field" unless mir.has_key? 'from'
+      raise "mirror missing 'to' field" unless mir.has_key? 'to'
+
+      get_from = mir['from']
+      save_to = File.expand_path mir['to']
+
+      raise "Directory not found: #{save_to}" unless File.exist? save_to
+      raise "Not a directory: #{save_to}" unless File.directory? save_to
+
+      mirror = Gem::Mirror.new(get_from, save_to)
+
+      say "Fetching: #{mirror.from(Gem::Mirror::SPECS_FILE_Z)}"
+      mirror.update_specs
+
+      say "Total gems: #{mirror.gems.size}"
+
+      progress = ui.progress_reporter mirror.gems_to_fetch.size,
+                                  "Fetching #{mirror.gems_to_fetch.size} gems"
+
+      mirror.update_gems { progress.updated true }
+
+      progress = ui.progress_reporter mirror.gems_to_delete.size,
+                                 "Deleting #{mirror.gems_to_delete.size} gems"
+
+      mirror.delete_gems { progress.updated true }
+    end
+  end
+end

lib/rubygems/mirror/fetcher.rb

@@ -0,0 +1,56 @@
+require 'net/http'
+require 'net/http/persistent'
+
+class Gem::Mirror::Fetcher
+  # TODO  beef
+  class Error < StandardError; end
+
+  def initialize
+    @http = Net::HTTP::Persistent.new
+  end
+
+  # Fetch a source path under the base uri, and put it in the same or given
+  # destination path under the base path.
+  def fetch(uri, path)
+    modified_time = File.exists?(path) && File.stat(path).mtime.rfc822
+
+    req = Net::HTTP::Get.new uri
+    req.add_field 'If-Modified-Since', modified_time if modified_time
+
+    @http.request URI(uri), req do |resp|
+      return handle_response(resp, path)
+    end
+  end
+
+  # Handle an http response, follow redirects, etc. returns true if a file was
+  # downloaded, false if a 304. Raise Error on unknown responses.
+  def handle_response(resp, path)
+    case resp.code.to_i
+    when 304
+    when 302
+      fetch resp['location'], path
+    when 200
+      write_file(resp, path)
+    when 403
+      warn "403 on #{File.basename(path)}"
+    else
+      raise Error, "unexpected response #{resp.inspect}"
+    end
+    # TODO rescue http errors and reraise cleanly
+  end
+
+  # Efficiently writes an http response object to a particular path. If there
+  # is an error, it will remove the target file.
+  def write_file(resp, path)
+    FileUtils.mkdir_p File.dirname(path)
+    File.open(path, 'wb') do |output|
+      resp.read_body { |chunk| output << chunk }
+    end
+    true
+  ensure
+    # cleanup incomplete files, rescue perm errors etc, they're being
+    # raised already.
+    File.delete(path) rescue nil if $!
+  end
+
+end

lib/rubygems/mirror/pool.rb

@@ -0,0 +1,20 @@
+class Gem::Mirror::Pool
+  def initialize(size)
+    @size = size
+    @queue = Queue.new
+  end
+
+  def job(&blk)
+    @queue << blk
+  end
+
+  def run_til_done
+    threads = Array.new(@size) do
+      Thread.new { @queue.pop.call while true }
+    end
+    until @queue.empty? && @queue.num_waiting == @size
+      threads.each { |t| t.join(0.1) }
+    end
+    threads.each { |t| t.kill }
+  end
+end