Add hashdir support

To support older filesystems (EG: AFS), we need to add support for
directory hashes, to deal with directory size limits.

Disabled by default to maintain backwards compatibility, when enabled
filenames are hashed into md5sums, and directories created.

Signed-off-by: Paul Belanger <pabelanger@redhat.com>

lib/rubygems/mirror.rb

@@ -14,8 +14,8 @@ class Gem::Mirror
 
   RUBY = 'ruby'
 
-  def initialize(from = DEFAULT_URI, to = DEFAULT_TO, parallelism = nil, retries = nil, skiperror = nil)
-    @from, @to = from, to
+  def initialize(from = DEFAULT_URI, to = DEFAULT_TO, parallelism = nil, retries = nil, skiperror = nil, hashdir = false)
+    @from, @to, @hashdir = from, to, hashdir
     @fetcher = Fetcher.new :retries => retries, :skiperror => skiperror
     @pool = Pool.new(parallelism || 10)
   end
@@ -24,27 +24,33 @@ def from(*args)
     File.join(@from, *args)
   end
 
+  def hash(path)
+    return File.join(path[0], path[0,2], path)
+  end
+
   def to(*args)
     File.join(@to, *args)
   end
 
   def update_specs
     SPECS_FILES.each do |sf|
       sfz = "#{sf}.gz"
-
       specz = to(sfz)
+      path = to(sf)
+
       @fetcher.fetch(from(sfz), specz)
-      open(to(sf), 'wb') { |f| f << Gem.gunzip(File.read(specz)) }
+      open(path, 'wb') { |f| f << Gem.gunzip(File.read(specz)) }
     end
   end
 
   def gems
     gems = []
 
     SPECS_FILES.each do |sf|
-      update_specs unless File.exist?(to(sf))
+      path = to(sf)
+      update_specs unless File.exist?(path)
 
-      gems += Marshal.load(File.read(to(sf)))
+      gems += Marshal.load(File.read(path))
     end
 
     gems.map! do |name, ver, plat|
@@ -55,11 +61,19 @@ def gems
   end
 
   def existing_gems
-    Dir[to('gems', '*.gem')].entries.map { |f| File.basename(f) }
+    path = to('gems', '*.gem')
+    if @hashdir
+        path = to('gems', '**', '*.gem')
+    end
+    Dir[path].entries.map { |f| File.basename(f) }
   end
 
   def existing_gemspecs
-    Dir[to("quick/Marshal.#{Gem.marshal_version}", '*.rz')].entries.map { |f| File.basename(f) }
+    path = to("quick/Marshal.#{Gem.marshal_version}", '*.rz')
+    if @hashdir
+        path = to("quick/Marshal.#{Gem.marshal_version}", '**', '*.rz')
+    end
+    Dir[path].entries.map { |f| File.basename(f) }
   end
 
   def gems_to_fetch
@@ -77,14 +91,22 @@ def gems_to_delete
   def update_gems
     gems_to_fetch.each do |g|
       @pool.job do
-        @fetcher.fetch(from('gems', g), to('gems', g))
+        path = to('gems', g)
+        if @hashdir
+            path = to('gems', hash(g))
+        end
+        @fetcher.fetch(from('gems', g), path)
         yield if block_given?
       end
     end
 
     gemspecs_to_fetch.each do |g_spec|
       @pool.job do
-        @fetcher.fetch(from("quick/Marshal.#{Gem.marshal_version}", g_spec), to("quick/Marshal.#{Gem.marshal_version}", g_spec))
+        path = to("quick/Marshal.#{Gem.marshal_version}", g_spec)
+        if @hashdir
+            path = to("quick/Marshal.#{Gem.marshal_version}", hash(g_spec))
+        end
+        @fetcher.fetch(from("quick/Marshal.#{Gem.marshal_version}", g_spec), path)
         yield if block_given?
       end
     end

lib/rubygems/mirror/command.rb

@@ -22,6 +22,8 @@ def description # :nodoc:
     retries: 3                    # retry 3 times if fail to download a gem, optional, def is 1. (no retry)
     delete: false                 # whether delete gems (if remote ones are removed),optional, default is false. 
     skiperror: true               # whether skip error, optional, def is true. will stop at error if set this to false.
+    hashdir: false                # store files by directory hashes, meaning that they can reside on a filesystem
+                                  # with directory size limits. Default is false.
 
 Multiple sources and destinations may be specified.
     EOF
@@ -45,12 +47,13 @@ def execute
       parallelism = mir['parallelism']
       retries = mir['retries'] || 1
       skiperror = mir['skiperror']
+      hashdir = mir['hashdir'] || false
       delete = mir['delete']
 
       raise "Directory not found: #{save_to}" unless File.exist? save_to
       raise "Not a directory: #{save_to}" unless File.directory? save_to
 
-      mirror = Gem::Mirror.new(get_from, save_to, parallelism, retries, skiperror)
+      mirror = Gem::Mirror.new(get_from, save_to, parallelism, retries, skiperror, hashdir)
 
       Gem::Mirror::SPECS_FILES.each do |sf|
         say "Fetching: #{mirror.from(sf)}"