From dfdb464ce187bfd1ee76774a465a704239257924 Mon Sep 17 00:00:00 2001
From: Paul Belanger <pabelanger@redhat.com>
Date: Sat, 25 Mar 2017 18:15:52 -0400
Subject: [PATCH] Add hashdir support

To support older filesystems (EG: AFS), we need to add support for
directory hashes, to deal with directory size limits.

Disabled by default to maintain backwards compatibility, when enabled
filenames are hashed into md5sums, and directories created.

Signed-off-by: Paul Belanger <pabelanger@redhat.com>
---
 lib/rubygems/mirror.rb         | 42 ++++++++++++++++++++++++++--------
 lib/rubygems/mirror/command.rb |  5 +++-
 2 files changed, 36 insertions(+), 11 deletions(-)

diff --git a/lib/rubygems/mirror.rb b/lib/rubygems/mirror.rb
index 8b9e609..2d72b74 100644
--- a/lib/rubygems/mirror.rb
+++ b/lib/rubygems/mirror.rb
@@ -14,8 +14,8 @@ class Gem::Mirror
 
   RUBY = 'ruby'
 
-  def initialize(from = DEFAULT_URI, to = DEFAULT_TO, parallelism = nil, retries = nil, skiperror = nil)
-    @from, @to = from, to
+  def initialize(from = DEFAULT_URI, to = DEFAULT_TO, parallelism = nil, retries = nil, skiperror = nil, hashdir = false)
+    @from, @to, @hashdir = from, to, hashdir
     @fetcher = Fetcher.new :retries => retries, :skiperror => skiperror
     @pool = Pool.new(parallelism || 10)
   end
@@ -24,6 +24,10 @@ def from(*args)
     File.join(@from, *args)
   end
 
+  def hash(path)
+    return File.join(path[0], path[0,2], path)
+  end
+
   def to(*args)
     File.join(@to, *args)
   end
@@ -31,10 +35,11 @@ def to(*args)
   def update_specs
     SPECS_FILES.each do |sf|
       sfz = "#{sf}.gz"
-
       specz = to(sfz)
+      path = to(sf)
+
       @fetcher.fetch(from(sfz), specz)
-      open(to(sf), 'wb') { |f| f << Gem.gunzip(File.read(specz)) }
+      open(path, 'wb') { |f| f << Gem.gunzip(File.read(specz)) }
     end
   end
 
@@ -42,9 +47,10 @@ def gems
     gems = []
 
     SPECS_FILES.each do |sf|
-      update_specs unless File.exist?(to(sf))
+      path = to(sf)
+      update_specs unless File.exist?(path)
 
-      gems += Marshal.load(File.read(to(sf)))
+      gems += Marshal.load(File.read(path))
     end
 
     gems.map! do |name, ver, plat|
@@ -55,11 +61,19 @@ def gems
   end
 
   def existing_gems
-    Dir[to('gems', '*.gem')].entries.map { |f| File.basename(f) }
+    path = to('gems', '*.gem')
+    if @hashdir
+        path = to('gems', '**', '*.gem')
+    end
+    Dir[path].entries.map { |f| File.basename(f) }
   end
 
   def existing_gemspecs
-    Dir[to("quick/Marshal.#{Gem.marshal_version}", '*.rz')].entries.map { |f| File.basename(f) }
+    path = to("quick/Marshal.#{Gem.marshal_version}", '*.rz')
+    if @hashdir
+        path = to("quick/Marshal.#{Gem.marshal_version}", '**', '*.rz')
+    end
+    Dir[path].entries.map { |f| File.basename(f) }
   end
 
   def gems_to_fetch
@@ -77,14 +91,22 @@ def gems_to_delete
   def update_gems
     gems_to_fetch.each do |g|
       @pool.job do
-        @fetcher.fetch(from('gems', g), to('gems', g))
+        path = to('gems', g)
+        if @hashdir
+            path = to('gems', hash(g))
+        end
+        @fetcher.fetch(from('gems', g), path)
         yield if block_given?
       end
     end
 
     gemspecs_to_fetch.each do |g_spec|
       @pool.job do
-        @fetcher.fetch(from("quick/Marshal.#{Gem.marshal_version}", g_spec), to("quick/Marshal.#{Gem.marshal_version}", g_spec))
+        path = to("quick/Marshal.#{Gem.marshal_version}", g_spec)
+        if @hashdir
+            path = to("quick/Marshal.#{Gem.marshal_version}", hash(g_spec))
+        end
+        @fetcher.fetch(from("quick/Marshal.#{Gem.marshal_version}", g_spec), path)
         yield if block_given?
       end
     end
diff --git a/lib/rubygems/mirror/command.rb b/lib/rubygems/mirror/command.rb
index 78d72f6..3db4623 100644
--- a/lib/rubygems/mirror/command.rb
+++ b/lib/rubygems/mirror/command.rb
@@ -22,6 +22,8 @@ def description # :nodoc:
     retries: 3                    # retry 3 times if fail to download a gem, optional, def is 1. (no retry)
     delete: false                 # whether delete gems (if remote ones are removed),optional, default is false. 
     skiperror: true               # whether skip error, optional, def is true. will stop at error if set this to false.
+    hashdir: false                # store files by directory hashes, meaning that they can reside on a filesystem
+                                  # with directory size limits. Default is false.
 
 Multiple sources and destinations may be specified.
     EOF
@@ -45,12 +47,13 @@ def execute
       parallelism = mir['parallelism']
       retries = mir['retries'] || 1
       skiperror = mir['skiperror']
+      hashdir = mir['hashdir'] || false
       delete = mir['delete']
 
       raise "Directory not found: #{save_to}" unless File.exist? save_to
       raise "Not a directory: #{save_to}" unless File.directory? save_to
 
-      mirror = Gem::Mirror.new(get_from, save_to, parallelism, retries, skiperror)
+      mirror = Gem::Mirror.new(get_from, save_to, parallelism, retries, skiperror, hashdir)
       
       Gem::Mirror::SPECS_FILES.each do |sf|
         say "Fetching: #{mirror.from(sf)}"