Add --jobs (parallel git fetch)

docs/shards.adoc

@@ -58,7 +58,7 @@ Exit status:
 *init*::
 Initializes a default _shard.yml_ in the current folder.
 
-*install* [--frozen] [--without-development] [--production] [--skip-postinstall] [--skip-executables]::
+*install* [--frozen] [--without-development] [--production] [--skip-postinstall] [--skip-executables] [--parallel-fetch=N]::
 Resolves and installs dependencies into the _lib_ folder. If not already
 present, generates a _shard.lock_ file from resolved dependencies, locking
 version numbers or Git commits.
@@ -74,6 +74,7 @@ doesn't generate a conflict, thus generating a new _shard.lock_ file.
 --production:: same as _--frozen_ and _--without-development_
 --skip-postinstall:: Does not run postinstall of dependencies.
 --skip-executables:: Does not install executables.
+--parallel-fetch:: Number of git fetches to perform in parallel (default: 8).
 --
 
 *list* [--tree]::

src/cli.cr

@@ -47,6 +47,7 @@ module Shards
         self.skip_executables = true
       end
       opts.on("--local", "Don't update remote repositories, use the local cache only.") { self.local = true }
+      opts.on("--parallel-fetch=N", "Number of git fetches to perform in parallel (default: 8)") { |n| self.parallel_fetch = n.to_i }
       # TODO: remove in the future
       opts.on("--ignore-crystal-version", "Has no effect. Kept for compatibility, to be removed in the future.") { }
       opts.on("-v", "--verbose", "Increase the log verbosity, printing all debug statements.") { self.set_debug_log_level }

src/config.cr

@@ -107,4 +107,7 @@ module Shards
   class_property? local = false
   class_property? skip_postinstall = false
   class_property? skip_executables = false
+
+  # Defaults to 1 (disabled) when running in `crystal spec`
+  class_property parallel_fetch : Int32 = {{ @top_level.has_constant?("Spec") ? 1 : 8 }}
 end

src/molinillo_solver.cr

@@ -47,7 +47,38 @@ module Shards
       end
     end
 
+    private def prefetch_local_caches(lock_index, deps)
+      count = 0
+      active = 0
+      ch = Channel(Exception?).new(deps.size + 1)
+      deps.each do |dep|
+        next unless lock = lock_index[dep.name]?
+        next unless dep.matches?(lock.version)
+        count += 1
+        active += 1
+        while active > Shards.parallel_fetch
+          sleep 0.1
+        end
+        spawn do
+          begin
+            dep.resolver.update_local_cache
+            ch.send(nil)
+            active -= 1
+          rescue ex : Exception
+            ch.send(ex)
+          end
+        end
+      end
+
+      count.times do
+        obj = ch.receive
+        raise obj if obj.is_a? Exception
+      end
+    end
+
     private def add_lock(base, lock_index, deps : Array(Dependency))
+      prefetch_local_caches(lock_index, deps) if Shards.parallel_fetch > 1
+
       deps.each do |dep|
         if lock = lock_index[dep.name]?
           next unless dep.matches?(lock.version)

src/resolvers/crystal.cr

@@ -21,5 +21,8 @@ module Shards
     def report_version(version : Version) : String
       version.value
     end
+
+    def update_local_cache
+    end
   end
 end

src/resolvers/git.cr

@@ -283,7 +283,7 @@ module Shards
       end
     end
 
-    private def update_local_cache
+    def update_local_cache
       if cloned_repository? && origin_changed?
         delete_repository
         @updated_cache = false
@@ -317,7 +317,7 @@ module Shards
       # This configuration can be overriden by defining the environment
       # variable `GIT_ASKPASS`.
       git_retry(err: "Failed to clone #{git_url}") do
-        run_in_current_folder "git clone -c core.askPass=true --mirror --quiet -- #{Process.quote(git_url)} #{Process.quote(local_path)}"
+        run_in_folder "git clone -c core.askPass=true --mirror --quiet -- #{Process.quote(git_url)} #{Process.quote(local_path)}"
       end
     end
 
@@ -411,12 +411,12 @@ module Shards
         dependency_name = File.basename(path, ".git")
         raise Error.new("Missing repository cache for #{dependency_name.inspect}. Please run without --local to fetch it.")
       end
-      Dir.cd(path) do
-        run_in_current_folder(command, capture)
-      end
+      run_in_folder(command, path, capture)
     end
 
-    private def run_in_current_folder(command, capture = false)
+    # Chdir to a folder and run command.
+    # Runs in current folder if `path` is nil.
+    private def run_in_folder(command, path : String? = nil, capture = false)
       unless GitResolver.has_git_command?
         raise Error.new("Error missing git command line tool. Please install Git first!")
       end
@@ -425,7 +425,11 @@ module Shards
 
       output = capture ? IO::Memory.new : Process::Redirect::Close
       error = IO::Memory.new
-      status = Process.run(command, shell: true, output: output, error: error)
+      if path
+        status = Process.run(command, shell: true, output: output, error: error, chdir: path)
+      else
+        status = Process.run(command, shell: true, output: output, error: error)
+      end
 
       if status.success?
         output.to_s if capture

src/resolvers/hg.cr

@@ -302,7 +302,7 @@ module Shards
       end
     end
 
-    private def update_local_cache
+    def update_local_cache
       if cloned_repository? && origin_changed?
         delete_repository
         @updated_cache = false

src/resolvers/path.cr

@@ -45,6 +45,9 @@ module Shards
       "#{version.value} at #{source}"
     end
 
+    def update_local_cache
+    end
+
     register_resolver "path", PathResolver
   end
 end

src/resolvers/resolver.cr

@@ -87,6 +87,7 @@ module Shards
     abstract def read_spec(version : Version) : String?
     abstract def install_sources(version : Version, install_dir : String)
     abstract def report_version(version : Version) : String
+    abstract def update_local_cache
 
     def parse_requirement(params : Hash(String, String)) : Requirement
       if version = params["version"]?