(BOLT-1589) Support installing any git-based module

This updates Bolt to support installing git-based modules from any
source. Now, Bolt will attempt to download metadata directly from GitHub
or GitLab for public repositories and fall back to cloning the git
repo's metadata if that fails.

This also updates the spec resolver to no longer calculate a SHA based
on the ref provided in project configuration. Previously, Bolt would
calculate a SHA and write it to the Puppetfile to pin the module to a
specific commit when modules are installed. However, doing this is
difficult and time-consuming with the move to supporting any git-based
module.

!feature

* **Support installing any git-based module**
  ([puppetlabs#3109](puppetlabs#3109))

  Bolt now supports installing any git-based module. Previously, Bolt
  only supported installing and resolving dependencies for modules
  hosted in a public GitHub repository. Now, Bolt will check both GitHub
  and GitLab before falling back to cloning a module's metadata using
  the `git` executable. This allows users to install modules from
  private repositories, or from locations other than GitHub.

bolt.gemspec

@@ -57,7 +57,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "net-ssh-krb", "~> 0.5"
   spec.add_dependency "orchestrator_client", "~> 0.5"
   spec.add_dependency "puppet", ">= 6.18.0"
-  spec.add_dependency "puppetfile-resolver", "~> 0.5"
+  spec.add_dependency "puppetfile-resolver", ">= 0.6.2", "< 1.0"
   spec.add_dependency "puppet-resource_api", ">= 1.8.1"
   spec.add_dependency "puppet-strings", "~> 2.3"
   spec.add_dependency "r10k", "~> 3.10"

lib/bolt/module_installer/resolver.rb

@@ -88,7 +88,7 @@ def resolve(specs, config = {})
             modules << Bolt::ModuleInstaller::Puppetfile::GitModule.new(
               spec.name,
               spec.git,
-              spec.sha
+              spec.ref
             )
           end
         end

lib/bolt/module_installer/specs/git_spec.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
 
 require 'json'
+require 'net/http'
+require 'open3'
 require 'set'
 
 require_relative '../../../bolt/error'
+require_relative '../../../bolt/logger'
 
 # This class represents a Git module specification.
 #
@@ -18,21 +21,27 @@ class GitSpec
         attr_reader :git, :ref, :resolve, :type
 
         def initialize(init_hash)
-          @resolve    = init_hash.key?('resolve') ? init_hash['resolve'] : true
-          @name       = parse_name(init_hash['name'])
-          @git, @repo = parse_git(init_hash['git'])
-          @ref        = init_hash['ref']
-          @type       = :git
+          @logger  = Bolt::Logger.logger(self)
+          @resolve = init_hash.key?('resolve') ? init_hash['resolve'] : true
+          @git     = init_hash['git']
+          @ref     = init_hash['ref']
+          @name    = parse_name(init_hash['name'])
+          @type    = :git
+
+          unless @resolve == true || @resolve == false
+            raise Bolt::ValidationError,
+                  "Option 'resolve' for module spec #{@git} must be a Boolean"
+          end
 
           if @name.nil? && @resolve == false
             raise Bolt::ValidationError,
                   "Missing name for Git module specification: #{@git}. Git module specifications "\
                   "must include a 'name' key when 'resolve' is false."
           end
 
-          unless @resolve == true || @resolve == false
+          unless valid_url?(@git)
             raise Bolt::ValidationError,
-                  "Option 'resolve' for module spec #{@git} must be a Boolean"
+                  "Invalid URI #{@git}. Valid URIs must begin with 'git@', 'http://', or 'https://'."
           end
         end
 
@@ -57,22 +66,6 @@ def self.implements?(hash)
           match[:name]
         end
 
-        # Gets the repo from the git URL.
-        #
-        private def parse_git(git)
-          return [git, nil] unless @resolve
-
-          repo = if git.start_with?('git@github.com:')
-                   git.split('git@github.com:').last.split('.git').first
-                 elsif git.start_with?('https://github.com')
-                   git.split('https://github.com/').last.split('.git').first
-                 else
-                   raise Bolt::ValidationError, invalid_git_msg(git)
-                 end
-
-          [git, repo]
-        end
-
         # Returns true if the specification is satisfied by the module.
         #
         def satisfied_by?(mod)
@@ -88,110 +81,175 @@ def to_hash
           }
         end
 
-        # Returns an error message that the provided repo is not a git repo or
-        # is private.
-        #
-        private def invalid_git_msg(repo_name)
-          "#{repo_name} is not a public GitHub repository. See https://pup.pt/no-resolve "\
-            "for information on how to install this module."
-        end
-
         # Returns a PuppetfileResolver::Model::GitModule object for resolving.
         #
         def to_resolver_module
           require 'puppetfile-resolver'
 
           PuppetfileResolver::Puppetfile::GitModule.new(name).tap do |mod|
             mod.remote = @git
-            mod.ref    = sha
+            mod.ref    = @ref
           end
         end
 
         # Resolves the module's title from the module metadata. This is lazily
-        # resolved since Bolt does not always need to know a Git module's name.
+        # resolved since Bolt does not always need to know a Git module's name,
+        # and fetching the metadata to figure it out is expensive.
         #
         def name
-          @name ||= begin
-            url      = "https://raw.githubusercontent.com/#{@repo}/#{sha}/metadata.json"
-            response = make_request(:Get, url)
+          @name ||= parse_name(metadata['name'])
+        end
 
-            case response
-            when Net::HTTPOK
-              body = JSON.parse(response.body)
+        # Fetches the module's metadata. Attempts to fetch metadata from either
+        # GitHub or GitLab and falls back to cloning the repo if that fails.
+        #
+        private def metadata
+          data = github_metadata || gitlab_metadata || clone_metadata
 
-              unless body.key?('name')
-                raise Bolt::Error.new(
-                  "Missing name in metadata.json at #{git}. This is not a valid module.",
-                  "bolt/missing-module-name-error"
-                )
-              end
+          unless data
+            raise Bolt::Error.new(
+              "Unable to locate metadata.json for module at #{@git}. This may not be a valid module. "\
+              "For more information about how Bolt attempted to locate the metadata file, check the "\
+              "debugging logs.",
+              'bolt/missing-module-metadata-error'
+            )
+          end
 
-              parse_name(body['name'])
-            else
-              raise Bolt::Error.new(
-                "Missing metadata.json at #{git}. This is not a valid module.",
-                "bolt/missing-module-metadata-error"
-              )
-            end
+          data = JSON.parse(data)
+
+          unless data.is_a?(Hash)
+            raise Bolt::Error.new(
+              "Invalid metadata.json at #{@git}. Expected a Hash, got a #{data.class}.",
+              'bolt/invalid-module-metadata-error'
+            )
           end
+
+          unless data.key?('name')
+            raise Bolt::Error.new(
+              "Invalid metadata.json at #{@git}. Metadata must include a 'name' key.",
+              'bolt/missing-module-name-error'
+            )
+          end
+
+          data
+        rescue JSON::ParserError => e
+          raise Bolt::Error.new(
+            "Unable to parse metadata.json for module at #{@git}: #{e.message}",
+            'bolt/metadata-parse-error'
+          )
         end
 
-        # Resolves the SHA for the specified ref. This is lazily resolved since
-        # Bolt does not always need to know a Git module's SHA.
+        # Returns the metadata for a GitHub-hosted module.
         #
-        def sha
-          @sha ||= begin
-            url      = "https://api.github.com/repos/#{@repo}/commits/#{ref}"
-            headers  = ENV['GITHUB_TOKEN'] ? { "Authorization" => "token #{ENV['GITHUB_TOKEN']}" } : {}
-            response = make_request(:Get, url, headers)
+        private def github_metadata
+          repo = if @git.start_with?('git@github.com:')
+                   @git.split('git@github.com:').last.split('.git').first
+                 elsif @git.start_with?('https://github.com')
+                   @git.split('https://github.com/').last.split('.git').first
+                 end
 
-            case response
-            when Net::HTTPOK
-              body = JSON.parse(response.body)
-              body['sha']
-            when Net::HTTPUnauthorized
-              raise Bolt::Error.new(
-                "Invalid token at GITHUB_TOKEN, unable to resolve git modules.",
-                "bolt/invalid-git-token-error"
-              )
-            when Net::HTTPForbidden
-              message = "GitHub API rate limit exceeded, unable to resolve git modules. "
-
-              unless ENV['GITHUB_TOKEN']
-                message += "To increase your rate limit, set the GITHUB_TOKEN environment "\
-                          "variable with a GitHub personal access token."
+          return nil if repo.nil?
+
+          request_metadata("https://raw.githubusercontent.com/#{repo}/#{@ref}/metadata.json")
+        end
+
+        # Returns the metadata for a GitLab-hosted module.
+        #
+        private def gitlab_metadata
+          repo = if @git.start_with?('git@gitlab.com:')
+                   @git.split('git@gitlab.com:').last.split('.git').first
+                 elsif @git.start_with?('https://gitlab.com')
+                   @git.split('https://gitlab.com/').last.split('.git').first
+                 end
+
+          return nil if repo.nil?
+
+          request_metadata("https://gitlab.com/#{repo}/-/raw/#{@ref}/metadata.json")
+        end
+
+        # Returns the metadata by cloning a git-based module.
+        # Because cloning is the last attempt to locate module metadata
+        #
+        private def clone_metadata
+          unless git?
+            @logger.debug("'git' executable not found, unable to use git clone resolution.")
+            return nil
+          end
+
+          # Clone the repo into a temp directory that will be automatically cleaned up.
+          Dir.mktmpdir do |dir|
+            command = %W[git clone --bare --depth=1 --single-branch --branch=#{@ref} #{@git} #{dir}]
+            @logger.debug("Executing command '#{command.join(' ')}'")
+
+            out, err, status = Open3.capture3(*command)
+
+            unless status.success?
+              @logger.debug("Unable to clone #{@git}: #{err}")
+              return nil
+            end
+
+            # Read the metadata.json file from the cloned repo.
+            Dir.chdir(dir) do
+              command = %W[git show #{@ref}:metadata.json]
+              @logger.debug("Executing command '#{command.join(' ')}'")
+
+              out, err, status = Open3.capture3(*command)
+
+              unless status.success?
+                @logger.debug("Unable to read metadata.json file for #{@git}: #{err}")
+                return nil
               end
 
-              raise Bolt::Error.new(message, 'bolt/github-api-rate-limit-error')
-            when Net::HTTPNotFound
-              raise Bolt::Error.new(invalid_git_msg(git), "bolt/missing-git-repository-error")
-            else
-              raise Bolt::Error.new(
-                "Ref #{ref} at #{git} is not a commit, tag, or branch.",
-                "bolt/invalid-git-ref-error"
-              )
+              out
             end
           end
         end
 
-        # Makes a generic HTTP request.
+        # Requests module metadata from the specified url.
         #
-        private def make_request(verb, url, headers = {})
-          require 'net/http'
+        private def request_metadata(url)
+          uri  = URI.parse(url)
+          opts = { use_ssl: uri.scheme == 'https' }
 
-          uri      = URI.parse(url)
-          opts     = { use_ssl: uri.scheme == 'https' }
+          @logger.debug("Requesting metadata file from #{url}")
 
           Net::HTTP.start(uri.host, uri.port, opts) do |client|
-            request = Net::HTTP.const_get(verb).new(uri, headers)
-            client.request(request)
+            response = client.request(Net::HTTP::Get.new(uri))
+
+            case response
+            when Net::HTTPOK
+              response.body
+            else
+              @logger.debug("Unable to locate metadata file at #{url}")
+              nil
+            end
           end
         rescue StandardError => e
           raise Bolt::Error.new(
             "Failed to connect to #{uri}: #{e.message}",
             "bolt/http-connect-error"
           )
         end
+
+        # Returns true if the 'git' executable is available.
+        #
+        private def git?
+          Open3.capture3('git', '--version')
+          true
+        rescue Errno::ENOENT
+          false
+        end
+
+        # Returns true if the URL is valid.
+        #
+        private def valid_url?(url)
+          return true if url.start_with?('git@')
+
+          uri = URI.parse(url)
+          uri.is_a?(URI::HTTP) && uri.host
+        rescue URI::InvalidURIError
+          false
+        end
       end
     end
   end