Add support for Cargo private registries (#8719)

* Add support for Rust alternative registries
* Fix handling of missing source info in dependencies
* Fixed calculation of crates dl URL
* Minor cleanups
* Fixes for crates-ms
* Update to get interface with private registry (tested with Cloudsmith) working
* Try to handle the case of updating versions for private registries
* Fix up some tests for fetching Cargo config file
* Sorbet typechecking adjustments
* Refactor sparse registry details to method
* Expect Cargo config file to be fetched in FileFetcher spec
* Revert changes to base Gitlab metadata finder made in feature base PR
* Enable using index URL to fetch crate metadata from private registry
* Remove special case for Microsoft; we can/need to authenticate normally
* Cargo prefers config.toml to have the extension
* Fix typo in method name
* Fix filename for fixture
* Merge main into cargo-private-registries
* Add a bit more spam in the DEBUG_HELPERS case.
* Fix issues found in local testing, especially setting `CARGO_REGISTRY_GLOBAL_CREDENTIAL_PROVIDER=cargo:token`.
* Style lint fixes
* Simplify conditional logic to satisfy ABC linter

---------

Co-authored-by: John Batty <john.batty@metaswitch.com>
Co-authored-by: John Batty <johnbatty@microsoft.com>
Co-authored-by: Ian Joiner <14581281+iajoiner@users.noreply.github.com>
Co-authored-by: Rob Jellinghaus <rjelling@microsoft.com>
Co-authored-by: AbdulFattaah Popoola <abdulapopoola@github.com>

cargo/lib/dependabot/cargo/file_fetcher.rb

@@ -46,7 +46,7 @@ def ecosystem_versions
 
       sig { override.returns(T::Array[DependencyFile]) }
       def fetch_files
-        fetched_files = []
+        fetched_files = T.let([], T::Array[DependencyFile])
         fetched_files << cargo_toml
         fetched_files << cargo_lock if cargo_lock
         fetched_files << cargo_config if cargo_config
@@ -327,7 +327,10 @@ def cargo_lock
       def cargo_config
         return @cargo_config if defined?(@cargo_config)
 
-        @cargo_config = fetch_file_if_present(".cargo/config.toml")
+        @cargo_config = fetch_support_file(".cargo/config.toml")
+
+        @cargo_config ||= fetch_support_file(".cargo/config")
+                          &.tap { |f| f.name = ".cargo/config.toml" }
       end
 
       def rust_toolchain

cargo/lib/dependabot/cargo/file_parser.rb

@@ -2,13 +2,15 @@
 # frozen_string_literal: true
 
 require "toml-rb"
+require "pathname"
 
 require "dependabot/dependency"
 require "dependabot/file_parsers"
 require "dependabot/file_parsers/base"
 require "dependabot/cargo/requirement"
 require "dependabot/cargo/version"
 require "dependabot/errors"
+require "dependabot/cargo/registry_fetcher"
 
 # Relevant Cargo docs can be found at:
 # - https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -162,8 +164,77 @@ def source_from_declaration(declaration)
         raise "Unexpected dependency declaration: #{declaration}" unless declaration.is_a?(Hash)
 
         return git_source_details(declaration) if declaration["git"]
+        return { type: "path" } if declaration["path"]
 
-        { type: "path" } if declaration["path"]
+        registry_source_details(declaration)
+      end
+
+      def registry_source_details(declaration)
+        registry_name = declaration["registry"]
+        return if registry_name.nil?
+
+        index_url = cargo_config_field("registries.#{registry_name}.index")
+        if index_url.nil?
+          raise "Registry index for #{registry_name} must be defined via " \
+                "cargo config"
+        end
+
+        if index_url.start_with?("sparse+")
+          sparse_registry_source_details(registry_name, index_url)
+        else
+          source = Source.from_url(index_url)
+          registry_fetcher = RegistryFetcher.new(
+            source: T.must(source),
+            credentials: credentials
+          )
+
+          {
+            type: "registry",
+            name: registry_name,
+            index: index_url,
+            dl: registry_fetcher.dl,
+            api: registry_fetcher.api
+          }
+        end
+      end
+
+      def sparse_registry_source_details(registry_name, index_url)
+        token = credentials.find do |cred|
+          cred["type"] == "cargo_registry" && cred["registry"] == registry_name
+        end&.fetch("token", nil)
+        # Fallback to configuration in the environment if available
+        token ||= cargo_config_from_env("registries.#{registry_name}.token")
+
+        headers = {}
+        headers["Authorization"] = "Token #{token}" if token
+
+        url = index_url.delete_prefix("sparse+")
+        url << "/" unless url.end_with?("/")
+        url << "config.json"
+        config_json = JSON.parse(RegistryClient.get(url: url, headers: headers).body)
+
+        {
+          type: "registry",
+          name: registry_name,
+          index: index_url,
+          dl: config_json["dl"],
+          api: config_json["api"]
+        }
+      end
+
+      # Looks up dotted key name in cargo config
+      # e.g. "registries.my_registry.index"
+      def cargo_config_field(key_name)
+        cargo_config_from_env(key_name) || cargo_config_from_file(key_name)
+      end
+
+      def cargo_config_from_env(key_name)
+        env_var = "CARGO_#{key_name.upcase.tr('-.', '_')}"
+        ENV.fetch(env_var, nil)
+      end
+
+      def cargo_config_from_file(key_name)
+        parsed_file(cargo_config).dig(*key_name.split("."))
       end
 
       def version_from_lockfile(name, declaration)
@@ -237,6 +308,10 @@ def lockfile
         @lockfile ||= get_original_file("Cargo.lock")
       end
 
+      def cargo_config
+        @cargo_config ||= get_original_file(".cargo/config.toml")
+      end
+
       def version_class
         Cargo::Version
       end

cargo/lib/dependabot/cargo/file_updater/lockfile_updater.rb

@@ -7,6 +7,7 @@
 require "dependabot/cargo/file_updater"
 require "dependabot/cargo/file_updater/manifest_updater"
 require "dependabot/cargo/file_parser"
+require "dependabot/cargo/helpers"
 require "dependabot/shared_helpers"
 module Dependabot
   module Cargo
@@ -33,7 +34,7 @@ def updated_lockfile_content
             SharedHelpers.with_git_configured(credentials: credentials) do
               # Shell out to Cargo, which handles everything for us, and does
               # so without doing an install (so it's fast).
-              run_shell_command("cargo update -p #{dependency_spec}", fingerprint: "cargo update -p <dependency_spec>")
+              run_cargo_command("cargo update -p #{dependency_spec}", fingerprint: "cargo update -p <dependency_spec>")
             end
 
             updated_lockfile = File.read("Cargo.lock")
@@ -112,7 +113,6 @@ def dependency_spec
             spec += ":#{git_previous_version}" if git_previous_version
           elsif dependency.previous_version
             spec += ":#{dependency.previous_version}"
-            spec = "https://github.com/rust-lang/crates.io-index#" + spec
           end
 
           spec
@@ -138,10 +138,14 @@ def desired_lockfile_content
           %(name = "#{dependency.name}"\nversion = "#{dependency.version}")
         end
 
-        def run_shell_command(command, fingerprint:)
+        def run_cargo_command(command, fingerprint:)
           start = Time.now
           command = SharedHelpers.escape_command(command)
-          stdout, process = Open3.capture2e(command)
+          Helpers.setup_credentials_in_environment(credentials)
+          # Pass through any registry tokens supplied via CARGO_REGISTRIES_...
+          # environment variables.
+          env = ENV.select { |key, _value| key.match(/^CARGO_REGISTRIES_/) }
+          stdout, process = Open3.capture2e(env, command)
           time_taken = Time.now - start
 
           # Raise an error with the output from the shell session if Cargo
@@ -187,6 +191,10 @@ def write_temporary_dependency_files
 
           File.write(lockfile.name, lockfile.content)
           File.write(toolchain.name, toolchain.content) if toolchain
+          return unless config
+
+          FileUtils.mkdir_p(File.dirname(config.name))
+          File.write(config.name, config.content)
         end
 
         def write_temporary_manifest_files
@@ -394,6 +402,10 @@ def toolchain
             dependency_files.find { |f| f.name == "rust-toolchain" }
         end
 
+        def config
+          @config ||= dependency_files.find { |f| f.name == ".cargo/config.toml" }
+        end
+
         def virtual_manifest?(file)
           !file.content.include?("[package]")
         end

cargo/lib/dependabot/cargo/helpers.rb

@@ -0,0 +1,38 @@
+# typed: true
+# frozen_string_literal: true
+
+require "yaml"
+
+module Dependabot
+  module Cargo
+    module Helpers
+      def self.setup_credentials_in_environment(credentials)
+        credentials.each do |cred|
+          next if cred["type"] != "cargo_registry"
+
+          # If there is a 'token' property, then apply it.
+          # If there is not, it probably means we are running under dependabot-cli which stripped
+          # all tokens. So in that case, we assume that the dependabot proxy will re-inject the
+          # actual correct token, and we just use 'token' as a placeholder at this point.
+          # (We must add these environment variables here, or 'cargo update' will not think it is
+          # configured properly for the private registries.)
+
+          token_env_var = "CARGO_REGISTRIES_#{cred['cargo_registry'].upcase.tr('-', '_')}_TOKEN"
+
+          token = "placeholder_token"
+          if cred["token"].nil?
+            puts "Setting #{token_env_var} to 'placeholder_token' because dependabot-cli proxy will override it anyway"
+          else
+            token = cred["token"]
+            puts "Setting #{token_env_var} to provided token value"
+          end
+
+          ENV[token_env_var] ||= token
+        end
+
+        # And set CARGO_REGISTRY_GLOBAL_CREDENTIAL_PROVIDERS here as well, so Cargo will expect tokens
+        ENV["CARGO_REGISTRY_GLOBAL_CREDENTIAL_PROVIDERS"] ||= "cargo:token"
+      end
+    end
+  end
+end

cargo/lib/dependabot/cargo/metadata_finder.rb

@@ -10,12 +10,14 @@ module Dependabot
   module Cargo
     class MetadataFinder < Dependabot::MetadataFinders::Base
       SOURCE_KEYS = %w(repository homepage documentation).freeze
+      CRATES_IO_API = "https://crates.io/api/v1/crates"
 
       private
 
       def look_up_source
         case new_source_type
         when "default" then find_source_from_crates_listing
+        when "registry" then find_source_from_crates_listing
         when "git" then find_source_from_git_url
         else raise "Unexpected source type: #{new_source_type}"
         end
@@ -44,9 +46,47 @@ def find_source_from_git_url
       def crates_listing
         return @crates_listing unless @crates_listing.nil?
 
-        response = Dependabot::RegistryClient.get(url: "https://crates.io/api/v1/crates/#{dependency.name}")
+        info = dependency.requirements.filter_map { |r| r[:source] }.first
+        index = (info && info[:index]) || CRATES_IO_API
+
+        # Default request headers
+        hdrs = { "User-Agent" => "Dependabot (dependabot.com)" }
+
+        if index != CRATES_IO_API
+          # Add authentication headers if credentials are present for this registry
+          credentials.find { |cred| cred["type"] == "cargo_registry" && cred["registry"] == info[:name] }&.tap do |cred|
+            hdrs["Authorization"] = "Token #{cred['token']}"
+          end
+        end
+
+        url = metadata_fetch_url(dependency, index)
+
+        response = Excon.get(
+          url,
+          idempotent: true,
+          **SharedHelpers.excon_defaults(headers: hdrs)
+        )
+
         @crates_listing = JSON.parse(response.body)
       end
+
+      def metadata_fetch_url(dependency, index)
+        return "#{index}/#{dependency.name}" if index == CRATES_IO_API
+
+        # Determine cargo's index file path for the dependency
+        index = index.delete_prefix("sparse+")
+        name_length = dependency.name.length
+        dependency_path = case name_length
+                          when 1, 2
+                            "#{name_length}/#{dependency.name}"
+                          when 3
+                            "#{name_length}/#{dependency.name[0..1]}/#{dependency.name}"
+                          else
+                            "#{dependency.name[0..1]}/#{dependency.name[2..3]}/#{dependency.name}"
+                          end
+
+        "#{index}#{'/' unless index.end_with?('/')}#{dependency_path}"
+      end
     end
   end
 end

cargo/lib/dependabot/cargo/registry_fetcher.rb

@@ -0,0 +1,42 @@
+# typed: true
+# frozen_string_literal: true
+
+require "dependabot/file_fetchers"
+require "dependabot/file_fetchers/base"
+
+module Dependabot
+  module Cargo
+    class RegistryFetcher < Dependabot::FileFetchers::Base
+      def self.required_files_in?(filenames)
+        filenames.include?("config.json")
+      end
+
+      def self.required_files_message
+        "Repo must contain a config.json"
+      end
+
+      def dl
+        parsed_config_json["dl"].chomp("/")
+      end
+
+      def api
+        parsed_config_json["api"].chomp("/")
+      end
+
+      private
+
+      def fetch_files
+        fetched_files = []
+        fetched_files << config_json
+      end
+
+      def parsed_config_json
+        @parsed_config_json ||= JSON.parse(config_json.content)
+      end
+
+      def config_json
+        @config_json ||= fetch_file_from_host("config.json")
+      end
+    end
+  end
+end