Cargo implementation must support Cargo sparse index file format

cargo/lib/dependabot/cargo/metadata_finder.rb

@@ -67,7 +67,12 @@ def crates_listing
           **SharedHelpers.excon_defaults(headers: hdrs)
         )
 
-        @crates_listing = JSON.parse(response.body)
+        if index.start_with?("sparse+")
+          parsed_response = response.body.lines.map { |line| JSON.parse(line) }
+          @crates_listing = { "versions" => parsed_response }
+        else
+          @crates_listing = JSON.parse(response.body)
+        end
       end
 
       def metadata_fetch_url(dependency, index)

cargo/lib/dependabot/cargo/update_checker/latest_version_finder.rb

@@ -97,46 +97,33 @@ def available_versions
           crates_listing
             .fetch("versions", [])
             .reject { |v| v["yanked"] }
-            .map { |v| version_class.new(v.fetch("num")) }
+            # Handle both default and sparse registry responses.
+            # Default registry uses "num" for version number.
+            # Sparse registry uses "vers" for version number.
+            .map do |v|
+              version_number = v["num"] || v["vers"]
+              version_class.new(version_number)
+            end
         end
 
         def crates_listing
           return @crates_listing unless @crates_listing.nil?
 
-          info = dependency.requirements.filter_map { |r| r[:source] }.first
-          index = (info && info[:index]) || CRATES_IO_API
-
-          # Default request headers
-          hdrs = { "User-Agent" => "Dependabot (dependabot.com)" }
-
-          if index != CRATES_IO_API
-            # Add authentication headers if credentials are present for this registry
-            registry_creds = credentials.find do |cred|
-              cred["type"] == "cargo_registry" && cred["registry"] == info[:name]
-            end
-
-            unless registry_creds.nil?
-              # If there is a credential, but no actual token at this point, it means that dependabot-cli
-              # stripped the token from our credentials. In this case, the dependabot proxy will reintroduce
-              # the correct token, so we just use 'placeholder_token' as the token value.
-              token = registry_creds["token"] || "placeholder_token"
+          info = fetch_dependency_info
+          index = fetch_index(info)
 
-              hdrs["Authorization"] = token
-            end
-          end
+          hdrs = default_headers
+          hdrs.merge!(auth_headers(info)) if index != CRATES_IO_API
 
           url = metadata_fetch_url(dependency, index)
 
           # B4PR
           puts "Calling #{url} to fetch metadata for #{dependency.name} from #{index}"
 
-          response = Excon.get(
-            url,
-            idempotent: true,
-            **SharedHelpers.excon_defaults(headers: hdrs)
-          )
+          response = fetch_response(url, hdrs)
+          return {} if response.status == 404
 
-          @crates_listing = JSON.parse(response.body)
+          @crates_listing = parse_response(response, index)
 
           # B4PR
           puts "Fetched metadata for #{dependency.name} from #{index} successfully"
@@ -145,6 +132,46 @@ def crates_listing
           @crates_listing
         end
 
+        def fetch_dependency_info
+          dependency.requirements.filter_map { |r| r[:source] }.first
+        end
+
+        def fetch_index(info)
+          (info && info[:index]) || CRATES_IO_API
+        end
+
+        def default_headers
+          { "User-Agent" => "Dependabot (dependabot.com)" }
+        end
+
+        def auth_headers(info)
+          registry_creds = credentials.find do |cred|
+            cred["type"] == "cargo_registry" && cred["registry"] == info[:name]
+          end
+
+          return {} if registry_creds.nil?
+
+          token = registry_creds["token"] || "placeholder_token"
+          { "Authorization" => token }
+        end
+
+        def fetch_response(url, headers)
+          Excon.get(
+            url,
+            idempotent: true,
+            **SharedHelpers.excon_defaults(headers: headers)
+          )
+        end
+
+        def parse_response(response, index)
+          if index.start_with?("sparse+")
+            parsed_response = response.body.lines.map { |line| JSON.parse(line) }
+            { "versions" => parsed_response }
+          else
+            JSON.parse(response.body)
+          end
+        end
+
         def metadata_fetch_url(dependency, index)
           return "#{index}/#{dependency.name}" if index == CRATES_IO_API
 

cargo/spec/dependabot/cargo/update_checker/latest_version_finder_spec.rb

@@ -7,13 +7,12 @@
 require "dependabot/cargo/update_checker/latest_version_finder"
 
 RSpec.describe Dependabot::Cargo::UpdateChecker::LatestVersionFinder do
-  before do
-    stub_request(:get, crates_url).to_return(status: 200, body: crates_response)
-  end
   let(:crates_url) { "https://crates.io/api/v1/crates/#{dependency_name}" }
   let(:crates_response) { fixture("crates_io_responses", crates_fixture_name) }
   let(:crates_fixture_name) { "#{dependency_name}.json" }
-
+  let(:ignored_versions) { [] }
+  let(:raise_on_ignored) { false }
+  let(:security_advisories) { [] }
   let(:finder) do
     described_class.new(
       dependency: dependency,
@@ -24,10 +23,6 @@
       security_advisories: security_advisories
     )
   end
-
-  let(:ignored_versions) { [] }
-  let(:raise_on_ignored) { false }
-  let(:security_advisories) { [] }
   let(:credentials) do
     [{
       "type" => "git_source",
@@ -66,6 +61,10 @@
 
   describe "#latest_version" do
     subject { finder.latest_version }
+    before do
+      stub_request(:get, crates_url).to_return(status: 200, body: crates_response)
+    end
+
     it { is_expected.to eq(Gem::Version.new("0.1.40")) }
 
     context "when the latest version is being ignored" do
@@ -177,6 +176,10 @@
   describe "#lowest_security_fix_version" do
     subject { finder.lowest_security_fix_version }
 
+    before do
+      stub_request(:get, crates_url).to_return(status: 200, body: crates_response)
+    end
+
     let(:dependency_name) { "time" }
     let(:dependency_version) { "0.1.12" }
     let(:security_advisories) do
@@ -243,4 +246,192 @@
       end
     end
   end
+
+  # Tests for sparse registry responses
+  describe "Sparse registry response handling" do
+    let(:sparse_registry_url) { "https://cargo.cloudsmith.io/honeyankit/test/he/ll/hello-world" }
+    let(:sparse_registry_response) { fixture("private_registry_responses", crates_fixture_name) }
+    let(:crates_fixture_name) { "#{dependency_name}.json" }
+
+    let(:credentials) do
+      [{
+        "type" => "cargo_registry",
+        "cargo_registry" => "honeyankit-test",
+        "url" => "https://cargo.cloudsmith.io/honeyankit/test/",
+        "token" => "token"
+      }]
+    end
+    let(:dependency_name) { "hello-world" }
+    let(:dependency_version) { "1.0.0" }
+    let(:requirements) do
+      [{
+        file: "Cargo.toml",
+        requirement: "1.0.0",
+        groups: ["dependencies"],
+        source: {
+          type: "registry",
+          name: "honeyankit-test",
+          index: "sparse+https://cargo.cloudsmith.io/honeyankit/test/",
+          dl: "https://dl.cloudsmith.io/basic/honeyankit/test/cargo/{crate}-{version}.crate",
+          api: "https://cargo.cloudsmith.io/honeyankit/test"
+        }
+      }]
+    end
+
+    describe "#latest_version" do
+      subject { finder.latest_version }
+      before do
+        stub_request(:get, sparse_registry_url).to_return(status: 200, body: sparse_registry_response)
+      end
+
+      it { is_expected.to eq(Gem::Version.new("1.0.1")) }
+
+      context "when the latest version is being ignored" do
+        let(:ignored_versions) { [">= 1.0.1, < 2.0"] }
+        it { is_expected.to eq(Gem::Version.new("1.0.0")) }
+      end
+
+      context "when the sparse registry link resolves to a 'Not Found' page" do
+        before do
+          stub_request(:get, sparse_registry_url)
+            .to_return(status: 404, body: sparse_registry_response)
+        end
+        let(:crates_fixture_name) { "not_found.json" }
+
+        it { is_expected.to be_nil }
+      end
+
+      context "when the latest version is a pre-release" do
+        let(:sparse_registry_response) do
+          <<~BODY
+            {"name": "hello-world", "vers": "1.0.0", "deps": [], "cksum": "b2c263921f1114820f4acc6b542d72bbc859ce7023c5b235346b157074dcccc7", "features": {}, "yanked": false, "links": null}
+            {"name": "hello-world", "vers": "2.0.0-pre1", "deps": [], "cksum": "8a55b58def1ecc7aa8590c7078f379ec9a85328363ffb81d4354314b132b95c4", "features": {}, "yanked": false, "links": null}
+          BODY
+        end
+        it { is_expected.to eq(Gem::Version.new("1.0.0")) }
+
+        context "with the user wants a pre-release" do
+          let(:requirements) do
+            [{
+              file: "Cargo.toml",
+              requirement: "~2.0.0-pre1",
+              groups: ["dependencies"],
+              source: {
+                type: "registry",
+                name: "honeyankit-test",
+                index: "sparse+https://cargo.cloudsmith.io/honeyankit/test/",
+                dl: "https://dl.cloudsmith.io/basic/honeyankit/test/cargo/{crate}-{version}.crate",
+                api: "https://cargo.cloudsmith.io/honeyankit/test"
+              }
+            }]
+          end
+          it { is_expected.to eq(Gem::Version.new("2.0.0-pre1")) }
+        end
+      end
+
+      context "when already on the latest version" do
+        let(:dependency_version) { "1.0.1" }
+        it { is_expected.to eq(Gem::Version.new("1.0.1")) }
+      end
+
+      context "when all later versions are being ignored" do
+        let(:ignored_versions) { ["> 1.0.0"] }
+        it { is_expected.to eq(Gem::Version.new("1.0.0")) }
+
+        context "with raise_on_ignored" do
+          let(:raise_on_ignored) { true }
+          it "raises an error" do
+            expect { subject }.to raise_error(Dependabot::AllVersionsIgnored)
+          end
+        end
+      end
+    end
+
+    describe "#lowest_security_fix_version" do
+      subject { finder.lowest_security_fix_version }
+
+      before do
+        stub_request(:get, sparse_registry_url).to_return(status: 200, body: sparse_registry_response)
+      end
+
+      let(:dependency_name) { "hello-world" }
+      let(:dependency_version) { "1.0.0" }
+      let(:security_advisories) do
+        [
+          Dependabot::SecurityAdvisory.new(
+            dependency_name: dependency_name,
+            package_manager: "cargo",
+            vulnerable_versions: ["<= 1.0.0"]
+          )
+        ]
+      end
+      it { is_expected.to eq(Gem::Version.new("1.0.1")) }
+
+      context "when the lowest version is being ignored" do
+        let(:ignored_versions) { [">= 1.0.0, < 1.0.1"] }
+        it { is_expected.to eq(Gem::Version.new("1.0.1")) }
+      end
+
+      context "when all versions are being ignored" do
+        let(:ignored_versions) { [">= 0"] }
+        it "returns nil" do
+          expect(subject).to be_nil
+        end
+
+        context "with raise_on_ignored" do
+          let(:raise_on_ignored) { true }
+          it "raises an error" do
+            expect { subject }.to raise_error(Dependabot::AllVersionsIgnored)
+          end
+        end
+      end
+
+      context "when the lowest fixed version is a pre-release" do
+        let(:sparse_registry_response) do
+          <<~BODY
+            {"name": "hello-world", "vers": "1.0.0", "deps": [], "cksum": "b2c263921f1114820f4acc6b542d72bbc859ce7023c5b235346b157074dcccc7", "features": {}, "yanked": false, "links": null}
+            {"name": "hello-world", "vers": "2.0.0", "deps": [], "cksum": "b2c263921f1114820f4acc6b542d72bbc859ce7023c5b235346b157074dcccc8", "features": {}, "yanked": false, "links": null}
+            {"name": "hello-world", "vers": "2.0.0-pre1", "deps": [], "cksum": "8a55b58def1ecc7aa8590c7078f379ec9a85328363ffb81d4354314b132b95c4", "features": {}, "yanked": false, "links": null}
+            {"name": "hello-world", "vers": "2.0.0-pre2", "deps": [], "cksum": "8a55b58def1ecc7aa8590c7078f379ec9a85328363ffb81d4354314b132b95f6", "features": {}, "yanked": false, "links": null}
+            {"name": "hello-world", "vers": "2.0.0-pre3", "deps": [], "cksum": "8a55b58def1ecc7aa8590c7078f379ec9a85328363ffb81d4354314b132b95d6", "features": {}, "yanked": false, "links": null}
+          BODY
+        end
+        let(:security_advisories) do
+          [
+            Dependabot::SecurityAdvisory.new(
+              dependency_name: dependency_name,
+              package_manager: "cargo",
+              vulnerable_versions: ["<= 2.0.0-pre2"]
+            )
+          ]
+        end
+        it { is_expected.to eq(Gem::Version.new("2.0.0")) }
+
+        context "with the user wants a pre-release" do
+          context "when their current version is a pre-release" do
+            let(:dependency_version) { "2.0.0-pre1" }
+            it { is_expected.to eq(Gem::Version.new("2.0.0-pre3")) }
+          end
+
+          context "when their requirements say they want pre-releases" do
+            let(:requirements) do
+              [{
+                file: "Cargo.toml",
+                requirement: "~2.0.0-pre1",
+                groups: ["dependencies"],
+                source: {
+                  type: "registry",
+                  name: "honeyankit-test",
+                  index: "sparse+https://cargo.cloudsmith.io/honeyankit/test/",
+                  dl: "https://dl.cloudsmith.io/basic/honeyankit/test/cargo/{crate}-{version}.crate",
+                  api: "https://cargo.cloudsmith.io/honeyankit/test"
+                }
+              }]
+            end
+            it { is_expected.to eq(Gem::Version.new("2.0.0-pre3")) }
+          end
+        end
+      end
+    end
+  end
 end

cargo/spec/fixtures/private_registry_responses/hello-world.json

@@ -0,0 +1,2 @@
+{"name": "hello-world", "vers": "1.0.0", "deps": [], "cksum": "b2c263921f1114820f4acc6b542d72bbc859ce7023c5b235346b157074dcccc7", "features": {}, "yanked": false, "links": null}
+{"name": "hello-world", "vers": "1.0.1", "deps": [], "cksum": "8a55b58def1ecc7aa8590c7078f379ec9a85328363ffb81d4354314b132b95c4", "features": {}, "yanked": false, "links": null}

cargo/spec/fixtures/private_registry_responses/not_found.json

@@ -0,0 +1 @@
+Invalid url.