NuGet efficiency

bin/dry-run.rb

@@ -138,6 +138,15 @@
   }
 end
 
+unless ENV["LOCAL_AZURE_ACCESS_TOKEN"].to_s.strip.empty?
+  $options[:credentials] << {
+    "type" => "nuget_feed",
+    "host" => "pkgs.dev.azure.com",
+    "url" => ENV.fetch("LOCAL_AZURE_FEED_URL", nil),
+    "token" => ":#{ENV.fetch('LOCAL_AZURE_ACCESS_TOKEN', nil)}"
+  }
+end
+
 unless ENV["LOCAL_CONFIG_VARIABLES"].to_s.strip.empty?
   # For example:
   # "[{\"type\":\"npm_registry\",\"registry\":\
@@ -387,8 +396,8 @@ def fetch_files(fetcher)
     else
       puts "=> cloning into #{$repo_contents_path}"
       FileUtils.rm_rf($repo_contents_path)
+      fetcher.clone_repo_contents
     end
-    fetcher.clone_repo_contents
     if $options[:commit]
       Dir.chdir($repo_contents_path) do
         puts "=> checking out commit #{$options[:commit]}"

nuget/lib/dependabot/nuget/file_fetcher.rb

@@ -3,6 +3,7 @@
 
 require "dependabot/file_fetchers"
 require "dependabot/file_fetchers/base"
+require "dependabot/nuget/cache_manager"
 require "set"
 require "sorbet-runtime"
 

nuget/lib/dependabot/nuget/file_parser/project_file_parser.rb

@@ -298,25 +298,24 @@ def dependency_has_search_results?(dependency)
         def dependency_url_has_matching_result?(dependency_name, dependency_url)
           repository_type = dependency_url.fetch(:repository_type)
           if repository_type == "v3"
-            dependency_url_has_matching_result_v3?(dependency_name, dependency_url)
+            dependency_url_has_matching_result_v3?(dependency_url)
           elsif repository_type == "v2"
             dependency_url_has_matching_result_v2?(dependency_name, dependency_url)
           else
             raise "Unknown repository type: #{repository_type}"
           end
         end
 
-        def dependency_url_has_matching_result_v3?(dependency_name, dependency_url)
-          url = dependency_url.fetch(:search_url)
+        def dependency_url_has_matching_result_v3?(dependency_url)
+          url = dependency_url.fetch(:versions_url)
           auth_header = dependency_url.fetch(:auth_header)
           response = execute_search_for_dependency_url(url, auth_header)
           return false unless response.status == 200
 
           body = JSON.parse(response.body)
-          data = body["data"]
-          return false unless data.length.positive?
+          versions = body["versions"]
 
-          data.any? { |result| result["id"].casecmp?(dependency_name) }
+          versions != nil
         end
 
         def dependency_url_has_matching_result_v2?(dependency_name, dependency_url)

nuget/lib/dependabot/nuget/update_checker/repository_finder.rb

@@ -54,6 +54,7 @@ def find_dependency_urls
             end.compact.uniq
         end
 
+        # rubocop:disable Metrics/AbcSize
         def build_url_for_details(repo_details)
           response = get_repo_metadata(repo_details)
           check_repo_response(response, repo_details)
@@ -62,7 +63,9 @@ def build_url_for_details(repo_details)
           body = remove_wrapping_zero_width_chars(response.body)
           base_url = base_url_from_v3_metadata(JSON.parse(body))
           resolved_base_url = base_url || repo_details.fetch(:url).gsub("/index.json", "-flatcontainer")
-          search_url = search_url_from_v3_metadata(JSON.parse(body))
+          parsed_json = JSON.parse(body)
+          search_url = search_url_from_v3_metadata(parsed_json)
+          registration_url = registration_url_from_v3_metadata(parsed_json)
 
           details = {
             base_url: resolved_base_url,
@@ -78,18 +81,30 @@ def build_url_for_details(repo_details)
             details[:search_url] =
               search_url + "?q=#{dependency.name.downcase}&prerelease=true&semVerLevel=2.0.0"
           end
+
+          details[:registration_url] = registration_url + dependency.name.downcase if registration_url
+
           details
         rescue JSON::ParserError
           build_v2_url(response, repo_details)
         rescue Excon::Error::Timeout, Excon::Error::Socket
           handle_timeout(repo_metadata_url: repo_details.fetch(:url))
         end
+        # rubocop:enable Metrics/AbcSize
 
         def get_repo_metadata(repo_details)
-          Dependabot::RegistryClient.get(
-            url: repo_details.fetch(:url),
-            headers: auth_header_for_token(repo_details.fetch(:token))
-          )
+          url = repo_details.fetch(:url)
+          cache = CacheManager.cache("repo_finder_metadatacache")
+          if !CacheManager.caching_disabled? && cache[url]
+            cache[url]
+          else
+            result = Dependabot::RegistryClient.get(
+              url: url,
+              headers: auth_header_for_token(repo_details.fetch(:token))
+            )
+            cache[url] = result
+            result
+          end
         end
 
         def base_url_from_v3_metadata(metadata)
@@ -99,6 +114,17 @@ def base_url_from_v3_metadata(metadata)
             &.fetch("@id")
         end
 
+        def registration_url_from_v3_metadata(metadata)
+          allowed_registration_types = %w(
+            RegistrationsBaseUrl/3.0.0-beta
+            RegistrationsBaseUrl
+          )
+          metadata
+            .fetch("resources", [])
+            .find { |r| allowed_registration_types.find { |s| r.fetch("@type") == s } }
+            &.fetch("@id")
+        end
+
         def search_url_from_v3_metadata(metadata)
           # allowable values from here: https://learn.microsoft.com/en-us/nuget/api/search-query-service-resource#versioning
           allowed_search_types = %w(

nuget/lib/dependabot/nuget/update_checker/version_finder.rb

@@ -296,7 +296,10 @@ def fetch_v2_next_link_href(xml_body)
         def versions_for_v3_repository(repository_details)
           # If we have a search URL that returns results we use it
           # (since it will exclude unlisted versions)
-          if repository_details[:search_url]
+
+          if repository_details[:registration_url]
+            get_nuget_versions_from_registration(repository_details)
+          elsif repository_details[:search_url]
             fetch_versions_from_search_url(repository_details)
           # Otherwise, use the versions URL
           elsif repository_details[:versions_url]
@@ -311,6 +314,64 @@ def versions_for_v3_repository(repository_details)
           end
         end
 
+        def get_nuget_versions_from_registration(repository_details)
+          response = Dependabot::RegistryClient.get(
+            url: repository_details[:registration_url],
+            headers: repository_details[:auth_header]
+          )
+          return unless response.status == 200
+
+          body = remove_wrapping_zero_width_chars(response.body)
+          pages = JSON.parse(body).fetch("items")
+          versions = Set.new
+          pages.each do |page|
+            items = page["items"]
+            if items
+              # inlined entries
+              items.each do |item|
+                catalog_entry = item["catalogEntry"]
+                if catalog_entry["listed"] == true
+                  vers = catalog_entry["version"]
+                  versions << vers
+                end
+              end
+            else
+              # paged entries
+              page_url = page["@id"]
+              page_versions = get_nuget_versions_from_registration_page(repository_details, page_url)
+              versions.merge(page_versions)
+            end
+          end
+
+          versions
+        rescue Excon::Error::Timeout, Excon::Error::Socket
+          repo_url = repository_details[:repository_url]
+          raise if repo_url == RepositoryFinder::DEFAULT_REPOSITORY_URL
+
+          raise PrivateSourceTimedOut, repo_url
+        end
+
+        def get_nuget_versions_from_registration_page(repository_details, page_url)
+          response = Dependabot::RegistryClient.get(
+            url: page_url,
+            headers: repository_details[:auth_header]
+          )
+          return unless response.status == 200
+
+          body = remove_wrapping_zero_width_chars(response.body)
+          items = JSON.parse(body).fetch("items")
+          versions = Set.new
+          items.each do |item|
+            catalog_entry = item.fetch("catalogEntry")
+            versions << item.fetch("version") if catalog_entry["listed"] == true
+          end
+        rescue Excon::Error::Timeout, Excon::Error::Socket
+          repo_url = repository_details[:repository_url]
+          raise if repo_url == RepositoryFinder::DEFAULT_REPOSITORY_URL
+
+          raise PrivateSourceTimedOut, repo_url
+        end
+
         def fetch_versions_from_search_url(repository_details)
           response = Dependabot::RegistryClient.get(
             url: repository_details[:search_url],

nuget/spec/dependabot/nuget/file_parser/project_file_parser_spec.rb

@@ -8,14 +8,58 @@
 
 module NuGetSearchStubs
   def stub_no_search_results(name)
-    stub_request(:get, "https://azuresearch-usnc.nuget.org/query?prerelease=true&q=#{name}&semVerLevel=2.0.0")
-      .to_return(status: 200, body: fixture("nuget_responses", "search_no_data.json"))
+    stub_request(:get, "https://api.nuget.org/v3-flatcontainer/#{name}/index.json")
+      .to_return(status: 404, body: "")
+    stub_request(:get, "https://api.nuget.org/v3/registration5-semver1/#{name}/index.json")
+      .to_return(status: 404, body: "")
+  end
+
+  def stub_registry_v3(name, versions)
+    registration_json = registration_results(name, versions)
+    stub_request(:get, "https://api.nuget.org/v3/registration5-semver1/#{name}/index.json")
+      .to_return(status: 200, body: registration_json)
   end
 
   def stub_search_results_with_versions_v3(name, versions)
-    json = search_results_with_versions_v3(name, versions)
-    stub_request(:get, "https://azuresearch-usnc.nuget.org/query?prerelease=true&q=#{name}&semVerLevel=2.0.0")
-      .to_return(status: 200, body: json)
+    stub_registry_v3(name, versions)
+    stub_versions_v3(name, versions)
+  end
+
+  def stub_versions_v3(name, versions)
+    versions_json = version_results_v3(versions)
+    stub_request(:get, "https://api.nuget.org/v3-flatcontainer/#{name}/index.json")
+      .to_return(status: 200, body: versions_json)
+  end
+
+  def version_results_v3(versions)
+    {
+      "versions" => versions
+    }.to_json
+  end
+
+  def registration_results(name, versions)
+    page = {
+      "@id": "https://api.nuget.org/v3/registration5-semver1/#{name}/index.json#page/PAGE1",
+      "@type": "catalog:CatalogPage",
+      "count" => versions.count,
+      "items" => versions.map do |version|
+        {
+          "catalogEntry" => {
+            "@type": "PackageDetails",
+            "id" => "#{name}",
+            "listed" => true,
+            "version" => version
+          }
+        }
+      end
+    }
+    pages = [page]
+    response = {
+      "@id": "https://api.nuget.org/v3/registration5-gz-semver1/#{name}/index.json",
+      "count" => versions.count,
+      "items" => pages
+    }
+    response.to_json
   end
 
   def search_results_with_versions_v3(name, versions)
@@ -189,7 +233,9 @@ def search_results_with_versions_v2(name, versions)
       end
       let(:parser) { described_class.new(dependency_files: files, credentials: credentials) }
       let(:dependencies) { dependency_set.dependencies }
-      subject(:transitive_dependencies) { dependencies.reject(&:top_level?) }
+      subject(:transitive_dependencies) do
+        dependencies.reject(&:top_level?)
+      end
 
       its(:length) { is_expected.to eq(20) }
 
@@ -687,13 +733,18 @@ def dependencies_from(dep_info)
             end
           end
 
+          # This is a bit of a noop now that we're moved off the query Nuget API,
+          # But we're keeping the test for completeness.
           describe "the dependency name is a partial, but not perfect match" do
             let(:file_body) do
               fixture("csproj", "dependency_with_name_that_does_not_exist.csproj")
             end
 
             before do
-              stub_request(:get, "https://azuresearch-usnc.nuget.org/query?prerelease=true&q=this.dependency.does.not.exist&semVerLevel=2.0.0")
+              stub_request(:get, "https://api.nuget.org/v3-flatcontainer/this.dependency.does.not.exist/index.json")
+                .to_return(status: 404, body: "")
+
+              stub_request(:get, "https://api.nuget.org/v3-flatcontainer/this.dependency.does.not.exist_but.this.one.does")
                 .to_return(status: 200, body: search_results_with_versions_v3(
                   "this.dependency.does.not.exist_but.this.one.does", ["1.0.0"]
                 ))
@@ -744,19 +795,18 @@ def dependencies_from(dep_info)
               stub_request(:get, "https://no-results.api.example.com/v3/index.json")
                 .to_return(status: 200, body: fixture("nuget_responses", "index.json",
                                                       "no-results.api.example.com.index.json"))
-              stub_request(:get, "https://no-results.api.example.com/query?prerelease=true&q=microsoft.extensions.dependencymodel&semVerLevel=2.0.0")
-                .to_return(status: 200, body: fixture("nuget_responses", "search_no_data.json"))
-              stub_request(:get, "https://no-results.api.example.com/query?prerelease=true&q=this.dependency.does.not.exist&semVerLevel=2.0.0")
-                .to_return(status: 200, body: fixture("nuget_responses", "search_no_data.json"))
+              stub_request(:get, "https://no-results.api.example.com/v3-flatcontainer/microsoft.extensions.dependencymodel/index.json")
+                .to_return(status: 404, body: "")
+              stub_request(:get, "https://no-results.api.example.com/v3-flatcontainer/this.dependency.does.not.exist/index.json")
+                .to_return(status: 404, body: "")
               # with results
               stub_request(:get, "https://with-results.api.example.com/v3/index.json")
                 .to_return(status: 200, body: fixture("nuget_responses", "index.json",
                                                       "with-results.api.example.com.index.json"))
-              stub_request(:get, "https://with-results.api.example.com/query?prerelease=true&q=microsoft.extensions.dependencymodel&semVerLevel=2.0.0")
-                .to_return(status: 200, body: search_results_with_versions_v3("microsoft.extensions.dependencymodel",
-                                                                              ["1.1.1", "1.1.0"]))
-              stub_request(:get, "https://with-results.api.example.com/query?prerelease=true&q=this.dependency.does.not.exist&semVerLevel=2.0.0")
-                .to_return(status: 200, body: fixture("nuget_responses", "search_no_data.json"))
+              stub_request(:get, "https://with-results.api.example.com/v3-flatcontainer/microsoft.extensions.dependencymodel/index.json")
+                .to_return(status: 200, body: version_results_v3(["1.1.1", "1.1.0"]))
+              stub_request(:get, "https://with-results.api.example.com/v3-flatcontainer/this.dependency.does.not.exist/index.json")
+                .to_return(status: 404, body: "")
             end
 
             it "has the right details" do
@@ -870,8 +920,9 @@ def dependencies_from(dep_info)
             end
 
             it "has the right details" do
-              query_stub = stub_search_results_with_versions_v3("microsoft.extensions.dependencymodel_cached",
-                                                                ["1.1.1", "1.1.0"])
+              versions_stub = stub_versions_v3("microsoft.extensions.dependencymodel_cached", ["1.1.1", "1.1.0"])
+              stub_registry_v3("microsoft.extensions.dependencymodel_cached", ["1.1.1", "1.1.0"])
+
               expect(top_level_dependencies.count).to eq(1)
               expect(top_level_dependencies.first).to be_a(Dependabot::Dependency)
               expect(top_level_dependencies.first.name).to eq("Microsoft.Extensions.DependencyModel_cached")
@@ -884,7 +935,7 @@ def dependencies_from(dep_info)
                   source: nil
                 }]
               )
-              expect(WebMock::RequestRegistry.instance.times_executed(query_stub.request_pattern)).to eq(1)
+              expect(WebMock::RequestRegistry.instance.times_executed(versions_stub.request_pattern)).to eq(1)
             end
           end
         end