Resilience

bin/dry-run.rb

@@ -138,6 +138,15 @@
   }
 end
 
+unless ENV["LOCAL_AZURE_ACCESS_TOKEN"].to_s.strip.empty?
+  $options[:credentials] << {
+    "type" => "nuget_feed",
+    "host" => "pkgs.dev.azure.com",
+    "url" => ENV.fetch("LOCAL_AZURE_FEED_URL", nil),
+    "token" => ":#{ENV.fetch("LOCAL_AZURE_ACCESS_TOKEN", nil)}"
+  }
+end
+
 unless ENV["LOCAL_CONFIG_VARIABLES"].to_s.strip.empty?
   # For example:
   # "[{\"type\":\"npm_registry\",\"registry\":\
@@ -387,8 +396,8 @@ def fetch_files(fetcher)
     else
       puts "=> cloning into #{$repo_contents_path}"
       FileUtils.rm_rf($repo_contents_path)
+      fetcher.clone_repo_contents
     end
-    fetcher.clone_repo_contents
     if $options[:commit]
       Dir.chdir($repo_contents_path) do
         puts "=> checking out commit #{$options[:commit]}"

nuget/lib/dependabot/nuget/file_fetcher.rb

@@ -32,6 +32,7 @@ def self.required_files_message
 
       sig { override.returns(T::Array[DependencyFile]) }
       def fetch_files
+        @files_fetched = {}
         fetched_files = []
         fetched_files += project_files
         fetched_files += directory_build_files
@@ -280,27 +281,31 @@ def imported_property_files
       end
 
       def fetch_imported_property_files(file:, previously_fetched_files:)
-        paths =
-          ImportPathsFinder.new(project_file: file).import_paths +
-          ImportPathsFinder.new(project_file: file).project_reference_paths +
-          ImportPathsFinder.new(project_file: file).project_file_paths
-
-        paths.flat_map do |path|
-          next if previously_fetched_files.map(&:name).include?(path)
-          next if file.name == path
-          next if path.include?("$(")
-
-          fetched_file = fetch_file_from_host(path)
-          grandchild_property_files = fetch_imported_property_files(
-            file: fetched_file,
-            previously_fetched_files: previously_fetched_files + [file]
-          )
-          [fetched_file, *grandchild_property_files]
-        rescue Dependabot::DependencyFileNotFound
-          # Don't worry about missing files too much for now (at least
-          # until we start resolving properties)
-          nil
-        end.compact
+        file_id = file.directory+'/'+file.name
+        if !@files_fetched.assoc(file_id)
+          @files_fetched[file_id] = 1;
+          paths =
+            ImportPathsFinder.new(project_file: file).import_paths +
+            ImportPathsFinder.new(project_file: file).project_reference_paths +
+            ImportPathsFinder.new(project_file: file).project_file_paths
+
+          paths.flat_map do |path|
+            next if previously_fetched_files.map(&:name).include?(path)
+            next if file.name == path
+            next if path.include?("$(")
+
+            fetched_file = fetch_file_from_host(path)
+            grandchild_property_files = fetch_imported_property_files(
+              file: fetched_file,
+              previously_fetched_files: previously_fetched_files + [file]
+            )
+            [fetched_file, *grandchild_property_files]
+          rescue Dependabot::DependencyFileNotFound
+            # Don't worry about missing files too much for now (at least
+            # until we start resolving properties)
+            nil
+          end.compact
+        end
       end
     end
   end

nuget/lib/dependabot/nuget/file_parser/project_file_parser.rb

@@ -156,14 +156,19 @@ def add_transitive_dependencies_from_project_references(project_file, doc, depen
             referenced_file = dependency_files.find { |f| f.name == full_path }
             next unless referenced_file
 
-            dependency_set(project_file: referenced_file).dependencies.each do |dep|
-              dependency = Dependency.new(
-                name: dep.name,
-                version: dep.version,
-                package_manager: dep.package_manager,
-                requirements: []
-              )
-              dependency_set << dependency
+            key = "#{referenced_file.directory.downcase}#{referenced_file.name.downcase}::#{referenced_file.content.hash}::transitive"
+            cache = ProjectFileParser.dependency_set_cache
+            if !cache.key?(key)
+              dependency_set(project_file: referenced_file).dependencies.each do |dep|
+                dependency = Dependency.new(
+                  name: dep.name,
+                  version: dep.version,
+                  package_manager: dep.package_manager,
+                  requirements: []
+                )
+                dependency_set << dependency
+              end
+              cache[key] = 1
             end
           end
         end
@@ -307,16 +312,21 @@ def dependency_url_has_matching_result?(dependency_name, dependency_url)
         end
 
         def dependency_url_has_matching_result_v3?(dependency_name, dependency_url)
-          url = dependency_url.fetch(:search_url)
+          url = dependency_url.fetch(:versions_url)
           auth_header = dependency_url.fetch(:auth_header)
           response = execute_search_for_dependency_url(url, auth_header)
           return false unless response.status == 200
 
-          body = JSON.parse(response.body)
-          data = body["data"]
-          return false unless data.length.positive?
+          begin
+            body = JSON.parse(response.body)
+            versions = body["versions"]
 
-          data.any? { |result| result["id"].casecmp?(dependency_name) }
+            versions.any?
+          rescue JSON::ParserError => e
+            Dependabot.logger.error("Error parsing JSON from #{url}: #{e.message}. Json was: '#{response.body}'")
+            record_security_update_error(e)
+            raise e
+          end
         end
 
         def dependency_url_has_matching_result_v2?(dependency_name, dependency_url)

nuget/lib/dependabot/nuget/update_checker/repository_finder.rb

@@ -14,6 +14,8 @@ class RepositoryFinder
         DEFAULT_REPOSITORY_URL = "https://api.nuget.org/v3/index.json"
         DEFAULT_REPOSITORY_API_KEY = "nuget.org"
 
+        @@metadata_cache = {}
+
         def initialize(dependency:, credentials:, config_files: [])
           @dependency  = dependency
           @credentials = credentials
@@ -62,7 +64,9 @@ def build_url_for_details(repo_details)
           body = remove_wrapping_zero_width_chars(response.body)
           base_url = base_url_from_v3_metadata(JSON.parse(body))
           resolved_base_url = base_url || repo_details.fetch(:url).gsub("/index.json", "-flatcontainer")
-          search_url = search_url_from_v3_metadata(JSON.parse(body))
+          parsed_json = JSON.parse(body) 
+          search_url = search_url_from_v3_metadata(parsed_json)
+          registration_url = registration_url_from_v3_metadata(parsed_json)
 
           details = {
             base_url: resolved_base_url,
@@ -78,6 +82,9 @@ def build_url_for_details(repo_details)
             details[:search_url] =
               search_url + "?q=#{dependency.name.downcase}&prerelease=true&semVerLevel=2.0.0"
           end
+          if registration_url
+            details[:registration_url] = registration_url + dependency.name.downcase
+          end
           details
         rescue JSON::ParserError
           build_v2_url(response, repo_details)
@@ -86,10 +93,13 @@ def build_url_for_details(repo_details)
         end
 
         def get_repo_metadata(repo_details)
-          Dependabot::RegistryClient.get(
-            url: repo_details.fetch(:url),
+          url = repo_details.fetch(:url)
+          @@metadata_cache[url] ||= Dependabot::RegistryClient.get(
+            url: url,
             headers: auth_header_for_token(repo_details.fetch(:token))
           )
+
+          @@metadata_cache[url]
         end
 
         def base_url_from_v3_metadata(metadata)
@@ -99,6 +109,17 @@ def base_url_from_v3_metadata(metadata)
             &.fetch("@id")
         end
 
+        def registration_url_from_v3_metadata(metadata)
+          allowed_registration_types = %w(
+            RegistrationsBaseUrl/3.0.0-beta
+            RegistrationsBaseUrl
+          )
+          metadata
+            .fetch("resources", [])
+            .find { |r| allowed_registration_types.find { |s| r.fetch("@type") == s } }
+            &.fetch("@id")
+        end
+
         def search_url_from_v3_metadata(metadata)
           # allowable values from here: https://learn.microsoft.com/en-us/nuget/api/search-query-service-resource#versioning
           allowed_search_types = %w(

nuget/lib/dependabot/nuget/update_checker/version_finder.rb

@@ -296,7 +296,10 @@ def fetch_v2_next_link_href(xml_body)
         def versions_for_v3_repository(repository_details)
           # If we have a search URL that returns results we use it
           # (since it will exclude unlisted versions)
-          if repository_details[:search_url]
+
+          if repository_details[:registration_url]
+            get_nuget_versions_from_registration(repository_details)
+          elsif repository_details[:search_url]
             fetch_versions_from_search_url(repository_details)
           # Otherwise, use the versions URL
           elsif repository_details[:versions_url]
@@ -311,7 +314,68 @@ def versions_for_v3_repository(repository_details)
           end
         end
 
+        def get_nuget_versions_from_registration(repository_details)
+          response = Dependabot::RegistryClient.get(
+            url: repository_details[:registration_url],
+            headers: repository_details[:auth_header]
+          )
+          return unless response.status == 200
+
+          body = remove_wrapping_zero_width_chars(response.body)
+          pages = JSON.parse(body).fetch("items")
+          versions = Set.new
+          pages.each do |page|
+            items = page["items"]
+            if items
+              # inlined entries
+              items.each do |item|
+                catalog_entry = item["catalogEntry"]
+                if catalog_entry['listed'] == true
+                  vers = catalog_entry['version']
+                  versions << vers
+                end
+              end
+            elsif
+              # paged entries
+              page_url = page["@id"]
+              page_versions = get_nuget_versions_from_registration_page(repository_details, page_url)
+              versions.merge(page_versions)
+            end
+          end
+
+          versions
+        rescue Excon::Error::Timeout, Excon::Error::Socket
+          repo_url = repository_details[:repository_url]
+          raise if repo_url == RepositoryFinder::DEFAULT_REPOSITORY_URL
+
+          raise PrivateSourceTimedOut, repo_url
+        end
+
+        def get_nuget_versions_from_registration_page(repository_details, page_url)
+          response = Dependabot::RegistryClient.get(
+            url: repository_details[:registration_url],
+            headers: repository_details[:auth_header]
+          )
+          return unless response.status == 200
+
+          body = remove_wrapping_zero_width_chars(response.body)
+          items = JSON.parse(body).fetch("items")
+          versions = Set.new
+          items.each do |item|
+            catalog_entry = item.fetch('catalogEntry')
+            if catalog_entry['listed'] == true
+              versions << item.fetch('version')
+            end
+          end
+        rescue Excon::Error::Timeout, Excon::Error::Socket
+          repo_url = repository_details[:repository_url]
+          raise if repo_url == RepositoryFinder::DEFAULT_REPOSITORY_URL
+
+          raise PrivateSourceTimedOut, repo_url
+        end
+
         def fetch_versions_from_search_url(repository_details)
+          puts "#{repository_details[:search_url]}"
           response = Dependabot::RegistryClient.get(
             url: repository_details[:search_url],
             headers: repository_details[:auth_header]

updater/lib/dependabot/updater/security_update_helpers.rb

@@ -16,6 +16,15 @@ def grouped_security_update_group(job)
         )
       end
 
+      def record_security_update_error(exception)
+        service.record_update_job_error(
+          error_type: "security_update_error",
+          error_details: {
+            "error": exception,
+          }
+        )
+      end
+
       def record_security_update_not_needed_error(checker)
         Dependabot.logger.info(
           "no security update needed as #{checker.dependency.name} " \