Merge branch 'main' into jonjanego-patch-1

common/lib/dependabot/file_fetchers/base.rb

@@ -354,11 +354,6 @@ def update_linked_paths(repo, path, commit, github_response)
         end
       end
 
-      sig { returns(T::Boolean) }
-      def recurse_submodules_when_cloning?
-        false
-      end
-
       sig do
         returns(
           T.any(
@@ -776,7 +771,6 @@ def _linked_dir_for(path)
       # rubocop:disable Metrics/MethodLength
       # rubocop:disable Metrics/PerceivedComplexity
       # rubocop:disable Metrics/BlockLength
-      # rubocop:disable Metrics/CyclomaticComplexity
       sig { params(target_directory: T.nilable(String)).returns(String) }
       def _clone_repo_contents(target_directory:)
         SharedHelpers.with_git_configured(credentials: credentials) do
@@ -789,11 +783,7 @@ def _clone_repo_contents(target_directory:)
 
           clone_options = StringIO.new
           clone_options << "--no-tags --depth 1"
-          clone_options << if recurse_submodules_when_cloning?
-                             " --recurse-submodules --shallow-submodules"
-                           else
-                             " --no-recurse-submodules"
-                           end
+          clone_options << " --recurse-submodules --shallow-submodules"
           clone_options << " --branch #{source.branch} --single-branch" if source.branch
 
           submodule_cloning_failed = false
@@ -805,7 +795,7 @@ def _clone_repo_contents(target_directory:)
               CMD
             )
 
-            @submodules = find_submodules(path) if recurse_submodules_when_cloning?
+            @submodules = find_submodules(path)
           rescue SharedHelpers::HelperSubprocessFailed => e
             if GIT_RETRYABLE_ERRORS.any? { |error| error.match?(e.message) } && retries < 5
               retries += 1
@@ -835,20 +825,20 @@ def _clone_repo_contents(target_directory:)
             Dir.chdir(path) do
               fetch_options = StringIO.new
               fetch_options << "--depth 1"
-              fetch_options << if recurse_submodules_when_cloning? && !submodule_cloning_failed
-                                 " --recurse-submodules=on-demand"
-                               else
+              fetch_options << if submodule_cloning_failed
                                  " --no-recurse-submodules"
+                               else
+                                 " --recurse-submodules=on-demand"
                                end
               # Need to fetch the commit due to the --depth 1 above.
               SharedHelpers.run_shell_command("git fetch #{fetch_options.string} origin #{source.commit}")
 
               reset_options = StringIO.new
               reset_options << "--hard"
-              reset_options << if recurse_submodules_when_cloning? && !submodule_cloning_failed
-                                 " --recurse-submodules"
-                               else
+              reset_options << if submodule_cloning_failed
                                  " --no-recurse-submodules"
+                               else
+                                 " --recurse-submodules"
                                end
               # Set HEAD to this commit so later calls so git reset HEAD will work.
               SharedHelpers.run_shell_command("git reset #{reset_options.string} #{source.commit}")
@@ -862,7 +852,6 @@ def _clone_repo_contents(target_directory:)
       # rubocop:enable Metrics/MethodLength
       # rubocop:enable Metrics/PerceivedComplexity
       # rubocop:enable Metrics/BlockLength
-      # rubocop:enable Metrics/CyclomaticComplexity
 
       sig { params(str: String).returns(String) }
       def decode_binary_string(str)

common/spec/dependabot/file_fetchers/base_spec.rb

@@ -1659,7 +1659,7 @@ def fetch_files
             )
 
           expect { subject }.to_not raise_error
-          expect(Dependabot::SharedHelpers).to have_received(:run_shell_command).twice
+          expect(Dependabot::SharedHelpers).to have_received(:run_shell_command).thrice
           expect(file_fetcher_instance).to have_received(:sleep).once
         end
 
@@ -1706,23 +1706,23 @@ def fetch_files
     after { FileUtils.rm_rf(repo_contents_path) }
 
     describe "#clone_repo_contents" do
-      it "does not clone submodules by default" do
+      it "clones submodules by default" do
         file_fetcher_instance.clone_repo_contents
 
-        expect(`ls -1 #{submodule_contents_path}`.split).to_not include("go.mod")
+        expect(`ls -1 #{submodule_contents_path}`.split).to include("go.mod")
       end
 
       context "with a source commit" do
         let(:source_commit) { "5c7e92a4860382fd31336872f0fe79a848669c4d" }
 
-        it "does not fetch/reset submodules by default" do
+        it "fetches/reset submodules by default" do
           file_fetcher_instance.clone_repo_contents
 
-          expect(`ls -1 #{submodule_contents_path}`.split).to_not include("go.mod")
+          expect(`ls -1 #{submodule_contents_path}`.split).to include("go.mod")
         end
       end
 
-      context "when #recurse_submodules_when_cloning? returns true" do
+      context "when there's a submodule" do
         let(:child_class) do
           Class.new(described_class) do
             def self.required_files_in?(filenames)
@@ -1738,10 +1738,6 @@ def self.required_files_message
             def fetch_files
               [fetch_file_from_host("go.mod")]
             end
-
-            def recurse_submodules_when_cloning?
-              true
-            end
           end
         end
 

go_modules/lib/dependabot/go_modules/file_fetcher.rb

@@ -55,10 +55,6 @@ def go_sum
 
         @go_sum = fetch_file_if_present("go.sum")
       end
-
-      def recurse_submodules_when_cloning?
-        true
-      end
     end
   end
 end

npm_and_yarn/lib/dependabot/npm_and_yarn/file_fetcher.rb

@@ -87,10 +87,6 @@ def fetch_files
 
       private
 
-      def recurse_submodules_when_cloning?
-        true
-      end
-
       def npm_files
         fetched_npm_files = []
         fetched_npm_files << package_lock if package_lock && !skip_package_lock?

nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Utilities/MSBuildHelperTests.cs

@@ -319,6 +319,116 @@ public async Task AllPackageDependencies_DoNotIncludeUpdateOnlyPackages()
         Assert.Equal(expectedDependencies, actualDependencies);
     }
 
+    [Fact]
+    public async Task GetAllPackageDependencies_NugetConfigInvalid_DoesNotThrow()
+    {
+        var nugetPackagesDirectory = Environment.GetEnvironmentVariable("NUGET_PACKAGES");
+        var nugetHttpCacheDirectory = Environment.GetEnvironmentVariable("NUGET_HTTP_CACHE_PATH");
+
+        try
+        {
+            using var temp = new TemporaryDirectory();
+
+            // It is important to have empty NuGet caches for this test, so override them with temp directories.
+            var tempNuGetPackagesDirectory = Path.Combine(temp.DirectoryPath, ".nuget", "packages");
+            Environment.SetEnvironmentVariable("NUGET_PACKAGES", tempNuGetPackagesDirectory);
+            var tempNuGetHttpCacheDirectory = Path.Combine(temp.DirectoryPath, ".nuget", "v3-cache");
+            Environment.SetEnvironmentVariable("NUGET_HTTP_CACHE_PATH", tempNuGetHttpCacheDirectory);
+
+            // Write the NuGet.config with a missing "/>"
+            await File.WriteAllTextAsync(
+                Path.Combine(temp.DirectoryPath, "NuGet.Config"), """
+                <?xml version="1.0" encoding="utf-8"?>
+                <configuration>
+                  <packageSources>
+                    <clear />
+                    <add key="contoso" value="https://contoso.com/v3/index.json"
+                  </packageSources>
+                </configuration>
+                """);
+
+            // Asserting it didn't throw
+            var actualDependencies = await MSBuildHelper.GetAllPackageDependenciesAsync(
+                temp.DirectoryPath,
+                temp.DirectoryPath,
+                "netstandard2.0",
+                [new Dependency("Newtonsoft.Json", "4.5.11", DependencyType.Unknown)]
+            );
+        }
+        finally
+        {
+            // Restore the NuGet caches.
+            Environment.SetEnvironmentVariable("NUGET_PACKAGES", nugetPackagesDirectory);
+            Environment.SetEnvironmentVariable("NUGET_HTTP_CACHE_PATH", nugetHttpCacheDirectory);
+        }
+    }
+
+    [Fact]
+    public async Task GetAllPackageDependencies_LocalNuGetRepos_AreCopiedToTempProject()
+    {
+        // If we end up using this EnvVar pattern again I think it'd be worth it to abstract it out into an IDisposable.
+        var nugetPackagesDirectory = Environment.GetEnvironmentVariable("NUGET_PACKAGES");
+        var nugetHttpCacheDirectory = Environment.GetEnvironmentVariable("NUGET_HTTP_CACHE_PATH");
+        var logger = new Logger(verbose: true);
+        try
+        {
+            // First create a fake local nuget repository
+            using var restoreDir = new TemporaryDirectory();
+
+            var restoreNuGetPackagesDirectory = Path.Combine(restoreDir.DirectoryPath, ".nuget", "packages");
+            Environment.SetEnvironmentVariable("NUGET_PACKAGES", restoreNuGetPackagesDirectory);
+            var restoreNuGetHttpCacheDirectory = Path.Combine(restoreDir.DirectoryPath, ".nuget", "v3-cache");
+            Environment.SetEnvironmentVariable("NUGET_HTTP_CACHE_PATH", restoreNuGetHttpCacheDirectory);
+
+            using var temp = new TemporaryDirectory();
+            using (var restoreProjectTemp = new TemporaryDirectory())
+            {
+                // dotnet restore .csproj with things we want
+                await MSBuildHelper.DependenciesAreCoherentAsync(restoreProjectTemp.DirectoryPath, restoreProjectTemp.DirectoryPath, "netstandard2.0",
+                    [new Dependency("Newtonsoft.Json", "4.5.11", DependencyType.Unknown)], logger);
+                Assert.True(Directory.Exists(restoreNuGetPackagesDirectory), "packages directory didn't exist");
+                PathHelper.CopyDirectory(restoreNuGetPackagesDirectory, Path.Combine(temp.DirectoryPath, "local_repo"));
+            }
+
+            // It is important to have empty NuGet caches for this test, so override them with temp directories.
+            var tempNuGetPackagesDirectory = Path.Combine(temp.DirectoryPath, ".nuget", "packages");
+            Environment.SetEnvironmentVariable("NUGET_PACKAGES", tempNuGetPackagesDirectory);
+            var tempNuGetHttpCacheDirectory = Path.Combine(temp.DirectoryPath, ".nuget", "v3-cache");
+            Environment.SetEnvironmentVariable("NUGET_HTTP_CACHE_PATH", tempNuGetHttpCacheDirectory);
+
+            // Write the NuGet.config.
+            await File.WriteAllTextAsync(
+                Path.Combine(temp.DirectoryPath, "NuGet.Config"), """
+                <?xml version="1.0" encoding="utf-8"?>
+                <configuration>
+                  <packageSources>
+                    <clear />
+                    <add key="local-repo" value="local_repo" />
+                  </packageSources>
+                </configuration>
+                """);
+            var expectedDependencies = new Dependency[]
+            {
+                new("Newtonsoft.Json", "4.5.11", DependencyType.Unknown),
+                new("NETStandard.Library", "2.0.3", DependencyType.Unknown),
+            };
+            var actualDependencies = await MSBuildHelper.GetAllPackageDependenciesAsync(
+                temp.DirectoryPath,
+                temp.DirectoryPath,
+                "netstandard2.0",
+                [new Dependency("Newtonsoft.Json", "4.5.11", DependencyType.Unknown)]
+            );
+            Assert.False(Directory.Exists(tempNuGetHttpCacheDirectory), "The .nuget/.v3-cache directory was created, meaning http was used.");
+            Assert.Equal(expectedDependencies, actualDependencies);
+        }
+        finally
+        {
+            // Restore the NuGet caches.
+            Environment.SetEnvironmentVariable("NUGET_PACKAGES", nugetPackagesDirectory);
+            Environment.SetEnvironmentVariable("NUGET_HTTP_CACHE_PATH", nugetHttpCacheDirectory);
+        }
+    }
+
     [Fact]
     public async Task AllPackageDependenciesCanBeFoundWithNuGetConfig()
     {

nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/MSBuildHelper.cs

@@ -17,6 +17,8 @@
 using Microsoft.Build.Locator;
 using Microsoft.Extensions.FileSystemGlobbing;
 
+using NuGet.Configuration;
+
 using NuGetUpdater.Core.Utilities;
 
 namespace NuGetUpdater.Core;
@@ -331,6 +333,25 @@ private static ProjectRootElement CreateProjectRootElement(ProjectBuildFile buil
         return projectRoot;
     }
 
+    private static IEnumerable<PackageSource>? LoadPackageSources(string nugetConfigPath)
+    {
+        try
+        {
+            var nugetConfigDir = Path.GetDirectoryName(nugetConfigPath);
+            var settings = Settings.LoadSpecificSettings(nugetConfigDir, Path.GetFileName(nugetConfigPath));
+            var packageSourceProvider = new PackageSourceProvider(settings);
+            return packageSourceProvider.LoadPackageSources();
+        }
+        catch (NuGetConfigurationException ex)
+        {
+            Console.WriteLine("Error while parsing NuGet.config");
+            Console.WriteLine(ex.Message);
+
+            // Nuget.config is invalid. Won't be able to do anything with specific sources.
+            return null;
+        }
+    }
+
     private static async Task<string> CreateTempProjectAsync(
         DirectoryInfo tempDir,
         string repoRoot,
@@ -341,10 +362,27 @@ private static async Task<string> CreateTempProjectAsync(
         var projectDirectory = Path.GetDirectoryName(projectPath);
         projectDirectory ??= repoRoot;
         var topLevelFiles = Directory.GetFiles(repoRoot);
-        var nugetConfigPath = PathHelper.GetFileInDirectoryOrParent(projectDirectory, repoRoot, "NuGet.Config", caseSensitive: false);
+        var nugetConfigPath = PathHelper.GetFileInDirectoryOrParent(projectPath, repoRoot, "NuGet.Config", caseSensitive: false);
         if (nugetConfigPath is not null)
         {
+            // Copy nuget.config to temp project directory
             File.Copy(nugetConfigPath, Path.Combine(tempDir.FullName, "NuGet.Config"));
+            var nugetConfigDir = Path.GetDirectoryName(nugetConfigPath);
+
+            var packageSources = LoadPackageSources(nugetConfigPath);
+            if (packageSources is not null)
+            {
+                // We need to copy local package sources from the NuGet.Config file to the temp directory
+                foreach (var localSource in packageSources.Where(p => p.IsLocal))
+                {
+                    var subDir = localSource.Source.Split(nugetConfigDir)[1];
+                    var destPath = Path.Join(tempDir.FullName, subDir);
+                    if (Directory.Exists(localSource.Source))
+                    {
+                        PathHelper.CopyDirectory(localSource.Source, destPath);
+                    }
+                }
+            }
         }
 
         var packageReferences = string.Join(

nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/PathHelper.cs

@@ -79,4 +79,31 @@ public static string GetFullPathFromRelative(string rootPath, string relativePat
 
         return null;
     }
+
+    public static void CopyDirectory(string sourceDirectory, string destinationDirectory)
+    {
+        var sourceDirInfo = new DirectoryInfo(sourceDirectory);
+        var destinationDirInfo = new DirectoryInfo(destinationDirectory);
+
+        if (!sourceDirInfo.Exists)
+        {
+            throw new DirectoryNotFoundException($"Source directory does not exist or could not be found: {sourceDirectory}");
+        }
+
+        if (!destinationDirInfo.Exists)
+        {
+            destinationDirInfo.Create();
+        }
+
+        foreach (var file in sourceDirInfo.EnumerateFiles())
+        {
+            file.CopyTo(Path.Combine(destinationDirectory, file.Name), true);
+        }
+
+        foreach (var subDir in sourceDirInfo.EnumerateDirectories())
+        {
+            var newDestinationDir = Path.Combine(destinationDirectory, subDir.Name);
+            CopyDirectory(subDir.FullName, newDestinationDir);
+        }
+    }
 }

nuget/lib/dependabot/nuget/nuget_client.rb

@@ -21,11 +21,33 @@ def self.get_package_versions(dependency_name, repository_details)
           get_package_versions_v3(dependency_name, repository_details)
         elsif repository_type == "v2"
           get_package_versions_v2(dependency_name, repository_details)
+        elsif repository_type == "local"
+          get_package_versions_local(dependency_name, repository_details)
         else
           raise "Unknown repository type: #{repository_type}"
         end
       end
 
+      sig do
+        params(dependency_name: String, repository_details: T::Hash[Symbol, String])
+          .returns(T.nilable(T::Set[String]))
+      end
+      private_class_method def self.get_package_versions_local(dependency_name, repository_details)
+        url = repository_details.fetch(:base_url)
+        raise "Local repo #{url} doesn't exist or isn't a directory" unless File.exist?(url) && File.directory?(url)
+
+        package_dir = File.join(url, dependency_name)
+
+        versions = Set.new
+        return versions unless File.exist?(package_dir) && File.directory?(package_dir)
+
+        Dir.each_child(package_dir) do |child|
+          versions.add(child) if File.directory?(File.join(package_dir, child))
+        end
+
+        versions
+      end
+
       sig do
         params(dependency_name: String, repository_details: T::Hash[Symbol, String])
           .returns(T.nilable(T::Set[String]))