C#: Check fallback nuget feeds before trying to use them in the fallb…

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -339,7 +339,7 @@ private void AddNetFrameworkDlls(ISet<AssemblyLookupLocation> dllLocations, ISet
                 {
                     logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
 
-                    if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, null))
+                    if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies))
                     {
                         runtimeLocation = GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory);
                     }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -95,9 +95,9 @@ public bool AddPackage(string folder, string package)
 
         public IList<string> GetListedSdks() => GetResultList("--list-sdks");
 
-        private IList<string> GetResultList(string args)
+        private IList<string> GetResultList(string args, string? workingDirectory = null)
         {
-            if (dotnetCliInvoker.RunCommand(args, out var results))
+            if (dotnetCliInvoker.RunCommand(args, workingDirectory, out var results))
             {
                 return results;
             }
@@ -111,7 +111,19 @@ public bool Exec(string execArgs)
             return dotnetCliInvoker.RunCommand(args);
         }
 
-        public IList<string> GetNugetFeeds(string nugetConfig) => GetResultList($"nuget list source --format Short --configfile \"{nugetConfig}\"");
+        private const string nugetListSourceCommand = "nuget list source --format Short";
+
+        public IList<string> GetNugetFeeds(string nugetConfig)
+        {
+            logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'...");
+            return GetResultList($"{nugetListSourceCommand} --configfile \"{nugetConfig}\"");
+        }
+
+        public IList<string> GetNugetFeedsFromFolder(string folderPath)
+        {
+            logger.LogInfo($"Getting Nuget feeds in folder '{folderPath}'...");
+            return GetResultList(nugetListSourceCommand, folderPath);
+        }
 
         // The version number should be kept in sync with the version .NET version used for building the application.
         public const string LatestDotNetSdkVersion = "8.0.101";

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs

@@ -21,41 +21,49 @@ public DotNetCliInvoker(ILogger logger, string exec)
             this.Exec = exec;
         }
 
-        private ProcessStartInfo MakeDotnetStartInfo(string args)
+        private ProcessStartInfo MakeDotnetStartInfo(string args, string? workingDirectory)
         {
             var startInfo = new ProcessStartInfo(Exec, args)
             {
                 UseShellExecute = false,
                 RedirectStandardOutput = true,
                 RedirectStandardError = true
             };
+            if (!string.IsNullOrWhiteSpace(workingDirectory))
+            {
+                startInfo.WorkingDirectory = workingDirectory;
+            }
             // Set the .NET CLI language to English to avoid localized output.
             startInfo.EnvironmentVariables["DOTNET_CLI_UI_LANGUAGE"] = "en";
             startInfo.EnvironmentVariables["MSBUILDDISABLENODEREUSE"] = "1";
             startInfo.EnvironmentVariables["DOTNET_SKIP_FIRST_TIME_EXPERIENCE"] = "true";
             return startInfo;
         }
 
-        private bool RunCommandAux(string args, out IList<string> output)
+        private bool RunCommandAux(string args, string? workingDirectory, out IList<string> output)
         {
-            logger.LogInfo($"Running {Exec} {args}");
-            var pi = MakeDotnetStartInfo(args);
+            var dirLog = string.IsNullOrWhiteSpace(workingDirectory) ? "" : $" in {workingDirectory}";
+            logger.LogInfo($"Running {Exec} {args}{dirLog}");
+            var pi = MakeDotnetStartInfo(args, workingDirectory);
             var threadId = Environment.CurrentManagedThreadId;
             void onOut(string s) => logger.LogInfo(s, threadId);
             void onError(string s) => logger.LogError(s, threadId);
             var exitCode = pi.ReadOutput(out output, onOut, onError);
             if (exitCode != 0)
             {
-                logger.LogError($"Command {Exec} {args} failed with exit code {exitCode}");
+                logger.LogError($"Command {Exec} {args}{dirLog} failed with exit code {exitCode}");
                 return false;
             }
             return true;
         }
 
         public bool RunCommand(string args) =>
-            RunCommandAux(args, out _);
+            RunCommandAux(args, null, out _);
 
         public bool RunCommand(string args, out IList<string> output) =>
-            RunCommandAux(args, out output);
+            RunCommandAux(args, null, out output);
+
+        public bool RunCommand(string args, string? workingDirectory, out IList<string> output) =>
+            RunCommandAux(args, workingDirectory, out output);
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -33,10 +33,28 @@ internal class EnvironmentVariableNames
         public const string NugetFeedResponsivenessInitialTimeout = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT";
 
         /// <summary>
-        /// Specifies how many requests to make to the NuGet feed to check its responsiveness.
+        /// Specifies the timeout (as an integer) in milliseconds for the initial check of fallback NuGet feeds responsiveness. The value is then doubled for each subsequent check.
+        /// This is primarily used in testing.
+        /// </summary>
+        internal const string NugetFeedResponsivenessInitialTimeoutForFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_TIMEOUT";
+
+        /// <summary>
+        /// Specifies how many requests to make to the NuGet feeds to check their responsiveness.
         /// </summary>
         public const string NugetFeedResponsivenessRequestCount = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT";
 
+        /// <summary>
+        /// Specifies how many requests to make to the fallback NuGet feeds to check their responsiveness.
+        /// This is primarily used in testing.
+        /// </summary>
+        internal const string NugetFeedResponsivenessRequestCountForFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_LIMIT";
+
+        /// <summary>
+        /// Specifies the NuGet feeds to use for fallback Nuget dependency fetching. The value is a space-separated list of feed URLs.
+        /// The default value is `https://api.nuget.org/v3/index.json`.
+        /// </summary>
+        public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK";
+
         /// <summary>
         /// Specifies the location of the diagnostic directory.
         /// </summary>

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -14,6 +14,7 @@ public interface IDotNet
         IList<string> GetListedSdks();
         bool Exec(string execArgs);
         IList<string> GetNugetFeeds(string nugetConfig);
+        IList<string> GetNugetFeedsFromFolder(string folderPath);
     }
 
     public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNetCliInvoker.cs

@@ -19,5 +19,11 @@ internal interface IDotNetCliInvoker
         /// The output of the command is returned in `output`.
         /// </summary>
         bool RunCommand(string args, out IList<string> output);
+
+        /// <summary>
+        /// Execute `dotnet <args>` in `<workingDirectory>` and return true if the command succeeded, otherwise false.
+        /// The output of the command is returned in `output`.
+        /// </summary>
+        bool RunCommand(string args, string? workingDirectory, out IList<string> output);
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs

@@ -243,7 +243,7 @@ private void RunMonoNugetCommand(string command, out IList<string> stdout)
         private void AddDefaultPackageSource(string nugetConfig)
         {
             logger.LogInfo("Adding default package source...");
-            RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source https://api.nuget.org/v3/index.json -ConfigFile \"{nugetConfig}\"", out var _);
+            RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source {DependencyManager.PublicNugetFeed} -ConfigFile \"{nugetConfig}\"", out var _);
         }
 
         public void Dispose()

csharp/extractor/Semmle.Extraction.Tests/DotNet.cs

@@ -10,6 +10,7 @@ internal class DotNetCliInvokerStub : IDotNetCliInvoker
     {
         private readonly IList<string> output;
         private string lastArgs = "";
+        public string WorkingDirectory { get; private set; } = "";
         public bool Success { get; set; } = true;
 
         public DotNetCliInvokerStub(IList<string> output)
@@ -32,6 +33,12 @@ public bool RunCommand(string args, out IList<string> output)
             return Success;
         }
 
+        public bool RunCommand(string args, string? workingDirectory, out IList<string> output)
+        {
+            WorkingDirectory = workingDirectory ?? "";
+            return RunCommand(args, out output);
+        }
+
         public string GetLastArgs() => lastArgs;
     }
 
@@ -262,5 +269,36 @@ public void TestDotnetExec()
             var lastArgs = dotnetCliInvoker.GetLastArgs();
             Assert.Equal("exec myarg1 myarg2", lastArgs);
         }
+
+        [Fact]
+        public void TestNugetFeeds()
+        {
+            // Setup
+            var dotnetCliInvoker = new DotNetCliInvokerStub(new List<string>());
+            var dotnet = MakeDotnet(dotnetCliInvoker);
+
+            // Execute
+            dotnet.GetNugetFeeds("abc");
+
+            // Verify
+            var lastArgs = dotnetCliInvoker.GetLastArgs();
+            Assert.Equal("nuget list source --format Short --configfile \"abc\"", lastArgs);
+        }
+
+        [Fact]
+        public void TestNugetFeedsFromFolder()
+        {
+            // Setup
+            var dotnetCliInvoker = new DotNetCliInvokerStub(new List<string>());
+            var dotnet = MakeDotnet(dotnetCliInvoker);
+
+            // Execute
+            dotnet.GetNugetFeedsFromFolder("abc");
+
+            // Verify
+            var lastArgs = dotnetCliInvoker.GetLastArgs();
+            Assert.Equal("nuget list source --format Short", lastArgs);
+            Assert.Equal("abc", dotnetCliInvoker.WorkingDirectory);
+        }
     }
 }

csharp/extractor/Semmle.Extraction.Tests/Runtime.cs

@@ -28,6 +28,8 @@ public DotNetStub(IList<string> runtimes, IList<string> sdks)
         public bool Exec(string execArgs) => true;
 
         public IList<string> GetNugetFeeds(string nugetConfig) => [];
+
+        public IList<string> GetNugetFeedsFromFolder(string folderPath) => [];
     }
 
     public class RuntimeTests

csharp/extractor/Semmle.Util/EnvironmentVariables.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Globalization;
 using System.Numerics;
 
@@ -34,5 +35,10 @@ public static bool GetBoolean(string name)
             var _ = bool.TryParse(env, out var value);
             return value;
         }
+
+        public static IEnumerable<string> GetURLs(string name)
+        {
+            return Environment.GetEnvironmentVariable(name)?.Split(" ", StringSplitOptions.RemoveEmptyEntries) ?? [];
+        }
     }
 }

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected

@@ -1,5 +1,6 @@
 | All Nuget feeds reachable | 0.0 |
 | Fallback nuget restore | 1.0 |
+| Inherited Nuget feed count | 1.0 |
 | Project files on filesystem | 1.0 |
 | Resolved assembly conflicts | 7.0 |
 | Restored .NET framework variants | 0.0 |

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/proj/nuget.config

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <configuration>
   <packageSources>
-    <clear />
+    <!-- <clear /> -->
     <add key="x" value="https://localhost:53/packages/" />
     <add key="y" value="https://localhost:80/packages/" />
   </packageSources>

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/test.py

@@ -6,5 +6,12 @@
 os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT"] = "1"     # 1ms, the GET request should fail with such short timeout
 os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT"] = "1"       # Limit the count of checks to 1
 os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_EXCLUDED"] = "https://abc.de:8000/packages/"   # Exclude this feed from check
+
+# Making sure the reachability test of `nuget.org` succeeds:
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_TIMEOUT"] = "1000"
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_LIMIT"] = "5"
+# The second feed is ignored in the fallback restore, because of network issues:
+os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK"] = "https://api.nuget.org/v3/index.json https://abc.def:8000/packages/"
+
 run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
 check_diagnostics()