diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs index 134b1857f8fb..d9a0eac4845a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs @@ -60,6 +60,11 @@ internal class EnvironmentVariableNames /// public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK"; + /// + /// Controls whether to include NuGet feeds from nuget.config files in the fallback restore logic. + /// + public const string AddNugetConfigFeedsToFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK_INCLUDE_NUGET_CONFIG_FEEDS"; + /// /// Specifies the path to the nuget executable to be used for package restoration. /// diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index 5e556682df21..0204e9b7c408 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -98,12 +98,14 @@ public HashSet Restore() logger.LogInfo($"Checking NuGet feed responsiveness: {checkNugetFeedResponsiveness}"); compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", checkNugetFeedResponsiveness ? "1" : "0")); + HashSet? explicitFeeds = null; + try { - if (checkNugetFeedResponsiveness && !CheckFeeds()) + if (checkNugetFeedResponsiveness && !CheckFeeds(out explicitFeeds)) { // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too. - var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds(); + var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds(explicitFeeds); return unresponsiveMissingPackageLocation is null ? [] : [unresponsiveMissingPackageLocation]; @@ -163,7 +165,7 @@ public HashSet Restore() LogAllUnusedPackages(dependencies); var missingPackageLocation = checkNugetFeedResponsiveness - ? DownloadMissingPackagesFromSpecificFeeds() + ? DownloadMissingPackagesFromSpecificFeeds(explicitFeeds) : DownloadMissingPackages(); if (missingPackageLocation is not null) @@ -173,13 +175,24 @@ public HashSet Restore() return assemblyLookupLocations; } - private List GetReachableFallbackNugetFeeds() + private List GetReachableFallbackNugetFeeds(HashSet? feedsFromNugetConfigs) { var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet(); if (fallbackFeeds.Count == 0) { fallbackFeeds.Add(PublicNugetOrgFeed); - logger.LogInfo($"No fallback Nuget feeds specified. Using default feed: {PublicNugetOrgFeed}"); + logger.LogInfo($"No fallback Nuget feeds specified. Adding default feed: {PublicNugetOrgFeed}"); + + var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback); + logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}"); + + if (shouldAddNugetConfigFeeds && feedsFromNugetConfigs?.Count > 0) + { + // There are some feeds in `feedsFromNugetConfigs` that have already been checked for reachability, we could skip those. + // But we might use different responsiveness testing settings when we try them in the fallback logic, so checking them again is safer. + fallbackFeeds.UnionWith(feedsFromNugetConfigs); + logger.LogInfo($"Using Nuget feeds from nuget.config files as fallback feeds: {string.Join(", ", feedsFromNugetConfigs.OrderBy(f => f))}"); + } } logger.LogInfo($"Checking fallback Nuget feed reachability on feeds: {string.Join(", ", fallbackFeeds.OrderBy(f => f))}"); @@ -194,6 +207,8 @@ private List GetReachableFallbackNugetFeeds() logger.LogInfo($"Reachable fallback Nuget feeds: {string.Join(", ", reachableFallbackFeeds.OrderBy(f => f))}"); } + compilationInfoContainer.CompilationInfos.Add(("Reachable fallback Nuget feed count", reachableFallbackFeeds.Count.ToString())); + return reachableFallbackFeeds; } @@ -272,9 +287,9 @@ private void RestoreProjects(IEnumerable projects, out ConcurrentBag? feedsFromNugetConfigs) { - var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(); + var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(feedsFromNugetConfigs); if (reachableFallbackFeeds.Count > 0) { return DownloadMissingPackages(fallbackNugetFeeds: reachableFallbackFeeds); @@ -623,10 +638,10 @@ private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount, return (timeoutMilliSeconds, tryCount); } - private bool CheckFeeds() + private bool CheckFeeds(out HashSet explicitFeeds) { logger.LogInfo("Checking Nuget feeds..."); - var (explicitFeeds, allFeeds) = GetAllFeeds(); + (explicitFeeds, var allFeeds) = GetAllFeeds(); var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck) .ToHashSet() ?? []; diff --git a/csharp/ql/integration-tests/all-platforms/standalone_resx/CompilationInfo.expected b/csharp/ql/integration-tests/all-platforms/standalone_resx/CompilationInfo.expected index 1fbab458c34a..48cca2534533 100644 --- a/csharp/ql/integration-tests/all-platforms/standalone_resx/CompilationInfo.expected +++ b/csharp/ql/integration-tests/all-platforms/standalone_resx/CompilationInfo.expected @@ -3,6 +3,7 @@ | Failed solution restore with package source error | 0.0 | | NuGet feed responsiveness checked | 1.0 | | Project files on filesystem | 1.0 | +| Reachable fallback Nuget feed count | 1.0 | | Resource extraction enabled | 1.0 | | Restored .NET framework variants | 1.0 | | Restored projects through solution files | 0.0 | diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/CompilationInfo.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/CompilationInfo.expected index 81a44b5f8fd2..53ebd1016fb7 100644 --- a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/CompilationInfo.expected +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/CompilationInfo.expected @@ -4,6 +4,7 @@ | Fallback nuget restore | 1.0 | | NuGet feed responsiveness checked | 1.0 | | Project files on filesystem | 1.0 | +| Reachable fallback Nuget feed count | 1.0 | | Resolved assembly conflicts | 7.0 | | Resource extraction enabled | 0.0 | | Restored .NET framework variants | 0.0 | diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected index 026a3d386e3c..777d615d99b9 100644 --- a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected @@ -3,6 +3,7 @@ | Inherited Nuget feed count | 1.0 | | NuGet feed responsiveness checked | 1.0 | | Project files on filesystem | 1.0 | +| Reachable fallback Nuget feed count | 1.0 | | Resolved assembly conflicts | 7.0 | | Resource extraction enabled | 0.0 | | Restored .NET framework variants | 0.0 | diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/Assemblies.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/Assemblies.expected new file mode 100644 index 000000000000..2a530060edb2 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/Assemblies.expected @@ -0,0 +1 @@ +| [...]/newtonsoft.json/13.0.3/lib/net6.0/Newtonsoft.Json.dll | diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/Assemblies.ql b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/Assemblies.ql new file mode 100644 index 000000000000..79cf92de7911 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/Assemblies.ql @@ -0,0 +1,11 @@ +import csharp + +private string getPath(Assembly a) { + not a.getCompilation().getOutputAssembly() = a and + exists(string s | s = a.getFile().getAbsolutePath() | + result = "[...]/" + s.substring(s.indexOf("newtonsoft.json"), s.length()) + ) +} + +from Assembly a +select getPath(a) diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/CompilationInfo.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/CompilationInfo.expected new file mode 100644 index 000000000000..9e869e1a6fb1 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/CompilationInfo.expected @@ -0,0 +1,16 @@ +| All Nuget feeds reachable | 0.0 | +| Fallback nuget restore | 1.0 | +| NuGet feed responsiveness checked | 1.0 | +| Project files on filesystem | 1.0 | +| Reachable fallback Nuget feed count | 2.0 | +| Resolved assembly conflicts | 7.0 | +| Resource extraction enabled | 0.0 | +| Restored .NET framework variants | 0.0 | +| Solution files on filesystem | 1.0 | +| Source files generated | 0.0 | +| Source files on filesystem | 1.0 | +| Successfully ran fallback nuget restore | 1.0 | +| Unresolved references | 0.0 | +| UseWPF set | 0.0 | +| UseWindowsForms set | 0.0 | +| WebView extraction enabled | 1.0 | diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/CompilationInfo.ql b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/CompilationInfo.ql new file mode 100644 index 000000000000..073ffe3b224d --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/CompilationInfo.ql @@ -0,0 +1,15 @@ +import csharp +import semmle.code.csharp.commons.Diagnostics + +query predicate compilationInfo(string key, float value) { + key != "Resolved references" and + not key.matches("Compiler diagnostic count for%") and + exists(Compilation c, string infoKey, string infoValue | infoValue = c.getInfo(infoKey) | + key = infoKey and + value = infoValue.toFloat() + or + not exists(infoValue.toFloat()) and + key = infoKey + ": " + infoValue and + value = 1 + ) +} diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/diagnostics.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/diagnostics.expected new file mode 100644 index 000000000000..5f298cd3a11f --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/diagnostics.expected @@ -0,0 +1,42 @@ +{ + "markdownMessage": "C# analysis with build-mode 'none' completed.", + "severity": "unknown", + "source": { + "extractorName": "csharp", + "id": "csharp/autobuilder/buildless/complete", + "name": "C# analysis with build-mode 'none' completed" + }, + "visibility": { + "cliSummaryTable": true, + "statusPage": false, + "telemetry": true + } +} +{ + "markdownMessage": "C# with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.", + "severity": "note", + "source": { + "extractorName": "csharp", + "id": "csharp/autobuilder/buildless/mode-active", + "name": "C# with build-mode set to 'none'" + }, + "visibility": { + "cliSummaryTable": true, + "statusPage": true, + "telemetry": true + } +} +{ + "markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.", + "severity": "warning", + "source": { + "extractorName": "csharp", + "id": "csharp/autobuilder/buildless/unreachable-feed", + "name": "Found unreachable Nuget feed in C# analysis with build-mode 'none'" + }, + "visibility": { + "cliSummaryTable": true, + "statusPage": true, + "telemetry": true + } +} diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/Program.cs b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/Program.cs new file mode 100644 index 000000000000..39a9e95bb6e3 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/Program.cs @@ -0,0 +1,6 @@ +class Program +{ + static void Main(string[] args) + { + } +} \ No newline at end of file diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/nuget.config b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/nuget.config new file mode 100644 index 000000000000..6e4302658a95 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/nuget.config @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/proj.csproj b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/proj.csproj new file mode 100644 index 000000000000..cef71796352c --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/proj/proj.csproj @@ -0,0 +1,16 @@ + + + + Exe + net8.0 + + + + + + + + + + + diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/standalone.sln b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/standalone.sln new file mode 100644 index 000000000000..493ab54b59a7 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/standalone.sln @@ -0,0 +1,19 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.002.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "proj", "proj\proj.csproj", "{6ED00460-7666-4AE9-A405-4B6C8B02279A}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {4ED55A1C-066C-43DF-B32E-7EAA035985EE} + EndGlobalSection +EndGlobal diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/test.py b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/test.py new file mode 100644 index 000000000000..630dbfc06d47 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/test.py @@ -0,0 +1,14 @@ +from create_database_utils import * +from diagnostics_test_utils import * +import os + +# os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK"] = "true" # Nuget feed check is enabled by default +os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT"] = "1" # 1ms, the GET request should fail with such short timeout +os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT"] = "1" # Limit the count of checks to 1 + +# Making sure the reachability test succeeds when doing a fallback restore: +os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_TIMEOUT"] = "1000" +os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_LIMIT"] = "5" + +run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"]) +check_diagnostics() \ No newline at end of file