diff --git a/csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs b/csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs index 8a02370bcf67..b7e3293a0f2e 100644 --- a/csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs +++ b/csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs @@ -75,7 +75,7 @@ public static string[] AsListWithExpandedEnvVars(this string? value, IBuildActio return defaultValue; return value. - Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries). + Split(FileUtils.NewLineCharacters, StringSplitOptions.RemoveEmptyEntries). Select(s => AsStringWithExpandedEnvVars(s, actions)).ToArray(); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index e24eb762aa7c..c59af42fe180 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -61,7 +61,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg try { this.dotnet = DotNet.Make(options, logger, tempWorkingDirectory); - runtimeLazy = new Lazy(() => new Runtime(dotnet)); + runtimeLazy = new Lazy(() => new Runtime(dotnet, logger)); } catch { @@ -111,7 +111,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg logger.LogInfo($"Unresolved reference {r.Key} in project {r.Value}"); } - var webViewExtractionOption = Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_STANDALONE_EXTRACT_WEB_VIEWS"); + var webViewExtractionOption = Environment.GetEnvironmentVariable(EnvironmentVariableNames.WebViewGeneration); if (webViewExtractionOption == null || bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) && shouldExtractWebViews) @@ -158,6 +158,53 @@ private HashSet AddFrameworkDlls(HashSet dllPaths) { var frameworkLocations = new HashSet(); + var frameworkReferences = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferences); + var frameworkReferencesUseSubfolders = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders); + _ = bool.TryParse(frameworkReferencesUseSubfolders, out var useSubfolders); + if (!string.IsNullOrWhiteSpace(frameworkReferences)) + { + RemoveFrameworkNugetPackages(dllPaths); + RemoveNugetPackageReference(FrameworkPackageNames.AspNetCoreFramework, dllPaths); + RemoveNugetPackageReference(FrameworkPackageNames.WindowsDesktopFramework, dllPaths); + + var frameworkPaths = frameworkReferences.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries); + + foreach (var path in frameworkPaths) + { + if (!Directory.Exists(path)) + { + logger.LogError($"Specified framework reference path '{path}' does not exist."); + continue; + } + + if (useSubfolders) + { + dllPaths.Add(path); + frameworkLocations.Add(path); + continue; + } + + try + { + var dlls = Directory.GetFiles(path, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive }); + if (dlls.Length == 0) + { + logger.LogError($"No DLLs found in specified framework reference path '{path}'."); + continue; + } + + dllPaths.UnionWith(dlls); + frameworkLocations.UnionWith(dlls); + } + catch (Exception e) + { + logger.LogError($"Error while searching for DLLs in '{path}': {e.Message}"); + } + } + + return frameworkLocations; + } + AddNetFrameworkDlls(dllPaths, frameworkLocations); AddAspNetCoreFrameworkDlls(dllPaths, frameworkLocations); AddMicrosoftWindowsDesktopDlls(dllPaths, frameworkLocations); @@ -203,9 +250,9 @@ private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable< nugetPackageDllPaths.ExceptWith(excludedPaths); dllPaths.UnionWith(nugetPackageDllPaths); } - catch (Exception) + catch (Exception exc) { - logger.LogError("Failed to restore Nuget packages with nuget.exe"); + logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}"); } var restoredProjects = RestoreSolutions(allSolutions, out var assets1); @@ -296,6 +343,15 @@ private static DirectoryInfo[] GetPackageVersionSubDirectories(string packagePat .ToArray(); } + private void RemoveFrameworkNugetPackages(ISet dllPaths, int fromIndex = 0) + { + var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks; + for (var i = fromIndex; i < packagesInPrioOrder.Length; i++) + { + RemoveNugetPackageReference(packagesInPrioOrder[i], dllPaths); + } + } + private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLocations) { // Multiple dotnet framework packages could be present. @@ -303,7 +359,7 @@ private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLo var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks; var frameworkPaths = packagesInPrioOrder - .Select((s, index) => (Index: index, Path: GetPackageDirectory(s))) + .Select((s, index) => (Index: index, Path: GetPackageDirectory(s, packageDirectory))) .Where(pair => pair.Path is not null) .ToArray(); @@ -317,12 +373,7 @@ private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLo } SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllPaths, frameworkLocations); - - for (var i = frameworkPath.Index + 1; i < packagesInPrioOrder.Length; i++) - { - RemoveNugetPackageReference(packagesInPrioOrder[i], dllPaths); - } - + RemoveFrameworkNugetPackages(dllPaths, frameworkPath.Index + 1); return; } @@ -335,6 +386,16 @@ private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLo else if (fileContent.IsLegacyProjectStructureUsed) { runtimeLocation = Runtime.DesktopRuntime; + + if (runtimeLocation is null) + { + logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually."); + + if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, null)) + { + runtimeLocation = GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory); + } + } } runtimeLocation ??= Runtime.ExecutingRuntime; @@ -374,7 +435,7 @@ private void AddAspNetCoreFrameworkDlls(ISet dllPaths, ISet fram } // First try to find ASP.NET Core assemblies in the NuGet packages - if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework) is string aspNetCorePackage) + if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework, packageDirectory) is string aspNetCorePackage) { SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllPaths, frameworkLocations); return; @@ -390,15 +451,15 @@ private void AddAspNetCoreFrameworkDlls(ISet dllPaths, ISet fram private void AddMicrosoftWindowsDesktopDlls(ISet dllPaths, ISet frameworkLocations) { - if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework) is string windowsDesktopApp) + if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework, packageDirectory) is string windowsDesktopApp) { SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllPaths, frameworkLocations); } } - private string? GetPackageDirectory(string packagePrefix) + private string? GetPackageDirectory(string packagePrefix, TemporaryDirectory root) { - return new DirectoryInfo(packageDirectory.DirInfo.FullName) + return new DirectoryInfo(root.DirInfo.FullName) .EnumerateDirectories(packagePrefix + "*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false }) .FirstOrDefault()? .FullName; @@ -434,19 +495,19 @@ private void GenerateSourceFileFromImplicitUsings() } // Hardcoded values from https://learn.microsoft.com/en-us/dotnet/core/project-sdk/overview#implicit-using-directives - usings.UnionWith(new[] { "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading", - "System.Threading.Tasks" }); + usings.UnionWith([ "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading", + "System.Threading.Tasks" ]); if (fileContent.UseAspNetCoreDlls) { - usings.UnionWith(new[] { "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting", + usings.UnionWith([ "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting", "Microsoft.AspNetCore.Http", "Microsoft.AspNetCore.Routing", "Microsoft.Extensions.Configuration", - "Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" }); + "Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" ]); } if (fileContent.UseWindowsForms) { - usings.UnionWith(new[] { "System.Drawing", "System.Windows.Forms" }); + usings.UnionWith(["System.Drawing", "System.Windows.Forms"]); } usings.UnionWith(fileContent.CustomImplicitUsings); @@ -828,47 +889,58 @@ private void DownloadMissingPackages(List allFiles, ISet dllPa Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package => { - logger.LogInfo($"Restoring package {package}..."); - using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir")); - var success = dotnet.New(tempDir.DirInfo.FullName); + var success = TryRestorePackageManually(package, nugetConfig); if (!success) { return; } - success = dotnet.AddPackage(tempDir.DirInfo.FullName, package); - if (!success) + lock (sync) { - return; + successCount++; } + }); - var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig)); - if (!res.Success) - { - if (res.HasNugetPackageSourceError) - { - // Restore could not be completed because the listed source is unavailable. Try without the nuget.config: - res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true)); - } + CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString())); - // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package. + dllPaths.Add(missingPackageDirectory.DirInfo.FullName); + } - if (!res.Success) - { - logger.LogInfo($"Failed to restore nuget package {package}"); - return; - } - } + private bool TryRestorePackageManually(string package, string? nugetConfig) + { + logger.LogInfo($"Restoring package {package}..."); + using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir")); + var success = dotnet.New(tempDir.DirInfo.FullName); + if (!success) + { + return false; + } - lock (sync) + success = dotnet.AddPackage(tempDir.DirInfo.FullName, package); + if (!success) + { + return false; + } + + var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig)); + if (!res.Success) + { + if (res.HasNugetPackageSourceError && nugetConfig is not null) { - successCount++; + // Restore could not be completed because the listed source is unavailable. Try without the nuget.config: + res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true)); } - }); - CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString())); + // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package. - dllPaths.Add(missingPackageDirectory.DirInfo.FullName); + if (!res.Success) + { + logger.LogInfo($"Failed to restore nuget package {package}"); + return false; + } + } + + return true; } public void Dispose(TemporaryDirectory? dir, string name) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs new file mode 100644 index 000000000000..65a4664e83e4 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs @@ -0,0 +1,20 @@ +namespace Semmle.Extraction.CSharp.DependencyFetching +{ + internal class EnvironmentVariableNames + { + /// + /// Controls whether to generate source files from Asp.Net Core views (`.cshtml`, `.razor`). + /// + public const string WebViewGeneration = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_WEB_VIEWS"; + + /// + /// Specifies the location of .Net framework references added to the compilation. + /// + public const string DotnetFrameworkReferences = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES"; + + /// + /// Controls whether to use framework dependencies from subfolders. + /// + public const string DotnetFrameworkReferencesUseSubfolders = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES_USE_SUBFOLDERS"; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs index 24d92e0f0683..d1a3ed011d40 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs @@ -31,7 +31,7 @@ private record class PathFilter(Regex Regex, bool Include); public IEnumerable Filter(IEnumerable files) { - var filters = (Environment.GetEnvironmentVariable("LGTM_INDEX_FILTERS") ?? string.Empty).Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); + var filters = (Environment.GetEnvironmentVariable("LGTM_INDEX_FILTERS") ?? string.Empty).Split(FileUtils.NewLineCharacters, StringSplitOptions.RemoveEmptyEntries); if (filters.Length == 0) { return files; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FrameworkPackageNames.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FrameworkPackageNames.cs index 7b4a076f99f2..5e4a856b88cb 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FrameworkPackageNames.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FrameworkPackageNames.cs @@ -1,25 +1,25 @@ using System.Collections.Generic; -using System.Linq; namespace Semmle.Extraction.CSharp.DependencyFetching { internal static class FrameworkPackageNames { + public static string LatestNetFrameworkReferenceAssemblies { get; } = "microsoft.netframework.referenceassemblies.net481"; + public static string AspNetCoreFramework { get; } = "microsoft.aspnetcore.app.ref"; public static string WindowsDesktopFramework { get; } = "microsoft.windowsdesktop.app.ref"; // The order of the packages is important. - public static string[] NetFrameworks { get; } = new string[] - { + public static string[] NetFrameworks { get; } = + [ "microsoft.netcore.app.ref", // net7.0, ... net5.0, netcoreapp3.1, netcoreapp3.0 "microsoft.netframework.referenceassemblies.", // net48, ..., net20 "netstandard.library.ref", // netstandard2.1 "netstandard.library" // netstandard2.0 - }; + ]; public static IEnumerable AllFrameworks { get; } = - NetFrameworks - .Union(new string[] { AspNetCoreFramework, WindowsDesktopFramework }); + [.. NetFrameworks, AspNetCoreFramework, WindowsDesktopFramework]; } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs index 598e3e7e1f7c..e46d67685db5 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs @@ -71,7 +71,7 @@ public IEnumerable GenerateFiles(IEnumerable cshtmls, IEnumerabl var args = new StringBuilder(); args.Append($"/target:exe /generatedfilesout:\"{outputFolder}\" /out:\"{dllPath}\" /analyzerconfig:\"{analyzerConfig}\" "); - foreach (var f in Directory.GetFiles(sourceGeneratorFolder, "*.dll")) + foreach (var f in Directory.GetFiles(sourceGeneratorFolder, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive })) { args.Append($"/analyzer:\"{f}\" "); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs index 5b6f47d22f3b..90c4af2c4d39 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs @@ -5,6 +5,7 @@ using System.Runtime.InteropServices; using System.Text.RegularExpressions; using Semmle.Util; +using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { @@ -17,12 +18,14 @@ internal partial class Runtime private const string aspNetCoreApp = "Microsoft.AspNetCore.App"; private readonly IDotNet dotNet; + private readonly ILogger logger; private readonly Lazy> newestRuntimes; private Dictionary NewestRuntimes => newestRuntimes.Value; - public Runtime(IDotNet dotNet) + public Runtime(IDotNet dotNet, ILogger logger) { this.dotNet = dotNet; + this.logger = logger; this.newestRuntimes = new(GetNewestRuntimes); } @@ -65,7 +68,7 @@ internal Dictionary GetNewestRuntimes() /// Locates .NET Desktop Runtimes. /// This includes Mono and Microsoft.NET. /// - private static IEnumerable DesktopRuntimes + private IEnumerable DesktopRuntimes { get { @@ -76,20 +79,19 @@ private static IEnumerable DesktopRuntimes } var monoPath = FileUtils.FindProgramOnPath(Win32.IsWindows() ? "mono.exe" : "mono"); - var monoDirs = monoPath is not null - ? new[] { Path.GetFullPath(Path.Combine(monoPath, "..", "lib", "mono")), monoPath } - : new[] { "/usr/lib/mono", "/usr/local/mono", "/usr/local/bin/mono", @"C:\Program Files\Mono\lib\mono" }; + string[] monoDirs = monoPath is not null + ? [Path.GetFullPath(Path.Combine(monoPath, "..", "lib", "mono")), monoPath] + : ["/usr/lib/mono", "/usr/local/mono", "/usr/local/bin/mono", @"C:\Program Files\Mono\lib\mono"]; - var dir = monoDirs.FirstOrDefault(Directory.Exists); - - if (dir is not null) + var monoDir = monoDirs.FirstOrDefault(Directory.Exists); + if (monoDir is not null) { - return Directory.EnumerateDirectories(dir) - .Where(d => Char.IsDigit(Path.GetFileName(d)[0])) + return Directory.EnumerateDirectories(monoDir) + .Where(d => char.IsDigit(Path.GetFileName(d)[0])) .OrderByDescending(Path.GetFileName); } - return Enumerable.Empty(); + return []; } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs index 5d5bc5860f42..86677f686206 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs @@ -386,7 +386,7 @@ private static ExitCode AnalyseTracing( if (compilerArguments.GeneratedFilesOutputDirectory is not null) { - paths.AddRange(Directory.GetFiles(compilerArguments.GeneratedFilesOutputDirectory, "*.cs", SearchOption.AllDirectories)); + paths.AddRange(Directory.GetFiles(compilerArguments.GeneratedFilesOutputDirectory, "*.cs", new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive })); } return ReadSyntaxTrees( diff --git a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs index 17bc477bde8a..0d4ed6c4b920 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs @@ -47,7 +47,7 @@ public void TestRuntime1() "Microsoft.NETCore.App 7.0.2 [/path/dotnet/shared/Microsoft.NETCore.App]" }; var dotnet = new DotNetStub(listedRuntimes, null!); - var runtime = new Runtime(dotnet); + var runtime = new Runtime(dotnet, new LoggerStub()); // Execute var runtimes = runtime.GetNewestRuntimes(); @@ -73,7 +73,7 @@ public void TestRuntime2() "Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]" }; var dotnet = new DotNetStub(listedRuntimes, null!); - var runtime = new Runtime(dotnet); + var runtime = new Runtime(dotnet, new LoggerStub()); // Execute var runtimes = runtime.GetNewestRuntimes(); @@ -96,7 +96,7 @@ public void TestRuntime3() "Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]" }; var dotnet = new DotNetStub(listedRuntimes, null!); - var runtime = new Runtime(dotnet); + var runtime = new Runtime(dotnet, new LoggerStub()); // Execute var runtimes = runtime.GetNewestRuntimes(); @@ -125,7 +125,7 @@ public void TestRuntime4() @"Microsoft.WindowsDesktop.App 7.0.4 [C:\Program Files\dotnet\shared\Microsoft.WindowsDesktop.App]" }; var dotnet = new DotNetStub(listedRuntimes, null!); - var runtime = new Runtime(dotnet); + var runtime = new Runtime(dotnet, new LoggerStub()); // Execute var runtimes = runtime.GetNewestRuntimes(); diff --git a/csharp/extractor/Semmle.Extraction/CsProjFile.cs b/csharp/extractor/Semmle.Extraction/CsProjFile.cs index 10deb2883a3b..bed9d746996b 100644 --- a/csharp/extractor/Semmle.Extraction/CsProjFile.cs +++ b/csharp/extractor/Semmle.Extraction/CsProjFile.cs @@ -112,7 +112,7 @@ private static (string[] csFiles, string[] references, string[] projectReference .Where(s => s is not null) ?? Enumerable.Empty(); - var additionalCsFiles = System.IO.Directory.GetFiles(directoryName, "*.cs", SearchOption.AllDirectories); + var additionalCsFiles = System.IO.Directory.GetFiles(directoryName, "*.cs", new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive }); var projectReferences = root .SelectNodes("/Project/ItemGroup/ProjectReference/@Include", mgr) diff --git a/csharp/extractor/Semmle.Util/FileUtils.cs b/csharp/extractor/Semmle.Util/FileUtils.cs index 3315c3e705e4..09269e37e8bd 100644 --- a/csharp/extractor/Semmle.Util/FileUtils.cs +++ b/csharp/extractor/Semmle.Util/FileUtils.cs @@ -13,6 +13,8 @@ public static class FileUtils { public const string NugetExeUrl = "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe"; + public static readonly char[] NewLineCharacters = ['\r', '\n']; + public static string ConvertToWindows(string path) { return path.Replace('/', '\\'); diff --git a/csharp/ql/integration-tests/all-platforms/cshtml_standalone_disabled/test.py b/csharp/ql/integration-tests/all-platforms/cshtml_standalone_disabled/test.py index c00c6f6e2e8e..0b433134e5a3 100644 --- a/csharp/ql/integration-tests/all-platforms/cshtml_standalone_disabled/test.py +++ b/csharp/ql/integration-tests/all-platforms/cshtml_standalone_disabled/test.py @@ -1,5 +1,5 @@ import os from create_database_utils import * -os.environ['CODEQL_EXTRACTOR_CSHARP_STANDALONE_EXTRACT_WEB_VIEWS'] = 'false' +os.environ['CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_WEB_VIEWS'] = 'false' run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true"]) diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/Assemblies.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/Assemblies.expected new file mode 100644 index 000000000000..7943fdd9c1be --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/Assemblies.expected @@ -0,0 +1 @@ +| /packages/newtonsoft.json/6.0.4/lib/net45/Newtonsoft.Json.dll | diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/Assemblies.ql b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/Assemblies.ql new file mode 100644 index 000000000000..18b4e12db2f7 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/Assemblies.ql @@ -0,0 +1,15 @@ +import csharp + +private string getPath(Assembly a) { + not a.getCompilation().getOutputAssembly() = a and + exists(string s | s = a.getFile().getAbsolutePath() | + result = + s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16, s.length()) + or + result = s and + not exists(s.indexOf("test-db/working/")) + ) +} + +from Assembly a +select getPath(a) diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/Program.cs b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/Program.cs new file mode 100644 index 000000000000..39a9e95bb6e3 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/Program.cs @@ -0,0 +1,6 @@ +class Program +{ + static void Main(string[] args) + { + } +} \ No newline at end of file diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/global.json b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/global.json new file mode 100644 index 000000000000..5c3fd64fbd12 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/global.json @@ -0,0 +1,5 @@ +{ + "sdk": { + "version": "8.0.101" + } +} diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/packages.config b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/packages.config new file mode 100644 index 000000000000..0f63b3daf6ca --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/packages.config @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/skip-on-platform-osx-arm b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/skip-on-platform-osx-arm new file mode 100644 index 000000000000..6ebb8d63fcc8 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/skip-on-platform-osx-arm @@ -0,0 +1 @@ +Skipping the test on the ARM runners, as we're running into trouble with Mono and nuget. diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/test.py b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/test.py new file mode 100644 index 000000000000..ddd46a7f1e29 --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/test.py @@ -0,0 +1,5 @@ +from create_database_utils import * +import os + +os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES"] = "/non-existent-path" +run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"]) diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/test_old.csproj b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/test_old.csproj new file mode 100644 index 000000000000..f7600103d99f --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/test_old.csproj @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/test_sdk.csproj b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/test_sdk.csproj new file mode 100644 index 000000000000..4425f1c1431b --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_no_framework/test_sdk.csproj @@ -0,0 +1,16 @@ + + + + Exe + net8.0 + + + + + + + + + + +