Skip to content

C#: Choose between .NET framework or core DLLs in standalone #14368

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 17 commits into from
Oct 16, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/csharp-qltest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ jobs:
run: |
# Generate (Asp)NetCore stubs
STUBS_PATH=stubs_output
python3 ql/src/Stubs/make_stubs_nuget.py webapp Swashbuckle.AspNetCore.Swagger latest "$STUBS_PATH"
python3 ql/src/Stubs/make_stubs_nuget.py webapp Swashbuckle.AspNetCore.Swagger 6.5.0 "$STUBS_PATH"
rm -rf ql/test/resources/stubs/_frameworks
# Update existing stubs in the repo with the freshly generated ones
mv "$STUBS_PATH/output/stubs/_frameworks" ql/test/resources/stubs/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ internal sealed partial class AssemblyInfo

/// <summary>
/// The version number of the .NET Core framework that this assembly targets.
///
///
/// This is extracted from the `TargetFrameworkAttribute` of the assembly, e.g.
/// ```
/// [assembly:TargetFramework(".NETCoreApp,Version=v7.0")]
Expand Down Expand Up @@ -160,11 +160,22 @@ public static AssemblyInfo ReadFromFile(string filename)
* loading the same assembly from different locations.
*/
using var pereader = new System.Reflection.PortableExecutable.PEReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read));
if (!pereader.HasMetadata)
{
throw new AssemblyLoadException();
}

using var sha1 = SHA1.Create();
var metadata = pereader.GetMetadata();

unsafe
{
var reader = new MetadataReader(metadata.Pointer, metadata.Length);
if (!reader.IsAssembly)
{
throw new AssemblyLoadException();
}

var def = reader.GetAssemblyDefinition();

// This is how you compute the public key token from the full public key.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,12 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
this.progressMonitor = new ProgressMonitor(logger);
this.sourceDir = new DirectoryInfo(srcDir);

packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
tempWorkingDirectory = new TemporaryDirectory(FileUtils.GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory));

try
{
this.dotnet = DotNet.Make(options, progressMonitor);
this.dotnet = DotNet.Make(options, progressMonitor, tempWorkingDirectory);
}
catch
{
Expand All @@ -59,8 +62,6 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg

this.progressMonitor.FindingFiles(srcDir);

packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
tempWorkingDirectory = new TemporaryDirectory(FileUtils.GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory));

var allFiles = GetAllFiles();
var binaryFileExtensions = new HashSet<string>(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions.
Expand All @@ -77,21 +78,6 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
? allFiles.SelectFileNamesByExtension(".dll").ToList()
: options.DllDirs.Select(Path.GetFullPath).ToList();

// Find DLLs in the .Net / Asp.Net Framework
if (options.ScanNetFrameworkDlls)
{
var runtime = new Runtime(dotnet);
var runtimeLocation = runtime.GetRuntime(options.UseSelfContainedDotnet);
progressMonitor.LogInfo($".NET runtime location selected: {runtimeLocation}");
dllDirNames.Add(runtimeLocation);

if (fileContent.UseAspNetDlls && runtime.GetAspRuntime() is string aspRuntime)
{
progressMonitor.LogInfo($"ASP.NET runtime location selected: {aspRuntime}");
dllDirNames.Add(aspRuntime);
}
}

if (options.UseNuGet)
{
dllDirNames.Add(packageDirectory.DirInfo.FullName);
Expand All @@ -111,6 +97,26 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
DownloadMissingPackages(allNonBinaryFiles);
}

var existsNetCoreRefNugetPackage = false;
var existsNetFrameworkRefNugetPackage = false;

// Find DLLs in the .Net / Asp.Net Framework
// This block needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
if (options.ScanNetFrameworkDlls)
{
existsNetCoreRefNugetPackage = IsNugetPackageAvailable("microsoft.netcore.app.ref");
existsNetFrameworkRefNugetPackage = IsNugetPackageAvailable("microsoft.netframework.referenceassemblies");

if (existsNetCoreRefNugetPackage || existsNetFrameworkRefNugetPackage)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The DCA changes show that this logic is not working great. If dotnet core 6 is installed, and there are net6.0 and net5.0 targets, then the reference assemblies of net5.0 are downloaded and preferred over the installed dotnet 6 DLLs. And this causes an issue if there are other references which are compiled against dotnet 6.0.

We could probably force the download of the ref assemblies for all target platforms, and we should choose the latest one from those.

{
progressMonitor.LogInfo("Found .NET Core/Framework DLLs in NuGet packages. Not adding installation directory.");
}
else
{
AddNetFrameworkDlls(dllDirNames);
}
}

assemblyCache = new AssemblyCache(dllDirNames, progressMonitor);
AnalyseSolutions(solutions);

Expand All @@ -119,7 +125,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
UseReference(filename);
}

RemoveRuntimeNugetPackageReferences();
RemoveUnnecessaryNugetPackages(existsNetCoreRefNugetPackage, existsNetFrameworkRefNugetPackage);
ResolveConflicts();

// Output the findings
Expand Down Expand Up @@ -154,38 +160,158 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
DateTime.Now - startTime);
}

private void RemoveRuntimeNugetPackageReferences()
private void RemoveUnnecessaryNugetPackages(bool existsNetCoreRefNugetPackage, bool existsNetFrameworkRefNugetPackage)
{
RemoveNugetAnalyzerReferences();
RemoveRuntimeNugetPackageReferences();

if (fileContent.IsNewProjectStructureUsed
&& !fileContent.UseAspNetCoreDlls)
{
// This might have been restored by the CLI even though the project isn't an asp.net core one.
RemoveNugetPackageReference("microsoft.aspnetcore.app.ref");
}

if (existsNetCoreRefNugetPackage && existsNetFrameworkRefNugetPackage)
{
// Multiple packages are available, we keep only one:
RemoveNugetPackageReference("microsoft.netframework.referenceassemblies.");
}

// TODO: There could be multiple `microsoft.netframework.referenceassemblies` packages,
// we could keep the newest one, but this is covered by the conflict resolution logic
// (if the file names match)
}

private void RemoveNugetAnalyzerReferences()
{
if (!options.UseNuGet)
{
return;
}

var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
var runtimePackageNamePrefixes = new[]
if (packageFolder == null)
{
return;
}

foreach (var filename in usedReferences.Keys)
{
Path.Combine(packageFolder, "microsoft.netcore.app.runtime"),
Path.Combine(packageFolder, "microsoft.aspnetcore.app.runtime"),
Path.Combine(packageFolder, "microsoft.windowsdesktop.app.runtime"),
var lowerFilename = filename.ToLowerInvariant();

if (lowerFilename.StartsWith(packageFolder))
{
var firstDirectorySeparatorCharIndex = lowerFilename.IndexOf(Path.DirectorySeparatorChar, packageFolder.Length + 1);
if (firstDirectorySeparatorCharIndex == -1)
{
continue;
}
var secondDirectorySeparatorCharIndex = lowerFilename.IndexOf(Path.DirectorySeparatorChar, firstDirectorySeparatorCharIndex + 1);
if (secondDirectorySeparatorCharIndex == -1)
{
continue;
}
var subFolderIndex = secondDirectorySeparatorCharIndex + 1;
var isInAnalyzersFolder = lowerFilename.IndexOf("analyzers", subFolderIndex) == subFolderIndex;
if (isInAnalyzersFolder)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this needed?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nuget packages can ship analyzers, code fixes, and source generators in the analyzers folder. These DLLs need to be passed in the analyzers csc argument and not in references. The base classes used in these DLLs are not part of the dotnet runtime (but the dotnet SDK), so if these are passed as references, they will produce compile errors, so we remove them from the reference list.

{
usedReferences.Remove(filename);
progressMonitor.RemovedReference(filename);
}
}
}
}
private void AddNetFrameworkDlls(List<string> dllDirNames)
{
var runtime = new Runtime(dotnet);
string? runtimeLocation = null;

if (options.UseSelfContainedDotnet)
{
runtimeLocation = runtime.ExecutingRuntime;
}
else if (fileContent.IsNewProjectStructureUsed)
{
runtimeLocation = runtime.NetCoreRuntime;
}
else if (fileContent.IsLegacyProjectStructureUsed)
{
runtimeLocation = runtime.DesktopRuntime;
}

runtimeLocation ??= runtime.ExecutingRuntime;

progressMonitor.LogInfo($".NET runtime location selected: {runtimeLocation}");
dllDirNames.Add(runtimeLocation);

if (fileContent.IsNewProjectStructureUsed
&& fileContent.UseAspNetCoreDlls
&& runtime.AspNetCoreRuntime is string aspRuntime)
{
progressMonitor.LogInfo($"ASP.NET runtime location selected: {aspRuntime}");
dllDirNames.Add(aspRuntime);
}
}

private void RemoveRuntimeNugetPackageReferences()
{
var runtimePackagePrefixes = new[]
{
"microsoft.netcore.app.runtime",
"microsoft.aspnetcore.app.runtime",
"microsoft.windowsdesktop.app.runtime",

// legacy runtime packages:
Path.Combine(packageFolder, "runtime.linux-x64.microsoft.netcore.app"),
Path.Combine(packageFolder, "runtime.osx-x64.microsoft.netcore.app"),
Path.Combine(packageFolder, "runtime.win-x64.microsoft.netcore.app"),
"runtime.linux-x64.microsoft.netcore.app",
"runtime.osx-x64.microsoft.netcore.app",
"runtime.win-x64.microsoft.netcore.app",

// Internal implementation packages not meant for direct consumption:
"runtime."
};
RemoveNugetPackageReference(runtimePackagePrefixes);
}

private void RemoveNugetPackageReference(params string[] packagePrefixes)
{
if (!options.UseNuGet)
{
return;
}

var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
if (packageFolder == null)
{
return;
}

var packagePathPrefixes = packagePrefixes.Select(p => Path.Combine(packageFolder, p.ToLowerInvariant()));

foreach (var filename in usedReferences.Keys)
{
var lowerFilename = filename.ToLowerInvariant();

if (runtimePackageNamePrefixes.Any(prefix => lowerFilename.StartsWith(prefix)))
if (packagePathPrefixes.Any(prefix => lowerFilename.StartsWith(prefix)))
{
usedReferences.Remove(filename);
progressMonitor.RemovedReference(filename);
}
}
}

private bool IsNugetPackageAvailable(string packagePrefix)
{
if (!options.UseNuGet)
{
return false;
}

return new DirectoryInfo(packageDirectory.DirInfo.FullName)
.EnumerateDirectories(packagePrefix + "*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.Any();
}

private void GenerateSourceFileFromImplicitUsings()
{
var usings = new HashSet<string>();
Expand All @@ -198,7 +324,7 @@ private void GenerateSourceFileFromImplicitUsings()
usings.UnionWith(new[] { "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading",
"System.Threading.Tasks" });

if (fileContent.UseAspNetDlls)
if (fileContent.UseAspNetCoreDlls)
{
usings.UnionWith(new[] { "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting",
"Microsoft.AspNetCore.Http", "Microsoft.AspNetCore.Routing", "Microsoft.Extensions.Configuration",
Expand Down Expand Up @@ -461,11 +587,11 @@ private void AnalyseProject(FileInfo project)

}

private bool RestoreProject(string project, string? pathToNugetConfig = null) =>
dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, pathToNugetConfig);
private bool RestoreProject(string project, bool forceDotnetRefAssemblyFetching, string? pathToNugetConfig = null) =>
dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching, pathToNugetConfig);

private bool RestoreSolution(string solution, out IEnumerable<string> projects) =>
dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, out projects);
dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out projects);

/// <summary>
/// Executes `dotnet restore` on all solution files in solutions.
Expand All @@ -491,7 +617,7 @@ private void RestoreProjects(IEnumerable<string> projects)
{
Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
{
RestoreProject(project);
RestoreProject(project, forceDotnetRefAssemblyFetching: true);
});
}

Expand Down Expand Up @@ -536,7 +662,7 @@ private void DownloadMissingPackages(List<FileInfo> allFiles)
return;
}

success = RestoreProject(tempDir.DirInfo.FullName, nugetConfig);
success = RestoreProject(tempDir.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, pathToNugetConfig: nugetConfig);
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
if (!success)
{
Expand Down Expand Up @@ -564,9 +690,25 @@ private void AnalyseSolutions(IEnumerable<string> solutions)

public void Dispose()
{
packageDirectory?.Dispose();
try
{
packageDirectory?.Dispose();
}
catch (Exception exc)
{
progressMonitor.LogInfo("Couldn't delete package directory: " + exc.Message);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This logs Couldn't delete package directory: The process cannot access the file 'System.EnterpriseServices.Wrapper.dll' because it is being used by another process. on Windows. The DLL can't be loaded as an assembly, and it looks like System.Reflection.PortableExecutable.PEReader doesn't release it.

The DLL is coming from the microsoft.netframework.referenceassemblies.net48.1.0.3 nuget package.

}
Comment on lines +697 to +700

Check notice

Code scanning / CodeQL

Generic catch clause

Generic catch clause.
if (cleanupTempWorkingDirectory)
tempWorkingDirectory?.Dispose();
{
try
{
tempWorkingDirectory?.Dispose();
}
catch (Exception exc)
{
progressMonitor.LogInfo("Couldn't delete temporary working directory: " + exc.Message);
}
Comment on lines +707 to +710

Check notice

Code scanning / CodeQL

Generic catch clause

Generic catch clause.
}
}
}
}
Loading