Skip to content

Commit

Permalink
update all packages with dependencies on the target package when poss…
Browse files Browse the repository at this point in the history 
…ible (#9507)
  • Loading branch information
@brettfo
brettfo committed Apr 18, 2024
1 parent 5b41afa commit 7311e21
Show file tree
Hide file tree
Showing 4 changed files with 237 additions and 21 deletions.
48 changes: 30 additions & 18 deletions nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.Sdk.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,24 +170,6 @@ public async Task PartialUpdate_InMultipleProjectFiles_ForVersionConstraint()
]);
}

[Fact]
public async Task NoChange_WhenPackageHasVersionConstraint()
{
// Dependency package has version constraint
await TestNoChangeforProject("AWSSDK.Core", "3.3.21.19", "3.7.300.20",
projectContents: $"""
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AWSSDK.S3" Version="3.3.17.3" />
<PackageReference Include="AWSSDK.Core" Version="3.3.21.19" />
</ItemGroup>
</Project>
""");
}

[Fact]
public async Task UpdateVersionAttribute_InProjectFile_ForPackageReferenceInclude_Windows()
{
Expand Down Expand Up @@ -2553,5 +2535,35 @@ public async Task UnresolvablePropertyDoesNotStopOtherUpdates()
"""
);
}

[Fact]
public async Task UpdatingPackageAlsoUpdatesAnythingWithADependencyOnTheUpdatedPackage()
{
// updating SpecFlow from 3.3.30 requires that SpecFlow.Tools.MsBuild.Generation also be updated
await TestUpdateForProject("SpecFlow", "3.3.30", "3.4.3",
projectContents: """
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="SpecFlow" Version="3.3.30" />
<PackageReference Include="SpecFlow.Tools.MsBuild.Generation" Version="3.3.30" />
</ItemGroup>
</Project>
""",
expectedProjectContents: """
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="SpecFlow" Version="3.4.3" />
<PackageReference Include="SpecFlow.Tools.MsBuild.Generation" Version="3.4.3" />
</ItemGroup>
</Project>
"""
);
}
}
}
39 changes: 39 additions & 0 deletions nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Utilities/MSBuildHelperTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,45 @@ public async Task AllPackageDependenciesCanBeFoundWithNuGetConfig()
}
}

[Fact]
public async Task DependencyConflictsCanBeResolved()
{
// the package `SpecFlow` was already updated from 3.3.30 to 3.4.3, but this causes a conflict with
// `SpecFlow.Tools.MsBuild.Generation` that needs to be resolved
var repoRoot = Directory.CreateTempSubdirectory($"test_{nameof(DependencyConflictsCanBeResolved)}_");
try
{
var projectPath = Path.Join(repoRoot.FullName, "project.csproj");
await File.WriteAllTextAsync(projectPath, """
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="SpecFlow" Version="3.4.3" />
<PackageReference Include="SpecFlow.Tools.MsBuild.Generation" Version="3.3.30" />
</ItemGroup>
</Project>
""");
var dependencies = new[]
{
new Dependency("SpecFlow", "3.4.3", DependencyType.PackageReference),
new Dependency("SpecFlow.Tools.MsBuild.Generation", "3.3.30", DependencyType.PackageReference),
};
var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(repoRoot.FullName, projectPath, "netstandard2.0", dependencies, new Logger(true));
Assert.NotNull(resolvedDependencies);
Assert.Equal(2, resolvedDependencies.Length);
Assert.Equal("SpecFlow", resolvedDependencies[0].Name);
Assert.Equal("3.4.3", resolvedDependencies[0].Version);
Assert.Equal("SpecFlow.Tools.MsBuild.Generation", resolvedDependencies[1].Name);
Assert.Equal("3.4.3", resolvedDependencies[1].Version);
}
finally
{
repoRoot.Delete(recursive: true);
}
}

public static IEnumerable<object[]> GetTopLevelPackageDependencyInfosTestData()
{
// simple case
Expand Down
45 changes: 42 additions & 3 deletions nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/SdkPackageUpdater.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ internal static class SdkPackageUpdater
// SDK-style project, modify the XML directly
logger.Log(" Running for SDK-style project");

var (buildFiles, tfms) = await MSBuildHelper.LoadBuildFilesAndTargetFrameworksAsync(repoRootPath, projectPath);
(ImmutableArray<ProjectBuildFile> buildFiles, string[] tfms) = await MSBuildHelper.LoadBuildFilesAndTargetFrameworksAsync(repoRootPath, projectPath);

// Get the set of all top-level dependencies in the current project
var topLevelDependencies = MSBuildHelper.GetTopLevelPackageDependencyInfos(buildFiles).ToArray();
Expand All @@ -41,7 +41,7 @@ internal static class SdkPackageUpdater
return;
}

UpdateTopLevelDepdendency(buildFiles, dependencyName, previousDependencyVersion, newDependencyVersion, peerDependencies, logger);
await UpdateTopLevelDepdendency(repoRootPath, buildFiles, tfms, dependencyName, previousDependencyVersion, newDependencyVersion, peerDependencies, logger);
}

if (!await AreDependenciesCoherentAsync(repoRootPath, projectPath, dependencyName, logger, buildFiles, tfms))
Expand Down Expand Up @@ -287,8 +287,10 @@ private static async Task AddTransitiveDependencyAsync(string projectPath, strin
return packagesAndVersions;
}

private static void UpdateTopLevelDepdendency(
private static async Task UpdateTopLevelDepdendency(
string repoRootPath,
ImmutableArray<ProjectBuildFile> buildFiles,
string[] targetFrameworks,
string dependencyName,
string previousDependencyVersion,
string newDependencyVersion,
Expand All @@ -306,6 +308,43 @@ private static async Task AddTransitiveDependencyAsync(string projectPath, strin
{
TryUpdateDependencyVersion(buildFiles, packageName, previousDependencyVersion: null, newDependencyVersion: packageVersion, logger);
}

// now make all dependency requirements coherent
Dependency[] updatedTopLevelDependencies = MSBuildHelper.GetTopLevelPackageDependencyInfos(buildFiles).ToArray();
foreach (ProjectBuildFile projectFile in buildFiles)
{
foreach (string tfm in targetFrameworks)
{
Dependency[]? resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(repoRootPath, projectFile.Path, tfm, updatedTopLevelDependencies, logger);
if (resolvedDependencies is null)
{
logger.Log($" Unable to resolve dependency conflicts for {projectFile.Path}.");
continue;
}

// ensure the originally requested dependency was resolved to the correct version
var specificResolvedDependency = resolvedDependencies.Where(d => d.Name.Equals(dependencyName, StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
if (specificResolvedDependency is null)
{
logger.Log($" Unable resolve requested dependency for {dependencyName} in {projectFile.Path}.");
continue;
}

if (!newDependencyVersion.Equals(specificResolvedDependency.Version, StringComparison.OrdinalIgnoreCase))
{
logger.Log($" Inconsistent resolution for {dependencyName}; attempted upgrade to {newDependencyVersion} but resolved {specificResolvedDependency.Version}.");
continue;
}

// update all other dependencies
foreach (Dependency resolvedDependency in resolvedDependencies
.Where(d => !d.Name.Equals(dependencyName, StringComparison.OrdinalIgnoreCase))
.Where(d => d.Version is not null))
{
TryUpdateDependencyVersion(buildFiles, resolvedDependency.Name, previousDependencyVersion: null, newDependencyVersion: resolvedDependency.Version!, logger);
}
}
}
}

private static UpdateResult TryUpdateDependencyVersion(
Expand Down
126 changes: 126 additions & 0 deletions nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/MSBuildHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Diagnostics.CodeAnalysis;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.RegularExpressions;
using System.Xml;

Expand All @@ -13,6 +14,7 @@
using Microsoft.Extensions.FileSystemGlobbing;

using NuGet.Configuration;
using NuGet.Versioning;

using NuGetUpdater.Core.Utilities;

Expand Down Expand Up @@ -307,6 +309,124 @@ internal static async Task<bool> DependenciesAreCoherentAsync(string repoRoot, s
}
}

internal static async Task<Dependency[]?> ResolveDependencyConflicts(string repoRoot, string projectPath, string targetFramework, Dependency[] packages, Logger logger)
{
var tempDirectory = Directory.CreateTempSubdirectory("package-dependency-coherence_");
try
{
var tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages);
var (exitCode, stdOut, stdErr) = await ProcessEx.RunAsync("dotnet", $"restore \"{tempProjectPath}\"", workingDirectory: tempDirectory.FullName);

// simple cases first
// if restore failed, nothing we can do
if (exitCode != 0)
{
return null;
}

// if no problems found, just return the current set
if (!stdOut.Contains("NU1608"))
{
return packages;
}

// now it gets complicated; look for the packages with issues
MatchCollection matches = PackageIncompatibilityWarningPattern().Matches(stdOut);
(string, NuGetVersion)[] badPackagesAndVersions = matches.Select(m => (m.Groups["PackageName"].Value, NuGetVersion.Parse(m.Groups["PackageVersion"].Value))).ToArray();
Dictionary<string, HashSet<NuGetVersion>> badPackagesAndCandidateVersionsDictionary = new(StringComparer.OrdinalIgnoreCase);

// and for each of those packages, find all versions greater than the one that's currently installed
foreach ((string packageName, NuGetVersion packageVersion) in badPackagesAndVersions)
{
// this command dumps a JSON object with all versions of the specified package from all package sources
(exitCode, stdOut, stdErr) = await ProcessEx.RunAsync("dotnet", $"package search {packageName} --exact-match --format json", workingDirectory: tempDirectory.FullName);
if (exitCode != 0)
{
continue;
}

// ensure collection exists
if (!badPackagesAndCandidateVersionsDictionary.ContainsKey(packageName))
{
badPackagesAndCandidateVersionsDictionary.Add(packageName, new HashSet<NuGetVersion>());
}

HashSet<NuGetVersion> foundVersions = badPackagesAndCandidateVersionsDictionary[packageName];

var json = JsonHelper.ParseNode(stdOut);
if (json?["searchResult"] is JsonArray searchResults)
{
foreach (var searchResult in searchResults)
{
if (searchResult?["packages"] is JsonArray packagesArray)
{
foreach (var package in packagesArray)
{
// in 8.0.xxx SDKs, the package version is in the `latestVersion` property, but in 9.0.xxx, it's `version`
var packageVersionProperty = package?["version"] ?? package?["latestVersion"];
if (packageVersionProperty is JsonValue latestVersion &&
latestVersion.GetValueKind() == JsonValueKind.String &&
NuGetVersion.TryParse(latestVersion.ToString(), out var nugetVersion) &&
nugetVersion > packageVersion)
{
foundVersions.Add(nugetVersion);
}
}
}
}
}
}

// generate all possible combinations
(string Key, NuGetVersion v)[][] expandedLists = badPackagesAndCandidateVersionsDictionary.Select(kvp => kvp.Value.Order().Select(v => (kvp.Key, v)).ToArray()).ToArray();
IEnumerable<(string PackageName, NuGetVersion PackageVersion)>[] product = expandedLists.CartesianProduct().ToArray();

// FUTURE WORK: pre-filter individual known package incompatibilities to reduce the number of combinations, e.g., if Package.A v1.0.0
// is incompatible with Package.B v2.0.0, then remove _all_ combinations with that pair

// this is the slow part
foreach (IEnumerable<(string PackageName, NuGetVersion PackageVersion)> candidateSet in product)
{
// rebuild candidate dependency list with the relevant versions
Dictionary<string, NuGetVersion> packageVersions = candidateSet.ToDictionary(candidateSet => candidateSet.PackageName, candidateSet => candidateSet.PackageVersion);
Dependency[] candidatePackages = packages.Select(p =>
{
if (packageVersions.TryGetValue(p.Name, out var version))
{
// create a new dependency with the updated version
return new Dependency(p.Name, version.ToString(), p.Type, IsDevDependency: p.IsDevDependency, IsOverride: p.IsOverride, IsUpdate: p.IsUpdate);
}
// not the dependency we're looking for, use whatever it already was in this set
return p;
}).ToArray();

if (await DependenciesAreCoherentAsync(repoRoot, projectPath, targetFramework, candidatePackages, logger))
{
// return as soon as we find a coherent set
return candidatePackages;
}
}

// no package resolution set found
return null;
}
finally
{
tempDirectory.Delete(recursive: true);
}
}

// fully expand all possible combinations using the algorithm from here:
// https://ericlippert.com/2010/06/28/computing-a-cartesian-product-with-linq/
private static IEnumerable<IEnumerable<T>> CartesianProduct<T>(this IEnumerable<IEnumerable<T>> sequences)
{
IEnumerable<IEnumerable<T>> emptyProduct = [[]];
return sequences.Aggregate(emptyProduct, (accumulator, sequence) => from accseq in accumulator
from item in sequence
select accseq.Concat([item]));
}

private static ProjectRootElement CreateProjectRootElement(ProjectBuildFile buildFile)
{
var xmlString = buildFile.Contents.ToFullString();
Expand Down Expand Up @@ -603,4 +723,10 @@ internal static async Task<(ImmutableArray<ProjectBuildFile> ProjectBuildFiles,

[GeneratedRegex("^\\s*NuGetData::Package=(?<PackageName>[^,]+), Version=(?<PackageVersion>.+)$")]
private static partial Regex PackagePattern();

// Example output:
// NU1608: Detected package version outside of dependency constraint: SpecFlow.Tools.MsBuild.Generation 3.3.30 requires SpecFlow(= 3.3.30) but version SpecFlow 3.9.74 was resolved.
// PackageName-|+++++++++++++++++++++++++++++++| |++++|-PackageVersion
[GeneratedRegex("NU1608: [^:]+: (?<PackageName>[^ ]+) (?<PackageVersion>[^ ]+)")]
private static partial Regex PackageIncompatibilityWarningPattern();
}

0 comments on commit 7311e21

Please sign in to comment.