Skip to content

Remove more usage of AbsolutePath to avoid long path prefix issues #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dfederm merged 1 commit into from
Jan 25, 2024
Merged

Remove more usage of AbsolutePath to avoid long path prefix issues #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 26 additions & 29 deletions src/AzurePipelines/PipelineCachingCacheClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
using Microsoft.VisualStudio.Services.Content.Common.Tracing;
using Microsoft.VisualStudio.Services.PipelineCache.Common;
using Microsoft.VisualStudio.Services.PipelineCache.WebApi;
using BxlContentHashWithPath = BuildXL.Cache.ContentStore.Interfaces.Sessions.ContentHashWithPath;
using FileInfo = System.IO.FileInfo;

namespace Microsoft.MSBuildCache.AzurePipelines;
Expand Down Expand Up @@ -63,7 +64,7 @@ namespace Microsoft.MSBuildCache.AzurePipelines;
internal sealed class PipelineCachingCacheClient : CacheClient
#pragma warning restore CS1570 // XML comment has badly formed XML
{
////private readonly record struct ContentHashWithPath(ContentHash Hash, string Path);
private readonly record struct ContentHashWithPath(ContentHash Hash, string Path);

private const string InternalMetadataPathPrefix = "/???";

Expand Down Expand Up @@ -236,7 +237,7 @@ protected override async Task AddNodeAsync(
});

List<ContentHashWithPath> tempFiles = tempFilesPerHash
.Select(kvp => new ContentHashWithPath(kvp.Key, new AbsolutePath(kvp.Value)))
.Select(kvp => new ContentHashWithPath(kvp.Key, kvp.Value))
.ToList();

Dictionary<string, PlaceFileResult> placeResults = await TryPlaceFilesFromCacheAsync(
Expand Down Expand Up @@ -403,13 +404,13 @@ private async Task<Dictionary<string, PlaceFileResult>> TryPlaceFilesFromCacheAs
// cache expects destination directories already exist
foreach (ContentHashWithPath file in files)
{
CreateParentDirectory(file.Path.Path);
CreateParentDirectory(file.Path);
}

Dictionary<string, PlaceFileResult> results = new();
List<ContentHashWithPath> places = new();
Dictionary<string, PlaceFileResult> results = new(files.Count);
List<BxlContentHashWithPath> places = new(files.Count);

var operationGroups = files.GroupBy(f => (GetAlgorithmId(f.Hash), realizationModeOverride ?? GetFileRealizationMode(f.Path.Path)));
var operationGroups = files.GroupBy(f => (GetAlgorithmId(f.Hash), realizationModeOverride ?? GetFileRealizationMode(f.Path)));

foreach (IGrouping<(byte algoId, FileRealizationMode mode), ContentHashWithPath>? filesGroup in operationGroups)
{
Expand All @@ -419,7 +420,10 @@ private async Task<Dictionary<string, PlaceFileResult>> TryPlaceFilesFromCacheAs
: FileAccessMode.ReadOnly;

places.Clear();
places.AddRange(filesGroup);
foreach (ContentHashWithPath file in filesGroup)
{
places.Add(new BxlContentHashWithPath(file.Hash, new AbsolutePath(file.Path)));
}

List<Task<Indexed<PlaceFileResult>>> groupResults = (await LocalCacheSession.PlaceFileAsync(
context, places, accessMode, FileReplacementMode.ReplaceExisting, realizationMode, cancellationToken)).ToList();
Expand All @@ -444,7 +448,8 @@ private async Task<Dictionary<string, PlaceFileResult>> TryPlaceFilesFromCacheAs
foreach (Task<Indexed<PlaceFileResult>> resultTask in groupResults)
{
Indexed<PlaceFileResult> result = await resultTask;
results.Add(places[result.Index].Path.Path, result.Item);
string path = PathHelper.RemoveLongPathPrefixes(places[result.Index].Path.Path);
results.Add(path, result.Item);
}
}

Expand Down Expand Up @@ -509,7 +514,7 @@ public async Task PlaceFilesAsync(Context context, IReadOnlyDictionary<string, C
ThrowIfDifferent(manifestFiles, requestFiles, $"Manifest `{_manifestId}` and PlaceFiles don't match:");

// try to pull whole files from the cache
var places = files.Select(f => new ContentHashWithPath(f.Value, new AbsolutePath(f.Key))).ToList();
var places = files.Select(f => new ContentHashWithPath(f.Value, f.Key)).ToList();

Dictionary<string, PlaceFileResult> placeResults = await _client.TryPlaceFilesFromCacheAsync(context, places, realizationModeOverride: null, cancellationToken);

Expand Down Expand Up @@ -611,18 +616,9 @@ private async Task<byte[]> GetBytes(Context context, DedupIdentifier dedupId, Ca
cancellationToken);
}

private sealed class RelativePathComparer : IComparer<RelativePath>
{
public static readonly RelativePathComparer Instance = new();
private RelativePathComparer() { }

public int Compare(RelativePath? x, RelativePath? y) =>
StringComparer.OrdinalIgnoreCase.Compare(x?.Path, y?.Path);
}

private static SortedDictionary<RelativePath, DedupIdentifier> CreateNormalizedManifest(Manifest m)
private static SortedDictionary<string, DedupIdentifier> CreateNormalizedManifest(Manifest m)
{
SortedDictionary<RelativePath, DedupIdentifier> sorted = new(RelativePathComparer.Instance);
SortedDictionary<string, DedupIdentifier> sorted = new(StringComparer.OrdinalIgnoreCase);

foreach (ManifestItem item in m.Items)
{
Expand All @@ -631,27 +627,28 @@ private static SortedDictionary<RelativePath, DedupIdentifier> CreateNormalizedM
continue;
}

sorted.Add(new RelativePath(item.Path), DedupIdentifier.Create(item.Blob.Id));
sorted.Add(item.Path, DedupIdentifier.Create(item.Blob.Id));
}

return sorted;
}

private SortedDictionary<RelativePath, DedupIdentifier> CreateNormalizedManifest(IReadOnlyDictionary<string, ContentHash> files)
private SortedDictionary<string, DedupIdentifier> CreateNormalizedManifest(IReadOnlyDictionary<string, ContentHash> files)
{
SortedDictionary<RelativePath, DedupIdentifier> sorted = new(RelativePathComparer.Instance);
SortedDictionary<string, DedupIdentifier> sorted = new(StringComparer.OrdinalIgnoreCase);

foreach (KeyValuePair<string, ContentHash> f in files)
{
sorted.Add(new RelativePath(f.Key.MakePathRelativeTo(RepoRoot)!), f.Value.ToBlobIdentifier().ToDedupIdentifier());
string relativePath = $"/{f.Key.MakePathRelativeTo(RepoRoot)!.Replace("\\", "/", StringComparison.Ordinal)}";
sorted.Add(relativePath, f.Value.ToBlobIdentifier().ToDedupIdentifier());
}

return sorted;
}

private static void ThrowIfDifferent(
SortedDictionary<RelativePath, DedupIdentifier> left,
SortedDictionary<RelativePath, DedupIdentifier> right,
SortedDictionary<string, DedupIdentifier> left,
SortedDictionary<string, DedupIdentifier> right,
string message
)
{
Expand All @@ -660,10 +657,10 @@ string message
return;
}

SortedSet<(RelativePath, DedupIdentifier)> leftOnly = new(left.Select(kvp => (kvp.Key, kvp.Value)));
SortedSet<(RelativePath, DedupIdentifier)> rightOnly = new(right.Select(kvp => (kvp.Key, kvp.Value)));
SortedSet<(string, DedupIdentifier)> leftOnly = new(left.Select(kvp => (kvp.Key, kvp.Value)));
SortedSet<(string, DedupIdentifier)> rightOnly = new(right.Select(kvp => (kvp.Key, kvp.Value)));

SortedSet<(RelativePath, DedupIdentifier)> both = new(leftOnly);
SortedSet<(string, DedupIdentifier)> both = new(leftOnly);
both.IntersectWith(rightOnly);

leftOnly.ExceptWith(both);
Expand Down
2 changes: 1 addition & 1 deletion src/Common/PathHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public static bool IsUnderDirectory(this string filePath, string directoryPath)
/// \\?\ - removes the file name limit of 260 chars. It makes it 32735 (+ a null terminator)
/// \??\ - this is a native Win32 FS path WinNt32
/// </summary>
internal static string RemoveLongPathPrefixes(string absolutePath)
public static string RemoveLongPathPrefixes(string absolutePath)
{
if (absolutePath.Length < 4 || absolutePath[0] != '\\')
{
Expand Down