diff --git a/src/Agent.Plugins/PipelineCache/FingerprintCreator.cs b/src/Agent.Plugins/PipelineCache/FingerprintCreator.cs index 71827ed2f4..ce10799a60 100644 --- a/src/Agent.Plugins/PipelineCache/FingerprintCreator.cs +++ b/src/Agent.Plugins/PipelineCache/FingerprintCreator.cs @@ -18,6 +18,12 @@ namespace Agent.Plugins.PipelineCache { + public enum FingerprintType + { + Key, + Path + } + public static class FingerprintCreator { private static readonly bool isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); @@ -32,7 +38,7 @@ public static class FingerprintCreator AllowWindowsPaths = isWindows, }; - private static readonly char[] GlobChars = new [] { '*', '?', '[', ']' }; + private static readonly char[] GlobChars = new[] { '*', '?', '[', ']' }; private const char ForceStringLiteral = '"'; @@ -45,21 +51,22 @@ private static bool IsPathyChar(char c) return !Path.GetInvalidFileNameChars().Contains(c); } - internal static bool IsPathyKeySegment(string keySegment) + internal static bool IsPathySegment(string segment) { - if (keySegment.First() == ForceStringLiteral && keySegment.Last() == ForceStringLiteral) return false; - if (keySegment.Any(c => !IsPathyChar(c))) return false; - if (!keySegment.Contains(".") && - !keySegment.Contains(Path.DirectorySeparatorChar) && - !keySegment.Contains(Path.AltDirectorySeparatorChar)) return false; - if (keySegment.Last() == '.') return false; + if (segment.First() == ForceStringLiteral && segment.Last() == ForceStringLiteral) return false; + if (segment.Any(c => !IsPathyChar(c))) return false; + if (!segment.Contains(".") && + !segment.Contains(Path.DirectorySeparatorChar) && + !segment.Contains(Path.AltDirectorySeparatorChar)) return false; + if (segment.Last() == '.') return false; return true; } internal static Func CreateMinimatchFilter(AgentTaskPluginExecutionContext context, string rule, bool invert) { - Func filter = Minimatcher.CreateFilter(rule, minimatchOptions); - Func tracedFilter = (path) => { + Func filter = Minimatcher.CreateFilter(rule, minimatchOptions); + Func tracedFilter = (path) => + { bool filterResult = filter(path); context.Verbose($"Path `{path}` is {(filterResult ? "" : "not ")}{(invert ? "excluded" : "included")} because of pattern `{(invert ? "!" : "")}{rule}`."); return invert ^ filterResult; @@ -81,16 +88,16 @@ internal static string MakePathCanonical(string defaultWorkingDirectory, string } } - internal static Func CreateFilter( + internal static Func CreateFilter( AgentTaskPluginExecutionContext context, IEnumerable includeRules, IEnumerable excludeRules) { - Func[] includeFilters = includeRules.Select(includeRule => - CreateMinimatchFilter(context, includeRule, invert: false)).ToArray(); - Func[] excludeFilters = excludeRules.Select(excludeRule => - CreateMinimatchFilter(context, excludeRule, invert: true)).ToArray(); - Func filter = (path) => includeFilters.Any(f => f(path)) && excludeFilters.All(f => f(path)); + Func[] includeFilters = includeRules.Select(includeRule => + CreateMinimatchFilter(context, includeRule, invert: false)).ToArray(); + Func[] excludeFilters = excludeRules.Select(excludeRule => + CreateMinimatchFilter(context, excludeRule, invert: true)).ToArray(); + Func filter = (path) => includeFilters.Any(f => f(path)) && excludeFilters.All(f => f(path)); return filter; } @@ -109,10 +116,10 @@ public MatchedFile(string displayPath, long fileLength, string hash) { this.DisplayPath = displayPath; this.FileLength = fileLength; - this.Hash = hash; + this.Hash = hash; } - public MatchedFile(string displayPath, FileStream fs): + public MatchedFile(string displayPath, FileStream fs) : this(displayPath, fs.Length, s_sha256.ComputeHash(fs).ToHex()) { } @@ -121,11 +128,13 @@ public MatchedFile(string displayPath, long fileLength, string hash) public long FileLength; public string Hash; - public string GetHash() { - return MatchedFile.GenerateHash(new [] { this }); + public string GetHash() + { + return MatchedFile.GenerateHash(new[] { this }); } - public static string GenerateHash(IEnumerable matches) { + public static string GenerateHash(IEnumerable matches) + { string s = matches.Aggregate(new StringBuilder(), (sb, file) => sb.Append($"\nSHA256({file.DisplayPath})=[{file.FileLength}]{file.Hash}"), sb => sb.ToString()); @@ -138,7 +147,9 @@ internal enum KeySegmentType { String = 0, FilePath = 1, - FilePattern = 2 + FilePattern = 2, + Directory = 3, + DirectoryPattern = 4 } // Given a globby path, figure out where to start enumerating. @@ -159,7 +170,8 @@ internal static Enumeration DetermineFileEnumerationFromGlob(string includeGlobP // no globbing if (firstGlob < 0) { - return new Enumeration() { + return new Enumeration() + { RootPath = Path.GetDirectoryName(includeGlobPathAbsolute), Pattern = Path.GetFileName(includeGlobPathAbsolute), Depth = SearchOption.TopDirectoryOnly @@ -167,105 +179,130 @@ internal static Enumeration DetermineFileEnumerationFromGlob(string includeGlobP } else { - int rootDirLength = includeGlobPathAbsolute.Substring(0,firstGlob).LastIndexOfAny( new [] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar}); - return new Enumeration() { - RootPath = includeGlobPathAbsolute.Substring(0,rootDirLength), + int rootDirLength = includeGlobPathAbsolute.Substring(0, firstGlob).LastIndexOfAny(new[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }); + return new Enumeration() + { + RootPath = includeGlobPathAbsolute.Substring(0, rootDirLength), Pattern = "*", Depth = hasRecursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly }; } } - internal static void CheckKeySegment(string keySegment) + internal static void CheckSegment(string segment, string segmentType) { - if (keySegment.Equals("*", StringComparison.Ordinal)) + if (segment.Equals("*", StringComparison.Ordinal)) { - throw new ArgumentException("`*` is a reserved key segment. For path glob, use `./*`."); + throw new ArgumentException($"`*` is a reserved {segmentType} segment. For path glob, use `./*`."); } - else if (keySegment.Equals(Fingerprint.Wildcard, StringComparison.Ordinal)) + else if (segment.Equals(Fingerprint.Wildcard, StringComparison.Ordinal)) { - throw new ArgumentException("`**` is a reserved key segment. For path glob, use `./**`."); + throw new ArgumentException($"`**` is a reserved {segmentType} segment. For path glob, use `./**`."); } - else if (keySegment.First() == '\'') + else if (segment.First() == '\'') { - throw new ArgumentException("A key segment cannot start with a single-quote character`."); + throw new ArgumentException($"A {segmentType} segment cannot start with a single-quote character`."); } - else if (keySegment.First() == '`') + else if (segment.First() == '`') { - throw new ArgumentException("A key segment cannot start with a backtick character`."); + throw new ArgumentException($"A {segmentType} segment cannot start with a backtick character`."); } } - public static Fingerprint EvaluateKeyToFingerprint( + private static void LogSegment( AgentTaskPluginExecutionContext context, - string filePathRoot, - IEnumerable keySegments) + IEnumerable segments, + string segment, + KeySegmentType type, + Object details) { - // Quickly validate all segments - foreach (string keySegment in keySegments) + string FormatForDisplay(string value, int displayLength) { - CheckKeySegment(keySegment); - } - - string defaultWorkingDirectory = context.Variables.GetValueOrDefault( - "system.defaultworkingdirectory" // Constants.Variables.System.DefaultWorkingDirectory - )?.Value; + if (value.Length > displayLength) + { + value = value.Substring(0, displayLength - 3) + "..."; + } - var resolvedSegments = new List(); - var exceptions = new List(); + return value.PadRight(displayLength); + }; - Action LogKeySegment = (segment, type, details) => + string formattedSegment = FormatForDisplay(segment, Math.Min(segments.Select(s => s.Length).Max(), 50)); + + void LogPatternSegment(object value, string title, Func getText, Func getSuffix = null) { - Func FormatForDisplay = (value, displayLength) => + var matches = (value as T[]) ?? Array.Empty(); + context.Output($" - {formattedSegment} [{title} pattern; matches: {matches.Length}]"); + if (matches.Any()) { - if (value.Length > displayLength) + int displayLength = Math.Min(matches.Select(d => getText(d).Length).Max(), 70); + foreach (var match in matches) { - value = value.Substring(0, displayLength - 3) + "..."; + context.Output($" - {FormatForDisplay(getText(match), displayLength)}{(getSuffix != null ? getSuffix(match) : "")}"); } + } + }; - return value.PadRight(displayLength); - }; + switch (type) + { + case KeySegmentType.String: + context.Output($" - {formattedSegment} [string]"); + break; + case KeySegmentType.Directory: + context.Output($" - {formattedSegment} [directory]"); + break; + case KeySegmentType.DirectoryPattern: + LogPatternSegment( + details, + "directory", + s => s + ); + break; + case KeySegmentType.FilePath: + var files = (details as MatchedFile[]) ?? new MatchedFile[0]; + string fileHash = files.Length > 0 ? files[0].Hash : null; + context.Output($" - {formattedSegment} [file] {(!string.IsNullOrWhiteSpace(fileHash) ? $"--> {fileHash}" : "(not found)")}"); + break; + case KeySegmentType.FilePattern: + LogPatternSegment( + details, + "file", + match => match.DisplayPath, + match => $" --> {match.Hash}" + ); + break; + } + } - string formattedSegment = FormatForDisplay(segment, Math.Min(keySegments.Select(s => s.Length).Max(), 50)); + public static Fingerprint EvaluateToFingerprint( + AgentTaskPluginExecutionContext context, + string filePathRoot, + IEnumerable segments, + FingerprintType fingerprintType) + { + // Quickly validate all segments + foreach (string segment in segments) + { + CheckSegment(segment, fingerprintType.ToString()); + } - if (type == KeySegmentType.String) - { - context.Output($" - {formattedSegment} [string]"); - } - else - { - var matches = (details as MatchedFile[]) ?? new MatchedFile[0]; - - if (type == KeySegmentType.FilePath) - { - string fileHash = matches.Length > 0 ? matches[0].Hash : null; - context.Output($" - {formattedSegment} [file] {(!string.IsNullOrWhiteSpace(fileHash) ? $"--> {fileHash}" : "(not found)")}"); - } - else if (type == KeySegmentType.FilePattern) - { - context.Output($" - {formattedSegment} [file pattern; matches: {matches.Length}]"); - if (matches.Any()) - { - int filePathDisplayLength = Math.Min(matches.Select(mf => mf.DisplayPath.Length).Max(), 70); - foreach (var match in matches) - { - context.Output($" - {FormatForDisplay(match.DisplayPath, filePathDisplayLength)} --> {match.Hash}"); - } - } - } - } - }; + string defaultWorkingDirectory = context.Variables.GetValueOrDefault( + "system.defaultworkingdirectory" // Constants.Variables.System.DefaultWorkingDirectory + )?.Value ?? filePathRoot; + + var resolvedSegments = new List(); + var exceptions = new List(); + var hasPatternSegments = false; - foreach (string keySegment in keySegments) + foreach (string segment in segments) { - if (!IsPathyKeySegment(keySegment)) + if (!IsPathySegment(segment)) { - LogKeySegment(keySegment, KeySegmentType.String, null); - resolvedSegments.Add(keySegment); + LogSegment(context, segments, segment, KeySegmentType.String, null); + resolvedSegments.Add(segment); } else { - string[] pathRules = keySegment.Split(new []{','}, StringSplitOptions.RemoveEmptyEntries).Select(s => s.Trim()).ToArray(); + string[] pathRules = segment.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(s => s.Trim()).ToArray(); string[] includeRules = pathRules.Where(p => !p.StartsWith('!')).ToArray(); if (!includeRules.Any()) @@ -273,75 +310,119 @@ internal static void CheckKeySegment(string keySegment) throw new ArgumentException("No include rules specified."); } - var enumerations = new Dictionary>(); - foreach(string includeRule in includeRules) + var enumerations = new Dictionary>(); + foreach (string includeRule in includeRules) { string absoluteRootRule = MakePathCanonical(defaultWorkingDirectory, includeRule); context.Verbose($"Expanded include rule is `{absoluteRootRule}`."); Enumeration enumeration = DetermineFileEnumerationFromGlob(absoluteRootRule); List globs; - if(!enumerations.TryGetValue(enumeration, out globs)) + if (!enumerations.TryGetValue(enumeration, out globs)) { - enumerations[enumeration] = globs = new List(); + enumerations[enumeration] = globs = new List(); } globs.Add(absoluteRootRule); } string[] excludeRules = pathRules.Where(p => p.StartsWith('!')).ToArray(); - string[] absoluteExcludeRules = excludeRules.Select(excludeRule => { + string[] absoluteExcludeRules = excludeRules.Select(excludeRule => + { excludeRule = excludeRule.Substring(1); return MakePathCanonical(defaultWorkingDirectory, excludeRule); }).ToArray(); var matchedFiles = new SortedDictionary(StringComparer.Ordinal); + var matchedDirectories = new SortedDictionary(StringComparer.Ordinal); - foreach(var kvp in enumerations) + foreach (var kvp in enumerations) { Enumeration enumerate = kvp.Key; List absoluteIncludeGlobs = kvp.Value; context.Verbose($"Enumerating starting at root `{enumerate.RootPath}` with pattern `{enumerate.Pattern}` and depth `{enumerate.Depth}`."); - IEnumerable files = Directory.EnumerateFiles(enumerate.RootPath, enumerate.Pattern, enumerate.Depth); - Func filter = CreateFilter(context, absoluteIncludeGlobs, absoluteExcludeRules); + IEnumerable files = fingerprintType == FingerprintType.Key + ? Directory.EnumerateFiles(enumerate.RootPath, enumerate.Pattern, enumerate.Depth) + : Directory.EnumerateDirectories(enumerate.RootPath, enumerate.Pattern, enumerate.Depth); + + Func filter = CreateFilter(context, absoluteIncludeGlobs, absoluteExcludeRules); files = files.Where(f => filter(f)).Distinct(); - foreach(string path in files) + foreach (string path in files) { - using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + // Path.GetRelativePath returns 'The relative path, or path if the paths don't share the same root.' + string displayPath = filePathRoot == null ? path : Path.GetRelativePath(filePathRoot, path); + + if (fingerprintType == FingerprintType.Key) + { + using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + matchedFiles.Add(path, new MatchedFile(displayPath, fs)); + } + } + else { - // Path.GetRelativePath returns 'The relative path, or path if the paths don't share the same root.' - string displayPath = filePathRoot == null ? path : Path.GetRelativePath(filePathRoot, path); - matchedFiles.Add(path, new MatchedFile(displayPath, fs)); + matchedDirectories.Add(path, displayPath); } } } - var patternSegment = keySegment.IndexOfAny(GlobChars) >= 0 || matchedFiles.Count() > 1; + var patternSegment = segment.IndexOfAny(GlobChars) >= 0 || matchedFiles.Count > 1; + hasPatternSegments |= patternSegment; - var displayKeySegment = keySegment; + var displaySegment = segment; if (context.Container != null) { - displayKeySegment = context.Container.TranslateToContainerPath(displayKeySegment); + displaySegment = context.Container.TranslateToContainerPath(displaySegment); } - LogKeySegment(displayKeySegment, - patternSegment ? KeySegmentType.FilePattern : KeySegmentType.FilePath, - matchedFiles.Values.ToArray()); - - if (!matchedFiles.Any()) + KeySegmentType segmentType; + object details; + if (fingerprintType == FingerprintType.Key) { - if (patternSegment) + segmentType = patternSegment ? KeySegmentType.FilePattern : KeySegmentType.FilePath; + details = matchedFiles.Values.ToArray(); + resolvedSegments.Add(MatchedFile.GenerateHash(matchedFiles.Values)); + + if (!matchedFiles.Any()) { - exceptions.Add(new FileNotFoundException($"No matching files found for pattern: {displayKeySegment}")); + var message = patternSegment ? $"No matching files found for pattern: {displaySegment}" : $"File not found: {displaySegment}"; + exceptions.Add(new FileNotFoundException(message)); } - else + } + else + { + segmentType = patternSegment ? KeySegmentType.DirectoryPattern : KeySegmentType.Directory; + details = matchedDirectories.Values.ToArray(); + resolvedSegments.AddRange(matchedDirectories.Values); + + if (!matchedDirectories.Any()) { - exceptions.Add(new FileNotFoundException($"File not found: {displayKeySegment}")); + var message = patternSegment ? $"No matching directories found for pattern: {displaySegment}" : $"Directory not found: {displaySegment}"; + exceptions.Add(new DirectoryNotFoundException(message)); } } - - resolvedSegments.Add(MatchedFile.GenerateHash(matchedFiles.Values)); - } + + LogSegment( + context, + segments, + displaySegment, + segmentType, + details + ); + } + } + + if (fingerprintType == FingerprintType.Path) + { + // If there are segments or contains a glob pattern, all resolved segments must be rooted within filePathRoot (e.g. Pipeline.Workspace) + // This limitation is mainly due to 7z not extracting backtraced paths + if (resolvedSegments.Count() > 1 || hasPatternSegments) + { + foreach (var backtracedPath in resolvedSegments.Where(x => x.StartsWith(".."))) + { + exceptions.Add(new ArgumentException($"Resolved path is not within `Pipeline.Workspace`: {backtracedPath}")); + } + } } if (exceptions.Any()) diff --git a/src/Agent.Plugins/PipelineCache/PipelineCacheServer.cs b/src/Agent.Plugins/PipelineCache/PipelineCacheServer.cs index 44dbea1524..006b3e8bc6 100644 --- a/src/Agent.Plugins/PipelineCache/PipelineCacheServer.cs +++ b/src/Agent.Plugins/PipelineCache/PipelineCacheServer.cs @@ -25,8 +25,9 @@ public class PipelineCacheServer [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1068: CancellationToken parameters must come last")] internal async Task UploadAsync( AgentTaskPluginExecutionContext context, - Fingerprint fingerprint, - string path, + Fingerprint keyFingerprint, + string[] pathSegments, + string workspaceRoot, CancellationToken cancellationToken, ContentFormat contentFormat) { @@ -40,7 +41,7 @@ public class PipelineCacheServer // Check if the key exists. PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context)); - PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new [] {fingerprint}, cancellationToken, cacheRecordGet); + PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new[] { keyFingerprint }, cancellationToken, cacheRecordGet); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet); //If cache exists, return. @@ -50,7 +51,12 @@ public class PipelineCacheServer return; } - string uploadPath = await this.GetUploadPathAsync(contentFormat, context, path, cancellationToken); + context.Output("Resolving path:"); + Fingerprint pathFp = FingerprintCreator.EvaluateToFingerprint(context, workspaceRoot, pathSegments, FingerprintType.Path); + context.Output($"Resolved to: {pathFp}"); + + string uploadPath = await this.GetUploadPathAsync(contentFormat, context, pathFp, pathSegments, workspaceRoot, cancellationToken); + //Upload the pipeline artifact. PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context)); @@ -60,10 +66,10 @@ public class PipelineCacheServer { return await dedupManifestClient.PublishAsync(uploadPath, cancellationToken); }); - + CreatePipelineCacheArtifactContract options = new CreatePipelineCacheArtifactContract { - Fingerprint = fingerprint, + Fingerprint = keyFingerprint, RootId = result.RootId, ManifestId = result.ManifestId, ProofNodes = result.ProofNodes.ToArray(), @@ -82,7 +88,7 @@ public class PipelineCacheServer } catch { } } - + // Cache the artifact PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context)); @@ -98,8 +104,9 @@ public class PipelineCacheServer internal async Task DownloadAsync( AgentTaskPluginExecutionContext context, Fingerprint[] fingerprints, - string path, + string[] pathSegments, string cacheHitVariable, + string workspaceRoot, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; @@ -126,12 +133,12 @@ public class PipelineCacheServer record: downloadRecord, actionAsync: async () => { - await this.DownloadPipelineCacheAsync(context, dedupManifestClient, result.ManifestId, path, Enum.Parse(result.ContentFormat), cancellationToken); + await this.DownloadPipelineCacheAsync(context, dedupManifestClient, result.ManifestId, pathSegments, workspaceRoot, Enum.Parse(result.ContentFormat), cancellationToken); }); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: downloadRecord); - + context.Output("Cache restored."); } @@ -146,11 +153,11 @@ public class PipelineCacheServer context.Verbose($"Exact fingerprint: `{result.Fingerprint.ToString()}`"); bool foundExact = false; - foreach(var fingerprint in fingerprints) + foreach (var fingerprint in fingerprints) { context.Verbose($"This fingerprint: `{fingerprint.ToString()}`"); - if (fingerprint == result.Fingerprint + if (fingerprint == result.Fingerprint || result.Fingerprint.Segments.Length == 1 && result.Fingerprint.Segments.Single() == fingerprint.SummarizeForV1()) { foundExact = true; @@ -177,21 +184,30 @@ public class PipelineCacheServer return pipelineCacheClient; } - private async Task GetUploadPathAsync(ContentFormat contentFormat, AgentTaskPluginExecutionContext context, string path, CancellationToken cancellationToken) + private Task GetUploadPathAsync(ContentFormat contentFormat, AgentTaskPluginExecutionContext context, Fingerprint pathFingerprint, string[] pathSegments, string workspaceRoot, CancellationToken cancellationToken) { - string uploadPath = path; - if(contentFormat == ContentFormat.SingleTar) + if (contentFormat == ContentFormat.SingleTar) { - uploadPath = await TarUtils.ArchiveFilesToTarAsync(context, path, cancellationToken); + var (tarWorkingDirectory, isWorkspaceContained) = GetTarWorkingDirectory(pathSegments, workspaceRoot); + + return TarUtils.ArchiveFilesToTarAsync( + context, + pathFingerprint, + tarWorkingDirectory, + isWorkspaceContained, + cancellationToken + ); } - return uploadPath; + + return Task.FromResult(pathFingerprint.Segments[0]); } private async Task DownloadPipelineCacheAsync( AgentTaskPluginExecutionContext context, DedupManifestArtifactClient dedupManifestClient, DedupIdentifier manifestId, - string targetDirectory, + string[] pathSegments, + string workspaceRoot, ContentFormat contentFormat, CancellationToken cancellationToken) { @@ -200,26 +216,42 @@ private async Task GetUploadPathAsync(ContentFormat contentFormat, Agent string manifestPath = Path.Combine(Path.GetTempPath(), $"{nameof(DedupManifestArtifactClient)}.{Path.GetRandomFileName()}.manifest"); await dedupManifestClient.DownloadFileToPathAsync(manifestId, manifestPath, proxyUri: null, cancellationToken: cancellationToken); Manifest manifest = JsonSerializer.Deserialize(File.ReadAllText(manifestPath)); - await TarUtils.DownloadAndExtractTarAsync (context, manifest, dedupManifestClient, targetDirectory, cancellationToken); + var (tarWorkingDirectory, _) = GetTarWorkingDirectory(pathSegments, workspaceRoot); + await TarUtils.DownloadAndExtractTarAsync(context, manifest, dedupManifestClient, tarWorkingDirectory, cancellationToken); try { - if(File.Exists(manifestPath)) + if (File.Exists(manifestPath)) { File.Delete(manifestPath); } } - catch {} + catch { } } else { DownloadDedupManifestArtifactOptions options = DownloadDedupManifestArtifactOptions.CreateWithManifestId( manifestId, - targetDirectory, + pathSegments[0], proxyUri: null, minimatchPatterns: null); await dedupManifestClient.DownloadAsync(options, cancellationToken); } + } + + private (string workingDirectory, bool isWorkspaceContained) GetTarWorkingDirectory(string[] segments, string workspaceRoot) + { + // If path segment is single directory outside of Pipeline.Workspace extract tarball directly to this path + if (segments.Count() == 1) + { + var workingDirectory = segments[0]; + if (FingerprintCreator.IsPathySegment(workingDirectory) && !workingDirectory.StartsWith(workspaceRoot)) + { + return (workingDirectory, false); + } + } + // All other scenarios means that paths must within and relative to Pipeline.Workspace + return (workspaceRoot.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), true); } } } diff --git a/src/Agent.Plugins/PipelineCache/PipelineCacheTaskPluginBase.cs b/src/Agent.Plugins/PipelineCache/PipelineCacheTaskPluginBase.cs index f7c805c26a..0be8ab55d1 100644 --- a/src/Agent.Plugins/PipelineCache/PipelineCacheTaskPluginBase.cs +++ b/src/Agent.Plugins/PipelineCache/PipelineCacheTaskPluginBase.cs @@ -31,26 +31,28 @@ public abstract class PipelineCacheTaskPluginBase : IAgentTaskPlugin public abstract String Stage { get; } public const string ResolvedFingerPrintVariableName = "RESTORE_STEP_RESOLVED_FINGERPRINT"; - internal static (bool isOldFormat, string[] keySegments,IEnumerable restoreKeys) ParseIntoSegments(string salt, string key, string restoreKeysBlock) + internal static (bool isOldFormat, string[] keySegments, IEnumerable restoreKeys, string[] pathSegments) ParseIntoSegments(string salt, string key, string restoreKeysBlock, string path) { - Func splitAcrossPipes = (s) => { - var segments = s.Split(new [] {'|'},StringSplitOptions.RemoveEmptyEntries).Select(segment => segment.Trim()); - if(!string.IsNullOrWhiteSpace(salt)) + Func splitAcrossPipes = (s) => + { + var segments = s.Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries).Select(segment => segment.Trim()); + if (!string.IsNullOrWhiteSpace(salt)) { - segments = (new [] { $"{SaltVariableName}={salt}"}).Concat(segments); + segments = (new[] { $"{SaltVariableName}={salt}" }).Concat(segments); } return segments.ToArray(); }; - Func splitAcrossNewlines = (s) => + Func splitAcrossNewlines = (s) => s.Replace("\r\n", "\n") //normalize newlines - .Split(new [] {'\n'}, StringSplitOptions.RemoveEmptyEntries) + .Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries) .Select(line => line.Trim()) .ToArray(); - + string[] keySegments; + string[] pathSegments; bool isOldFormat = key.Contains('\n'); - + IEnumerable restoreKeys; bool hasRestoreKeys = !string.IsNullOrWhiteSpace(restoreKeysBlock); @@ -58,7 +60,7 @@ internal static (bool isOldFormat, string[] keySegments,IEnumerable re { throw new ArgumentException(OldKeyFormatMessage); } - + if (isOldFormat) { keySegments = splitAcrossNewlines(key); @@ -67,7 +69,9 @@ internal static (bool isOldFormat, string[] keySegments,IEnumerable re { keySegments = splitAcrossPipes(key); } - + + // Path can be a list of path segments to include in cache + pathSegments = splitAcrossPipes(path); if (hasRestoreKeys) { @@ -78,9 +82,9 @@ internal static (bool isOldFormat, string[] keySegments,IEnumerable re restoreKeys = Enumerable.Empty(); } - return (isOldFormat, keySegments, restoreKeys); + return (isOldFormat, keySegments, restoreKeys, pathSegments); } - + public async virtual Task RunAsync(AgentTaskPluginExecutionContext context, CancellationToken token) { ArgUtil.NotNull(context, nameof(context)); @@ -93,8 +97,10 @@ public async virtual Task RunAsync(AgentTaskPluginExecutionContext context, Canc string key = context.GetInput(PipelineCacheTaskPluginConstants.Key, required: true); string restoreKeysBlock = context.GetInput(PipelineCacheTaskPluginConstants.RestoreKeys, required: false); + // TODO: Translate path from container to host (Ting) + string path = context.GetInput(PipelineCacheTaskPluginConstants.Path, required: true); - (bool isOldFormat, string[] keySegments, IEnumerable restoreKeys) = ParseIntoSegments(salt, key, restoreKeysBlock); + (bool isOldFormat, string[] keySegments, IEnumerable restoreKeys, string[] pathSegments) = ParseIntoSegments(salt, key, restoreKeysBlock, path); if (isOldFormat) { @@ -102,26 +108,26 @@ public async virtual Task RunAsync(AgentTaskPluginExecutionContext context, Canc } context.Output("Resolving key:"); - Fingerprint keyFp = FingerprintCreator.EvaluateKeyToFingerprint(context, workspaceRoot, keySegments); + Fingerprint keyFp = FingerprintCreator.EvaluateToFingerprint(context, workspaceRoot, keySegments, FingerprintType.Key); context.Output($"Resolved to: {keyFp}"); - Func restoreKeysGenerator = () => - restoreKeys.Select(restoreKey => { + Func restoreKeysGenerator = () => + restoreKeys.Select(restoreKey => + { context.Output("Resolving restore key:"); - Fingerprint f = FingerprintCreator.EvaluateKeyToFingerprint(context, workspaceRoot, restoreKey); - f.Segments = f.Segments.Concat(new [] { Fingerprint.Wildcard} ).ToArray(); + Fingerprint f = FingerprintCreator.EvaluateToFingerprint(context, workspaceRoot, restoreKey, FingerprintType.Key); + f.Segments = f.Segments.Concat(new[] { Fingerprint.Wildcard }).ToArray(); context.Output($"Resolved to: {f}"); return f; }).ToArray(); - // TODO: Translate path from container to host (Ting) - string path = context.GetInput(PipelineCacheTaskPluginConstants.Path, required: true); await ProcessCommandInternalAsync( context, keyFp, restoreKeysGenerator, - path, + pathSegments, + workspaceRoot, token); } @@ -130,7 +136,8 @@ public async virtual Task RunAsync(AgentTaskPluginExecutionContext context, Canc AgentTaskPluginExecutionContext context, Fingerprint fingerprint, Func restoreKeysGenerator, - string path, + string[] pathSegments, + string workspaceRoot, CancellationToken token); // Properties set by tasks diff --git a/src/Agent.Plugins/PipelineCache/RestorePipelineCacheV0.cs b/src/Agent.Plugins/PipelineCache/RestorePipelineCacheV0.cs index f454cceba5..003fb1eea1 100644 --- a/src/Agent.Plugins/PipelineCache/RestorePipelineCacheV0.cs +++ b/src/Agent.Plugins/PipelineCache/RestorePipelineCacheV0.cs @@ -9,7 +9,7 @@ using Microsoft.VisualStudio.Services.PipelineCache.WebApi; namespace Agent.Plugins.PipelineCache -{ +{ public class RestorePipelineCacheV0 : PipelineCacheTaskPluginBase { public override string Stage => "main"; @@ -18,7 +18,8 @@ public class RestorePipelineCacheV0 : PipelineCacheTaskPluginBase AgentTaskPluginExecutionContext context, Fingerprint fingerprint, Func restoreKeysGenerator, - string path, + string[] pathSegments, + string workspaceRoot, CancellationToken token) { context.SetTaskVariable(RestoreStepRanVariableName, RestoreStepRanVariableValue); @@ -27,10 +28,11 @@ public class RestorePipelineCacheV0 : PipelineCacheTaskPluginBase var server = new PipelineCacheServer(); Fingerprint[] restoreFingerprints = restoreKeysGenerator(); await server.DownloadAsync( - context, - (new [] { fingerprint}).Concat(restoreFingerprints).ToArray(), - path, + context, + (new[] { fingerprint }).Concat(restoreFingerprints).ToArray(), + pathSegments, context.GetInput(PipelineCacheTaskPluginConstants.CacheHitVariable, required: false), + workspaceRoot, token); } } diff --git a/src/Agent.Plugins/PipelineCache/SavePipelineCacheV0.cs b/src/Agent.Plugins/PipelineCache/SavePipelineCacheV0.cs index 40365423bb..addef477f3 100644 --- a/src/Agent.Plugins/PipelineCache/SavePipelineCacheV0.cs +++ b/src/Agent.Plugins/PipelineCache/SavePipelineCacheV0.cs @@ -57,20 +57,21 @@ public override async Task RunAsync(AgentTaskPluginExecutionContext context, Can protected override async Task ProcessCommandInternalAsync( AgentTaskPluginExecutionContext context, - Fingerprint fingerprint, + Fingerprint keyFingerprint, Func restoreKeysGenerator, - string path, + string[] pathSegments, + string workspaceRoot, CancellationToken token) { - string contentFormatValue = context.Variables.GetValueOrDefault(ContentFormatVariableName)?.Value ?? string.Empty; + string contentFormatValue = context.Variables.GetValueOrDefault(ContentFormatVariableName)?.Value ?? string.Empty; string calculatedFingerPrint = context.TaskVariables.GetValueOrDefault(ResolvedFingerPrintVariableName)?.Value ?? string.Empty; - if(!string.IsNullOrWhiteSpace(calculatedFingerPrint) && !fingerprint.ToString().Equals(calculatedFingerPrint, StringComparison.Ordinal)) + if (!string.IsNullOrWhiteSpace(calculatedFingerPrint) && !keyFingerprint.ToString().Equals(calculatedFingerPrint, StringComparison.Ordinal)) { - context.Warning($"The given cache key has changed in its resolved value between restore and save steps;\n"+ - $"original key: {calculatedFingerPrint}\n"+ - $"modified key: {fingerprint}\n"); - } + context.Warning($"The given cache key has changed in it's resolved value between restore and save steps;\n" + + $"original key: {calculatedFingerPrint}\n" + + $"modified key: {keyFingerprint}\n"); + } ContentFormat contentFormat; if (string.IsNullOrWhiteSpace(contentFormatValue)) @@ -85,8 +86,9 @@ public override async Task RunAsync(AgentTaskPluginExecutionContext context, Can PipelineCacheServer server = new PipelineCacheServer(); await server.UploadAsync( context, - fingerprint, - path, + keyFingerprint, + pathSegments, + workspaceRoot, token, contentFormat); } diff --git a/src/Agent.Plugins/PipelineCache/TarUtils.cs b/src/Agent.Plugins/PipelineCache/TarUtils.cs index e050b653b4..e55cb24b3d 100644 --- a/src/Agent.Plugins/PipelineCache/TarUtils.cs +++ b/src/Agent.Plugins/PipelineCache/TarUtils.cs @@ -13,6 +13,7 @@ using Microsoft.TeamFoundation.DistributedTask.WebApi; using Microsoft.VisualStudio.Services.BlobStore.Common; using Microsoft.VisualStudio.Services.BlobStore.WebApi; +using Microsoft.VisualStudio.Services.PipelineCache.WebApi; namespace Agent.Plugins.PipelineCache { @@ -29,18 +30,23 @@ public static class TarUtils /// The path to the TAR. public static async Task ArchiveFilesToTarAsync( AgentTaskPluginExecutionContext context, - string inputPath, + Fingerprint pathFingerprint, + string tarWorkingDirectory, + bool isWorkspaceContained, CancellationToken cancellationToken) { - if(File.Exists(inputPath)) + foreach (var inputPath in pathFingerprint.Segments) { - throw new DirectoryNotFoundException($"Please specify path to a directory, File path is not allowed. {inputPath} is a file."); + if (File.Exists(inputPath)) + { + throw new DirectoryNotFoundException($"Please specify path to a directory, File path is not allowed. {inputPath} is a file."); + } } var archiveFileName = CreateArchiveFileName(); var archiveFile = Path.Combine(Path.GetTempPath(), archiveFileName); - ProcessStartInfo processStartInfo = GetCreateTarProcessInfo(context, archiveFileName, inputPath); + ProcessStartInfo processStartInfo = GetCreateTarProcessInfo(context, archiveFileName, tarWorkingDirectory); Action actionOnFailure = () => { @@ -48,13 +54,43 @@ public static class TarUtils TryDeleteFile(archiveFile); }; + Func inputFilesTask = + (process, ct) => + Task.Run(async () => + { + try + { + // If path segment is single directory outside of Pipeline.Workspace, inputPaths is simply `.` + var inputPaths = isWorkspaceContained ? + pathFingerprint.Segments.Select(i => i.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)) + : new[]{ "." }; + + // Stream input paths to tar to avoid command length limitations + foreach (var inputPath in inputPaths) + { + await process.StandardInput.WriteLineAsync(inputPath); + } + + process.StandardInput.BaseStream.Close(); + } + catch (Exception e) + { + try + { + process.Kill(); + } + catch { } + ExceptionDispatchInfo.Capture(e).Throw(); + } + }); + await RunProcessAsync( context, processStartInfo, - // no additional tasks on create are required to run whilst running the TAR process - (Process process, CancellationToken ct) => Task.CompletedTask, + inputFilesTask, actionOnFailure, cancellationToken); + return archiveFile; } @@ -70,20 +106,25 @@ public static class TarUtils AgentTaskPluginExecutionContext context, Manifest manifest, DedupManifestArtifactClient dedupManifestClient, - string targetDirectory, + string tarWorkingDirectory, CancellationToken cancellationToken) { ValidateTarManifest(manifest); - Directory.CreateDirectory(targetDirectory); - - DedupIdentifier dedupId = DedupIdentifier.Create(manifest.Items.Single(i => i.Path.EndsWith(archive, StringComparison.OrdinalIgnoreCase)).Blob.Id); + DedupIdentifier dedupId = DedupIdentifier.Create(manifest.Items.Single(i => i.Path.EndsWith(archive, StringComparison.OrdinalIgnoreCase)).Blob.Id); - ProcessStartInfo processStartInfo = GetExtractStartProcessInfo(context, targetDirectory); + // We now can simply specify the working directory as the tarball will contain paths relative to it + ProcessStartInfo processStartInfo = GetExtractStartProcessInfo(context, tarWorkingDirectory); + + if (!Directory.Exists(tarWorkingDirectory)) + { + Directory.CreateDirectory(tarWorkingDirectory); + } Func downloadTaskFunc = (process, ct) => - Task.Run(async () => { + Task.Run(async () => + { try { await dedupManifestClient.DownloadToStreamAsync(dedupId, process.StandardInput.BaseStream, proxyUri: null, cancellationToken: ct); @@ -95,7 +136,7 @@ public static class TarUtils { process.Kill(); } - catch {} + catch { } ExceptionDispatchInfo.Capture(e).Throw(); } }); @@ -165,11 +206,13 @@ private static void CreateProcessStartInfo(ProcessStartInfo processStartInfo, st processStartInfo.WorkingDirectory = processWorkingDirectory; } - private static ProcessStartInfo GetCreateTarProcessInfo(AgentTaskPluginExecutionContext context, string archiveFileName, string inputPath) + private static ProcessStartInfo GetCreateTarProcessInfo(AgentTaskPluginExecutionContext context, string archiveFileName, string tarWorkingDirectory) { var processFileName = GetTar(context); - inputPath = inputPath.TrimEnd(Path.DirectorySeparatorChar).TrimEnd(Path.AltDirectorySeparatorChar); - var processArguments = $"-cf \"{archiveFileName}\" -C \"{inputPath}\" ."; // If given the absolute path for the '-cf' option, the GNU tar fails. The workaround is to start the tarring process in the temp directory, and simply speficy 'archive.tar' for that option. + + // If given the absolute path for the '-cf' option, the GNU tar fails. The workaround is to start the tarring process in the temp directory, and simply speficy 'archive.tar' for that option. + // The list of input files is piped in through the 'additionalTaskToExecuteWhilstRunningProcess' parameter + var processArguments = $"-cf \"{archiveFileName}\" -C \"{tarWorkingDirectory}\" -T -"; if (context.IsSystemDebugTrue()) { @@ -179,26 +222,28 @@ private static ProcessStartInfo GetCreateTarProcessInfo(AgentTaskPluginExecution { processArguments = "-h " + processArguments; } - + ProcessStartInfo processStartInfo = new ProcessStartInfo(); - CreateProcessStartInfo(processStartInfo, processFileName, processArguments, processWorkingDirectory: Path.GetTempPath()); // We want to create the archiveFile in temp folder, and hence starting the tar process from TEMP to avoid absolute paths in tar cmd line. + // We want to create the archiveFile in temp folder, and hence starting the tar process from TEMP to avoid absolute paths in tar cmd line. + CreateProcessStartInfo(processStartInfo, processFileName, processArguments, processWorkingDirectory: Path.GetTempPath()); return processStartInfo; } private static string GetTar(AgentTaskPluginExecutionContext context) - { + { // check if the user specified the tar executable to use: string location = Environment.GetEnvironmentVariable(TarLocationEnvironmentVariableName); - return String.IsNullOrWhiteSpace(location) ? "tar" : location; + return String.IsNullOrWhiteSpace(location) ? "tar" : location; } - private static ProcessStartInfo GetExtractStartProcessInfo(AgentTaskPluginExecutionContext context, string targetDirectory) + private static ProcessStartInfo GetExtractStartProcessInfo(AgentTaskPluginExecutionContext context, string tarWorkingDirectory) { string processFileName, processArguments; + if (isWindows && CheckIf7ZExists()) { processFileName = "7z"; - processArguments = $"x -si -aoa -o\"{targetDirectory}\" -ttar"; + processArguments = $"x -si -aoa -o\"{tarWorkingDirectory}\" -ttar"; if (context.IsSystemDebugTrue()) { processArguments = "-bb1 " + processArguments; @@ -207,7 +252,8 @@ private static ProcessStartInfo GetExtractStartProcessInfo(AgentTaskPluginExecut else { processFileName = GetTar(context); - processArguments = $"-xf - -C ."; // Instead of targetDirectory, we are providing . to tar, because the tar process is being started from targetDirectory. + // Instead of targetDirectory, we are providing . to tar, because the tar process is being started from workingDirectory. + processArguments = $"-xf - -C ."; if (context.IsSystemDebugTrue()) { processArguments = "-v " + processArguments; @@ -215,7 +261,8 @@ private static ProcessStartInfo GetExtractStartProcessInfo(AgentTaskPluginExecut } ProcessStartInfo processStartInfo = new ProcessStartInfo(); - CreateProcessStartInfo(processStartInfo, processFileName, processArguments, processWorkingDirectory: targetDirectory); + // Tar is started in the working directory because the tarball contains paths relative to it + CreateProcessStartInfo(processStartInfo, processFileName, processArguments, processWorkingDirectory: tarWorkingDirectory); return processStartInfo; } @@ -236,7 +283,7 @@ private static void TryDeleteFile(string fileName) File.Delete(fileName); } } - catch {} + catch { } } private static string CreateArchiveFileName() diff --git a/src/Test/L0/Plugin/FingerprintCreatorTests.cs b/src/Test/L0/Plugin/FingerprintCreatorTests.cs index 808b05d436..b7133003c4 100644 --- a/src/Test/L0/Plugin/FingerprintCreatorTests.cs +++ b/src/Test/L0/Plugin/FingerprintCreatorTests.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -21,11 +21,17 @@ public class FingerprintCreatorTests private static readonly byte[] hash1; private static readonly byte[] hash2; - private static readonly string directory; private static readonly string path1; private static readonly string path2; + private static readonly string workspaceRoot; + private static readonly string directory1; + private static readonly string directory2; + + private static readonly string directory1Name; + private static readonly string directory2Name; + static FingerprintCreatorTests() { var r = new Random(0); @@ -38,9 +44,28 @@ static FingerprintCreatorTests() directory = Path.GetDirectoryName(path1); Assert.Equal(directory, Path.GetDirectoryName(path2)); + var workspace = Guid.NewGuid().ToString(); + + var path3 = Path.Combine(directory, workspace, Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); + var path4 = Path.Combine(directory, workspace, Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); + + directory1 = Path.GetDirectoryName(path3); + directory2 = Path.GetDirectoryName(path4); + + workspaceRoot = Path.GetDirectoryName(directory1); + + directory1Name = Path.GetFileName(directory1); + directory2Name = Path.GetFileName(directory2); + File.WriteAllBytes(path1, content1); File.WriteAllBytes(path2, content2); + Directory.CreateDirectory(directory1); + Directory.CreateDirectory(directory2); + + File.WriteAllBytes(path3, content1); + File.WriteAllBytes(path4, content2); + using (var hasher = new SHA256Managed()) { hash1 = hasher.ComputeHash(content1); @@ -53,14 +78,20 @@ static FingerprintCreatorTests() [Trait("Category", "Plugin")] public void Fingerprint_ReservedFails() { - using(var hostContext = new TestHostContext(this)) + using (var hostContext = new TestHostContext(this)) { var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace()); Assert.Throws( - () => FingerprintCreator.EvaluateKeyToFingerprint(context, directory, new [] {"*"}) + () => FingerprintCreator.EvaluateToFingerprint(context, directory, new[] { "*" }, FingerprintType.Key) ); Assert.Throws( - () => FingerprintCreator.EvaluateKeyToFingerprint(context, directory, new [] {"**"}) + () => FingerprintCreator.EvaluateToFingerprint(context, directory, new[] { "**" }, FingerprintType.Key) + ); + Assert.Throws( + () => FingerprintCreator.EvaluateToFingerprint(context, directory, new[] { "*" }, FingerprintType.Path) + ); + Assert.Throws( + () => FingerprintCreator.EvaluateToFingerprint(context, directory, new[] { "**" }, FingerprintType.Path) ); } } @@ -68,9 +99,9 @@ public void Fingerprint_ReservedFails() [Fact] [Trait("Level", "L0")] [Trait("Category", "Plugin")] - public void Fingerprint_ExcludeExactMatches() + public void Fingerprint_Key_ExcludeExactMatches() { - using(var hostContext = new TestHostContext(this)) + using (var hostContext = new TestHostContext(this)) { var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace()); var segments = new[] @@ -78,7 +109,156 @@ public void Fingerprint_ExcludeExactMatches() $"{Path.GetDirectoryName(path1)},!{path1}", }; Assert.Throws( - () => FingerprintCreator.EvaluateKeyToFingerprint(context, directory, segments) + () => FingerprintCreator.EvaluateToFingerprint(context, directory, segments, FingerprintType.Key) + ); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Plugin")] + public void Fingerprint_Path_IncludeFullPathMatches() + { + using (var hostContext = new TestHostContext(this)) + { + var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace()); + var segments = new[] + { + directory1, + directory2 + }; + + Fingerprint f = FingerprintCreator.EvaluateToFingerprint(context, workspaceRoot, segments, FingerprintType.Path); + Assert.Equal( + new[] { directory1Name, directory2Name }.OrderBy(x => x), + f.Segments.OrderBy(x => x) + ); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Plugin")] + public void Fingerprint_Path_IncludeRelativePathMatches() + { + using (var hostContext = new TestHostContext(this)) + { + var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace()); + var segments = new[] + { + directory1Name, + directory2Name + }; + + Fingerprint f = FingerprintCreator.EvaluateToFingerprint(context, workspaceRoot, segments, FingerprintType.Path); + Assert.Equal( + new[] { directory1Name, directory2Name }.OrderBy(x => x), + f.Segments.OrderBy(x => x) + ); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Plugin")] + public void Fingerprint_Path_IncludeGlobMatches() + { + using (var hostContext = new TestHostContext(this)) + { + var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace()); + var segments = new[] + { + $"**/{directory1Name},**/{directory2Name}", + }; + + Fingerprint f = FingerprintCreator.EvaluateToFingerprint(context, workspaceRoot, segments, FingerprintType.Path); + Assert.Equal( + new[] { directory1Name, directory2Name }.OrderBy(x => x), + f.Segments.OrderBy(x => x) + ); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Plugin")] + public void Fingerprint_Path_ExcludeGlobMatches() + { + using (var hostContext = new TestHostContext(this)) + { + var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace()); + var segments = new[] + { + $"**/{directory1Name},!**/{directory2Name}", + }; + + Fingerprint f = FingerprintCreator.EvaluateToFingerprint(context, workspaceRoot, segments, FingerprintType.Path); + Assert.Equal( + new[] { directory1Name }, + f.Segments + ); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Plugin")] + public void Fingerprint_Path_NoIncludePatterns() + { + using (var hostContext = new TestHostContext(this)) + { + var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace()); + var segments = new[] + { + $"!**/{directory1Name},!**/{directory2Name}", + }; + + Assert.Throws( + () => FingerprintCreator.EvaluateToFingerprint(context, workspaceRoot, segments, FingerprintType.Path) + ); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Plugin")] + public void Fingerprint_Path_NoMatches() + { + using (var hostContext = new TestHostContext(this)) + { + var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace()); + var segments = new[] + { + $"**/{Guid.NewGuid().ToString()},!**/{directory2Name}" + }; + + // TODO: Should this really be throwing an exception? + Assert.Throws( + () => FingerprintCreator.EvaluateToFingerprint(context, workspaceRoot, segments, FingerprintType.Path) + ); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Plugin")] + public void Fingerprint_Path_SinglePathOutsidePipelineWorkspace() + { + using (var hostContext = new TestHostContext(this)) + { + var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace()); + var directoryInfo = new DirectoryInfo(workspaceRoot); + var segments = new[] + { + directoryInfo.Parent.FullName, + }; + + Fingerprint f = FingerprintCreator.EvaluateToFingerprint(context, workspaceRoot, segments, FingerprintType.Path); + + Assert.Equal(1, f.Segments.Count()); + Assert.Equal( + new[] { Path.GetRelativePath(workspaceRoot, directoryInfo.Parent.FullName) }, + f.Segments ); } } @@ -86,16 +266,57 @@ public void Fingerprint_ExcludeExactMatches() [Fact] [Trait("Level", "L0")] [Trait("Category", "Plugin")] - public void Fingerprint_ExcludeExactMisses() + public void Fingerprint_Path_MultiplePathOutsidePipelineWorkspace() { - using(var hostContext = new TestHostContext(this)) + using (var hostContext = new TestHostContext(this)) + { + var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace()); + var directoryInfo = new DirectoryInfo(workspaceRoot); + var segments = new[] + { + directoryInfo.Parent.FullName, + directory1Name, + }; + + Assert.Throws( + () => FingerprintCreator.EvaluateToFingerprint(context, workspaceRoot, segments, FingerprintType.Path) + ); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Plugin")] + public void Fingerprint_Path_BacktracedGlobPattern() + { + using (var hostContext = new TestHostContext(this)) + { + var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace()); + var directoryInfo = new DirectoryInfo(workspaceRoot); + var segments = new[] + { + $"{directoryInfo.Parent.FullName}/*", + }; + + Assert.Throws( + () => FingerprintCreator.EvaluateToFingerprint(context, workspaceRoot, segments, FingerprintType.Path) + ); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Plugin")] + public void Fingerprint_Key_ExcludeExactMisses() + { + using (var hostContext = new TestHostContext(this)) { var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace()); var segments = new[] { $"{path1},!{path2}", }; - Fingerprint f = FingerprintCreator.EvaluateKeyToFingerprint(context, directory, segments); + Fingerprint f = FingerprintCreator.EvaluateToFingerprint(context, directory, segments, FingerprintType.Key); Assert.Equal(1, f.Segments.Length); @@ -107,9 +328,9 @@ public void Fingerprint_ExcludeExactMisses() [Fact] [Trait("Level", "L0")] [Trait("Category", "Plugin")] - public void Fingerprint_FileAbsolute() + public void Fingerprint_Key_FileAbsolute() { - using(var hostContext = new TestHostContext(this)) + using (var hostContext = new TestHostContext(this)) { var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace()); var segments = new[] @@ -117,7 +338,7 @@ public void Fingerprint_FileAbsolute() $"{path1}", $"{path2}", }; - Fingerprint f = FingerprintCreator.EvaluateKeyToFingerprint(context, directory, segments); + Fingerprint f = FingerprintCreator.EvaluateToFingerprint(context, directory, segments, FingerprintType.Key); var file1 = new FingerprintCreator.MatchedFile(Path.GetFileName(path1), content1.Length, hash1.ToHex()); var file2 = new FingerprintCreator.MatchedFile(Path.GetFileName(path2), content2.Length, hash2.ToHex()); @@ -131,13 +352,13 @@ public void Fingerprint_FileAbsolute() [Fact] [Trait("Level", "L0")] [Trait("Category", "Plugin")] - public void Fingerprint_FileRelative() + public void Fingerprint_Key_FileRelative() { string workingDir = Path.GetDirectoryName(path1); string relPath1 = Path.GetFileName(path1); string relPath2 = Path.GetFileName(path2); - using(var hostContext = new TestHostContext(this)) + using (var hostContext = new TestHostContext(this)) { var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace()); context.SetVariable( @@ -151,7 +372,7 @@ public void Fingerprint_FileRelative() $"{relPath2}", }; - Fingerprint f = FingerprintCreator.EvaluateKeyToFingerprint(context, directory, segments); + Fingerprint f = FingerprintCreator.EvaluateToFingerprint(context, directory, segments, FingerprintType.Key); var file1 = new FingerprintCreator.MatchedFile(relPath1, content1.Length, hash1.ToHex()); var file2 = new FingerprintCreator.MatchedFile(relPath2, content2.Length, hash2.ToHex()); @@ -165,9 +386,9 @@ public void Fingerprint_FileRelative() [Fact] [Trait("Level", "L0")] [Trait("Category", "Plugin")] - public void Fingerprint_Str() + public void Fingerprint_Key_Str() { - using(var hostContext = new TestHostContext(this)) + using (var hostContext = new TestHostContext(this)) { var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace()); var segments = new[] @@ -175,7 +396,7 @@ public void Fingerprint_Str() $"hello", }; - Fingerprint f = FingerprintCreator.EvaluateKeyToFingerprint(context, directory, segments); + Fingerprint f = FingerprintCreator.EvaluateToFingerprint(context, directory, segments, FingerprintType.Key); Assert.Equal(1, f.Segments.Length); Assert.Equal($"hello", f.Segments[0]); @@ -187,12 +408,13 @@ public void Fingerprint_Str() [Trait("Category", "Plugin")] public void ParseMultilineKeyAsOld() { - (bool isOldFormat, string[] keySegments, IEnumerable restoreKeys) = PipelineCacheTaskPluginBase.ParseIntoSegments( + (bool isOldFormat, string[] keySegments, IEnumerable restoreKeys, string[] pathSegments) = PipelineCacheTaskPluginBase.ParseIntoSegments( string.Empty, "gems\n$(Agent.OS)\n$(Build.SourcesDirectory)/my.gemspec", + string.Empty, string.Empty); Assert.True(isOldFormat); - Assert.Equal(new [] {"gems", "$(Agent.OS)", "$(Build.SourcesDirectory)/my.gemspec"}, keySegments); + Assert.Equal(new[] { "gems", "$(Agent.OS)", "$(Build.SourcesDirectory)/my.gemspec" }, keySegments); Assert.Equal(0, restoreKeys.Count()); } @@ -201,12 +423,13 @@ public void ParseMultilineKeyAsOld() [Trait("Category", "Plugin")] public void ParseSingleLineAsNew() { - (bool isOldFormat, string[] keySegments, IEnumerable restoreKeys) = PipelineCacheTaskPluginBase.ParseIntoSegments( + (bool isOldFormat, string[] keySegments, IEnumerable restoreKeys, string[] pathSegments) = PipelineCacheTaskPluginBase.ParseIntoSegments( string.Empty, "$(Agent.OS)", + string.Empty, string.Empty); Assert.False(isOldFormat); - Assert.Equal(new [] {"$(Agent.OS)"}, keySegments); + Assert.Equal(new[] { "$(Agent.OS)" }, keySegments); Assert.Equal(0, restoreKeys.Count()); } @@ -215,13 +438,28 @@ public void ParseSingleLineAsNew() [Trait("Category", "Plugin")] public void ParseMultilineWithRestoreKeys() { - (bool isOldFormat, string[] keySegments, IEnumerable restoreKeys) = PipelineCacheTaskPluginBase.ParseIntoSegments( + (bool isOldFormat, string[] keySegments, IEnumerable restoreKeys, string[] pathSegments) = PipelineCacheTaskPluginBase.ParseIntoSegments( string.Empty, "$(Agent.OS) | Gemfile.lock | **/*.gemspec,!./junk/**", - "$(Agent.OS) | Gemfile.lock\n$(Agent.OS)"); + "$(Agent.OS) | Gemfile.lock\n$(Agent.OS)", + string.Empty); + Assert.False(isOldFormat); + Assert.Equal(new[] { "$(Agent.OS)", "Gemfile.lock", "**/*.gemspec,!./junk/**" }, keySegments); + Assert.Equal(new[] { new[] { "$(Agent.OS)", "Gemfile.lock" }, new[] { "$(Agent.OS)" } }, restoreKeys); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Plugin")] + public void ParsePathSegments() + { + (bool isOldFormat, string[] keySegments, IEnumerable restoreKeys, string[] pathSegments) = PipelineCacheTaskPluginBase.ParseIntoSegments( + string.Empty, + string.Empty, + string.Empty, + "node_modules | dist | **/globby.*,!**.exclude"); Assert.False(isOldFormat); - Assert.Equal(new [] {"$(Agent.OS)","Gemfile.lock","**/*.gemspec,!./junk/**"}, keySegments); - Assert.Equal(new [] {new []{ "$(Agent.OS)","Gemfile.lock"}, new[] {"$(Agent.OS)"}}, restoreKeys); + Assert.Equal(new[] { "node_modules", "dist", "**/globby.*,!**.exclude" }, pathSegments); } } } \ No newline at end of file diff --git a/src/Test/L0/Plugin/IsPathyTests.cs b/src/Test/L0/Plugin/IsPathyTests.cs index 5c60006afd..5c09a7f678 100644 --- a/src/Test/L0/Plugin/IsPathyTests.cs +++ b/src/Test/L0/Plugin/IsPathyTests.cs @@ -15,7 +15,7 @@ public class IsPathyTests public void Fingerprint_IsPath() { Action assertPath = (path, isPath) => - Assert.True(isPath == FingerprintCreator.IsPathyKeySegment(path), $"IsPathy({path}) should have returned {isPath}."); + Assert.True(isPath == FingerprintCreator.IsPathySegment(path), $"IsPathy({path}) should have returned {isPath}."); assertPath(@"''", false); assertPath(@"Windows_NT", false); assertPath(@"README.md", true); diff --git a/src/Test/L0/Plugin/TarUtilsL0.cs b/src/Test/L0/Plugin/TarUtilsL0.cs index 52213ee369..7ffac3e9a6 100644 --- a/src/Test/L0/Plugin/TarUtilsL0.cs +++ b/src/Test/L0/Plugin/TarUtilsL0.cs @@ -8,7 +8,7 @@ namespace Microsoft.VisualStudio.Services.Agent.Tests.PipelineCaching { - public class TarUtilsL0 { + public class TarUtilsL0 { [Fact] [Trait("Level", "L0")] [Trait("Category", "Plugin")]