Multiple directories and path glob pattern support for pipeline caching

src/Agent.Plugins/PipelineCache/PipelineCacheServer.cs

@@ -25,8 +25,9 @@ public class PipelineCacheServer
         [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1068: CancellationToken parameters must come last")]
         internal async Task UploadAsync(
             AgentTaskPluginExecutionContext context,
-            Fingerprint fingerprint,
-            string path,
+            Fingerprint keyFingerprint,
+            string[] pathSegments,
+            string workspaceRoot,
             CancellationToken cancellationToken,
             ContentFormat contentFormat)
         {
@@ -40,7 +41,7 @@ public class PipelineCacheServer
                 // Check if the key exists.
                 PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord<PipelineCacheActionRecord>((level, uri, type) =>
                         new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
-                PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new [] {fingerprint}, cancellationToken, cacheRecordGet);
+                PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new[] { keyFingerprint }, cancellationToken, cacheRecordGet);
                 // Send results to CustomerIntelligence
                 context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet);
                 //If cache exists, return.
@@ -50,7 +51,12 @@ public class PipelineCacheServer
                     return;
                 }
 
-                string uploadPath = await this.GetUploadPathAsync(contentFormat, context, path, cancellationToken);
+                context.Output("Resolving path:");
+                Fingerprint pathFp = FingerprintCreator.EvaluateToFingerprint(context, workspaceRoot, pathSegments, FingerprintType.Path);
+                context.Output($"Resolved to: {pathFp}");
+
+                string uploadPath = await this.GetUploadPathAsync(contentFormat, context, pathFp, pathSegments, workspaceRoot, cancellationToken);
+
                 //Upload the pipeline artifact.
                 PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord<PipelineCacheActionRecord>((level, uri, type) =>
                     new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context));
@@ -60,10 +66,10 @@ public class PipelineCacheServer
                     {
                         return await dedupManifestClient.PublishAsync(uploadPath, cancellationToken);
                     });
-        
+
                 CreatePipelineCacheArtifactContract options = new CreatePipelineCacheArtifactContract
                 {
-                    Fingerprint = fingerprint,
+                    Fingerprint = keyFingerprint,
                     RootId = result.RootId,
                     ManifestId = result.ManifestId,
                     ProofNodes = result.ProofNodes.ToArray(),
@@ -82,7 +88,7 @@ public class PipelineCacheServer
                     }
                     catch { }
                 }
-                
+
                 // Cache the artifact
                 PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord<PipelineCacheActionRecord>((level, uri, type) =>
                     new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context));
@@ -98,8 +104,9 @@ public class PipelineCacheServer
         internal async Task DownloadAsync(
             AgentTaskPluginExecutionContext context,
             Fingerprint[] fingerprints,
-            string path,
+            string[] pathSegments,
             string cacheHitVariable,
+            string workspaceRoot,
             CancellationToken cancellationToken)
         {
             VssConnection connection = context.VssConnection;
@@ -126,12 +133,12 @@ public class PipelineCacheServer
                         record: downloadRecord,
                         actionAsync: async () =>
                         {
-                            await this.DownloadPipelineCacheAsync(context, dedupManifestClient, result.ManifestId, path, Enum.Parse<ContentFormat>(result.ContentFormat), cancellationToken);
+                            await this.DownloadPipelineCacheAsync(context, dedupManifestClient, result.ManifestId, pathSegments, workspaceRoot, Enum.Parse<ContentFormat>(result.ContentFormat), cancellationToken);
                         });
 
                     // Send results to CustomerIntelligence
                     context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: downloadRecord);
-                    
+
                     context.Output("Cache restored.");
                 }
 
@@ -146,11 +153,11 @@ public class PipelineCacheServer
                         context.Verbose($"Exact fingerprint: `{result.Fingerprint.ToString()}`");
 
                         bool foundExact = false;
-                        foreach(var fingerprint in fingerprints)
+                        foreach (var fingerprint in fingerprints)
                         {
                             context.Verbose($"This fingerprint: `{fingerprint.ToString()}`");
 
-                            if (fingerprint == result.Fingerprint 
+                            if (fingerprint == result.Fingerprint
                                 || result.Fingerprint.Segments.Length == 1 && result.Fingerprint.Segments.Single() == fingerprint.SummarizeForV1())
                             {
                                 foundExact = true;
@@ -177,21 +184,30 @@ public class PipelineCacheServer
             return pipelineCacheClient;
         }
 
-        private async Task<string> GetUploadPathAsync(ContentFormat contentFormat, AgentTaskPluginExecutionContext context, string path, CancellationToken cancellationToken)
+        private Task<string> GetUploadPathAsync(ContentFormat contentFormat, AgentTaskPluginExecutionContext context, Fingerprint pathFingerprint, string[] pathSegments, string workspaceRoot, CancellationToken cancellationToken)
         {
-            string uploadPath = path;
-            if(contentFormat == ContentFormat.SingleTar)
+            if (contentFormat == ContentFormat.SingleTar)
             {
-                uploadPath = await TarUtils.ArchiveFilesToTarAsync(context, path, cancellationToken);
+                var (tarWorkingDirectory, isWorkspaceContained) = GetTarWorkingDirectory(pathSegments, workspaceRoot);
+
+                return TarUtils.ArchiveFilesToTarAsync(
+                    context, 
+                    pathFingerprint, 
+                    tarWorkingDirectory, 
+                    isWorkspaceContained, 
+                    cancellationToken
+                );
             }
-            return uploadPath;
+
+            return Task.FromResult(pathFingerprint.Segments[0]);
         }
 
         private async Task DownloadPipelineCacheAsync(
             AgentTaskPluginExecutionContext context,
             DedupManifestArtifactClient dedupManifestClient,
             DedupIdentifier manifestId,
-            string targetDirectory,
+            string[] pathSegments,
+            string workspaceRoot,
             ContentFormat contentFormat,
             CancellationToken cancellationToken)
         {
@@ -200,26 +216,42 @@ private async Task<string> GetUploadPathAsync(ContentFormat contentFormat, Agent
                 string manifestPath = Path.Combine(Path.GetTempPath(), $"{nameof(DedupManifestArtifactClient)}.{Path.GetRandomFileName()}.manifest");
                 await dedupManifestClient.DownloadFileToPathAsync(manifestId, manifestPath, proxyUri: null, cancellationToken: cancellationToken);
                 Manifest manifest = JsonSerializer.Deserialize<Manifest>(File.ReadAllText(manifestPath));
-                await TarUtils.DownloadAndExtractTarAsync (context, manifest, dedupManifestClient, targetDirectory, cancellationToken);
+                var (tarWorkingDirectory, _) = GetTarWorkingDirectory(pathSegments, workspaceRoot);
+                await TarUtils.DownloadAndExtractTarAsync(context, manifest, dedupManifestClient, tarWorkingDirectory, cancellationToken);
                 try
                 {
-                    if(File.Exists(manifestPath))
+                    if (File.Exists(manifestPath))
                     {
                         File.Delete(manifestPath);
                     }
                 }
-                catch {}
+                catch { }
             }
             else
             {
                 DownloadDedupManifestArtifactOptions options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
                     manifestId,
-                    targetDirectory,
+                    pathSegments[0],
                     proxyUri: null,
                     minimatchPatterns: null);
                 await dedupManifestClient.DownloadAsync(options, cancellationToken);
             }
+        }
+
+        private (string workingDirectory, bool isWorkspaceContained) GetTarWorkingDirectory(string[] segments, string workspaceRoot)
+        {
+            // If path segment is single directory outside of Pipeline.Workspace extract tarball directly to this path
+            if (segments.Count() == 1) 
+            {
+                var workingDirectory = segments[0];
+                if (FingerprintCreator.IsPathySegment(workingDirectory) && !workingDirectory.StartsWith(workspaceRoot))
+                {
+                    return (workingDirectory, false);
+                }
+            }
 
+            // All other scenarios means that paths must within and relative to Pipeline.Workspace
+            return (workspaceRoot.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), true);
         }
     }
 }

src/Agent.Plugins/PipelineCache/PipelineCacheTaskPluginBase.cs

@@ -31,34 +31,36 @@ public abstract class PipelineCacheTaskPluginBase : IAgentTaskPlugin
         public abstract String Stage { get; }
         public const string ResolvedFingerPrintVariableName = "RESTORE_STEP_RESOLVED_FINGERPRINT";
 
-        internal static (bool isOldFormat, string[] keySegments,IEnumerable<string[]> restoreKeys) ParseIntoSegments(string salt, string key, string restoreKeysBlock)
+        internal static (bool isOldFormat, string[] keySegments, IEnumerable<string[]> restoreKeys, string[] pathSegments) ParseIntoSegments(string salt, string key, string restoreKeysBlock, string path)
         {
-            Func<string,string[]> splitAcrossPipes = (s) => {
-                var segments = s.Split(new [] {'|'},StringSplitOptions.RemoveEmptyEntries).Select(segment => segment.Trim());
-                if(!string.IsNullOrWhiteSpace(salt))
+            Func<string, string[]> splitAcrossPipes = (s) =>
+            {
+                var segments = s.Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries).Select(segment => segment.Trim());
+                if (!string.IsNullOrWhiteSpace(salt))
                 {
-                    segments = (new [] { $"{SaltVariableName}={salt}"}).Concat(segments);
+                    segments = (new[] { $"{SaltVariableName}={salt}" }).Concat(segments);
                 }
                 return segments.ToArray();
             };
 
-            Func<string,string[]> splitAcrossNewlines = (s) => 
+            Func<string, string[]> splitAcrossNewlines = (s) =>
                 s.Replace("\r\n", "\n") //normalize newlines
-                 .Split(new [] {'\n'}, StringSplitOptions.RemoveEmptyEntries)
+                 .Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries)
                  .Select(line => line.Trim())
                  .ToArray();
-            
+
             string[] keySegments;
+            string[] pathSegments;
             bool isOldFormat = key.Contains('\n');
-            
+
             IEnumerable<string[]> restoreKeys;
             bool hasRestoreKeys = !string.IsNullOrWhiteSpace(restoreKeysBlock);
 
             if (isOldFormat && hasRestoreKeys)
             {
                 throw new ArgumentException(OldKeyFormatMessage);
             }
-            
+
             if (isOldFormat)
             {
                 keySegments = splitAcrossNewlines(key);
@@ -67,7 +69,9 @@ internal static (bool isOldFormat, string[] keySegments,IEnumerable<string[]> re
             {
                 keySegments = splitAcrossPipes(key);
             }
-
+
+            // Path can be a list of path segments to include in cache
+            pathSegments = splitAcrossPipes(path);
 
             if (hasRestoreKeys)
             {
@@ -78,9 +82,9 @@ internal static (bool isOldFormat, string[] keySegments,IEnumerable<string[]> re
                 restoreKeys = Enumerable.Empty<string[]>();
             }
 
-            return (isOldFormat, keySegments, restoreKeys);
+            return (isOldFormat, keySegments, restoreKeys, pathSegments);
         }
-        
+
         public async virtual Task RunAsync(AgentTaskPluginExecutionContext context, CancellationToken token)
         {
             ArgUtil.NotNull(context, nameof(context));
@@ -93,35 +97,37 @@ public async virtual Task RunAsync(AgentTaskPluginExecutionContext context, Canc
 
             string key = context.GetInput(PipelineCacheTaskPluginConstants.Key, required: true);
             string restoreKeysBlock = context.GetInput(PipelineCacheTaskPluginConstants.RestoreKeys, required: false);
+            // TODO: Translate path from container to host (Ting)
+            string path = context.GetInput(PipelineCacheTaskPluginConstants.Path, required: true);
 
-            (bool isOldFormat, string[] keySegments, IEnumerable<string[]> restoreKeys) = ParseIntoSegments(salt, key, restoreKeysBlock);
+            (bool isOldFormat, string[] keySegments, IEnumerable<string[]> restoreKeys, string[] pathSegments) = ParseIntoSegments(salt, key, restoreKeysBlock, path);
 
             if (isOldFormat)
             {
                 context.Warning(OldKeyFormatMessage);
             }
 
             context.Output("Resolving key:");
-            Fingerprint keyFp = FingerprintCreator.EvaluateKeyToFingerprint(context, workspaceRoot, keySegments);
+            Fingerprint keyFp = FingerprintCreator.EvaluateToFingerprint(context, workspaceRoot, keySegments, FingerprintType.Key);
             context.Output($"Resolved to: {keyFp}");
 
-            Func<Fingerprint[]> restoreKeysGenerator = () => 
-                restoreKeys.Select(restoreKey => {
+            Func<Fingerprint[]> restoreKeysGenerator = () =>
+                restoreKeys.Select(restoreKey =>
+                {
                     context.Output("Resolving restore key:");
-                    Fingerprint f = FingerprintCreator.EvaluateKeyToFingerprint(context, workspaceRoot, restoreKey);
-                    f.Segments = f.Segments.Concat(new [] { Fingerprint.Wildcard} ).ToArray();
+                    Fingerprint f = FingerprintCreator.EvaluateToFingerprint(context, workspaceRoot, restoreKey, FingerprintType.Key);
+                    f.Segments = f.Segments.Concat(new[] { Fingerprint.Wildcard }).ToArray();
                     context.Output($"Resolved to: {f}");
                     return f;
                 }).ToArray();
 
-            // TODO: Translate path from container to host (Ting)
-            string path = context.GetInput(PipelineCacheTaskPluginConstants.Path, required: true);
 
             await ProcessCommandInternalAsync(
                 context,
                 keyFp,
                 restoreKeysGenerator,
-                path,
+                pathSegments,
+                workspaceRoot,
                 token);
         }
 
@@ -130,7 +136,8 @@ public async virtual Task RunAsync(AgentTaskPluginExecutionContext context, Canc
             AgentTaskPluginExecutionContext context,
             Fingerprint fingerprint,
             Func<Fingerprint[]> restoreKeysGenerator,
-            string path,
+            string[] pathSegments,
+            string workspaceRoot,
             CancellationToken token);
 
         // Properties set by tasks

src/Agent.Plugins/PipelineCache/RestorePipelineCacheV0.cs

@@ -9,7 +9,7 @@
 using Microsoft.VisualStudio.Services.PipelineCache.WebApi;
 
 namespace Agent.Plugins.PipelineCache
-{    
+{
     public class RestorePipelineCacheV0 : PipelineCacheTaskPluginBase
     {
         public override string Stage => "main";
@@ -18,7 +18,8 @@ public class RestorePipelineCacheV0 : PipelineCacheTaskPluginBase
             AgentTaskPluginExecutionContext context,
             Fingerprint fingerprint,
             Func<Fingerprint[]> restoreKeysGenerator,
-            string path,
+            string[] pathSegments,
+            string workspaceRoot,
             CancellationToken token)
         {
             context.SetTaskVariable(RestoreStepRanVariableName, RestoreStepRanVariableValue);
@@ -27,10 +28,11 @@ public class RestorePipelineCacheV0 : PipelineCacheTaskPluginBase
             var server = new PipelineCacheServer();
             Fingerprint[] restoreFingerprints = restoreKeysGenerator();
             await server.DownloadAsync(
-                context, 
-                (new [] { fingerprint}).Concat(restoreFingerprints).ToArray(),
-                path,
+                context,
+                (new[] { fingerprint }).Concat(restoreFingerprints).ToArray(),
+                pathSegments,
                 context.GetInput(PipelineCacheTaskPluginConstants.CacheHitVariable, required: false),
+                workspaceRoot,
                 token);
         }
     }

src/Agent.Plugins/PipelineCache/SavePipelineCacheV0.cs

@@ -57,20 +57,21 @@ public override async Task RunAsync(AgentTaskPluginExecutionContext context, Can
 
         protected override async Task ProcessCommandInternalAsync(
             AgentTaskPluginExecutionContext context,
-            Fingerprint fingerprint,
+            Fingerprint keyFingerprint,
             Func<Fingerprint[]> restoreKeysGenerator,
-            string path,
+            string[] pathSegments,
+            string workspaceRoot,
             CancellationToken token)
         {
-            string contentFormatValue  = context.Variables.GetValueOrDefault(ContentFormatVariableName)?.Value ?? string.Empty;
+            string contentFormatValue = context.Variables.GetValueOrDefault(ContentFormatVariableName)?.Value ?? string.Empty;
             string calculatedFingerPrint = context.TaskVariables.GetValueOrDefault(ResolvedFingerPrintVariableName)?.Value ?? string.Empty;
 
-            if(!string.IsNullOrWhiteSpace(calculatedFingerPrint) && !fingerprint.ToString().Equals(calculatedFingerPrint, StringComparison.Ordinal))
+            if (!string.IsNullOrWhiteSpace(calculatedFingerPrint) && !keyFingerprint.ToString().Equals(calculatedFingerPrint, StringComparison.Ordinal))
             {
-                context.Warning($"The given cache key has changed in its resolved value between restore and save steps;\n"+
-                                $"original key: {calculatedFingerPrint}\n"+
-                                $"modified key: {fingerprint}\n");
-            } 
+                context.Warning($"The given cache key has changed in it's resolved value between restore and save steps;\n" +
+                                $"original key: {calculatedFingerPrint}\n" +
+                                $"modified key: {keyFingerprint}\n");
+            }
 
             ContentFormat contentFormat;
             if (string.IsNullOrWhiteSpace(contentFormatValue))
@@ -85,8 +86,9 @@ public override async Task RunAsync(AgentTaskPluginExecutionContext context, Can
             PipelineCacheServer server = new PipelineCacheServer();
             await server.UploadAsync(
                 context,
-                fingerprint, 
-                path,
+                keyFingerprint,
+                pathSegments,
+                workspaceRoot,
                 token,
                 contentFormat);
         }