Support for retention policies on containers

src/ApiService/ApiService/Functions/QueueFileChanges.cs

@@ -54,14 +54,16 @@ public class QueueFileChanges {
             return;
         }
 
+        var storageAccount = new ResourceIdentifier(topicElement.GetString()!);
+
         try {
             // Setting isLastRetryAttempt to false will rethrow any exceptions
             // With the intention that the azure functions runtime will handle requeing
             // the message for us. The difference is for the poison queue, we're handling the
             // requeuing ourselves because azure functions doesn't support retry policies
             // for queue based functions.
 
-            var result = await FileAdded(fileChangeEvent, isLastRetryAttempt: false);
+            var result = await FileAdded(storageAccount, fileChangeEvent, isLastRetryAttempt: false);
             if (!result.IsOk && result.ErrorV.Code == ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED) {
                 await RequeueMessage(msg, TimeSpan.FromDays(1));
             }
@@ -71,16 +73,37 @@ public class QueueFileChanges {
         }
     }
 
-    private async Async.Task<OneFuzzResultVoid> FileAdded(JsonDocument fileChangeEvent, bool isLastRetryAttempt) {
+    private async Async.Task<OneFuzzResultVoid> FileAdded(ResourceIdentifier storageAccount, JsonDocument fileChangeEvent, bool isLastRetryAttempt) {
         var data = fileChangeEvent.RootElement.GetProperty("data");
         var url = data.GetProperty("url").GetString()!;
         var parts = url.Split("/").Skip(3).ToList();
 
-        var container = parts[0];
+        var container = Container.Parse(parts[0]);
         var path = string.Join('/', parts.Skip(1));
 
         _log.LogInformation("file added : {Container} - {Path}", container, path);
-        return await _notificationOperations.NewFiles(Container.Parse(container), path, isLastRetryAttempt);
+
+        await ApplyRetentionPolicy(storageAccount, container, path);
+
+        return await _notificationOperations.NewFiles(container, path, isLastRetryAttempt);
+    }
+
+    private async Async.Task ApplyRetentionPolicy(ResourceIdentifier storageAccount, Container container, string path) {
+        // default retention period can be applied to the container
+        // if one exists, we will set the expiry date on the newly-created blob, if it doesn't already have one
+        var account = await _storage.GetBlobServiceClientForAccount(storageAccount);
+        var containerClient = account.GetBlobContainerClient(container.String);
+        var containerProps = await containerClient.GetPropertiesAsync();
+        var retentionPeriod = RetentionPolicyUtils.GetRetentionPeriodFromMetadata(containerProps.Value.Metadata);
+        if (retentionPeriod.HasValue) {
+            var blobClient = containerClient.GetBlobClient(path);
+            var tags = (await blobClient.GetTagsAsync()).Value.Tags;
+            var expiryDate = DateTime.UtcNow + retentionPeriod.Value;
+            var tag = RetentionPolicyUtils.CreateExpiryDateTag(DateOnly.FromDateTime(expiryDate));
+            if (tags.TryAdd(tag.Key, tag.Value)) {
+                await blobClient.SetTagsAsync(tags);
+            }
+        }
     }
 
     private async Async.Task RequeueMessage(string msg, TimeSpan? visibilityTimeout = null) {

src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs

@@ -22,13 +22,12 @@ public NotificationOperations(ILogger<NotificationOperations> log, IOnefuzzConte
 
     }
     public async Async.Task<OneFuzzResultVoid> NewFiles(Container container, string filename, bool isLastRetryAttempt) {
-        var result = OneFuzzResultVoid.Ok;
-
         // We don't want to store file added events for the events container because that causes an infinite loop
         if (container == WellKnownContainers.Events) {
-            return result;
+            return Result.Ok();
         }
 
+        var result = OneFuzzResultVoid.Ok;
         var notifications = GetNotifications(container);
         var hasNotifications = await notifications.AnyAsync();
         var reportOrRegression = await _context.Reports.GetReportOrRegression(container, filename, expectReports: hasNotifications);

src/ApiService/ApiService/onefuzzlib/RententionPolicy.cs → src/ApiService/ApiService/onefuzzlib/RetentionPolicy.cs

@@ -1,4 +1,6 @@
-namespace Microsoft.OneFuzz.Service;
+using System.Xml;
+
+namespace Microsoft.OneFuzz.Service;
 
 
 public interface IRetentionPolicy {
@@ -21,4 +23,21 @@ public class RetentionPolicyUtils {
     }
 
     public static string CreateExpiredBlobTagFilter() => $@"""{EXPIRY_TAG}"" <= '{DateOnly.FromDateTime(DateTime.UtcNow)}'";
+
+    public const string RETENTION_KEY = "RetentionPeriod";
+
+    public static TimeSpan? GetRetentionPeriodFromMetadata(IDictionary<string, string>? containerMetadata) {
+        if (containerMetadata is not null &&
+            containerMetadata.TryGetValue(RETENTION_KEY, out var retentionString) &&
+            !string.IsNullOrWhiteSpace(retentionString)) {
+            try {
+                return XmlConvert.ToTimeSpan(retentionString);
+            } catch {
+                // Log error: unable to convert xxx 
+                return null;
+            }
+        }
+
+        return null;
+    }
 }