Get image and thumbnail channels to work in engine

docs/storage-keys.md

@@ -6,21 +6,23 @@ The DLCS uses a number of S3 keys in various buckets to store assets. These gene
 > The default 'storage-key' is `$"{customer}/{space}/{assetKey}"`.
 > The various bucketnames below equate to those in `S3Settings`
 
-| Name               | Format                                                                    | Example                                                | Description                                                                                       |
-| ------------------ | ------------------------------------------------------------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------------- |
-| Storage            | `"{StorageBucket}/{storage-key}"`                                         | `dlcs-storage/1/2/foo`                                 | Default location where generated derivatives are stored                                           |
-| Storage Original   | `"{StorageBucket}/{storage-key}/original"`                                | `dlcs-storage/1/2/foo/original`                        | Where direct-copy of origin is stored. For `/file/` delivery or images with `use-original` policy |
-| InfoJson           | `"{StorageBucket}/{storage-key}/info/{image-server}/{version}/info.json"` | `dlcs-storage/1/2/foo/info/cantaloupe/v3/info.json`    | Location where pregenerated info.json stored                                                      |
-| Audio output       | `"{StorageBucket}/{storage-key}/full/max/default.{extension}"`            | `dlcs-storage/1/2/foo/full/max/default.mp3`            | Location where transcoded audio stored                                                            |
-| Video output       | `"{StorageBucket}/{storage-key}/full/full/max/max/0/default.{extension}"` | `dlcs-storage/1/2/foo/full/full/max/max/0/default.mp4` | Location where transcoded video stored                                                            |
-| Timebased Metadata | `"{StorageBucket}/{storage-key}/metadata"`                                | `dlcs-storage/1/2/foo/metadata`                        | XML blob storing ElasticTranscoder JobId                                                          |
-| Thumbnail          | `"{ThumbsBucket}/{storage-key}/{access}/{longestEdge}.jpg"`               | `dlcs-thumbs/1/2/foo/open/100.jpg`                     | Location of specific thumbnail                                                                    |
-| Legacy Thumbnail   | `"{ThumbsBucket}/{storage-key}/full/{w},{h}/0/default.jpg"`               | `dlcs-thumbs/1/2/foo/full/100,200/0/default.jpg`       | Location of specific thumbnail using legacy layout                                                |
-| Thumbnail Sizes    | `"{ThumbsBucket}/{storage-key}/s.json"`                                   | `dlcs-thumbs/1/2/foo/s.json`                           | JSON blob storing knowng thumbnails                                                               |
-| Largest Thumbnail  | `"{ThumbsBucket}/{storage-key}/low.jpg"`                                  | `dlcs-thumbs/1/2/foo/low.jpg`                          | The location of the largest generated thumbnail                                                   |
-| Thumbnail Root     | `"{ThumbsBucket}/{storage-key}/"`                                         | `dlcs-thumbs/1/2/foo/`                                 | Root key where thumbnails will reside                                                             |
-| Output Location    | `"{OutputBucket}/{storage-key}/"`                                         | `dlcs-output/1/2/foo/`                                 | Root key where DLCS 'output' is stored (e.g. projected NQ to PDF or Zip)                          |
-| Origin Location    | `"{OriginBucket}/{storage-key}"`                                          | `dlcs-origin/1/2/foo`                                  | Location where directly uploaded bytes are stored                                                 |
+| Name                        | Format                                                                    | Example                                                | Description                                                                                       |
+| --------------------------- | ------------------------------------------------------------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------------- |
+| Storage                     | `"{StorageBucket}/{storage-key}"`                                         | `dlcs-storage/1/2/foo`                                 | Default location where generated derivatives are stored                                           |
+| Storage Original            | `"{StorageBucket}/{storage-key}/original"`                                | `dlcs-storage/1/2/foo/original`                        | Where direct-copy of origin is stored. For `/file/` delivery or images with `use-original` policy |
+| InfoJson                    | `"{StorageBucket}/{storage-key}/info/{image-server}/{version}/info.json"` | `dlcs-storage/1/2/foo/info/cantaloupe/v3/info.json`    | Location where pregenerated info.json stored                                                      |
+| Audio output                | `"{StorageBucket}/{storage-key}/full/max/default.{extension}"`            | `dlcs-storage/1/2/foo/full/max/default.mp3`            | Location where transcoded audio stored                                                            |
+| Video output                | `"{StorageBucket}/{storage-key}/full/full/max/max/0/default.{extension}"` | `dlcs-storage/1/2/foo/full/full/max/max/0/default.mp4` | Location where transcoded video stored                                                            |
+| Timebased Metadata          | `"{StorageBucket}/{storage-key}/metadata"`                                | `dlcs-storage/1/2/foo/metadata`                        | XML blob storing ElasticTranscoder JobId                                                          |
+| Thumbnail                   | `"{ThumbsBucket}/{storage-key}/{access}/{longestEdge}.jpg"`               | `dlcs-thumbs/1/2/foo/open/100.jpg`                     | Location of specific thumbnail                                                                    |
+| Legacy Thumbnail            | `"{ThumbsBucket}/{storage-key}/full/{w},{h}/0/default.jpg"`               | `dlcs-thumbs/1/2/foo/full/100,200/0/default.jpg`       | Location of specific thumbnail using legacy layout                                                |
+| Thumbnail Sizes             | `"{ThumbsBucket}/{storage-key}/s.json"`                                   | `dlcs-thumbs/1/2/foo/s.json`                           | JSON blob storing known thumbnails                                                                |
+| Largest Thumbnail           | `"{ThumbsBucket}/{storage-key}/low.jpg"`                                  | `dlcs-thumbs/1/2/foo/low.jpg`                          | The location of the largest generated thumbnail                                                   |
+| Thumbnail Root              | `"{ThumbsBucket}/{storage-key}/"`                                         | `dlcs-thumbs/1/2/foo/`                                 | Root key where thumbnails will reside                                                             |
+| Output Location             | `"{OutputBucket}/{storage-key}/"`                                         | `dlcs-output/1/2/foo/`                                 | Root key where DLCS 'output' is stored (e.g. projected NQ to PDF or Zip)                          |
+| Origin Location             | `"{OriginBucket}/{storage-key}"`                                          | `dlcs-origin/1/2/foo`                                  | Location where directly uploaded bytes are stored                                                 |
+| Transient Images            | `"{StorageBucket}/transient/{storage-key}"`                               | `dlcs-thumbs/1/2/foo/open/100.jpg`                     | Location of transient images, that will be cleaned up by lifecycle policies                       |
+
 
 ## Timebased
 

src/protagonist/DLCS.AWS/S3/IStorageKeyGenerator.cs

@@ -166,4 +166,12 @@ public interface IStorageKeyGenerator
     /// </summary>
     /// <returns><see cref="ObjectInBucket"/> for JSON object containing credentials for a user's origin strategy</returns>
     ObjectInBucket GetOriginStrategyCredentialsLocation(int customerId, string originStrategyId);
+
+    /// <summary>
+    /// Get <see cref="RegionalisedObjectInBucket"/> root location for a transient image in the origin bucket
+    /// </summary>
+    /// <param name="assetId">asset id the request is for</param>
+    /// <returns><see cref="RegionalisedObjectInBucket"/> for specified transient asset's that will be cleared up after
+    /// a period of time</returns>
+    RegionalisedObjectInBucket GetTransientImageLocation(AssetId assetId);
 }

src/protagonist/DLCS.AWS/S3/S3StorageKeyGenerator.cs

@@ -191,4 +191,10 @@ public ObjectInBucket GetOriginStrategyCredentialsLocation(int customerId, strin
         var key = $"{customerId}/origin-strategy/{originStrategyId}/credentials.json";
         return new ObjectInBucket(s3Options.SecurityObjectsBucket, key);
     }
+
+    public RegionalisedObjectInBucket GetTransientImageLocation(AssetId assetId)
+    {
+        var key = GetStorageKey(assetId);
+        return new RegionalisedObjectInBucket(s3Options.StorageBucket, $"transient/{key}", awsSettings.Region);
+    }
 }

src/protagonist/DLCS.Core/FileSystem/FileSystem.cs

@@ -1,5 +1,7 @@
 using System;
 using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
 
 namespace DLCS.Core.FileSystem;
 
@@ -45,4 +47,10 @@ public long GetFileSize(string path)
     }
 
     public void SetLastWriteTimeUtc(string path, DateTime dateTime) => File.SetLastWriteTimeUtc(path, dateTime);
+
+    public async Task CreateFileFromStream(string path, Stream stream, CancellationToken cancellationToken = default)
+    {
+        await using var fileStream = new FileStream(path, FileMode.Create);
+        await stream.CopyToAsync(fileStream, cancellationToken);
+    }
 }

src/protagonist/DLCS.Core/FileSystem/IFileSystem.cs

@@ -1,4 +1,7 @@
 using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
 
 namespace DLCS.Core.FileSystem;
 
@@ -14,4 +17,5 @@ public interface IFileSystem
     bool FileExists(string path);
     long GetFileSize(string path);
     void SetLastWriteTimeUtc(string path, DateTime dateTime);
+    Task CreateFileFromStream(string path, Stream stream, CancellationToken cancellationToken = default);
 }

src/protagonist/DLCS.Model.Tests/Assets/AssetXTests.cs

@@ -14,14 +14,14 @@ public class AssetXTests
     public void GetAvailableThumbSizes_IncludeUnavailable_Correct_MaxUnauthorisedNoRoles()
     {
         // Arrange
+        var asset = new Asset {Width = 5000, Height = 2500, MaxUnauthorised = 500};
+
         var thumbnailPolicy = new ThumbnailPolicy
         {
             Id = "TestPolicy",
             Name = "TestPolicy",
             Sizes = "800,400,200,100"
         };
-
-        var asset = new Asset {Width = 5000, Height = 2500, MaxUnauthorised = 500};
 
         // Act
         var sizes = asset.GetAvailableThumbSizes(thumbnailPolicy, out var maxDimensions, true);
@@ -43,14 +43,13 @@ public void GetAvailableThumbSizes_IncludeUnavailable_Correct_MaxUnauthorisedNoR
     public void GetAvailableThumbSizes_NotIncludeUnavailable_Correct_MaxUnauthorisedNoRoles()
     {
         // Arrange
+        var asset = new Asset { Width = 5000, Height = 2500, MaxUnauthorised = 500};
         var thumbnailPolicy = new ThumbnailPolicy
         {
             Id = "TestPolicy",
             Name = "TestPolicy",
-            Sizes = "800,400,200,100"
+            Sizes = "800,400,200,100",
         };
-
-        var asset = new Asset {Width = 5000, Height = 2500, MaxUnauthorised = 500};
 
         // Act
         var sizes = asset.GetAvailableThumbSizes(thumbnailPolicy, out var maxDimensions, false);
@@ -71,14 +70,13 @@ public void GetAvailableThumbSizes_NotIncludeUnavailable_Correct_MaxUnauthorised
     public void GetAvailableThumbSizes_IncludeUnavailable_Correct_IfRolesNoMaxUnauthorised()
     {
         // Arrange
+        var asset = new Asset {Width = 5000, Height = 2500, Roles = "GoodGuys", MaxUnauthorised = -1};
         var thumbnailPolicy = new ThumbnailPolicy
         {
             Id = "TestPolicy",
             Name = "TestPolicy",
             Sizes = "800,400,200,100",
         };
-
-        var asset = new Asset {Width = 5000, Height = 2500, Roles = "GoodGuys", MaxUnauthorised = -1};
 
         // Act
         var sizes = asset.GetAvailableThumbSizes(thumbnailPolicy, out var maxDimensions, true);
@@ -110,7 +108,7 @@ public void GetAvailableThumbSizes_NotIncludeUnavailable_Correct_IfRolesNoMaxUna
         var asset = new Asset {Width = 5000, Height = 2500, Roles = "GoodGuys", MaxUnauthorised = -1};
 
         // Act
-        var sizes = asset.GetAvailableThumbSizes(thumbnailPolicy, out var maxDimensions, false);
+        var sizes = asset.GetAvailableThumbSizes(thumbnailPolicy,out var maxDimensions, false);
 
         // Assert
         sizes.Should().BeNullOrEmpty();
@@ -129,7 +127,7 @@ public void GetAvailableThumbSizes_RestrictsAvailableSizes_IfHasRolesAndMaxUnaut
             Name = "TestPolicy",
             Sizes = "800,400,200,100",
         };
-
+        
         var asset = new Asset {Width = 2500, Height = 5000, Roles = "GoodGuys", MaxUnauthorised = 399};
 
         // Act
@@ -156,7 +154,7 @@ public void GetAvailableThumbSizes_ReturnsAvailableAndUnavailableSizes_ButReturn
             Name = "TestPolicy",
             Sizes = "800,400,200,100",
         };
-
+        
         var asset = new Asset {Width = 2500, Height = 5000, Roles = "GoodGuys", MaxUnauthorised = 399};
 
         // Act
@@ -183,10 +181,10 @@ public void GetAvailableThumbSizes_HandlesImageBeingSmallerThanThumbnail()
         {
             Id = "TestPolicy",
             Name = "TestPolicy",
-            Sizes = "800,400,200,100"
+            Sizes = "800,400,200,100",
         };
-
-        var asset = new Asset { Width = 300, Height = 150 };
+        
+        var asset = new Asset { Width = 300, Height = 150};
 
         // Act
         var sizes = asset.GetAvailableThumbSizes(thumbnailPolicy, out var maxDimensions, true);
@@ -202,7 +200,7 @@ public void GetAvailableThumbSizes_HandlesImageBeingSmallerThanThumbnail()
         maxDimensions.maxAvailableWidth.Should().Be(300);
         maxDimensions.maxAvailableHeight.Should().Be(150);
     }
-    
+
     [Fact]
     public void SetFieldsForIngestion_ClearsFields()
     {

src/protagonist/DLCS.Model/Assets/Asset.cs

@@ -3,9 +3,7 @@
 using System.ComponentModel.DataAnnotations.Schema;
 using System.Linq;
 using DLCS.Core.Collections;
-using DLCS.Core.Guard;
 using DLCS.Core.Types;
-using DLCS.Model.Policies;
 
 namespace DLCS.Model.Assets;
 
@@ -97,23 +95,11 @@ public IEnumerable<string> TagsList
     /// </summary>
     public bool RequiresAuth => !string.IsNullOrWhiteSpace(Roles) || MaxUnauthorised >= 0;
 
-    /// <summary>
-    /// Full thumbnail policy object for Asset
-    /// </summary>
-    [NotMapped]
-    public ThumbnailPolicy? FullThumbnailPolicy { get; private set; }
-
     /// <summary>
     /// A list of image delivery channels attached to this asset
     /// </summary>
     public ICollection<ImageDeliveryChannel> ImageDeliveryChannels { get; set; }
 
-    /// <summary>
-    /// Full image optimisation policy object for Asset
-    /// </summary>
-    [NotMapped]
-    public ImageOptimisationPolicy FullImageOptimisationPolicy { get; private set; } = new();
-
     public Asset()
     {
     }
@@ -124,18 +110,4 @@ public Asset(AssetId assetId)
         Customer = assetId.Customer;
         Space = assetId.Space;
     }
-
-    public Asset WithThumbnailPolicy(ThumbnailPolicy? thumbnailPolicy)
-    {
-        FullThumbnailPolicy = Family == AssetFamily.Image
-            ? thumbnailPolicy.ThrowIfNull(nameof(thumbnailPolicy))
-            : thumbnailPolicy;
-        return this;
-    }
-
-    public Asset WithImageOptimisationPolicy(ImageOptimisationPolicy imageOptimisationPolicy)
-    {
-        FullImageOptimisationPolicy = imageOptimisationPolicy.ThrowIfNull(nameof(imageOptimisationPolicy));
-        return this;
-    }
 }

src/protagonist/DLCS.Model/Assets/AssetX.cs

@@ -40,11 +40,11 @@ public static class AssetX
         {
             var assetIsUnavailableForSize = AssetIsUnavailableForSize(asset, boundingSize);
             if (!includeUnavailable && assetIsUnavailableForSize) continue;
-
+            
             Size bounded = Size.Confine(boundingSize, size);
-
-            var boundedMaxDimension = bounded.MaxDimension;
 
+            var boundedMaxDimension = bounded.MaxDimension;
+
             // If image < thumb-size then boundedMax may already have been processed (it'll be the same as imageMax)
             if (generatedMax.Contains(boundedMaxDimension)) continue;
 

src/protagonist/DLCS.Repository/Assets/DapperAssetRepository.cs

@@ -88,7 +88,7 @@ private List<ImageDeliveryChannel> GenerateImageDeliveryChannels(List<dynamic> r
                 imageDeliveryChannels.Add(new ImageDeliveryChannel()
                 {
                     Channel = rawDeliveryChannel.Channel,
-                    DeliveryChannelPolicyId = rawDeliveryChannel.DeliveryChannelPolicyId
+                    DeliveryChannelPolicyId = rawDeliveryChannel.DeliveryChannelPolicyId,
                 });
             }
         }
@@ -101,7 +101,7 @@ private List<ImageDeliveryChannel> GenerateImageDeliveryChannels(List<dynamic> r
 ""PreservedUri"", ""Reference1"", ""Reference2"", ""Reference3"", ""MaxUnauthorised"", 
 ""NumberReference1"", ""NumberReference2"", ""NumberReference3"", ""Width"", 
 ""Height"", ""Error"", ""Batch"", ""Finished"", ""Ingesting"", ""ImageOptimisationPolicy"", 
-""ThumbnailPolicy"", ""Family"", ""MediaType"", ""Duration"", ""NotForDelivery"", ""DeliveryChannels"",  
+""ThumbnailPolicy"", ""Family"", ""MediaType"", ""Duration"", ""NotForDelivery"", ""DeliveryChannels"", 
 IDC.""Channel"", IDC.""DeliveryChannelPolicyId""
   FROM ""Images""
   LEFT OUTER JOIN ""ImageDeliveryChannels"" IDC on ""Images"".""Id"" = IDC.""ImageId""