Merge branch 'master' into net6

* master: feat: release v1.11 feat: PublishSlides: reduced memory consumption (#53) # Conflicts: # Clippit/PtOpenXmlUtil.cs # RELEASE_NOTES.md # paket.lock
sergey-tihon · May 29, 2022 · 601d3db · 601d3db
2 parents 776c327 + 2aa5425
commit 601d3db
Show file tree

Hide file tree

Showing 7 changed files with 69 additions and 110 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macOS-latest]
-        dotnet: [6.0.100]
+        dotnet: [6.0.300]
     runs-on: ${{ matrix.os }}
 
     steps:

diff --git a/Clippit/PowerPoint/FluentPresentationBuilder.cs b/Clippit/PowerPoint/FluentPresentationBuilder.cs
@@ -18,8 +18,7 @@ internal sealed class FluentPresentationBuilder : IDisposable
         private SlideSize _slideSize;
         private bool _isDocumentInitialized;
 
-        private readonly List<ImageData> _images = new();
-        private readonly List<MediaData> _mediaList = new();
+        private readonly List<ContentData> _mediaCache = new();
         private readonly List<SlideMasterData> _slideMasterList = new();
 
         internal FluentPresentationBuilder(PresentationDocument presentationDocument)
@@ -885,7 +884,8 @@ private void CopyRelatedImage(OpenXmlPart oldContentPart, OpenXmlPart newContent
                     var id = newContentPart.GetIdOfPart(newPart);
                     temp.AddContentPartRelTypeResourceIdTupple(newContentPart, newPart.RelationshipType, id);
 
-                    temp.WriteImage(newPart);
+                    using (var stream = oldPart.GetStream())
+                        newPart.FeedData(stream);
                     imageReference.SetAttributeValue(attributeName, id);
                 }
                 else
@@ -1064,27 +1064,25 @@ private void CopyRelatedMedia(OpenXmlPart oldContentPart, OpenXmlPart newContent
         // General function for handling images that tries to use an existing image if they are the same
         private ImageData ManageImageCopy(ImagePart oldImage)
         {
-            var oldImageData = new ImageData(oldImage);
-            foreach (var item in _images)
-            {
-                if (item.Compare(oldImageData))
-                    return item;
-            }
-            _images.Add(oldImageData);
-            return oldImageData;
+            return GetOrAddCachedMedia(new ImageData(oldImage));
         }
 
         // General function for handling media that tries to use an existing media item if they are the same
         private MediaData ManageMediaCopy(DataPart oldMedia)
         {
-            var oldMediaData = new MediaData(oldMedia);
-            foreach (var item in _mediaList)
+            return GetOrAddCachedMedia(new MediaData(oldMedia));
+        }
+
+        private T GetOrAddCachedMedia<T>(T contentData) where T : ContentData
+        {
+            var duplicateItem = _mediaCache.FirstOrDefault(x => x.Compare(contentData));
+            if (duplicateItem != null)
             {
-                if (item.Compare(oldMediaData))
-                    return item;
+                return (T)duplicateItem;
             }
-            _mediaList.Add(oldMediaData);
-            return oldMediaData;
+
+            _mediaCache.Add(contentData);
+            return contentData;
         }
 
         private ThemePart CopyThemePart(SlideMasterPart slideMasterPart, ThemePart oldThemePart, double scaleFactor)

diff --git a/Clippit/PtOpenXmlUtil.cs b/Clippit/PtOpenXmlUtil.cs
@@ -27,6 +27,12 @@ public static byte[] ReadToArray(this Stream stream)
             stream.CopyTo(ms);
             return ms.ToArray();
         }
+
+        public static byte[] ComputeHash(this Stream stream)
+        {
+            using var hashAlgo = System.Security.Cryptography.SHA256.Create();
+            return hashAlgo.ComputeHash(stream);
+        }
     }
 
     public static class PtOpenXmlExtensions
@@ -1798,101 +1804,52 @@ internal class ContentPartRelTypeIdTuple
         public string RelationshipId { get; set; }
     }
 
-    // This class is used to prevent duplication of images
-    internal class ImageData
+    abstract class ContentData
     {
-        private string ContentType { get; set; }
-        private byte[] Image { get; set; }
-        public OpenXmlPart ImagePart { get; set; }
-        public List<ContentPartRelTypeIdTuple> ContentPartRelTypeIdList = new List<ContentPartRelTypeIdTuple>();
-
-        public ImageData(ImagePart part)
-        {
-            ContentType = part.ContentType;
-            using var s = part.GetStream(FileMode.Open, FileAccess.Read);
-            Image = s.ReadToArray();
-        }
-
+        protected string ContentType { get; set; }
+        protected byte[] Hash { get; set; }
+
+        public List<ContentPartRelTypeIdTuple> ContentPartRelTypeIdList = new();
+
         public void AddContentPartRelTypeResourceIdTupple(OpenXmlPart contentPart, string relationshipType, string relationshipId)
         {
-            ContentPartRelTypeIdList.Add(
-                new ContentPartRelTypeIdTuple()
-                {
-                    ContentPart = contentPart,
-                    RelationshipType = relationshipType,
-                    RelationshipId = relationshipId,
-                });
+            ContentPartRelTypeIdList.Add(new ContentPartRelTypeIdTuple
+            {
+                ContentPart = contentPart,
+                RelationshipType = relationshipType,
+                RelationshipId = relationshipId,
+            });
         }
 
-        public void WriteImage(ImagePart part)
+        public bool Compare(ContentData arg)
         {
-            using var s = part.GetStream(FileMode.Create, FileAccess.ReadWrite);
-            s.Write(Image, 0, Image.GetUpperBound(0) + 1);
+            return ContentType == arg.ContentType && Hash.SequenceEqual(arg.Hash);
         }
+    }
+
+    // This class is used to prevent duplication of images
+    class ImageData : ContentData
+    {
+        public OpenXmlPart ImagePart { get; set; }
 
-        public bool Compare(ImageData arg)
+        public ImageData(ImagePart part)
         {
-            if (ContentType != arg.ContentType)
-                return false;
-            if (Image.GetLongLength(0) != arg.Image.GetLongLength(0))
-                return false;
-            // Compare the arrays byte by byte
-            var length = Image.GetLongLength(0);
-            var image1 = Image;
-            var image2 = arg.Image;
-            for (long n = 0; n < length; n++)
-                if (image1[n] != image2[n])
-                    return false;
-            return true;
+            ContentType = part.ContentType;
+            using var s = part.GetStream();
+            Hash = s.ComputeHash();
         }
     }
 
     // This class is used to prevent duplication of media
-    internal class MediaData
+    class MediaData : ContentData
     {
-        private string ContentType { get; set; }
-        private byte[] Media { get; set; }
         public DataPart DataPart { get; set; }
-        public List<ContentPartRelTypeIdTuple> ContentPartRelTypeIdList = new List<ContentPartRelTypeIdTuple>();
 
         public MediaData(DataPart part)
         {
             ContentType = part.ContentType;
-            using var s = part.GetStream(FileMode.Open, FileAccess.Read);
-            Media = s.ReadToArray();
-        }
-
-        public void AddContentPartRelTypeResourceIdTupple(OpenXmlPart contentPart, string relationshipType, string relationshipId)
-        {
-            ContentPartRelTypeIdList.Add(
-                new ContentPartRelTypeIdTuple()
-                {
-                    ContentPart = contentPart,
-                    RelationshipType = relationshipType,
-                    RelationshipId = relationshipId,
-                });
-        }
-
-        public void WriteMedia(DataPart part)
-        {
-            using var s = part.GetStream(FileMode.Create, FileAccess.ReadWrite);
-            s.Write(Media, 0, Media.GetUpperBound(0) + 1);
-        }
-
-        public bool Compare(MediaData arg)
-        {
-            if (ContentType != arg.ContentType)
-                return false;
-            if (Media.GetLongLength(0) != arg.Media.GetLongLength(0))
-                return false;
-            // Compare the arrays byte by byte
-            var length = Media.GetLongLength(0);
-            var media1 = Media;
-            var media2 = arg.Media;
-            for (long n = 0; n < length; n++)
-                if (media1[n] != media2[n])
-                    return false;
-            return true;
+            using var s = part.GetStream();
+            Hash = s.ComputeHash();
         }
     }
 

diff --git a/Clippit/Word/DocumentBuilder.cs b/Clippit/Word/DocumentBuilder.cs
@@ -3200,7 +3200,8 @@ private class FromPreviousSourceSemaphore { };
                     var id = newContentPart.GetIdOfPart(newPart);
                     temp.AddContentPartRelTypeResourceIdTupple(newContentPart, newPart.RelationshipType, id);
                     imageReference.SetAttributeValue(attributeName, id);
-                    temp.WriteImage(newPart);
+                    using (var stream = oldPart.GetStream())
+                        newPart.FeedData(stream);
                 }
                 else
                 {

diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
@@ -1,9 +1,12 @@
-#### 2.0.0-beta2 - Mar 16, 2022
+#### 2.0.0-beta3 - May 29, 2022
 - Migration to .NET 6.0
 - `System.Drawing.Common` replaced by `SixLabors.ImageSharp.Drawing`
 - Removed `libgdiplus` dependency
 - Drop old `WmlComparer`
 
+#### 1.11.0 - May 29, 2022
+- PublishSlides: reduced memory consumption [#53](https://github.com/sergey-tihon/Clippit/pull/53)
+
 #### 1.10.2 - Mar 16, 2022
 - DocumentFormat.OpenXml (2.16.0)
 - Lock `System.Drawing.Common` version to `v5`

diff --git a/global.json b/global.json
@@ -1,6 +1,6 @@
 {
     "sdk": {
-        "version": "6.0.100",
+        "version": "6.0.300",
         "rollForward": "minor"
     }
 }
diff --git a/paket.lock b/paket.lock
@@ -5,29 +5,29 @@ NUGET
     DocumentFormat.OpenXml (2.16)
       System.IO.Packaging (>= 4.7)
     IDisposableAnalyzers (4.0.2)
-    Microsoft.CodeCoverage (17.1) - restriction: || (== net6.0) (&& (== netstandard2.0) (>= net45)) (&& (== netstandard2.0) (>= netcoreapp1.0))
-    Microsoft.NET.Test.Sdk (17.1)
-      Microsoft.CodeCoverage (>= 17.1) - restriction: || (== net6.0) (&& (== netstandard2.0) (>= net45)) (&& (== netstandard2.0) (>= netcoreapp1.0))
-      Microsoft.TestPlatform.TestHost (>= 17.1) - restriction: || (== net6.0) (&& (== netstandard2.0) (>= netcoreapp1.0))
-    Microsoft.NETCore.Platforms (6.0.2)
+    Microsoft.CodeCoverage (17.2) - restriction: || (== net6.0) (&& (== netstandard2.0) (>= net45)) (&& (== netstandard2.0) (>= netcoreapp1.0))
+    Microsoft.NET.Test.Sdk (17.2)
+      Microsoft.CodeCoverage (>= 17.2) - restriction: || (== net6.0) (&& (== netstandard2.0) (>= net45)) (&& (== netstandard2.0) (>= netcoreapp1.0))
+      Microsoft.TestPlatform.TestHost (>= 17.2) - restriction: || (== net6.0) (&& (== netstandard2.0) (>= netcoreapp1.0))
+    Microsoft.NETCore.Platforms (6.0.3)
     Microsoft.NETCore.Targets (5.0) - restriction: || (&& (== net6.0) (< netcoreapp3.1)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.0)
-    Microsoft.TestPlatform.ObjectModel (17.1) - restriction: || (== net6.0) (&& (== netstandard2.0) (>= netcoreapp1.0))
+    Microsoft.TestPlatform.ObjectModel (17.2) - restriction: || (== net6.0) (&& (== netstandard2.0) (>= netcoreapp1.0))
       NuGet.Frameworks (>= 5.11)
       System.Reflection.Metadata (>= 1.6)
-    Microsoft.TestPlatform.TestHost (17.1) - restriction: || (== net6.0) (&& (== netstandard2.0) (>= netcoreapp1.0))
-      Microsoft.TestPlatform.ObjectModel (>= 17.1) - restriction: || (== net6.0) (&& (== netstandard2.0) (>= netcoreapp1.0)) (&& (== netstandard2.0) (>= uap10.0))
+    Microsoft.TestPlatform.TestHost (17.2) - restriction: || (== net6.0) (&& (== netstandard2.0) (>= netcoreapp1.0))
+      Microsoft.TestPlatform.ObjectModel (>= 17.2) - restriction: || (== net6.0) (&& (== netstandard2.0) (>= netcoreapp1.0)) (&& (== netstandard2.0) (>= uap10.0))
       Newtonsoft.Json (>= 9.0.1) - restriction: || (== net6.0) (&& (== netstandard2.0) (>= netcoreapp1.0)) (&& (== netstandard2.0) (>= uap10.0))
     NETStandard.Library (2.0.3)
       Microsoft.NETCore.Platforms (>= 1.1)
     Newtonsoft.Json (13.0.1) - restriction: || (== net6.0) (&& (== netstandard2.0) (>= netcoreapp1.0))
-    NuGet.Frameworks (6.1) - restriction: || (== net6.0) (&& (== netstandard2.0) (>= netcoreapp1.0)) (&& (== netstandard2.0) (>= netcoreapp2.1))
+    NuGet.Frameworks (6.2) - restriction: || (== net6.0) (&& (== netstandard2.0) (>= netcoreapp1.0)) (&& (== netstandard2.0) (>= netcoreapp2.1))
     runtime.native.System (4.3.1) - restriction: || (&& (== net6.0) (< netcoreapp3.1)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.0)
       Microsoft.NETCore.Platforms (>= 1.1.1)
       Microsoft.NETCore.Targets (>= 1.1.3)
     runtime.native.System.IO.Compression (4.3.2) - restriction: || (&& (== net6.0) (< netcoreapp3.1)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.0)
       Microsoft.NETCore.Platforms (>= 1.1.1)
       Microsoft.NETCore.Targets (>= 1.1.3)
-    SixLabors.Fonts (1.0.0-beta16)
+    SixLabors.Fonts (1.0.0-beta17)
       System.Buffers (>= 4.5.1) - restriction: || (&& (== net6.0) (< netstandard2.1)) (== netstandard2.0)
       System.IO.Compression (>= 4.3) - restriction: || (&& (== net6.0) (< netcoreapp3.1)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.0)
       System.IO.UnmanagedMemoryStream (>= 4.3) - restriction: || (&& (== net6.0) (< netcoreapp3.1)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.0)
@@ -36,7 +36,7 @@ NUGET
       System.Runtime.CompilerServices.Unsafe (>= 4.7) - restriction: || (&& (== net6.0) (< netcoreapp3.1)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.0)
       System.Threading.Tasks.Parallel (>= 4.3) - restriction: || (&& (== net6.0) (< netcoreapp3.1)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.0)
       System.ValueTuple (>= 4.5) - restriction: || (&& (== net6.0) (< netcoreapp3.1)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.0)
-    SixLabors.ImageSharp (2.1)
+    SixLabors.ImageSharp (2.1.2)
       System.Buffers (>= 4.5.1) - restriction: || (&& (== net6.0) (>= net472)) (&& (== net6.0) (< netcoreapp2.1)) (== netstandard2.0)
       System.Memory (>= 4.5.4) - restriction: || (&& (== net6.0) (>= net472)) (&& (== net6.0) (< netcoreapp2.1)) (== netstandard2.0)
       System.Numerics.Vectors (>= 4.5) - restriction: || (&& (== net6.0) (>= net472)) (&& (== net6.0) (< netcoreapp2.1)) (== netstandard2.0)
@@ -110,7 +110,7 @@ NUGET
       System.Runtime.InteropServices (>= 4.3)
       System.Threading (>= 4.3)
       System.Threading.Tasks (>= 4.3)
-    System.Memory (4.5.4) - restriction: || (&& (== net6.0) (>= net472)) (&& (== net6.0) (< netcoreapp2.1)) (== netstandard2.0)
+    System.Memory (4.5.5) - restriction: || (&& (== net6.0) (>= net472)) (&& (== net6.0) (< netcoreapp2.1)) (== netstandard2.0)
       System.Buffers (>= 4.5.1) - restriction: || (&& (== net6.0) (>= monotouch)) (&& (== net6.0) (>= net461)) (&& (== net6.0) (< netcoreapp2.0)) (&& (== net6.0) (< netstandard1.1)) (&& (== net6.0) (< netstandard2.0)) (&& (== net6.0) (>= xamarinios)) (&& (== net6.0) (>= xamarinmac)) (&& (== net6.0) (>= xamarintvos)) (&& (== net6.0) (>= xamarinwatchos)) (== netstandard2.0)
       System.Numerics.Vectors (>= 4.4) - restriction: || (&& (== net6.0) (< netcoreapp2.0)) (== netstandard2.0)
       System.Runtime.CompilerServices.Unsafe (>= 4.5.3) - restriction: || (&& (== net6.0) (>= monotouch)) (&& (== net6.0) (>= net461)) (&& (== net6.0) (< netcoreapp2.0)) (&& (== net6.0) (< netcoreapp2.1)) (&& (== net6.0) (< netstandard1.1)) (&& (== net6.0) (< netstandard2.0)) (&& (== net6.0) (>= uap10.1)) (&& (== net6.0) (>= xamarinios)) (&& (== net6.0) (>= xamarinmac)) (&& (== net6.0) (>= xamarintvos)) (&& (== net6.0) (>= xamarinwatchos)) (== netstandard2.0)
@@ -194,4 +194,4 @@ NUGET
       NETStandard.Library (>= 1.6.1)
       xunit.extensibility.core (2.4.1)
     xunit.runner.console (2.4.1)
-    xunit.runner.visualstudio (2.4.3)
+    xunit.runner.visualstudio (2.4.5)