From bc49979c121294a79c16aa5f1af7b7d430b37b18 Mon Sep 17 00:00:00 2001 From: Temtaime Date: Tue, 2 Nov 2021 12:47:31 +0300 Subject: [PATCH 1/6] Fix #682 --- .../Zip/ZipOutputStream.cs | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs b/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs index 0b292fb3f..269bd934a 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs @@ -313,6 +313,8 @@ internal void PutNextEntry(Stream stream, ZipEntry entry, long streamOffset = 0) throw new InvalidOperationException("The Password property must be set before AES encrypted entries can be added"); } + entryIsPrecompressed = string.IsNullOrEmpty(Password) && method == CompressionMethod.Deflated && entry.Size >= 0 && entry.HasCrc && entry.CompressedSize >= 0; + int compressionLevel = defaultCompressionLevel; // Clear flags that the library manages internally @@ -322,7 +324,7 @@ internal void PutNextEntry(Stream stream, ZipEntry entry, long streamOffset = 0) bool headerInfoAvailable; // No need to compress - definitely no data. - if (entry.Size == 0) + if (entry.Size == 0 && !entryIsPrecompressed) { entry.CompressedSize = entry.Size; entry.Crc = 0; @@ -406,14 +408,17 @@ internal void PutNextEntry(Stream stream, ZipEntry entry, long streamOffset = 0) // Activate the entry. curEntry = entry; + size = 0; + + if(entryIsPrecompressed) + return; + crc.Reset(); if (method == CompressionMethod.Deflated) { deflater_.Reset(); deflater_.SetLevel(compressionLevel); } - size = 0; - } /// @@ -506,6 +511,17 @@ internal void WriteEntryFooter(Stream stream) throw new InvalidOperationException("No open entry"); } + if(entryIsPrecompressed) + { + if(curEntry.CompressedSize != size) + { + throw new ZipException($"compressed size was {size}, but {curEntry.CompressedSize} expected"); + } + + offset += size; + return; + } + long csize = size; // First finish the deflater, if appropriate @@ -695,6 +711,13 @@ public override void Write(byte[] buffer, int offset, int count) throw new ArgumentException("Invalid offset/count combination"); } + if(entryIsPrecompressed) + { + size += count; + baseOutputStream_.Write(buffer, offset, count); + return; + } + if (curEntry.AESKeySize == 0) { // Only update CRC if AES is not enabled @@ -844,6 +867,8 @@ public override void Flush() /// private ZipEntry curEntry; + private bool entryIsPrecompressed; + private int defaultCompressionLevel = Deflater.DEFAULT_COMPRESSION; private CompressionMethod curMethod = CompressionMethod.Deflated; From 1204e30686c75ff2f11a957dc59decb37cda35dd Mon Sep 17 00:00:00 2001 From: Temtaime Date: Tue, 2 Nov 2021 13:50:04 +0300 Subject: [PATCH 2/6] Add test --- .../Zip/GeneralHandling.cs | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/test/ICSharpCode.SharpZipLib.Tests/Zip/GeneralHandling.cs b/test/ICSharpCode.SharpZipLib.Tests/Zip/GeneralHandling.cs index ad97563aa..fbe9699b1 100644 --- a/test/ICSharpCode.SharpZipLib.Tests/Zip/GeneralHandling.cs +++ b/test/ICSharpCode.SharpZipLib.Tests/Zip/GeneralHandling.cs @@ -1,5 +1,7 @@ -using ICSharpCode.SharpZipLib.Tests.TestSupport; +using ICSharpCode.SharpZipLib.Checksum; +using ICSharpCode.SharpZipLib.Tests.TestSupport; using ICSharpCode.SharpZipLib.Zip; +using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using NUnit.Framework; using System; using System.IO; @@ -384,6 +386,43 @@ public void StoredNonSeekableKnownSizeNoCrc() Assert.IsTrue(ZipTesting.TestArchive(ms.ToArray())); } + [Test] + [Category("Zip")] + public void StoredNonSeekablePrecompressed() + { + var data = Encoding.UTF8.GetBytes("Hello, world"); + + var crc = new Crc32(); + crc.Update(data); + + var compressedData = new MemoryStream(); + using(var gz = new System.IO.Compression.DeflateStream(compressedData, System.IO.Compression.CompressionMode.Compress, true)) + { + gz.Write(data, 0, data.Length); + } + compressedData.Position = 0; + + MemoryStream ms = new MemoryStreamWithoutSeek(); + + using (ZipOutputStream outStream = new ZipOutputStream(ms)) + { + outStream.IsStreamOwner = false; + + var entry = new ZipEntry("dummyfile.tst"); + + entry.CompressionMethod = CompressionMethod.Deflated; + entry.Size = data.Length; + entry.Crc = (uint)crc.Value; + entry.CompressedSize = compressedData.Length; + + outStream.PutNextEntry(entry); + + compressedData.CopyTo(outStream); + } + + Assert.IsTrue(ZipTesting.TestArchive(ms.ToArray())); + } + [Test] [Category("Zip")] public void StoredNonSeekableKnownSizeNoCrcEncrypted() From 2e4a62b297e344526b9c63a64ca11c39a5dea693 Mon Sep 17 00:00:00 2001 From: Temtaime Date: Wed, 3 Nov 2021 16:36:45 +0300 Subject: [PATCH 3/6] Fixes --- .../Zip/ZipOutputStream.cs | 117 +++++++++++++----- .../Zip/GeneralHandling.cs | 2 +- 2 files changed, 89 insertions(+), 30 deletions(-) diff --git a/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs b/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs index 269bd934a..208b73786 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs @@ -270,7 +270,76 @@ public void PutNextEntry(ZipEntry entry) WriteOutput(GetEntryEncryptionHeader(entry)); } } - + + /// + /// Starts a new passthrough Zip entry. It automatically closes the previous + /// entry if present. + /// Passthrough entry is an entry that is created from compressed data. + /// It is useful to avoid recompression to save CPU resources if compressed data is already disposable. + /// All entry elements bar name, crc, size and compressed size are optional, but must be correct if present. + /// Compression should be set to Deflated. + /// + /// + /// the entry. + /// + /// + /// if entry passed is null. + /// + /// + /// if an I/O error occurred. + /// + /// + /// if stream was finished. + /// + /// + /// Crc is not set
+ /// Size is not set
+ /// CompressedSize is not set
+ /// CompressionMethod is not Deflate
+ /// Too many entries in the Zip file
+ /// Entry name is too long
+ /// Finish has already been called
+ /// + /// + /// The Compression method specified for the entry is unsupported
+ /// Entry is encrypted
+ /// + public void PutNextPassthroughEntry(ZipEntry entry) + { + if(curEntry != null) + { + CloseEntry(); + } + + if(entry.Crc < 0) + { + throw new ZipException("Crc must be set for passthrough entry"); + } + + if(entry.Size < 0) + { + throw new ZipException("Size must be set for passthrough entry"); + } + + if(entry.CompressedSize < 0) + { + throw new ZipException("CompressedSize must be set for passthrough entry"); + } + + if(entry.CompressionMethod != CompressionMethod.Deflated) + { + throw new ZipException("Only Deflated entries are supported for passthrough"); + } + + if(!string.IsNullOrEmpty(Password)) + { + throw new NotImplementedException("Encrypted passthrough entries are not supported"); + } + + PutNextEntry(baseOutputStream_, entry, 0, true); + } + + private void WriteOutput(byte[] bytes) => baseOutputStream_.Write(bytes, 0, bytes.Length); @@ -282,7 +351,7 @@ private Task WriteOutputAsync(byte[] bytes) ? InitializeAESPassword(entry, Password) : CreateZipCryptoHeader(entry.Crc < 0 ? entry.DosTime << 16 : entry.Crc); - internal void PutNextEntry(Stream stream, ZipEntry entry, long streamOffset = 0) + internal void PutNextEntry(Stream stream, ZipEntry entry, long streamOffset = 0, bool passthroughEntry = false) { if (entry == null) { @@ -313,7 +382,7 @@ internal void PutNextEntry(Stream stream, ZipEntry entry, long streamOffset = 0) throw new InvalidOperationException("The Password property must be set before AES encrypted entries can be added"); } - entryIsPrecompressed = string.IsNullOrEmpty(Password) && method == CompressionMethod.Deflated && entry.Size >= 0 && entry.HasCrc && entry.CompressedSize >= 0; + entryIsPassthrough = passthroughEntry; int compressionLevel = defaultCompressionLevel; @@ -324,7 +393,7 @@ internal void PutNextEntry(Stream stream, ZipEntry entry, long streamOffset = 0) bool headerInfoAvailable; // No need to compress - definitely no data. - if (entry.Size == 0 && !entryIsPrecompressed) + if (entry.Size == 0 && !entryIsPassthrough) { entry.CompressedSize = entry.Size; entry.Crc = 0; @@ -410,7 +479,7 @@ internal void PutNextEntry(Stream stream, ZipEntry entry, long streamOffset = 0) curEntry = entry; size = 0; - if(entryIsPrecompressed) + if(entryIsPassthrough) return; crc.Reset(); @@ -511,7 +580,7 @@ internal void WriteEntryFooter(Stream stream) throw new InvalidOperationException("No open entry"); } - if(entryIsPrecompressed) + if(entryIsPassthrough) { if(curEntry.CompressedSize != size) { @@ -711,14 +780,7 @@ public override void Write(byte[] buffer, int offset, int count) throw new ArgumentException("Invalid offset/count combination"); } - if(entryIsPrecompressed) - { - size += count; - baseOutputStream_.Write(buffer, offset, count); - return; - } - - if (curEntry.AESKeySize == 0) + if (curEntry.AESKeySize == 0 && !entryIsPassthrough) { // Only update CRC if AES is not enabled crc.Update(new ArraySegment(buffer, offset, count)); @@ -726,22 +788,19 @@ public override void Write(byte[] buffer, int offset, int count) size += count; - switch (curMethod) + if(curMethod == CompressionMethod.Deflated) { - case CompressionMethod.Deflated: - base.Write(buffer, offset, count); - break; + base.Write(buffer, offset, count); + return; + } - case CompressionMethod.Stored: - if (Password != null) - { - CopyAndEncrypt(buffer, offset, count); - } - else - { - baseOutputStream_.Write(buffer, offset, count); - } - break; + if (Password != null) + { + CopyAndEncrypt(buffer, offset, count); + } + else + { + baseOutputStream_.Write(buffer, offset, count); } } @@ -867,7 +926,7 @@ public override void Flush() /// private ZipEntry curEntry; - private bool entryIsPrecompressed; + private bool entryIsPassthrough; private int defaultCompressionLevel = Deflater.DEFAULT_COMPRESSION; diff --git a/test/ICSharpCode.SharpZipLib.Tests/Zip/GeneralHandling.cs b/test/ICSharpCode.SharpZipLib.Tests/Zip/GeneralHandling.cs index fbe9699b1..cce910789 100644 --- a/test/ICSharpCode.SharpZipLib.Tests/Zip/GeneralHandling.cs +++ b/test/ICSharpCode.SharpZipLib.Tests/Zip/GeneralHandling.cs @@ -415,7 +415,7 @@ public void StoredNonSeekablePrecompressed() entry.Crc = (uint)crc.Value; entry.CompressedSize = compressedData.Length; - outStream.PutNextEntry(entry); + outStream.PutNextPassthroughEntry(entry); compressedData.CopyTo(outStream); } From af7187d6e3ad261faf9cac42b3529a5d986a3e95 Mon Sep 17 00:00:00 2001 From: Temtaime Date: Mon, 8 Nov 2021 15:17:50 +0300 Subject: [PATCH 4/6] Fix wrong entry write handling --- .../Zip/ZipOutputStream.cs | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs b/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs index 208b73786..76e74023e 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs @@ -782,25 +782,26 @@ public override void Write(byte[] buffer, int offset, int count) if (curEntry.AESKeySize == 0 && !entryIsPassthrough) { - // Only update CRC if AES is not enabled + // Only update CRC if AES is not enabled and entry is not a passthrough one crc.Update(new ArraySegment(buffer, offset, count)); } size += count; - if(curMethod == CompressionMethod.Deflated) + if(curMethod == CompressionMethod.Stored || entryIsPassthrough) { - base.Write(buffer, offset, count); - return; - } - - if (Password != null) - { - CopyAndEncrypt(buffer, offset, count); + if (Password != null) + { + CopyAndEncrypt(buffer, offset, count); + } + else + { + baseOutputStream_.Write(buffer, offset, count); + } } else { - baseOutputStream_.Write(buffer, offset, count); + base.Write(buffer, offset, count); } } From 9f19c8fd7ec9275e95c5986ff895584a8a1c5aac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nils=20m=C3=A5s=C3=A9n?= Date: Fri, 12 Nov 2021 13:56:19 +0100 Subject: [PATCH 5/6] change the exception for non-supported methods allowing other compression methods that are not implemented inside sharpziplib itself could be a good future use case for this. "not implemented" better conveys that it might be supported in the future. --- src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs b/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs index 76e74023e..36349c886 100644 --- a/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs +++ b/src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs @@ -328,7 +328,7 @@ public void PutNextPassthroughEntry(ZipEntry entry) if(entry.CompressionMethod != CompressionMethod.Deflated) { - throw new ZipException("Only Deflated entries are supported for passthrough"); + throw new NotImplementedException("Only Deflated entries are supported for passthrough"); } if(!string.IsNullOrEmpty(Password)) From 89f925d723933c9cad799eb0eaac3ea2dc66e01c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nils=20m=C3=A5s=C3=A9n?= Date: Fri, 12 Nov 2021 13:57:11 +0100 Subject: [PATCH 6/6] move tests to it's own file and add tests for exceptions --- .../Zip/GeneralHandling.cs | 37 ----- .../Zip/PassthroughTests.cs | 140 ++++++++++++++++++ 2 files changed, 140 insertions(+), 37 deletions(-) create mode 100644 test/ICSharpCode.SharpZipLib.Tests/Zip/PassthroughTests.cs diff --git a/test/ICSharpCode.SharpZipLib.Tests/Zip/GeneralHandling.cs b/test/ICSharpCode.SharpZipLib.Tests/Zip/GeneralHandling.cs index cce910789..e16a82967 100644 --- a/test/ICSharpCode.SharpZipLib.Tests/Zip/GeneralHandling.cs +++ b/test/ICSharpCode.SharpZipLib.Tests/Zip/GeneralHandling.cs @@ -386,43 +386,6 @@ public void StoredNonSeekableKnownSizeNoCrc() Assert.IsTrue(ZipTesting.TestArchive(ms.ToArray())); } - [Test] - [Category("Zip")] - public void StoredNonSeekablePrecompressed() - { - var data = Encoding.UTF8.GetBytes("Hello, world"); - - var crc = new Crc32(); - crc.Update(data); - - var compressedData = new MemoryStream(); - using(var gz = new System.IO.Compression.DeflateStream(compressedData, System.IO.Compression.CompressionMode.Compress, true)) - { - gz.Write(data, 0, data.Length); - } - compressedData.Position = 0; - - MemoryStream ms = new MemoryStreamWithoutSeek(); - - using (ZipOutputStream outStream = new ZipOutputStream(ms)) - { - outStream.IsStreamOwner = false; - - var entry = new ZipEntry("dummyfile.tst"); - - entry.CompressionMethod = CompressionMethod.Deflated; - entry.Size = data.Length; - entry.Crc = (uint)crc.Value; - entry.CompressedSize = compressedData.Length; - - outStream.PutNextPassthroughEntry(entry); - - compressedData.CopyTo(outStream); - } - - Assert.IsTrue(ZipTesting.TestArchive(ms.ToArray())); - } - [Test] [Category("Zip")] public void StoredNonSeekableKnownSizeNoCrcEncrypted() diff --git a/test/ICSharpCode.SharpZipLib.Tests/Zip/PassthroughTests.cs b/test/ICSharpCode.SharpZipLib.Tests/Zip/PassthroughTests.cs new file mode 100644 index 000000000..1a4b266f2 --- /dev/null +++ b/test/ICSharpCode.SharpZipLib.Tests/Zip/PassthroughTests.cs @@ -0,0 +1,140 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Text; +using ICSharpCode.SharpZipLib.Checksum; +using ICSharpCode.SharpZipLib.Tests.TestSupport; +using ICSharpCode.SharpZipLib.Zip; +using NUnit.Framework; + +namespace ICSharpCode.SharpZipLib.Tests.Zip +{ + [TestFixture] + public class PassthroughTests + { + [Test] + [Category("Zip")] + public void AddingValidPrecompressedEntryToZipOutputStream() + { + using var ms = new MemoryStream(); + + using (var outStream = new ZipOutputStream(ms){IsStreamOwner = false}) + { + var (compressedData, crc, size) = CreateDeflatedData(); + var entry = new ZipEntry("dummyfile.tst") + { + CompressionMethod = CompressionMethod.Deflated, + Size = size, + Crc = (uint)crc.Value, + CompressedSize = compressedData.Length, + }; + + outStream.PutNextPassthroughEntry(entry); + + compressedData.CopyTo(outStream); + } + + Assert.IsTrue(ZipTesting.TestArchive(ms.ToArray())); + } + + private static (MemoryStream, Crc32, int) CreateDeflatedData() + { + var data = Encoding.UTF8.GetBytes("Hello, world"); + + var crc = new Crc32(); + crc.Update(data); + + var compressedData = new MemoryStream(); + using(var gz = new DeflateStream(compressedData, CompressionMode.Compress, leaveOpen: true)) + { + gz.Write(data, 0, data.Length); + } + compressedData.Position = 0; + + return (compressedData, crc, data.Length); + } + + [Test] + [Category("Zip")] + public void AddingPrecompressedEntryToZipOutputStreamWithInvalidSize() + { + using var outStream = new ZipOutputStream(new MemoryStream()); + var (compressedData, crc, size) = CreateDeflatedData(); + outStream.Password = "mockpassword"; + var entry = new ZipEntry("dummyfile.tst") + { + CompressionMethod = CompressionMethod.Stored, + Crc = (uint)crc.Value, + CompressedSize = compressedData.Length, + }; + + Assert.Throws(() => + { + outStream.PutNextPassthroughEntry(entry); + }); + } + + + [Test] + [Category("Zip")] + public void AddingPrecompressedEntryToZipOutputStreamWithInvalidCompressedSize() + { + using var outStream = new ZipOutputStream(new MemoryStream()); + var (compressedData, crc, size) = CreateDeflatedData(); + outStream.Password = "mockpassword"; + var entry = new ZipEntry("dummyfile.tst") + { + CompressionMethod = CompressionMethod.Stored, + Size = size, + Crc = (uint)crc.Value, + }; + + Assert.Throws(() => + { + outStream.PutNextPassthroughEntry(entry); + }); + } + + [Test] + [Category("Zip")] + public void AddingPrecompressedEntryToZipOutputStreamWithNonSupportedMethod() + { + using var outStream = new ZipOutputStream(new MemoryStream()); + var (compressedData, crc, size) = CreateDeflatedData(); + outStream.Password = "mockpassword"; + var entry = new ZipEntry("dummyfile.tst") + { + CompressionMethod = CompressionMethod.LZMA, + Size = size, + Crc = (uint)crc.Value, + CompressedSize = compressedData.Length, + }; + + Assert.Throws(() => + { + outStream.PutNextPassthroughEntry(entry); + }); + } + + [Test] + [Category("Zip")] + public void AddingPrecompressedEntryToZipOutputStreamWithEncryption() + { + using var outStream = new ZipOutputStream(new MemoryStream()); + var (compressedData, crc, size) = CreateDeflatedData(); + outStream.Password = "mockpassword"; + var entry = new ZipEntry("dummyfile.tst") + { + CompressionMethod = CompressionMethod.Deflated, + Size = size, + Crc = (uint)crc.Value, + CompressedSize = compressedData.Length, + }; + + Assert.Throws(() => + { + outStream.PutNextPassthroughEntry(entry); + }); + } + } +}