Skip to content
This repository has been archived by the owner on Jan 23, 2023. It is now read-only.

Commit

Permalink
Merge pull request #2906 from ianhays/compression
Browse files Browse the repository at this point in the history 
Added ZLib decompression support to DeflateStream
  • Loading branch information
Ian Hays committed Aug 24, 2015
2 parents e296c69 + 94ba7b8 commit 6941118
Show file tree
Hide file tree
Showing 11 changed files with 487 additions and 68 deletions.
4 changes: 3 additions & 1 deletion src/System.IO.Compression/src/System.IO.Compression.csproj
View file
Expand Up @@ -41,8 +41,10 @@
<Compile Include="System\IO\Compression\GZipUtils.cs" />
<Compile Include="System\IO\Compression\HuffmanTree.cs" />
<Compile Include="System\IO\Compression\IDeflater.cs" />
<Compile Include="System\IO\Compression\Inflater.cs" />
<Compile Include="System\IO\Compression\IInflater.cs" />
<Compile Include="System\IO\Compression\InflaterManaged.cs" />
<Compile Include="System\IO\Compression\InflaterState.cs" />
<Compile Include="System\IO\Compression\InflaterZlib.cs" />
<Compile Include="System\IO\Compression\InputBuffer.cs" />
<Compile Include="System\IO\Compression\Match.cs" />
<Compile Include="System\IO\Compression\MatchState.cs" />
Expand Down
58 changes: 49 additions & 9 deletions src/System.IO.Compression/src/System/IO/Compression/DeflateStream.cs
View file
Expand Up @@ -15,7 +15,7 @@ public partial class DeflateStream : Stream
private Stream _stream;
private CompressionMode _mode;
private bool _leaveOpen;
private Inflater _inflater;
private IInflater _inflater;
private IDeflater _deflater;
private byte[] _buffer;

Expand All @@ -38,6 +38,23 @@ public DeflateStream(Stream stream, CompressionMode mode)
{
}

// Since a reader is being taken, CompressionMode.Decompress is implied
internal DeflateStream(Stream stream, bool leaveOpen, IFileFormatReader reader)
{
Debug.Assert(reader != null, "The IFileFormatReader passed to the internal DeflateStream constructor must be non-null");
if (stream == null)
throw new ArgumentNullException("stream");
if (!stream.CanRead)
throw new ArgumentException(SR.NotReadableStream, "stream");

_inflater = CreateInflater(reader);
_stream = stream;
_mode = CompressionMode.Decompress;
_leaveOpen = leaveOpen;
_buffer = new byte[DefaultBufferSize];
}


public DeflateStream(Stream stream, CompressionMode mode, bool leaveOpen)
{
if (stream == null)
Expand All @@ -50,7 +67,7 @@ public DeflateStream(Stream stream, CompressionMode mode, bool leaveOpen)
{
throw new ArgumentException(SR.NotReadableStream, "stream");
}
_inflater = new Inflater();
_inflater = CreateInflater();
break;

case CompressionMode.Compress:
Expand Down Expand Up @@ -126,11 +143,34 @@ private static IDeflater CreateDeflater(CompressionLevel? compressionLevel)
}
}

internal void SetFileFormatReader(IFileFormatReader reader)
private static IInflater CreateInflater(IFileFormatReader reader = null)
{
if (reader != null)
// The deflator type (zlib or managed) is normally determined by s_deflatorType,
// which is initialized by the provider based on what's available on the system.
// But for testing purposes, we sometimes want to override this, forcing
// compression/decompression to use a particular type.
WorkerType deflatorType = s_deflaterType;
#if DEBUG
if (s_forcedTestingDeflaterType != WorkerType.Unknown)
deflatorType = s_forcedTestingDeflaterType;
#endif

if (deflatorType == WorkerType.ZLib)
{
_inflater.SetFileFormatReader(reader);
// Rather than reading raw data and using a FormatReader to interpret
// headers/footers manually, we instead set the zlib stream to parse
// that information for us.
if (reader == null)
return new InflaterZlib(ZLibNative.Deflate_DefaultWindowBits);
else
{
Debug.Assert(reader.ZLibWindowSize == 47, "A GZip reader must be designated with ZLibWindowSize == 47. Other header formats aren't supported by ZLib.");
return new InflaterZlib(reader.ZLibWindowSize);
}
}
else
{
return new InflaterManaged(reader);
}
}

Expand Down Expand Up @@ -256,8 +296,6 @@ public override int Read(byte[] array, int offset, int count)
break;
}

Debug.Assert(_inflater.NeedsInput(), "We can only run into this case if we are short of input");

int bytes = _stream.Read(_buffer, 0, _buffer.Length);
if (bytes <= 0)
{
Expand Down Expand Up @@ -421,7 +459,6 @@ public override void Write(byte[] array, int offset, int count)
InternalWrite(array, offset, count);
}

// isAsync always seems to be false. why do we have it?
internal void InternalWrite(byte[] array, int offset, int count)
{
DoMaintenance(array, offset, count);
Expand Down Expand Up @@ -496,7 +533,7 @@ private void PurgeBuffers(bool disposing)
if (_mode != CompressionMode.Compress)
return;

// Some deflaters (e.g. ZLib write more than zero bytes for zero bytes inpuits.
// Some deflaters (e.g. ZLib) write more than zero bytes for zero byte inputs.
// This round-trips and we should be ok with this, but our legacy managed deflater
// always wrote zero output for zero input and upstack code (e.g. GZipStream)
// took dependencies on it. Thus, make sure to only "flush" when we actually had
Expand Down Expand Up @@ -564,10 +601,13 @@ protected override void Dispose(bool disposing)
{
if (_deflater != null)
_deflater.Dispose();
if (_inflater != null)
_inflater.Dispose();
}
finally
{
_deflater = null;
_inflater = null;
base.Dispose(disposing);
}
} // finally
Expand Down
54 changes: 23 additions & 31 deletions src/System.IO.Compression/src/System/IO/Compression/DeflaterZLib.cs
View file
Expand Up @@ -14,7 +14,7 @@ namespace System.IO.Compression
internal class DeflaterZLib : IDeflater
{
private ZLibNative.ZLibStreamHandle _zlibStream;
private GCHandle? _inputBufferHandle;
private GCHandle _inputBufferHandle;
private bool _isDisposed;

// Note, DeflateStream or the deflater do not try to be thread safe.
Expand Down Expand Up @@ -81,31 +81,32 @@ internal DeflaterZLib(CompressionLevel compressionLevel)
DeflateInit(zlibCompressionLevel, windowBits, memLevel, strategy);
}

~DeflaterZLib()
{
Dispose(false);
}

void IDisposable.Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}

[SecuritySafeCritical]
protected virtual void Dispose(bool disposing)
{
if (disposing && !_isDisposed)
if (!_isDisposed)
{
if (_inputBufferHandle.HasValue)
DeallocateInputBufferHandle();
if (disposing)
_zlibStream.Dispose();

_zlibStream.Dispose();
if (_inputBufferHandle.IsAllocated)
DeallocateInputBufferHandle();
_isDisposed = true;
}
}

private bool NeedsInput()
{
// Convenience method to call NeedsInput privately without a cast.
return ((IDeflater)this).NeedsInput();
}

bool IDeflater.NeedsInput()
public bool NeedsInput()
{
return 0 == _zlibStream.AvailIn;
}
Expand All @@ -115,7 +116,7 @@ void IDeflater.SetInput(byte[] inputBuffer, int startIndex, int count)
Debug.Assert(NeedsInput(), "We have something left in previous input!");
Debug.Assert(null != inputBuffer);
Debug.Assert(startIndex >= 0 && count >= 0 && count + startIndex <= inputBuffer.Length);
Debug.Assert(!_inputBufferHandle.HasValue);
Debug.Assert(!_inputBufferHandle.IsAllocated);

if (0 == count)
return;
Expand All @@ -124,7 +125,7 @@ void IDeflater.SetInput(byte[] inputBuffer, int startIndex, int count)
{
_inputBufferHandle = GCHandle.Alloc(inputBuffer, GCHandleType.Pinned);

_zlibStream.NextIn = _inputBufferHandle.Value.AddrOfPinnedObject() + startIndex;
_zlibStream.NextIn = _inputBufferHandle.AddrOfPinnedObject() + startIndex;
_zlibStream.AvailIn = (uint)count;
}
}
Expand All @@ -135,8 +136,7 @@ int IDeflater.GetDeflateOutput(byte[] outputBuffer)

Debug.Assert(null != outputBuffer, "Can't pass in a null output buffer!");
Debug.Assert(!NeedsInput(), "GetDeflateOutput should only be called after providing input");
Debug.Assert(_inputBufferHandle.HasValue);
Debug.Assert(_inputBufferHandle.Value.IsAllocated);
Debug.Assert(_inputBufferHandle.IsAllocated);

try
{
Expand All @@ -147,39 +147,33 @@ int IDeflater.GetDeflateOutput(byte[] outputBuffer)
finally
{
// Before returning, make sure to release input buffer if necesary:
if (0 == _zlibStream.AvailIn && _inputBufferHandle.HasValue)
if (0 == _zlibStream.AvailIn && _inputBufferHandle.IsAllocated)
DeallocateInputBufferHandle();
}
}

private ZErrorCode ReadDeflateOutput(byte[] outputBuffer, ZFlushCode flushCode, out int bytesRead)
private unsafe ZErrorCode ReadDeflateOutput(byte[] outputBuffer, ZFlushCode flushCode, out int bytesRead)
{
lock (_syncLock)
{
GCHandle outputBufferHndl = GCHandle.Alloc(outputBuffer, GCHandleType.Pinned);

try
fixed (byte* bufPtr = outputBuffer)
{
_zlibStream.NextOut = outputBufferHndl.AddrOfPinnedObject();
_zlibStream.NextOut = (IntPtr)bufPtr;
_zlibStream.AvailOut = (uint)outputBuffer.Length;

ZErrorCode errC = Deflate(flushCode);
bytesRead = outputBuffer.Length - (int)_zlibStream.AvailOut;

return errC;
}
finally
{
outputBufferHndl.Free();
}
}
}

bool IDeflater.Finish(byte[] outputBuffer, out int bytesRead)
{
Debug.Assert(null != outputBuffer, "Can't pass in a null output buffer!");
Debug.Assert(NeedsInput(), "We have something left in previous input!");
Debug.Assert(!_inputBufferHandle.HasValue);
Debug.Assert(!_inputBufferHandle.IsAllocated);

// Note: we require that NeedsInput() == true, i.e. that 0 == _zlibStream.AvailIn.
// If there is still input left we should never be getting here; instead we
Expand All @@ -196,15 +190,13 @@ bool IDeflater.Finish(byte[] outputBuffer, out int bytesRead)

private void DeallocateInputBufferHandle()
{
Debug.Assert(_inputBufferHandle.HasValue);
Debug.Assert(_inputBufferHandle.Value.IsAllocated);
Debug.Assert(_inputBufferHandle.IsAllocated);

lock (_syncLock)
{
_zlibStream.AvailIn = 0;
_zlibStream.NextIn = ZLibNative.ZNullPtr;
_inputBufferHandle.Value.Free();
_inputBufferHandle = null;
_inputBufferHandle.Free();
}
}

Expand Down
24 changes: 24 additions & 0 deletions src/System.IO.Compression/src/System/IO/Compression/FileFormats.cs
View file
Expand Up @@ -16,6 +16,30 @@ internal interface IFileFormatReader
bool ReadFooter(InputBuffer input);
void UpdateWithBytesRead(byte[] buffer, int offset, int bytesToCopy);
void Validate();

/// <summary>
/// A reader corresponds to an expected file format and contains methods
/// to read header/footer data from a file of that format. If the Zlib library
/// is instead being used and the file format is supported, we can simply pass
/// a supported WindowSize and let Zlib do the header/footer parsing for us.
///
/// This Property allows getting of a ZLibWindowSize that can be used in place
/// of manually parsing the raw data stream.
/// </summary>
/// <return>
/// For raw data, return -8..-15
/// For GZip header detection and decoding, return 16..31
/// For GZip and Zlib header detection and decoding, return 32..47
/// </return>
/// <remarks>
/// The windowBits parameter for inflation must be greater than or equal to the
/// windowBits parameter used in deflation.
/// </remarks>
/// <remarks>
/// If the incorrect header information is used, zlib inflation will likely throw a
/// Z_DATA_ERROR exception.
///</remarks>
int ZLibWindowSize { get; }
}
}

11 changes: 11 additions & 0 deletions src/System.IO.Compression/src/System/IO/Compression/GZipDecoder.cs
View file
Expand Up @@ -21,6 +21,17 @@ internal class GZipDecoder : IFileFormatReader
private uint _actualCrc32;
private long _actualStreamSizeModulo;

public int ZLibWindowSize
{
get
{
// Since we don't necessarily know what WindowSize the stream was compressed with,
// we use the upper bound of (32..47) as the WindowSize for decompression. This enables
// detection for Zlib and GZip headers
return 47;
}
}

public GZipDecoder()
{
Reset();
Expand Down
29 changes: 10 additions & 19 deletions src/System.IO.Compression/src/System/IO/Compression/GZipStream.cs
View file
Expand Up @@ -21,8 +21,15 @@ public GZipStream(Stream stream, CompressionMode mode)

public GZipStream(Stream stream, CompressionMode mode, bool leaveOpen)
{
_deflateStream = new DeflateStream(stream, mode, leaveOpen);
SetDeflateStreamFileFormatter(mode);
if (mode == CompressionMode.Decompress)
{
_deflateStream = new DeflateStream(stream, leaveOpen, new GZipDecoder());
}
else
{
_deflateStream = new DeflateStream(stream, mode, leaveOpen);
_deflateStream.SetFileFormatWriter(new GZipFormatter());
}
}


Expand All @@ -38,25 +45,9 @@ public GZipStream(Stream stream, CompressionLevel compressionLevel)
public GZipStream(Stream stream, CompressionLevel compressionLevel, bool leaveOpen)
{
_deflateStream = new DeflateStream(stream, compressionLevel, leaveOpen);
SetDeflateStreamFileFormatter(CompressionMode.Compress);
_deflateStream.SetFileFormatWriter(new GZipFormatter());
}


private void SetDeflateStreamFileFormatter(CompressionMode mode)
{
if (mode == CompressionMode.Compress)
{
IFileFormatWriter writeCommand = new GZipFormatter();
_deflateStream.SetFileFormatWriter(writeCommand);
}
else
{
IFileFormatReader readCommand = new GZipDecoder();
_deflateStream.SetFileFormatReader(readCommand);
}
}


public override bool CanRead
{
get
Expand Down

0 comments on commit 6941118

Please sign in to comment.