Skip to content

Commit

Permalink
COMMON: Add decompressDeflateChunk()
Browse files Browse the repository at this point in the history 
Useful for decompressing and reassembling files that have been split
into individually zlib-compressed chunks.
  • Loading branch information
@DrMcCoy
DrMcCoy committed Jul 21, 2018
1 parent b3a3230 commit 1382039
Show file tree
Hide file tree
Showing 3 changed files with 175 additions and 4 deletions.
61 changes: 57 additions & 4 deletions src/common/deflate.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,19 +34,26 @@

namespace Common {

static void initZStream(z_stream &strm, int windowBits, size_t size, const byte *data) {
/* Initialize the zlib data stream for decompression with our input data.
static void setZStreamInput(z_stream &strm, size_t size, const byte *data) {
/* Set the input data for the zlib data stream.
*
* This ugly const cast is necessary because the zlib API wants a non-const
* next_in pointer by default. Unless we define ZLIB_CONST, but that only
* appeared in zlib 1.2.5.3. Not really worth bumping our required zlib
* version for, IMHO. */

strm.avail_in = size;
strm.next_in = const_cast<byte *>(data);
}

static void initZStream(z_stream &strm, int windowBits, size_t size, const byte *data) {
/* Initialize the zlib data stream for decompression with our input data. */

strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;

strm.avail_in = size;
strm.next_in = const_cast<byte *>(data);
setZStreamInput(strm, size, data);

int zResult = inflateInit2(&strm, windowBits);
if (zResult != Z_OK)
Expand Down Expand Up @@ -150,4 +157,50 @@ SeekableReadStream *decompressDeflateWithoutOutputSize(ReadStream &input, size_t
return new MemoryReadStream(decompressedData, size, true);
}

size_t decompressDeflateChunk(SeekableReadStream &input, int windowBits,
byte *output, size_t outputSize, unsigned int frameSize) {

z_stream strm;
BOOST_SCOPE_EXIT( (&strm) ) {
inflateEnd(&strm);
} BOOST_SCOPE_EXIT_END

initZStream(strm, windowBits, 0, 0);

strm.avail_out = outputSize;
strm.next_out = output;

ScopedArray<byte> inputData(new byte[frameSize]);

/* As long as the zlib stream has not ended, the chunk end was not reached.
* Read a frame from the input buffer and decompress. */

int zResult = 0;
do {
if (strm.avail_in == 0) {
const size_t inputSize = MIN<size_t>(input.size() - input.pos(), frameSize);
if (inputSize == 0)
throw Exception("Failed to inflate: input buffer empty, stream not ended");

if (input.read(inputData.get(), inputSize) != inputSize)
throw Exception(kReadError);

setZStreamInput(strm, inputSize, inputData.get());
}

// Decompress. Z_SYNC_FLUSH, because we want to decompress partwise.
zResult = inflate(&strm, Z_SYNC_FLUSH);
if (zResult != Z_STREAM_END && zResult != Z_OK)
throw Exception("Failed to inflate: %s (%d)", zError(zResult), zResult);

} while (zResult != Z_STREAM_END);

/* Since we don't know where the chunk ends beforehand, we probably have
* read past the chunk. Now that we know where the zlib stream ends, we
* know where the chunk ended, so we can seek back to that place. */
input.seek(- static_cast<ptrdiff_t>(strm.avail_in), SeekableReadStream::kOriginCurrent);

return strm.total_out;
}

} // End of namespace Common
17 changes: 17 additions & 0 deletions src/common/deflate.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,23 @@ SeekableReadStream *decompressDeflate(ReadStream &input, size_t inputSize,
SeekableReadStream *decompressDeflateWithoutOutputSize(ReadStream &input, size_t inputSize,
int windowBits, unsigned int frameSize = 4096);

/** Decompress (inflate) using zlib's DEFLATE algorithm, until a stream end marker was reached.
*
* Used for decompressing and reassembling a file that was split into multiple,
* individually compressed chunks. This function decompresses a single chunk.
*
* @param input The compressed input data.
* @param windowBits The base two logarithm of the window size (the size of
* the history buffer). See the zlib documentation on
* inflateInit2() for details.
* @param output The output of the current chunk is stored here.
* @param outputSize Maximum number of bytes to write into output.
* @param frameSize The size of frame for reading from the input stream.
* @return The number of bytes written to output.
*/
size_t decompressDeflateChunk(SeekableReadStream &input, int windowBits, byte *output, size_t outputSize,
unsigned int frameSize = 4096);

} // End of namespace Common

#endif // COMMON_DEFLATE_H
101 changes: 101 additions & 0 deletions tests/common/deflate.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,3 +165,104 @@ GTEST_TEST(DEFLATE, decompressFailInputCut) {
kSizeDecompressed, Common::kWindowBitsMaxRaw),
Common::Exception);
}

GTEST_TEST(DEFLATE, decompressChunked) {
static const byte kDataChunked[] = {
0x78,0x9C,0x25,0x8B,0x31,0x0E,0xC2,0x40,0x0C,0x04,0xFB,0xBC,0x62,0x1F,0x80,0xF2,
0x00,0x7E,0x40,0x43,0x03,0x52,0x6A,0x4B,0xB7,0x21,0x27,0x7C,0xB6,0x38,0x3B,0xE1,
0xFB,0x5C,0x84,0x34,0xD5,0x8C,0xE6,0x86,0xC6,0x84,0x20,0xBB,0x1C,0x54,0x65,0xC7,
0xDA,0xBD,0x41,0x6C,0x90,0xF5,0xB3,0x13,0x2A,0x56,0xA6,0x65,0x73,0x84,0xD4,0x72,
0xC5,0xF3,0xEB,0x38,0x24,0xC6,0x63,0x65,0x5C,0xBB,0xBD,0x95,0x11,0x50,0xBE,0x02,
0xBE,0x22,0xD2,0x8D,0xD3,0x23,0xCF,0x5A,0x0D,0xB9,0x11,0x85,0xC1,0x9E,0x33,0xEE,
0x94,0x7E,0x8A,0x76,0x81,0xFF,0x4B,0x88,0xFD,0x00,0x6B,0xA0,0x2D,0x1E,0x78,0x9C,
0x25,0xCC,0xC1,0x0D,0xC3,0x30,0x0C,0x03,0xC0,0xBF,0xA7,0xE0,0x00,0x5A,0x22,0xBF,
0x0E,0xD0,0x05,0x8C,0x58,0x6E,0x84,0x28,0x52,0x60,0x39,0xF5,0xFA,0x15,0xD0,0x0F,
0x1F,0x07,0x92,0x8D,0xCA,0xAB,0x6A,0x47,0x3C,0x76,0x12,0x2A,0xE2,0xA8,0x73,0xF2,
0xE0,0x86,0xAF,0x44,0xFD,0x30,0x54,0x38,0x08,0xEB,0xF0,0x60,0xF4,0xE1,0xCB,0xA8,
0x6C,0xD6,0xB0,0x86,0xD8,0xA9,0x59,0x53,0xB9,0x73,0x97,0x12,0xC6,0x3C,0xE0,0x1D,
0xBB,0x6B,0xCB,0xB8,0xAE,0x54,0x2A,0x6F,0x56,0xC5,0xCC,0x57,0xC8,0x0C,0xC4,0xFE,
0xE8,0x3D,0x7D,0x60,0xFD,0xD9,0x7F,0x30,0xA0,0x2D,0x35,0x78,0x9C,0x25,0x8C,0x41,
0x0A,0xC3,0x30,0x0C,0x04,0xEF,0x7E,0xC5,0x3E,0x20,0x2F,0xE8,0xAD,0x7F,0x08,0xF4,
0x2C,0xE2,0x4D,0x2D,0x1A,0xCB,0xC1,0x52,0x03,0xFD,0x7D,0x5C,0x7C,0xDB,0x99,0x81,
0x75,0xE2,0x14,0x77,0x6D,0xE6,0xE8,0x94,0x9C,0x5E,0x45,0xB7,0x82,0x1F,0x03,0xFE,
0xED,0x97,0x5E,0x5C,0xE0,0x21,0xF5,0x64,0x46,0x33,0x44,0xA1,0x13,0x87,0xEE,0x3C,
0xE8,0x3E,0x50,0xED,0xED,0x4B,0x5A,0x0B,0x51,0xC4,0xF2,0x10,0x12,0xA8,0x6D,0xFB,
0xF0,0xBF,0x59,0x31,0xE5,0xA8,0x94,0x1E,0x33,0xEF,0xCC,0x8F,0xF4,0xB4,0xF9,0x77,
0x03,0xBB,0x83,0x2D,0x90,0x78,0x9C,0x1D,0xCD,0xB1,0x0D,0xC3,0x30,0x0C,0x44,0xD1,
0x5E,0x53,0x5C,0x2A,0x37,0x9E,0xC0,0x33,0xC4,0xC9,0x0C,0x0C,0xC4,0x58,0x84,0x22,
0x51,0x10,0x05,0x04,0xCC,0xF4,0x91,0x5D,0x5D,0xF3,0x1F,0x2E,0x31,0x1A,0x47,0xB6,
0x41,0x1F,0x8C,0xC4,0xC6,0xF8,0x6A,0x8F,0x06,0x6A,0x8D,0xA9,0x6F,0x61,0xD9,0x1D,
0x95,0x0A,0x43,0x0C,0xCF,0x9F,0x17,0xAA,0x51,0xC8,0x56,0x64,0xA9,0x07,0xF4,0x7D,
0xAD,0x6D,0xE1,0xAE,0x9A,0xA1,0x15,0xC5,0x4F,0x9F,0x67,0xE0,0x8C,0x5D,0x8E,0x34,
0x7C,0xC5,0x34,0x98,0x17,0x8D,0xA4,0xDF,0x96,0xF0,0xD0,0x91,0x4E,0xFC,0x62,0x93,
0xC8,0x7F,0x65,0x65,0x2C,0xDE,0x78,0x9C,0x1D,0x8B,0xC1,0x09,0xC3,0x30,0x10,0x04,
0xFF,0xAE,0x62,0x0B,0x08,0x6E,0x25,0x10,0xD2,0xC0,0x5A,0x5A,0x63,0xE3,0xB3,0x04,
0x77,0x4A,0x84,0xBB,0x8F,0xE2,0xDF,0xCE,0x30,0x0B,0xD7,0xC9,0xBD,0xC4,0x8C,0x57,
0xFD,0x94,0x8C,0xB6,0x09,0x59,0x89,0xD7,0xF4,0x5C,0x07,0xB0,0x21,0x55,0xAB,0x11,
0x34,0x74,0x57,0x3A,0x1E,0x58,0xFE,0x9D,0x29,0x02,0x1C,0xFD,0x42,0xD7,0xF4,0x1E,
0x27,0xAB,0x45,0xB7,0x31,0x7D,0x65,0x88,0x31,0x03,0xD1,0x5C,0x2D,0x6D,0x58,0xE9,
0x60,0xE7,0x35,0xFF,0x00,0x9C,0x78,0x27,0x94
};

static const size_t kSizeDecompressed = strlen(kDataUncompressed);
static const size_t kSizeChunked = sizeof(kDataChunked);

Common::MemoryReadStream input(kDataChunked, kSizeChunked);

byte *output = new byte[kSizeDecompressed];

size_t offset = 0;
size_t bytesLeft = kSizeDecompressed;

size_t bytesChunk;

bytesChunk = Common::decompressDeflateChunk(input, Common::kWindowBitsMax,
output + offset, bytesLeft, 16);

EXPECT_EQ(bytesChunk, 128);
EXPECT_EQ(input.pos(), 110);

offset += bytesChunk;
bytesLeft -= bytesChunk;

bytesChunk = Common::decompressDeflateChunk(input, Common::kWindowBitsMax,
output + offset, bytesLeft, 16);

EXPECT_EQ(bytesChunk, 128);
EXPECT_EQ(input.pos(), 219);

offset += bytesChunk;
bytesLeft -= bytesChunk;

bytesChunk = Common::decompressDeflateChunk(input, Common::kWindowBitsMax,
output + offset, bytesLeft, 16);

EXPECT_EQ(bytesChunk, 128);
EXPECT_EQ(input.pos(), 325);

offset += bytesChunk;
bytesLeft -= bytesChunk;

bytesChunk = Common::decompressDeflateChunk(input, Common::kWindowBitsMax,
output + offset, bytesLeft, 16);

EXPECT_EQ(bytesChunk, 128);
EXPECT_EQ(input.pos(), 438);

offset += bytesChunk;
bytesLeft -= bytesChunk;

bytesChunk = Common::decompressDeflateChunk(input, Common::kWindowBitsMax,
output + offset, bytesLeft, 16);

EXPECT_EQ(bytesChunk, 111);
EXPECT_EQ(input.pos(), input.size());

offset += bytesChunk;
bytesLeft -= bytesChunk;

for (size_t i = 0; i < kSizeDecompressed; i++)
EXPECT_EQ(output[i], kDataUncompressed[i]) << "At index " << i;

delete[] output;
}

0 comments on commit 1382039

Please sign in to comment.