Skip to content

Commit

Permalink
Merge pull request #2914 from fgda/issue_2138
Browse files Browse the repository at this point in the history 
Fix Issue 2138 - Allow more than 65535 files in Zip archives
  • Loading branch information
@andralex
andralex committed Jan 27, 2015
2 parents cd60676 + 8ef18e6 commit ad5ae87
Showing 1 changed file with 152 additions and 10 deletions.
162 changes: 152 additions & 10 deletions std/zip.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,11 @@ final class ZipArchive
private uint _diskStartDir;
private uint _numEntries;
private uint _totalEntries;
private bool _isZip64;
static const ushort zip64ExtractVersion = 45;
static const int digiSignLength = 6;
static const int eocd64LocLength = 20;
static const int eocd64Length = 56;

/// Read Only: array representing the entire contents of the archive.
@property ubyte[] data() { return _data; }
Expand All @@ -283,6 +288,12 @@ final class ZipArchive
/// Read Only: number of ArchiveMembers in the directory.
@property uint numEntries() { return _numEntries; }
@property uint totalEntries() { return _totalEntries; } /// ditto

/// True when the archive is in Zip64 format.
@property bool isZip64() { return _isZip64; }

/// Set this to true to force building a Zip64 archive.
@property void isZip64(bool value) { _isZip64 = value; }
/**
* Read Only: array indexed by the name of each member of the archive.
* All the members of the archive can be accessed with a foreach loop:
Expand Down Expand Up @@ -377,6 +388,11 @@ final class ZipArchive
}
assert(de._compressedData.length == de._compressedSize);

if (to!ulong(archiveSize) + 30 + de.name.length + de.extra.length + de.compressedSize
+ directorySize + 46 + de.name.length + de.extra.length + de.comment.length
+ 22 + comment.length + eocd64LocLength + eocd64Length > uint.max)
throw new ZipException("zip files bigger than 4 GB are unsupported");

archiveSize += 30 + de.name.length +
de.extra.length +
de.compressedSize;
Expand All @@ -385,7 +401,13 @@ final class ZipArchive
de.comment.length;
}

_data = new ubyte[archiveSize + directorySize + 22 + comment.length];
if (!isZip64 && _directory.length > ushort.max)
_isZip64 = true;
uint dataSize = archiveSize + directorySize + 22 + cast(uint)comment.length;
if (isZip64)
dataSize += eocd64LocLength + eocd64Length;

_data = new ubyte[dataSize];

// Populate the data[]

Expand Down Expand Up @@ -447,13 +469,37 @@ final class ZipArchive
}
_totalEntries = numEntries;

if (isZip64)
{
// Write zip64 end of central directory record
uint eocd64Offset = i;
_data[i .. i + 4] = cast(ubyte[])"PK\x06\x06";
putUlong (i + 4, eocd64Length - 12);
putUshort(i + 12, zip64ExtractVersion);
putUshort(i + 14, zip64ExtractVersion);
putUint (i + 16, diskNumber);
putUint (i + 20, diskStartDir);
putUlong (i + 24, numEntries);
putUlong (i + 32, totalEntries);
putUlong (i + 40, directorySize);
putUlong (i + 48, directoryOffset);
i += eocd64Length;

// Write zip64 end of central directory record locator
_data[i .. i + 4] = cast(ubyte[])"PK\x06\x07";
putUint (i + 4, diskNumber);
putUlong (i + 8, eocd64Offset);
putUint (i + 16, 1);
i += eocd64LocLength;
}

// Write end record
endrecOffset = i;
_data[i .. i + 4] = cast(ubyte[])"PK\x05\x06";
putUshort(i + 4, cast(ushort)diskNumber);
putUshort(i + 6, cast(ushort)diskStartDir);
putUshort(i + 8, cast(ushort)numEntries);
putUshort(i + 10, cast(ushort)totalEntries);
putUshort(i + 8, (numEntries > ushort.max ? ushort.max : cast(ushort)numEntries));
putUshort(i + 10, (totalEntries > ushort.max ? ushort.max : cast(ushort)totalEntries));
putUint (i + 12, directorySize);
putUint (i + 16, directoryOffset);
putUshort(i + 20, cast(ushort)comment.length);
Expand Down Expand Up @@ -484,34 +530,85 @@ final class ZipArchive
*/

this(void[] buffer)
{ int iend;
int i;
{ uint iend;
uint i;
int endcommentlength;
uint directorySize;
uint directoryOffset;

this._data = cast(ubyte[]) buffer;

if (data.length > uint.max - 2)
throw new ZipException("zip files bigger than 4 GB are unsupported");

// Find 'end record index' by searching backwards for signature
iend = to!uint(data.length) - 66000;
if (iend < 0)
iend = 0;
iend = (data.length > 66000 ? to!uint(data.length - 66000) : 0);
for (i = to!uint(data.length) - 22; 1; i--)
{
if (i < iend)
if (i < iend || i >= data.length)
throw new ZipException("no end record");

if (_data[i .. i + 4] == cast(ubyte[])"PK\x05\x06")
{
endcommentlength = getUshort(i + 20);
if (i + 22 + endcommentlength > data.length)
if (i + 22 + endcommentlength > data.length
|| i + 22 + endcommentlength < i)
continue;
comment = cast(string)(_data[i + 22 .. i + 22 + endcommentlength]);
endrecOffset = i;

uint k = i - eocd64LocLength;
if (k < i && _data[k .. k + 4] == cast(ubyte[])"PK\x06\x07")
{
_isZip64 = true;
i = k;
}

break;
}
}

if (isZip64)
{
// Read Zip64 record data
uint eocd64LocStart = i;
ulong eocdOffset = getUlong(i + 8);
if (eocdOffset + eocd64Length > _data.length)
throw new ZipException("corrupted directory");

i = to!uint(eocdOffset);
if (_data[i .. i + 4] != cast(ubyte[])"PK\x06\x06")
throw new ZipException("invalid Zip EOCD64 signature");

ulong eocd64Size = getUlong(i + 4);
if (eocd64Size + i - 12 > data.length)
throw new ZipException("invalid Zip EOCD64 size");

_diskNumber = getUint(i + 16);
_diskStartDir = getUint(i + 20);

ulong numEntriesUlong = getUlong(i + 24);
ulong totalEntriesUlong = getUlong(i + 32);
ulong directorySizeUlong = getUlong(i + 40);
ulong directoryOffsetUlong = getUlong(i + 48);

if (numEntriesUlong > uint.max)
throw new ZipException("supposedly more than 4294967296 files in archive");

if (numEntriesUlong != totalEntriesUlong)
throw new ZipException("multiple disk zips not supported");

if (directorySizeUlong > i || directoryOffsetUlong > i
|| directorySizeUlong + directoryOffsetUlong > i)
throw new ZipException("corrupted directory");

_numEntries = to!uint(numEntriesUlong);
_totalEntries = to!uint(totalEntriesUlong);
directorySize = to!uint(directorySizeUlong);
directoryOffset = to!uint(directoryOffsetUlong);
}
else
{
// Read end record data
_diskNumber = getUshort(i + 4);
_diskStartDir = getUshort(i + 6);
Expand All @@ -527,6 +624,7 @@ final class ZipArchive

if (directoryOffset + directorySize > i)
throw new ZipException("corrupted directory");
}

i = directoryOffset;
for (int n = 0; n < numEntries; n++)
Expand Down Expand Up @@ -662,6 +760,12 @@ final class ZipArchive
return littleEndianToNative!uint(result);
}

ulong getUlong(int i)
{
ubyte[8] result = data[i .. i + 8];
return littleEndianToNative!ulong(result);
}

void putUshort(int i, ushort us)
{
data[i .. i + 2] = nativeToLittleEndian(us);
Expand All @@ -671,6 +775,11 @@ final class ZipArchive
{
data[i .. i + 4] = nativeToLittleEndian(ui);
}

void putUlong(int i, ulong ul)
{
data[i .. i + 8] = nativeToLittleEndian(ul);
}
}

debug(print)
Expand Down Expand Up @@ -705,4 +814,37 @@ unittest
auto zip3 = new ZipArchive(data1);
zip3.build();
assert(zip3.directory["foo"].compressedSize == am1.compressedSize);

// Test if packing and unpacking produces the original data
import std.random : uniform, MinstdRand0;
import std.stdio, std.conv;
MinstdRand0 gen;
const uint itemCount = 20, minSize = 10, maxSize = 500;
foreach (variant; 0..2)
{
bool useZip64 = !!variant;
zip1 = new ZipArchive();
zip1.isZip64 = useZip64;
ArchiveMember[itemCount] ams;
foreach (i; 0..itemCount)
{
ams[i] = new ArchiveMember();
ams[i].name = to!string(i);
ams[i].expandedData = new ubyte[](uniform(minSize, maxSize));
foreach (ref ubyte c; ams[i].expandedData)
c = cast(ubyte)(uniform(0, 256));
ams[i].compressionMethod(CompressionMethod.deflate);
zip1.addMember(ams[i]);
}
auto zippedData = zip1.build();
zip2 = new ZipArchive(zippedData);
assert(zip2.isZip64 == useZip64);
foreach (am; ams)
{
am2 = zip2.directory[am.name];
zip2.expand(am2);
assert(am.crc32 == am2.crc32);
assert(am.expandedData == am2.expandedData);
}
}
}

0 comments on commit ad5ae87

Please sign in to comment.