Skip to content

Commit

Permalink
- zip64 implementation
Browse files Browse the repository at this point in the history 
Mainly for number of files, which is limited to 65535 for standard Zip.
  • Loading branch information
@coelckers
coelckers committed Apr 4, 2022
1 parent 16c81f0 commit 167e043
Show file tree
Hide file tree
Showing 4 changed files with 112 additions and 31 deletions.
104 changes: 80 additions & 24 deletions source/common/filesystem/file_zip.cpp
View file
Expand Up @@ -115,7 +115,7 @@ bool FCompressedBuffer::Decompress(char *destbuffer)
//
//-----------------------------------------------------------------------

static uint32_t Zip_FindCentralDir(FileReader &fin)
static uint32_t Zip_FindCentralDir(FileReader &fin, bool* zip64)
{
unsigned char buf[BUFREADCOMMENT + 4];
uint32_t FileSize;
Expand Down Expand Up @@ -145,8 +145,9 @@ static uint32_t Zip_FindCentralDir(FileReader &fin)

for (i = (int)uReadSize - 3; (i--) > 0;)
{
if (buf[i] == 'P' && buf[i+1] == 'K' && buf[i+2] == 5 && buf[i+3] == 6)
if (buf[i] == 'P' && buf[i+1] == 'K' && (buf[i+2] == 5 || buf[i+2] == 6) && buf[i+3] == 6)
{
*zip64 = buf[i+2] == 6;
uPosFound = uReadPos + i;
break;
}
Expand All @@ -172,8 +173,8 @@ FZipFile::FZipFile(const char * filename, FileReader &file)

bool FZipFile::Open(bool quiet, LumpFilterInfo* filter)
{
uint32_t centraldir = Zip_FindCentralDir(Reader);
FZipEndOfCentralDirectory info;
bool zip64 = false;
uint32_t centraldir = Zip_FindCentralDir(Reader, &zip64);
int skipped = 0;

Lumps = NULL;
Expand All @@ -184,25 +185,50 @@ bool FZipFile::Open(bool quiet, LumpFilterInfo* filter)
return false;
}

// Read the central directory info.
Reader.Seek(centraldir, FileReader::SeekSet);
Reader.Read(&info, sizeof(FZipEndOfCentralDirectory));

// No multi-disk zips!
if (info.NumEntries != info.NumEntriesOnAllDisks ||
info.FirstDisk != 0 || info.DiskNumber != 0)
uint64_t dirsize, DirectoryOffset;
if (!zip64)
{
if (!quiet) Printf(TEXTCOLOR_RED "\n%s: Multipart Zip files are not supported.\n", FileName.GetChars());
return false;
FZipEndOfCentralDirectory info;
// Read the central directory info.
Reader.Seek(centraldir, FileReader::SeekSet);
Reader.Read(&info, sizeof(FZipEndOfCentralDirectory));

// No multi-disk zips!
if (info.NumEntries != info.NumEntriesOnAllDisks ||
info.FirstDisk != 0 || info.DiskNumber != 0)
{
if (!quiet) Printf(TEXTCOLOR_RED "\n%s: Multipart Zip files are not supported.\n", FileName.GetChars());
return false;
}

NumLumps = LittleShort(info.NumEntries);
dirsize = LittleLong(info.DirectorySize);
DirectoryOffset = LittleLong(info.DirectoryOffset);
}
else
{
FZipEndOfCentralDirectory64 info;
// Read the central directory info.
Reader.Seek(centraldir, FileReader::SeekSet);
Reader.Read(&info, sizeof(FZipEndOfCentralDirectory));

// No multi-disk zips!
if (info.NumEntries != info.NumEntriesOnAllDisks ||
info.FirstDisk != 0 || info.DiskNumber != 0)
{
if (!quiet) Printf(TEXTCOLOR_RED "\n%s: Multipart Zip files are not supported.\n", FileName.GetChars());
return false;
}

NumLumps = LittleLong(info.NumEntries);
dirsize = info.DirectorySize;
DirectoryOffset = info.DirectoryOffset;
}

NumLumps = LittleShort(info.NumEntries);
Lumps = new FZipLump[NumLumps];

// Load the entire central directory. Too bad that this contains variable length entries...
int dirsize = LittleLong(info.DirectorySize);
void *directory = malloc(dirsize);
Reader.Seek(LittleLong(info.DirectoryOffset), FileReader::SeekSet);
Reader.Seek(DirectoryOffset, FileReader::SeekSet);
Reader.Read(directory, dirsize);

char *dirptr = (char*)directory;
Expand Down Expand Up @@ -305,7 +331,7 @@ bool FZipFile::Open(bool quiet, LumpFilterInfo* filter)
if (name0.IsNotEmpty()) name = name.Mid(name0.Len());

// skip Directories
if (name.IsEmpty() || (name.Back() == '/' && LittleLong(zip_fh->UncompressedSize) == 0))
if (name.IsEmpty() || (name.Back() == '/' && LittleLong(zip_fh->UncompressedSize32) == 0))
{
skipped++;
continue;
Expand Down Expand Up @@ -335,9 +361,39 @@ bool FZipFile::Open(bool quiet, LumpFilterInfo* filter)

FixPathSeperator(name);
name.ToLower();

uint32_t UncompressedSize =LittleLong(zip_fh->UncompressedSize32);
uint32_t CompressedSize = LittleLong(zip_fh->CompressedSize32);
uint64_t LocalHeaderOffset = LittleLong(zip_fh->LocalHeaderOffset32);
if (zip_fh->ExtraLength > 0)
{
uint8_t* rawext = (uint8_t*)zip_fh + zip_fh->NameLength;
uint32_t ExtraLength = LittleLong(zip_fh->ExtraLength);

while (ExtraLength > 0)
{
auto zip_64 = (FZipCentralDirectoryInfo64BitExt*)rawext;
uint32_t BlockLength = LittleLong(zip_64->Length);
if (LittleLong(zip_64->Type) == 1 && BlockLength >= 0x18)
{
if (zip_64->CompressedSize > 0xffffffff || zip_64->UncompressedSize > 0xffffffff)
{
// The file system is limited to 32 bit file sizes;
if (!quiet) Printf(TEXTCOLOR_YELLOW "\n%s: '%s' is too large.\n", FileName.GetChars(), name.GetChars());
skipped++;
continue;
}
UncompressedSize = (uint32_t)zip_64->UncompressedSize;
CompressedSize = (uint32_t)zip_64->CompressedSize;
LocalHeaderOffset = zip_64->LocalHeaderOffset;
}
rawext += BlockLength + 4;
ExtraLength -= BlockLength + 4;
}
}

lump_p->LumpNameSetup(name);
lump_p->LumpSize = LittleLong(zip_fh->UncompressedSize);
lump_p->LumpSize = UncompressedSize;
lump_p->Owner = this;
// The start of the Reader will be determined the first time it is accessed.
lump_p->Flags = LUMPF_FULLPATH;
Expand All @@ -346,8 +402,8 @@ bool FZipFile::Open(bool quiet, LumpFilterInfo* filter)
if (lump_p->Method != METHOD_STORED) lump_p->Flags |= LUMPF_COMPRESSED;
lump_p->GPFlags = zip_fh->Flags;
lump_p->CRC32 = zip_fh->CRC32;
lump_p->CompressedSize = LittleLong(zip_fh->CompressedSize);
lump_p->Position = LittleLong(zip_fh->LocalHeaderOffset);
lump_p->CompressedSize = CompressedSize;
lump_p->Position = LocalHeaderOffset;
lump_p->CheckEmbedded(filter);

lump_p++;
Expand Down Expand Up @@ -585,15 +641,15 @@ int AppendCentralDirectory(FileWriter *zip_file, const char *filename, FCompress
dir.ModTime = LittleShort(dostime.first);
dir.ModDate = LittleShort(dostime.second);
dir.CRC32 = content.mCRC32;
dir.CompressedSize = LittleLong(content.mCompressedSize);
dir.UncompressedSize = LittleLong(content.mSize);
dir.CompressedSize32 = LittleLong(content.mCompressedSize);
dir.UncompressedSize32 = LittleLong(content.mSize);
dir.NameLength = LittleShort((unsigned short)strlen(filename));
dir.ExtraLength = 0;
dir.CommentLength = 0;
dir.StartingDiskNumber = 0;
dir.InternalAttributes = 0;
dir.ExternalAttributes = 0;
dir.LocalHeaderOffset = LittleLong(position);
dir.LocalHeaderOffset32 = LittleLong(position);

if (zip_file->Write(&dir, sizeof(dir)) != sizeof(dir) ||
zip_file->Write(filename, strlen(filename)) != strlen(filename))
Expand Down
4 changes: 2 additions & 2 deletions source/common/filesystem/file_zip.h
View file
Expand Up @@ -15,7 +15,7 @@ struct FZipLump : public FResourceLump
uint8_t Method;
bool NeedFileStart;
int CompressedSize;
int Position;
int64_t Position;
unsigned CRC32;

virtual FileReader *GetReader();
Expand Down Expand Up @@ -46,4 +46,4 @@ class FZipFile : public FResourceFile
};


#endif
#endif
33 changes: 29 additions & 4 deletions source/common/filesystem/w_zip.h
View file
Expand Up @@ -17,6 +17,22 @@ struct FZipEndOfCentralDirectory
uint16_t ZipCommentLength;
} FORCE_PACKED;

struct FZipEndOfCentralDirectory64
{
uint32_t Magic;
uint32_t StructSize;
uint16_t VersionMadeBy;
uint16_t VersionNeeded;
uint32_t DiskNumber;
uint32_t FirstDisk;
uint32_t NumEntries;
uint32_t NumEntriesOnAllDisks;
uint64_t DirectorySize;
uint64_t DirectoryOffset;
uint16_t ZipCommentLength;
} FORCE_PACKED;


// FZipFileInfo
struct FZipCentralDirectoryInfo
{
Expand All @@ -28,18 +44,28 @@ struct FZipCentralDirectoryInfo
uint16_t ModTime;
uint16_t ModDate;
uint32_t CRC32;
uint32_t CompressedSize;
uint32_t UncompressedSize;
uint32_t CompressedSize32;
uint32_t UncompressedSize32;
uint16_t NameLength;
uint16_t ExtraLength;
uint16_t CommentLength;
uint16_t StartingDiskNumber;
uint16_t InternalAttributes;
uint32_t ExternalAttributes;
uint32_t LocalHeaderOffset;
uint32_t LocalHeaderOffset32;
// file name and other variable length info follows
} FORCE_PACKED;

struct FZipCentralDirectoryInfo64BitExt
{
uint16_t Type;
uint16_t Length;
uint64_t UncompressedSize;
uint64_t CompressedSize;
uint64_t LocalHeaderOffset;
uint32_t DiskNo;
} FORCE_PACKED;

// FZipLocalHeader
struct FZipLocalFileHeader
{
Expand All @@ -57,7 +83,6 @@ struct FZipLocalFileHeader
// file name and other variable length info follows
} FORCE_PACKED;


#pragma pack()

#define ZIP_LOCALFILE MAKE_ID('P','K',3,4)
Expand Down
2 changes: 1 addition & 1 deletion source/core/rendering/hw_sections.cpp
View file
Expand Up @@ -481,7 +481,7 @@ static void GroupData(TArray<loopcollect>& collect, TArray<sectionbuildsector>&
builder.sections.Last().loops.Push(std::move(loop));
for (auto c: outside[a])
{
if (inside[c] == a)
if (inside[c] == int(a))
{
auto& iloop = sectloops[c];
builder.sections.Last().wallcount += iloop.Size() - 1;
Expand Down

0 comments on commit 167e043

Please sign in to comment.