diff --git a/doomsday/engine/portable/include/zipfile.h b/doomsday/engine/portable/include/zipfile.h index 9cac029366..7569e806ae 100644 --- a/doomsday/engine/portable/include/zipfile.h +++ b/doomsday/engine/portable/include/zipfile.h @@ -8,8 +8,8 @@ * * @see abstractfile.h, AbstractFile * - * @authors Copyright © 2003-2012 Jaakko Keränen - * @authors Copyright © 2005-2012 Daniel Swanson + * @author Copyright © 2003-2012 Jaakko Keränen + * @author Copyright © 2005-2012 Daniel Swanson * * @par License * GPL: http://www.gnu.org/licenses/gpl.html @@ -31,23 +31,35 @@ #include "abstractfile.h" #include "lumpinfo.h" -#include "lumpdirectory.h" +#ifdef __cplusplus +extern "C" { +#endif + +struct lumpdirectory_s; struct pathdirectorynode_s; -struct zipfile_s; + +#ifdef __cplusplus +} // extern "C" +#endif /** - * ZipFile instance. Constructed with ZipFile_New(). + * WadFile. Runtime representation of an opened WAD file. */ +#ifdef __cplusplus +extern "C" { +#endif + +struct zipfile_s; // The zipfile instance (opaque) typedef struct zipfile_s ZipFile; /** * Constructs a new zipfile. The zipfile has to be destroyed with ZipFile_Delete() * after it is not needed any more. * - * @param file Virtual file handle to the underlying file resource. - * @param path Virtual file system path to associate with the resultant zipfile. - * @param info File info descriptor for the resultant zipfile. A copy is made. + * @param file Virtual file handle to the underlying file resource. + * @param path Virtual file system path to associate with the resultant zipfile. + * @param info File info descriptor for the resultant zipfile. A copy is made. */ ZipFile* ZipFile_New(DFile* file, const char* path, const LumpInfo* info); @@ -59,18 +71,20 @@ void ZipFile_Delete(ZipFile* zip); /** * Publish lumps to the end of the specified @a directory. * - * @param zip ZipFile instance. - * @param directory Directory to publish to. + * @param zip ZipFile instance. + * @param directory Directory to publish to. + * * @return Number of lumps published to the directory (not necessarily the same as * ZipFile_LumpCount()). */ -int ZipFile_PublishLumpsToDirectory(ZipFile* zip, LumpDirectory* directory); +int ZipFile_PublishLumpsToDirectory(ZipFile* zip, struct lumpdirectory_s* directory); /** * Lookup a directory node for a lump contained by this zipfile. * - * @param zip ZipFile instance. - * @param lumpIdx Logical index for the lump within the zipfile's internal directory. + * @param zip ZipFile instance. + * @param lumpIdx Logical index for the lump within the zipfile's internal directory. + * * @return Found directory node else @c NULL if @a lumpIdx is not valid. */ struct pathdirectorynode_s* ZipFile_LumpDirectoryNode(ZipFile* zip, int lumpIdx); @@ -78,11 +92,12 @@ struct pathdirectorynode_s* ZipFile_LumpDirectoryNode(ZipFile* zip, int lumpIdx) /** * Lookup a lump info descriptor for a lump contained by this zipfile. * - * @param zip ZipFile instance. - * @param lumpIdx Logical index for the lump within the zipfile's internal directory. + * @param zip ZipFile instance. + * @param lumpIdx Logical index for the lump within the zipfile's internal directory. + * * @return Found lump info else @c NULL if @a lumpIdx is not valid. */ -const LumpInfo* ZipFile_LumpInfo(ZipFile* zip, int lumpIdx); +LumpInfo const* ZipFile_LumpInfo(ZipFile* zip, int lumpIdx); /** * Compose the full virtual file system path to a lump contained by this zipfile. @@ -90,9 +105,10 @@ const LumpInfo* ZipFile_LumpInfo(ZipFile* zip, int lumpIdx); * @note Always returns a valid string object. In the case of an invalid @a lumpIdx * a zero-length string is returned. * - * @param zip ZipFile instance. - * @param lumpIdx Logical index for the lump. - * @param delimiter Delimit directory separators using this character (default: '/'). + * @param zip ZipFile instance. + * @param lumpIdx Logical index for the lump. + * @param delimiter Delimit directory separators using this character (default: '/'). + * @return String containing the full path. */ AutoStr* ZipFile_ComposeLumpPath(ZipFile* zip, int lumpIdx, char delimiter); @@ -100,10 +116,11 @@ AutoStr* ZipFile_ComposeLumpPath(ZipFile* zip, int lumpIdx, char delimiter); /** * Read the data associated with the specified lump index into @a buffer. * - * @param zip ZipFile instance. - * @param lumpIdx Lump index associated with the data being read. - * @param buffer Buffer to read into. Must be at least W_LumpLength() bytes. - * @param tryCache @c true = try the lump cache first. + * @param zip ZipFile instance. + * @param lumpIdx Lump index associated with the data being read. + * @param buffer Buffer to read into. Must be at least W_LumpLength() bytes. + * @param tryCache @c true= try the lump cache first. + * * @return Number of bytes read. */ size_t ZipFile_ReadLump2(ZipFile* zip, int lumpIdx, uint8_t* buffer, boolean tryCache); @@ -112,35 +129,35 @@ size_t ZipFile_ReadLump(ZipFile* zip, int lumpIdx, uint8_t* buffer); /** * Read a subsection of the data associated with the specified lump index into @a buffer. * - * @param zip ZipFile instance. - * @param lumpIdx Lump index associated with the data being read. - * @param buffer Buffer to read into. Must be at least W_LumpLength() bytes. + * @param zip ZipFile instance. + * @param lumpIdx Lump index associated with the data being read. + * @param buffer Buffer to read into. Must be at least W_LumpLength() bytes. * @param startOffset Offset from the beginning of the lump to start reading. - * @param length Number of bytes to be read. - * @param tryCache @c true = try the lump cache first. + * @param length Number of bytes to be read. + * @param tryCache @c true= try the lump cache first. + * * @return Number of bytes read. */ -size_t ZipFile_ReadLumpSection2(ZipFile* zip, int lumpIdx, uint8_t* buffer, - size_t startOffset, size_t length, boolean tryCache); -size_t ZipFile_ReadLumpSection(ZipFile* zip, int lumpIdx, uint8_t* buffer, - size_t startOffset, size_t length); +size_t ZipFile_ReadLumpSection2(ZipFile* zip, int lumpIdx, uint8_t* buffer, size_t startOffset, size_t length, boolean tryCache); +size_t ZipFile_ReadLumpSection(ZipFile* zip, int lumpIdx, uint8_t* buffer, size_t startOffset, size_t length); /** * Read the data associated with the specified lump index into the cache. * - * @param zip ZipFile instance. - * @param lumpIdx Lump index associated with the data being read. - * @param tag Zone purge level/cache tag to use. + * @param zip ZipFile instance. + * @param lumpIdx Lump index associated with the data being read. + * @param tag Zone purge level/cache tag to use. + * * @return Ptr to the cached copy of the associated data. */ -const uint8_t* ZipFile_CacheLump(ZipFile* zip, int lumpIdx, int tag); +uint8_t const* ZipFile_CacheLump(ZipFile* zip, int lumpIdx, int tag); /** * Change the Zone purge level/cache tag associated with a cached data lump. * - * @param zip ZipFile instance. - * @param lumpIdx Lump index associated with the cached data being changed. - * @param tag Zone purge level/cache tag to use. + * @param zip ZipFile instance. + * @param lumpIdx Lump index associated with the cached data being changed. + * @param tag Zone purge level/cache tag to use. */ void ZipFile_ChangeLumpCacheTag(ZipFile* zip, int lumpIdx, int tag); @@ -156,7 +173,8 @@ int ZipFile_LumpCount(ZipFile* zip); /** * Determines whether the specified file appears to be in a format recognised by * ZipFile. - * @param file Stream file handle/wrapper to the file being interpreted. + * + * @param file Stream file handle/wrapper to the file being interpreted. * * @return @c true iff this is a file that can be represented using ZipFile. */ @@ -226,4 +244,8 @@ uint8_t* ZipFile_Compress(uint8_t* in, size_t inSize, size_t* outSize); */ uint8_t* ZipFile_CompressAtLevel(uint8_t* in, size_t inSize, size_t* outSize, int level); -#endif // LIBDENG_FILESYS_ZIPFILE_H +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* LIBDENG_FILESYS_ZIPFILE_H */ diff --git a/doomsday/engine/portable/src/zipfile.cpp b/doomsday/engine/portable/src/zipfile.cpp index 44c8bd24c8..9cdaa0f723 100644 --- a/doomsday/engine/portable/src/zipfile.cpp +++ b/doomsday/engine/portable/src/zipfile.cpp @@ -1,60 +1,41 @@ -/**\file zipfile.c - *\section License - * License: GPL - * Online License Link: http://www.gnu.org/licenses/gpl.html - * - *\author Copyright © 2003-2012 Jaakko Keränen - *\author Copyright © 2006-2012 Daniel Swanson - *\author Copyright © 2006-2007 Jamie Jones +/** + * @file zipfile.cpp + * ZIP archives. @ingroup fs * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. + * @author Copyright © 2003-2012 Jaakko Keränen + * @author Copyright © 2006-2012 Daniel Swanson + * @author Copyright © 2006-2007 Jamie Jones * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * @par License + * GPL: http://www.gnu.org/licenses/gpl.html * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, - * Boston, MA 02110-1301 USA + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. This program is distributed in the hope that it + * will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. You should have received a copy of the GNU + * General Public License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA */ #include #include "de_base.h" -#include "de_console.h" #include "de_filesys.h" +#include "game.h" #include "lumpdirectory.h" #include "pathdirectory.h" #include "zipfile.h" -#include "m_misc.h" - -typedef struct { - size_t baseOffset; - LumpInfo info; -} zipfile_lumprecord_t; - -struct zipfile_s { - /// Base file instance. - abstractfile_t base; - - /// Directory containing structure and info records for all lumps. - PathDirectory* lumpDirectory; - - /// LUT which maps logical lump indices to PathDirectoryNodes. - PathDirectoryNode** lumpDirectoryMap; - - /// Vector of lump records. - zipfile_lumprecord_t* lumpRecords; - /// Lump cache data pointers. - void** lumpCache; -}; +#include +#include +#include +#include +#include #define SIG_LOCAL_FILE_HEADER 0x04034b50 #define SIG_CENTRAL_FILE_HEADER 0x02014b50 @@ -87,6 +68,7 @@ enum { ZFC_PKWARE_DCL_IMPLODED }; +/// The following structures are used to read data directly from ZIP files. #pragma pack(1) typedef struct localfileheader_s { uint32_t signature; @@ -145,11 +127,759 @@ typedef struct centralend_s { } centralend_t; #pragma pack() +static void ApplyPathMappings(ddstring_t* dest, const ddstring_t* src); + +struct LumpRecord +{ + size_t baseOffset; + LumpInfo info; +}; + +struct zipfile_s +{ + /// Base file instance. + abstractfile_t base; + + /// Directory containing structure and info records for all lumps. + PathDirectory* lumpDirectory; + + /// LUT which maps logical lump indices to PathDirectoryNodes. + PathDirectoryNode** lumpDirectoryMap; + + /// Vector of lump records. + LumpRecord* lumpRecords; + + /// Lump cache data pointers. + void** lumpCache; + + zipfile_s(DFile& file, const char* path, LumpInfo const& info) + : lumpDirectory(0), + lumpDirectoryMap(0), + lumpRecords(0), + lumpCache(0) + { + AbstractFile_Init(reinterpret_cast(this), FT_ZIPFILE, path, &file, &info); + } + + ~zipfile_s() + { + F_ReleaseFile(reinterpret_cast(this)); + clearLumpCache(); + + if(lumpDirectory) + { + if(PathDirectory_Size(lumpDirectory) > 1 && lumpCache) + { + M_Free(lumpCache); + } + + PathDirectory_Iterate(lumpDirectory, PCF_NO_BRANCH, NULL, PATHDIRECTORY_NOHASH, clearLumpRecordWorker); + PathDirectory_Delete(lumpDirectory); + } + + if(lumpDirectoryMap) M_Free(lumpDirectoryMap); + if(lumpRecords) M_Free(lumpRecords); + + AbstractFile_Destroy(reinterpret_cast(this)); + } + + static bool recognise(DFile& file) + { + localfileheader_t hdr; + if(!zipfile_s::readArchiveHeader(file, hdr)) return false; + return hdr.signature == SIG_LOCAL_FILE_HEADER; + } + + int lumpCount() + { + return lumpDirectory? PathDirectory_Size(lumpDirectory) : 0; + } + + LumpRecord* lumpRecord(int lumpIdx) + { + if(lumpIdx < 0 || lumpIdx >= lumpCount()) return NULL; + buildLumpDirectoryMap(); + return reinterpret_cast( PathDirectoryNode_UserData(lumpDirectoryMap[lumpIdx]) ); + } + + PathDirectoryNode* lumpDirectoryNode(int lumpIdx) + { + if(lumpIdx < 0 || lumpIdx >= lumpCount()) return NULL; + buildLumpDirectoryMap(); + return lumpDirectoryMap[lumpIdx]; + } + + LumpInfo const* lumpInfo(int lumpIdx) + { + LOG_AS("ZipFile"); + LumpRecord* lrec = lumpRecord(lumpIdx); + if(!lrec) throw de::Error("ZipFile::lumpInfo", QString("Invalid lump index %1 (valid range: [0..%2])").arg(lumpIdx).arg(lumpCount())); + return &lrec->info; + } + + AutoStr* composeLumpPath(int lumpIdx, char delimiter) + { + PathDirectoryNode* node = lumpDirectoryNode(lumpIdx); + if(node) + { + return PathDirectoryNode_ComposePath2(node, AutoStr_NewStd(), NULL, delimiter); + } + return AutoStr_NewStd(); + } + + static int clearLumpRecordWorker(PathDirectoryNode* node, void* /*parameters*/) + { + LumpRecord* rec = reinterpret_cast(PathDirectoryNode_UserData(node)); + if(rec) + { + // Detach our user data from this node. + PathDirectoryNode_SetUserData(node, 0); + F_DestroyLumpInfo(&rec->info); + // The record itself is free'd later. + } + return 0; // Continue iteration. + } + + /// @todo Do not position the stream here. + static bool readArchiveHeader(DFile& file, localfileheader_t& hdr) + { + size_t readBytes, initPos = DFile_Tell(&file); + // Seek to the start of the header. + DFile_Seek(&file, 0, SEEK_SET); + readBytes = DFile_Read(&file, (uint8_t*)&hdr, sizeof(localfileheader_t)); + // Return the stream to its original position. + DFile_Seek(&file, initPos, SEEK_SET); + if(!(readBytes < sizeof(localfileheader_t))) + { + hdr.signature = de::littleEndianByteOrder.toNative(hdr.signature); + hdr.requiredVersion = de::littleEndianByteOrder.toNative(hdr.requiredVersion); + hdr.flags = de::littleEndianByteOrder.toNative(hdr.flags); + hdr.compression = de::littleEndianByteOrder.toNative(hdr.compression); + hdr.lastModTime = de::littleEndianByteOrder.toNative(hdr.lastModTime); + hdr.lastModDate = de::littleEndianByteOrder.toNative(hdr.lastModDate); + hdr.crc32 = de::littleEndianByteOrder.toNative(hdr.crc32); + hdr.compressedSize = de::littleEndianByteOrder.toNative(hdr.compressedSize); + hdr.size = de::littleEndianByteOrder.toNative(hdr.size); + hdr.fileNameSize = de::littleEndianByteOrder.toNative(hdr.fileNameSize); + hdr.extraFieldSize = de::littleEndianByteOrder.toNative(hdr.extraFieldSize); + return true; + } + return false; + } + + static bool readCentralEnd(DFile& file, centralend_t& end) + { + size_t readBytes = DFile_Read(&file, (uint8_t*)&end, sizeof(centralend_t)); + if(!(readBytes < sizeof(centralend_t))) + { + end.disk = de::littleEndianByteOrder.toNative(end.disk); + end.centralStartDisk= de::littleEndianByteOrder.toNative(end.centralStartDisk); + end.diskEntryCount = de::littleEndianByteOrder.toNative(end.diskEntryCount); + end.totalEntryCount = de::littleEndianByteOrder.toNative(end.totalEntryCount); + end.size = de::littleEndianByteOrder.toNative(end.size); + end.offset = de::littleEndianByteOrder.toNative(end.offset); + end.commentSize = de::littleEndianByteOrder.toNative(end.commentSize); + return true; + } + return false; + } + + /** + * Finds the central directory end record in the end of the file. + * + * @note: This gets awfully slow if the comment is long. + * + * @return @c true= successful. + */ + bool locateCentralDirectory() + { + uint32_t signature; + // Start from the earliest location where the signature might be. + int pos = CENTRAL_END_SIZE; // Offset from the end. + while(pos < MAXIMUM_COMMENT_SIZE) + { + DFile_Seek(base._file, -pos, SEEK_END); + + // Is this the signature? + DFile_Read(base._file, (uint8_t*)&signature, sizeof(signature)); + if(de::littleEndianByteOrder.toNative(signature) == SIG_END_OF_CENTRAL_DIR) + return true; // Yes, this is it. + + // Move backwards. + pos++; + } + return false; + } + + void readLumpDirectory() + { + LOG_AS("ZipFile"); + + // Scan the end of the file for the central directory end record. + if(!locateCentralDirectory()) + throw de::Error("ZipFile::readLumpDirectory", QString("Central directory in %1 not found").arg(Str_Text(AbstractFile_Path(reinterpret_cast(this))))); + + // Read the central directory end record. + centralend_t summary; + readCentralEnd(*base._file, summary); + + // Does the summary say something we don't like? + if(summary.diskEntryCount != summary.totalEntryCount) + throw de::Error("ZipFile::readLumpDirectory", QString("Multipart zip file \"%1\" not supported").arg(Str_Text(AbstractFile_Path(reinterpret_cast(this))))); + + // We'll load the file directory using one continous read into a temporary + // local buffer before we process it into our runtime representation. + // Read the entire central directory into memory. + void* centralDirectory = M_Malloc(summary.size); + if(!centralDirectory) throw de::Error("ZipFile::readLumpDirectory", QString("Failed on allocation of %1 bytes for temporary copy of the central centralDirectory").arg(summary.size)); + + DFile_Seek(base._file, summary.offset, SEEK_SET); + DFile_Read(base._file, (uint8_t*)centralDirectory, summary.size); + + /** + * Pass 1: Validate support and count the number of lump records we need. + * Pass 2: Read all zip entries and populate the lump directory. + */ + char* pos; + LumpRecord* record; + int entryCount = 0; + ddstring_t entryPath; + Str_Init(&entryPath); + for(int pass = 0; pass < 2; ++pass) + { + if(pass == 1) + { + if(entryCount == 0) break; + + // We can now allocate the records. + lumpRecords = (LumpRecord*) M_Malloc(entryCount * sizeof(*lumpRecords)); + if(!lumpRecords) throw de::Error("ZipFile::readLumpDirectory", QString("Failed on allocation of %1 bytes for the lump record vector").arg(entryCount * sizeof(*lumpRecords))); + + // Get the first record. + record = lumpRecords; + } + + // Position the read cursor at the start of the buffered central centralDirectory. + pos = (char*)centralDirectory; + + // Read all the entries. + uint lumpIdx = 0; + for(int index = 0; index < summary.totalEntryCount; ++index, pos += sizeof(centralfileheader_t)) + { + centralfileheader_t const* header = (centralfileheader_t*) pos; + char const* nameStart = pos + sizeof(centralfileheader_t); + localfileheader_t localHeader; + + // Advance the cursor past the variable sized fields. + pos += USHORT(header->fileNameSize) + USHORT(header->extraFieldSize) + USHORT(header->commentSize); + + // Copy characters up to fileNameSize. + Str_Clear(&entryPath); + Str_PartAppend(&entryPath, nameStart, 0, USHORT(header->fileNameSize)); + + // Directories are skipped. + if(ULONG(header->size) == 0 && Str_RAt(&entryPath, 0) == '/') + { + continue; + } + + // Do we support the format of this lump? + if(USHORT(header->compression) != ZFC_NO_COMPRESSION && + USHORT(header->compression) != ZFC_DEFLATED) + { + if(pass != 0) continue; + LOG_WARNING("Zip %s:'%s' uses an unsupported compression algorithm, ignoring.") + << Str_Text(AbstractFile_Path(reinterpret_cast(this))) + << Str_Text(&entryPath); + } + + if(USHORT(header->flags) & ZFH_ENCRYPTED) + { + if(pass != 0) continue; + LOG_WARNING("Zip %s:'%s' is encrypted.\n Encryption is not supported, ignoring.") + << Str_Text(AbstractFile_Path(reinterpret_cast(this))) + << Str_Text(&entryPath); + } + + if(pass == 0) + { + // Another record will be needed. + ++entryCount; + continue; + } + + // Convert all slashes to our internal separator. + F_FixSlashes(&entryPath, &entryPath); + + // In some cases the path inside the file is mapped to another virtual location. + ApplyPathMappings(&entryPath, &entryPath); + + // Make it absolute. + F_PrependBasePath(&entryPath, &entryPath); + + // Have we yet to intialize the directory? + if(!lumpDirectory) + { + lumpDirectory = PathDirectory_NewWithFlags(PDF_ALLOW_DUPLICATE_LEAF); + } + + F_InitLumpInfo(&record->info); + PathDirectoryNode* node = PathDirectory_Insert2(lumpDirectory, Str_Text(&entryPath), '/'); + PathDirectoryNode_SetUserData(node, record); + + record->info.lumpIdx = lumpIdx++; + record->info.size = ULONG(header->size); + if(USHORT(header->compression) == ZFC_DEFLATED) + { + // Compressed using the deflate algorithm. + record->info.compressedSize = ULONG(header->compressedSize); + } + else // No compression. + { + record->info.compressedSize = record->info.size; + } + + // The modification date is inherited from the real file (note recursion). + record->info.lastModified = AbstractFile_LastModified(reinterpret_cast(this)); + record->info.container = reinterpret_cast(this); + + // Read the local file header, which contains the extra field size (Info-ZIP!). + DFile_Seek(base._file, ULONG(header->relOffset), SEEK_SET); + DFile_Read(base._file, (uint8_t*)&localHeader, sizeof(localHeader)); + + record->baseOffset = ULONG(header->relOffset) + sizeof(localfileheader_t) + + USHORT(header->fileNameSize) + USHORT(localHeader.extraFieldSize); + + // Next record please! + record++; + } + } + + // The file centralDirectory is no longer needed. + M_Free(centralDirectory); + Str_Free(&entryPath); + } + + static int buildLumpDirectoryMapWorker(PathDirectoryNode* node, void* parameters) + { + ZipFile* zip = (ZipFile*)parameters; + LumpRecord* lumpRecord = (LumpRecord*)PathDirectoryNode_UserData(node); + DENG2_ASSERT(lumpRecord && lumpRecord->info.lumpIdx >= 0 && lumpRecord->info.lumpIdx < zip->lumpCount()); + zip->lumpDirectoryMap[lumpRecord->info.lumpIdx] = node; + return 0; // Continue iteration. + } + + void buildLumpDirectoryMap() + { + LOG_AS("ZipFile"); + // Been here already? + if(lumpDirectoryMap) return; + + lumpDirectoryMap = (PathDirectoryNode**) M_Malloc(sizeof(*lumpDirectoryMap) * lumpCount()); + if(!lumpDirectoryMap) throw de::Error("ZipFile::buildLumpDirectoryMap", QString("Failed on allocation of %1 bytes for the lumpdirectory map").arg(sizeof(*lumpDirectoryMap) * lumpCount())); + + PathDirectory_Iterate2(lumpDirectory, PCF_NO_BRANCH, NULL, PATHDIRECTORY_NOHASH, + buildLumpDirectoryMapWorker, (void*)this); + } + + /** + * @param buffer Must be large enough to hold the entire uncompressed data lump. + */ + size_t bufferLump(LumpRecord const* lumpRecord, uint8_t* buffer) + { + DENG2_ASSERT(lumpRecord && buffer); + LOG_AS("ZipFile"); + + DFile_Seek(base._file, lumpRecord->baseOffset, SEEK_SET); + + if(lumpRecord->info.compressedSize != lumpRecord->info.size) + { + bool result; + uint8_t* compressedData = (uint8_t*) M_Malloc(lumpRecord->info.compressedSize); + if(!compressedData) throw de::Error("ZipFile::bufferLump", QString("Failed on allocation of %1 bytes for decompression buffer").arg(lumpRecord->info.compressedSize)); + + // Read the compressed data into a temporary buffer for decompression. + DFile_Read(base._file, compressedData, lumpRecord->info.compressedSize); + + // Uncompress into the buffer provided by the caller. + result = uncompressRaw(compressedData, lumpRecord->info.compressedSize, + buffer, lumpRecord->info.size); + + M_Free(compressedData); + if(!result) return 0; // Inflate failed. + } + else + { + // Read the uncompressed data directly to the buffer provided by the caller. + DFile_Read(base._file, buffer, lumpRecord->info.size); + } + return lumpRecord->info.size; + } + + zipfile_s& clearCachedLump(int lumpIdx, bool* retCleared = 0) + { + LOG_AS("WadFile"); + void** cacheAdr = lumpCacheAddress(lumpIdx); + bool isCached = (cacheAdr && *cacheAdr); + if(isCached) + { + // If the block has a user, it must be explicitly freed. + if(Z_GetTag(*cacheAdr) < PU_MAP) + { + Z_ChangeTag2(*cacheAdr, PU_MAP); + } + + // Mark the memory pointer in use, but unowned. + Z_ChangeUser(*cacheAdr, (void*) 0x2); + } + + if(retCleared) *retCleared = isCached; + return *this; + } + + int publishLumpsToDirectory(LumpDirectory* directory) + { + int numPublished = 0; + if(directory) + { + readLumpDirectory(); + if(lumpCount() > 0) + { + // Insert the lumps into their rightful places in the directory. + LumpDirectory_CatalogLumps(directory, reinterpret_cast(this), 0, lumpCount()); + numPublished += lumpCount(); + } + } + return numPublished; + } + + void** lumpCacheAddress(int lumpIdx) + { + if(!lumpCache || lumpIdx < 0 || lumpIdx >= lumpCount()) return 0; + if(lumpCount() > 1) + { + const uint cacheIdx = lumpIdx; + return &lumpCache[cacheIdx]; + } + else + { + return (void**)&lumpCache; + } + } + + zipfile_s& clearLumpCache() + { + LOG_AS("ZipFile"); + const int numLumps = lumpCount(); + for(int i = 0; i < numLumps; ++i) + { + clearCachedLump(i); + } + return *this; + } + + uint8_t const* cacheLump(int lumpIdx, int tag) + { + LOG_AS("ZipFile::cacheLump"); + LumpInfo const* info = lumpInfo(lumpIdx); + + LOG_TRACE("\"%s:%s\" (%lu bytes%s)") + << F_PrettyPath(Str_Text(AbstractFile_Path(reinterpret_cast(this)))) + << F_PrettyPath(Str_Text(composeLumpPath(lumpIdx, '/'))) + << (unsigned long) info->size + << (info->compressedSize != info->size? ", compressed" : ""); + + // Time to create the lump cache? + if(!lumpCache) + { + if(lumpCount() > 1) + { + lumpCache = (void**) M_Calloc(lumpCount() * sizeof(*lumpCache)); + if(!lumpCache) throw de::Error("ZipFile::cacheLump", QString("Failed on allocation of %1 bytes for lump cache data list").arg(sizeof(*lumpCache))); + } + } + + void** cacheAdr = lumpCacheAddress(lumpIdx); + bool isCached = cacheAdr && *cacheAdr; + if(!isCached) + { + uint8_t* buffer = (uint8_t*) Z_Malloc(info->size, tag, cacheAdr); + if(!buffer) throw de::Error("ZipFile::cacheLump", QString("Failed on allocation of %1 bytes for cache copy of lump #%2").arg(info->size).arg(lumpIdx)); + + readLump(lumpIdx, buffer, false); + } + else + { + Z_ChangeTag2(*cacheAdr, tag); + } + + return (uint8_t*)(*cacheAdr); + } + + zipfile_s& changeLumpCacheTag(int lumpIdx, int tag) + { + LOG_AS("ZipFile::changeLumpCacheTag"); + LOG_TRACE("\"%s:%s\" tag=%i") + << F_PrettyPath(Str_Text(AbstractFile_Path(reinterpret_cast(this)))) + << F_PrettyPath(Str_Text(composeLumpPath(lumpIdx, '/'))) + << tag; + + void** cacheAdr = lumpCacheAddress(lumpIdx); + bool isCached = cacheAdr && *cacheAdr; + if(isCached) + { + Z_ChangeTag2(*cacheAdr, tag); + } + return *this; + } + + size_t readLumpSection(int lumpIdx, uint8_t* buffer, size_t startOffset, size_t length, + bool tryCache = true) + { + LOG_AS("ZipFile::readLumpSection"); + LumpRecord const* lrec = lumpRecord(lumpIdx); + if(!lrec) return 0; + + LOG_TRACE("\"%s:%s\" (%lu bytes%s) [%lu +%lu]") + << F_PrettyPath(Str_Text(AbstractFile_Path(reinterpret_cast(this)))) + << F_PrettyPath(Str_Text(composeLumpPath(lumpIdx, '/'))) + << (unsigned long) lrec->info.size + << (lrec->info.compressedSize != lrec->info.size? ", compressed" : "") + << (unsigned long) startOffset + << (unsigned long)length; + + // Try to avoid a file system read by checking for a cached copy. + if(tryCache) + { + void** cacheAdr = lumpCacheAddress(lumpIdx); + bool isCached = cacheAdr && *cacheAdr; + LOG_DEBUG("Cache %s on #%i") << (isCached? "hit" : "miss") << lumpIdx; + if(isCached) + { + size_t readBytes = MIN_OF(lrec->info.size, length); + memcpy(buffer, (char*)*cacheAdr + startOffset, readBytes); + return readBytes; + } + } + + size_t readBytes; + if(!startOffset && length == lrec->info.size) + { + // Read it straight to the caller's data buffer. + readBytes = bufferLump(lrec, buffer); + } + else + { + // Allocate a temporary buffer and read the whole lump into it(!). + uint8_t* lumpData = (uint8_t*) M_Malloc(lrec->info.size); + if(!lumpData) throw de::Error("ZipFile::readLumpSection", QString("Failed on allocation of %1 bytes for work buffer").arg(lrec->info.size)); + + if(bufferLump(lrec, lumpData)) + { + readBytes = MIN_OF(lrec->info.size, length); + memcpy(buffer, lumpData + startOffset, readBytes); + } + else + { + readBytes = 0; + } + M_Free(lumpData); + } + + /// @todo Do not check the read length here. + if(readBytes < MIN_OF(lrec->info.size, length)) + throw de::Error("ZipFile::readLumpSection", QString("Only read %1 of %2 bytes of lump #%3").arg(readBytes).arg(length).arg(lumpIdx)); + + return readBytes; + } + + size_t readLump(int lumpIdx, uint8_t* buffer, bool tryCache = true) + { + LOG_AS("ZipFile::readLump"); + LumpInfo const* info = lumpInfo(lumpIdx); + if(!info) return 0; + return readLumpSection(lumpIdx, buffer, 0, info->size, tryCache); + } + + static uint8_t* compressAtLevel(uint8_t* in, size_t inSize, size_t* outSize, int level) + { +#define CHUNK_SIZE 32768 + + LOG_AS("ZipFile::compressAtLevel"); + + z_stream stream; + uint8_t chunk[CHUNK_SIZE]; + size_t allocSize = CHUNK_SIZE; + uint8_t* output = (uint8_t*) M_Malloc(allocSize); // some initial space + int result; + int have; + + DENG2_ASSERT(outSize); + *outSize = 0; + + memset(&stream, 0, sizeof(stream)); + stream.next_in = (Bytef*) in; + stream.avail_in = (uInt) inSize; + stream.zalloc = Z_NULL; + stream.zfree = Z_NULL; + stream.opaque = Z_NULL; + + if(level < Z_NO_COMPRESSION) + { + level = Z_NO_COMPRESSION; + } + if(level > Z_BEST_COMPRESSION) + { + level = Z_BEST_COMPRESSION; + } + result = deflateInit(&stream, level); + if(result != Z_OK) + { + M_Free(output); + return 0; + } + + // Compress until all the data has been exhausted. + do + { + stream.next_out = chunk; + stream.avail_out = CHUNK_SIZE; + result = deflate(&stream, Z_FINISH); + if(result == Z_STREAM_ERROR) + { + M_Free(output); + *outSize = 0; + return 0; + } + have = CHUNK_SIZE - stream.avail_out; + if(have) + { + // Need more memory? + if(*outSize + have > allocSize) + { + // Need more memory. + allocSize *= 2; + output = (uint8_t*) M_Realloc(output, allocSize); + } + // Append. + memcpy(output + *outSize, chunk, have); + *outSize += have; + } + } while(!stream.avail_out); // output chunk full, more data may follow + + DENG2_ASSERT(result == Z_STREAM_END); + DENG2_ASSERT(stream.total_out == *outSize); + + deflateEnd(&stream); + return output; + +#undef CHUNK_SIZE + } + + static uint8_t* uncompress(uint8_t* in, size_t inSize, size_t* outSize) + { +#define INF_CHUNK_SIZE 4096 // Uncompress in 4KB chunks. + + LOG_AS("ZipFile::uncompress"); + + z_stream stream; + uint8_t chunk[INF_CHUNK_SIZE]; + size_t allocSize = INF_CHUNK_SIZE; + uint8_t* output = (uint8_t*) M_Malloc(allocSize); // some initial space + int result; + int have; + + DENG2_ASSERT(outSize); + *outSize = 0; + + memset(&stream, 0, sizeof(stream)); + stream.next_in = (Bytef*) in; + stream.avail_in = (uInt) inSize; + + result = inflateInit(&stream); + if(result != Z_OK) + { + M_Free(output); + return 0; + } + + // Uncompress until all the input data has been exhausted. + do + { + stream.next_out = chunk; + stream.avail_out = INF_CHUNK_SIZE; + result = inflate(&stream, Z_FINISH); + if(result == Z_STREAM_ERROR) + { + M_Free(output); + *outSize = 0; + return 0; + } + have = INF_CHUNK_SIZE - stream.avail_out; + if(have) + { + // Need more memory? + if(*outSize + have > allocSize) + { + // Need more memory. + allocSize *= 2; + output = (uint8_t*) M_Realloc(output, allocSize); + } + // Append. + memcpy(output + *outSize, chunk, have); + *outSize += have; + } + } while(!stream.avail_out); // output chunk full, more data may follow + + // We should now be at the end. + DENG2_ASSERT(result == Z_STREAM_END); + + inflateEnd(&stream); + return output; + +#undef INF_CHUNK_SIZE + } + + static bool uncompressRaw(uint8_t* in, size_t inSize, uint8_t* out, size_t outSize) + { + LOG_AS("ZipFile::uncompressRaw"); + z_stream stream; + int result; + + memset(&stream, 0, sizeof(stream)); + stream.next_in = (Bytef*) in; + stream.avail_in = (uInt) inSize; + stream.zalloc = Z_NULL; + stream.zfree = Z_NULL; + stream.next_out = (Bytef*) out; + stream.avail_out = (uInt) outSize; + + if(inflateInit2(&stream, -MAX_WBITS) != Z_OK) + return false; + + // Do the inflation in one call. + result = inflate(&stream, Z_FINISH); + + if(stream.total_out != outSize) + { + inflateEnd(&stream); + LOG_WARNING("Failure due to %s (result code %i).") + << (result == Z_DATA_ERROR ? "corrupt data" : "zlib error") + << result; + return false; + } + + // We're done. + inflateEnd(&stream); + return true; + } +}; + /** * The path inside the zip might be mapped to another virtual location. * * @todo This is clearly implemented in the wrong place. Path mapping - * should be done at LumpDirectory level. + * should be done at a higher level. * * Data files (pk3, zip, lmp, wad, deh) in the root are mapped to Data/Game/Auto. * Definition files (ded) in the root are mapped to Defs/Game/Auto. @@ -157,7 +887,7 @@ typedef struct centralend_s { * Paths that begin with a '#' are mapped to Data/Game/Auto. * Key-named directories at the root are mapped to another location. */ -static void ZipFile_ApplyPathMappings(ddstring_t* dest, const ddstring_t* src) +static void ApplyPathMappings(ddstring_t* dest, const ddstring_t* src) { // Manually mapped to Defs? if(Str_At(src, 0) == '@') @@ -273,365 +1003,75 @@ static void ZipFile_ApplyPathMappings(ddstring_t* dest, const ddstring_t* src) } /** - * Finds the central directory end record in the end of the file. - * Note: This gets awfully slow if the comment is long. - * - * @return @c true, if successful. + * C Wrapper API: */ -static boolean ZipFile_LocateCentralDirectory(ZipFile* zip) -{ - int pos = CENTRAL_END_SIZE; // Offset from the end. - uint32_t signature; - assert(zip); - // Start from the earliest location where the signature might be. - while(pos < MAXIMUM_COMMENT_SIZE) - { - DFile_Seek(zip->base._file, -pos, SEEK_END); +#define TOINTERNAL(inst) \ + (inst) != 0? reinterpret_cast(inst) : NULL - // Is this the signature? - DFile_Read(zip->base._file, (uint8_t*)&signature, 4); - if(ULONG(signature) == SIG_END_OF_CENTRAL_DIR) - { - // This is it! - return true; - } - // Move backwards. - pos++; - } - // Scan was not successful. - return false; -} +#define TOINTERNAL_CONST(inst) \ + (inst) != 0? reinterpret_cast(inst) : NULL -static void ZipFile_ReadLumpDirectory(ZipFile* zip) -{ - zipfile_lumprecord_t* record; - int entryCount, index, pass; - PathDirectoryNode* node; - void* centralDirectory; - ddstring_t entryPath; - centralend_t summary; - uint lumpIdx; - char* pos; - assert(zip); - - VERBOSE( Con_Message("ZipFile::readLumpDirectory: \"%s\"\n", - F_PrettyPath(Str_Text(AbstractFile_Path((abstractfile_t*)zip)))) ); - - // Scan the end of the file for the central centralDirectory end record. - if(!ZipFile_LocateCentralDirectory(zip)) - { - Con_Error("ZipFile::readLumpDirectory: Central centralDirectory in %s not found!", - Str_Text(AbstractFile_Path((abstractfile_t*)zip))); - } +#define SELF(inst) \ + DENG2_ASSERT(inst); \ + zipfile_s* self = TOINTERNAL(inst) - // Read the central centralDirectory end record. - DFile_Read(zip->base._file, (uint8_t*)&summary, sizeof(summary)); +#define SELF_CONST(inst) \ + DENG2_ASSERT(inst); \ + zipfile_s const* self = TOINTERNAL_CONST(inst) - // Does the summary say something we don't like? - if(USHORT(summary.diskEntryCount) != USHORT(summary.totalEntryCount)) +ZipFile* ZipFile_New(DFile* file, const char* path, const LumpInfo* info) +{ + if(!info) LegacyCore_FatalError("ZipFile_New: Received invalid LumpInfo (=NULL)."); + try { - Con_Error("ZipFile::readLumpDirectory: Multipart Zip file \"%s\" not supported.", - Str_Text(AbstractFile_Path((abstractfile_t*)zip))); + return reinterpret_cast(new zipfile_s(*file, path, *info)); } - - // Read the entire central centralDirectory into memory. - centralDirectory = malloc(ULONG(summary.size)); - if(!centralDirectory) - Con_Error("ZipFile::readLumpDirectory: Failed on allocation of %lu bytes for " - "temporary copy of the central centralDirectory.", (unsigned long) ULONG(summary.size)); - DFile_Seek(zip->base._file, ULONG(summary.offset), SEEK_SET); - DFile_Read(zip->base._file, (uint8_t*)centralDirectory, ULONG(summary.size)); - - /** - * Pass 1: Validate support and count the number of lump records we need. - * Pass 2: Read all zip entries and populate the lump directory. - */ - entryCount = 0; - Str_Init(&entryPath); - for(pass = 0; pass < 2; ++pass) + catch(de::Error& er) { - if(pass == 1) - { - if(entryCount == 0) break; - - // We can now allocate the records. - zip->lumpRecords = (zipfile_lumprecord_t*)malloc(entryCount * sizeof(*zip->lumpRecords)); - if(!zip->lumpRecords) - Con_Error("ZipFile::readLumpDirectory: Failed on allocation of %lu bytes for the lump record vector.", (unsigned long) (entryCount * sizeof(*zip->lumpRecords))); - - // Get the first record. - record = zip->lumpRecords; - } - - // Position the read cursor at the start of the buffered central centralDirectory. - pos = centralDirectory; - - // Read all the entries. - lumpIdx = 0; - for(index = 0; index < USHORT(summary.totalEntryCount); - ++index, pos += sizeof(centralfileheader_t)) - { - const centralfileheader_t* header = (void*) pos; - const char* nameStart = pos + sizeof(centralfileheader_t); - localfileheader_t localHeader; - - // Advance the cursor past the variable sized fields. - pos += USHORT(header->fileNameSize) + USHORT(header->extraFieldSize) + USHORT(header->commentSize); - - // Copy characters up to fileNameSize. - Str_Clear(&entryPath); - Str_PartAppend(&entryPath, nameStart, 0, USHORT(header->fileNameSize)); - - // Directories are skipped. - if(ULONG(header->size) == 0 && Str_RAt(&entryPath, 0) == '/') - { - continue; - } - - // Do we support the format of this lump? - if(USHORT(header->compression) != ZFC_NO_COMPRESSION && - USHORT(header->compression) != ZFC_DEFLATED) - { - if(pass != 0) continue; - Con_Message("Warning: Zip %s:'%s' uses an unsupported compression algorithm, ignoring.\n", - Str_Text(AbstractFile_Path((abstractfile_t*)zip)), Str_Text(&entryPath)); - } - - if(USHORT(header->flags) & ZFH_ENCRYPTED) - { - if(pass != 0) continue; - Con_Message("Warning: Zip %s:'%s' is encrypted.\n Encryption is not supported, ignoring.\n", - Str_Text(AbstractFile_Path((abstractfile_t*)zip)), Str_Text(&entryPath)); - } - - if(pass == 0) - { - // Another record will be needed. - ++entryCount; - continue; - } - - // Convert all slashes to our internal separator. - F_FixSlashes(&entryPath, &entryPath); - - // In some cases the path inside the file is mapped to another virtual location. - ZipFile_ApplyPathMappings(&entryPath, &entryPath); - - // Make it absolute. - F_PrependBasePath(&entryPath, &entryPath); - - // Have we yet to intialize the directory? - if(!zip->lumpDirectory) - { - zip->lumpDirectory = PathDirectory_NewWithFlags(PDF_ALLOW_DUPLICATE_LEAF); - } - - F_InitLumpInfo(&record->info); - node = PathDirectory_Insert2(zip->lumpDirectory, Str_Text(&entryPath), '/'); - PathDirectoryNode_SetUserData(node, record); - - record->info.lumpIdx = lumpIdx++; - record->info.size = ULONG(header->size); - if(USHORT(header->compression) == ZFC_DEFLATED) - { - // Compressed using the deflate algorithm. - record->info.compressedSize = ULONG(header->compressedSize); - } - else // No compression. - { - record->info.compressedSize = record->info.size; - } - - // The modification date is inherited from the real file (note recursion). - record->info.lastModified = AbstractFile_LastModified((abstractfile_t*)zip); - record->info.container = (abstractfile_t*)zip; - - // Read the local file header, which contains the extra field size (Info-ZIP!). - DFile_Seek(zip->base._file, ULONG(header->relOffset), SEEK_SET); - DFile_Read(zip->base._file, (uint8_t*)&localHeader, sizeof(localHeader)); - - record->baseOffset = ULONG(header->relOffset) + sizeof(localfileheader_t) + USHORT(header->fileNameSize) + USHORT(localHeader.extraFieldSize); - - // Next record please! - record++; - } + QString msg = QString("ZipFile_New: Failed to instantiate new ZipFile. ") + er.asText(); + LegacyCore_FatalError(msg.toUtf8().constData()); + exit(1); // Unreachable. } - - // The file centralDirectory is no longer needed. - free(centralDirectory); - Str_Free(&entryPath); -} - -static int insertNodeInLumpDirectoryMap(PathDirectoryNode* node, void* parameters) -{ - ZipFile* zip = (ZipFile*)parameters; - zipfile_lumprecord_t* lumpRecord = (zipfile_lumprecord_t*)PathDirectoryNode_UserData(node); - assert(lumpRecord && lumpRecord->info.lumpIdx >= 0 && lumpRecord->info.lumpIdx < ZipFile_LumpCount(zip)); - zip->lumpDirectoryMap[lumpRecord->info.lumpIdx] = node; - return 0; // Continue iteration. } -static void buildLumpDirectoryMap(ZipFile* zip) +void ZipFile_Delete(ZipFile* zip) { - assert(zip); - // Time to build the lump directory map? - if(!zip->lumpDirectoryMap) + if(zip) { - int lumpCount = ZipFile_LumpCount(zip); - - zip->lumpDirectoryMap = malloc(sizeof(*zip->lumpDirectoryMap) * lumpCount); - if(!zip->lumpDirectoryMap) - Con_Error("ZipFile::buildLumpDirectoryMap: Failed on allocation of %lu bytes for the lumpdirectory map.", (unsigned long) (sizeof(*zip->lumpDirectoryMap) * lumpCount)); - - PathDirectory_Iterate2(zip->lumpDirectory, PCF_NO_BRANCH, NULL, PATHDIRECTORY_NOHASH, - insertNodeInLumpDirectoryMap, (void*)zip); + SELF(zip); + delete self; } } -static zipfile_lumprecord_t* ZipFile_LumpRecord(ZipFile* zip, int lumpIdx) -{ - assert(zip); - if(lumpIdx < 0 || lumpIdx >= ZipFile_LumpCount(zip)) return NULL; - buildLumpDirectoryMap(zip); - return (zipfile_lumprecord_t*)PathDirectoryNode_UserData(zip->lumpDirectoryMap[lumpIdx]); -} - -ZipFile* ZipFile_New(DFile* file, const char* path, const LumpInfo* info) -{ - ZipFile* zip; - - if(!info) Con_Error("ZipFile::New: Received invalid LumpInfo."); - - zip = (ZipFile*)malloc(sizeof(*zip)); - if(!zip) Con_Error("ZipFile::New: Failed on allocation of %lu bytes for new ZipFile.", - (unsigned long) sizeof *zip); - - AbstractFile_Init((abstractfile_t*)zip, FT_ZIPFILE, path, file, info); - zip->lumpDirectory = NULL; - zip->lumpDirectoryMap = NULL; - zip->lumpRecords = NULL; - zip->lumpCache = NULL; - return zip; -} - int ZipFile_PublishLumpsToDirectory(ZipFile* zip, LumpDirectory* directory) { - int numPublished = 0; - assert(zip); - - if(directory) - { - ZipFile_ReadLumpDirectory(zip); - if(ZipFile_LumpCount(zip) > 0) - { - // Insert the lumps into their rightful places in the directory. - LumpDirectory_CatalogLumps(directory, (abstractfile_t*)zip, 0, ZipFile_LumpCount(zip)); - numPublished += ZipFile_LumpCount(zip); - } - } - return numPublished; + SELF(zip); + return self->publishLumpsToDirectory(directory); } PathDirectoryNode* ZipFile_LumpDirectoryNode(ZipFile* zip, int lumpIdx) { - if(lumpIdx < 0 || lumpIdx >= ZipFile_LumpCount(zip)) return NULL; - buildLumpDirectoryMap(zip); - return zip->lumpDirectoryMap[lumpIdx]; + SELF(zip); + return self->lumpDirectoryNode(lumpIdx); } AutoStr* ZipFile_ComposeLumpPath(ZipFile* zip, int lumpIdx, char delimiter) { - PathDirectoryNode* node = ZipFile_LumpDirectoryNode(zip, lumpIdx); - if(node) - { - return PathDirectoryNode_ComposePath2(node, AutoStr_NewStd(), NULL, delimiter); - } - return AutoStr_NewStd(); + SELF(zip); + return self->composeLumpPath(lumpIdx, delimiter); } -const LumpInfo* ZipFile_LumpInfo(ZipFile* zip, int lumpIdx) +LumpInfo const* ZipFile_LumpInfo(ZipFile* zip, int lumpIdx) { - zipfile_lumprecord_t* lumpRecord = ZipFile_LumpRecord(zip, lumpIdx); - if(!lumpRecord) - { - Con_Error("ZipFile::LumpInfo: Invalid lump index %i (valid range: [0..%i]).", lumpIdx, ZipFile_LumpCount(zip)); - exit(1); // Unreachable. - } - return &lumpRecord->info; -} - -static int destroyRecord(PathDirectoryNode* node, void* parameters) -{ - zipfile_lumprecord_t* rec = PathDirectoryNode_UserData(node); - - DENG_UNUSED(parameters); - - if(rec) - { - // Detach our user data from this node. - PathDirectoryNode_SetUserData(node, 0); - F_DestroyLumpInfo(&rec->info); - // The record itself is free'd later. - } - return 0; // Continue iteration. -} - -void ZipFile_Delete(ZipFile* zip) -{ - assert(zip); - - F_ReleaseFile((abstractfile_t*)zip); - ZipFile_ClearLumpCache(zip); - - if(zip->lumpDirectory) - { - if(PathDirectory_Size(zip->lumpDirectory) > 1 && zip->lumpCache) - { - free(zip->lumpCache); - } - - PathDirectory_Iterate(zip->lumpDirectory, PCF_NO_BRANCH, NULL, PATHDIRECTORY_NOHASH, destroyRecord); - PathDirectory_Delete(zip->lumpDirectory); - } - - if(zip->lumpDirectoryMap) free(zip->lumpDirectoryMap); - if(zip->lumpRecords) free(zip->lumpRecords); - - AbstractFile_Destroy((abstractfile_t*)zip); - free(zip); + SELF(zip); + return self->lumpInfo(lumpIdx); } void ZipFile_ClearLumpCache(ZipFile* zip) { - int i, lumpCount = ZipFile_LumpCount(zip); - assert(zip); - - if(lumpCount == 1) - { - if(zip->lumpCache) - { - // If the block has a user, it must be explicitly freed. - if(Z_GetTag(zip->lumpCache) < PU_MAP) - Z_ChangeTag(zip->lumpCache, PU_MAP); - // Mark the memory pointer in use, but unowned. - Z_ChangeUser(zip->lumpCache, (void*) 0x2); - } - return; - } - - if(!zip->lumpCache) return; - - for(i = 0; i < lumpCount; ++i) - { - if(!zip->lumpCache[i]) continue; - - // If the block has a user, it must be explicitly freed. - if(Z_GetTag(zip->lumpCache[i]) < PU_MAP) - Z_ChangeTag(zip->lumpCache[i], PU_MAP); - // Mark the memory pointer in use, but unowned. - Z_ChangeUser(zip->lumpCache[i], (void*) 0x2); - } + SELF(zip); + self->clearLumpCache(); } uint8_t* ZipFile_Compress(uint8_t* in, size_t inSize, size_t* outSize) @@ -641,399 +1081,65 @@ uint8_t* ZipFile_Compress(uint8_t* in, size_t inSize, size_t* outSize) uint8_t* ZipFile_CompressAtLevel(uint8_t* in, size_t inSize, size_t* outSize, int level) { -#define CHUNK_SIZE 32768 - z_stream stream; - uint8_t chunk[CHUNK_SIZE]; - size_t allocSize = CHUNK_SIZE; - uint8_t* output = M_Malloc(allocSize); // some initial space - int result; - int have; - - assert(outSize); - *outSize = 0; - - memset(&stream, 0, sizeof(stream)); - stream.next_in = (Bytef*) in; - stream.avail_in = (uInt) inSize; - stream.zalloc = Z_NULL; - stream.zfree = Z_NULL; - stream.opaque = Z_NULL; - - if(level < Z_NO_COMPRESSION) - { - level = Z_NO_COMPRESSION; - } - if(level > Z_BEST_COMPRESSION) - { - level = Z_BEST_COMPRESSION; - } - result = deflateInit(&stream, level); - if(result != Z_OK) - { - free(output); - return 0; - } - - // Compress until all the data has been exhausted. - do { - stream.next_out = chunk; - stream.avail_out = CHUNK_SIZE; - result = deflate(&stream, Z_FINISH); - if(result == Z_STREAM_ERROR) - { - free(output); - *outSize = 0; - return 0; - } - have = CHUNK_SIZE - stream.avail_out; - if(have) - { - // Need more memory? - if(*outSize + have > allocSize) - { - // Need more memory. - allocSize *= 2; - output = M_Realloc(output, allocSize); - } - // Append. - memcpy(output + *outSize, chunk, have); - *outSize += have; - } - } while(!stream.avail_out); // output chunk full, more data may follow - - assert(result == Z_STREAM_END); - assert(stream.total_out == *outSize); - - deflateEnd(&stream); - return output; -#undef CHUNK_SIZE + return zipfile_s::compressAtLevel(in, inSize, outSize, level); } uint8_t* ZipFile_Uncompress(uint8_t* in, size_t inSize, size_t* outSize) { -#define INF_CHUNK_SIZE 4096 // Uncompress in 4KB chunks. - z_stream stream; - uint8_t chunk[INF_CHUNK_SIZE]; - size_t allocSize = INF_CHUNK_SIZE; - uint8_t* output = M_Malloc(allocSize); // some initial space - int result; - int have; - - assert(outSize); - *outSize = 0; - - memset(&stream, 0, sizeof(stream)); - stream.next_in = (Bytef*) in; - stream.avail_in = (uInt) inSize; - - result = inflateInit(&stream); - if(result != Z_OK) - { - free(output); - return 0; - } - - // Uncompress until all the input data has been exhausted. - do { - stream.next_out = chunk; - stream.avail_out = INF_CHUNK_SIZE; - result = inflate(&stream, Z_FINISH); - if(result == Z_STREAM_ERROR) - { - free(output); - *outSize = 0; - return 0; - } - have = INF_CHUNK_SIZE - stream.avail_out; - if(have) - { - // Need more memory? - if(*outSize + have > allocSize) - { - // Need more memory. - allocSize *= 2; - output = M_Realloc(output, allocSize); - } - // Append. - memcpy(output + *outSize, chunk, have); - *outSize += have; - } - } while(!stream.avail_out); // output chunk full, more data may follow - - // We should now be at the end. - assert(result == Z_STREAM_END); - - inflateEnd(&stream); - return output; -#undef INF_CHUNK_SIZE + return zipfile_s::uncompress(in, inSize, outSize); } boolean ZipFile_UncompressRaw(uint8_t* in, size_t inSize, uint8_t* out, size_t outSize) { - z_stream stream; - int result; - - memset(&stream, 0, sizeof(stream)); - stream.next_in = (Bytef*) in; - stream.avail_in = (uInt) inSize; - stream.zalloc = Z_NULL; - stream.zfree = Z_NULL; - stream.next_out = (Bytef*) out; - stream.avail_out = (uInt) outSize; - - if(inflateInit2(&stream, -MAX_WBITS) != Z_OK) - return false; - - // Do the inflation in one call. - result = inflate(&stream, Z_FINISH); - - if(stream.total_out != outSize) - { - inflateEnd(&stream); - Con_Message("ZipFile::Uncompress: Failure due to %s (result code %i).\n", - (result == Z_DATA_ERROR ? "corrupt data" : "zlib error"), result); - return false; - } - - // We're done. - inflateEnd(&stream); - return true; -} - -/** - * @param buffer Must be large enough to hold the entire uncompressed data lump. - */ -static size_t ZipFile_BufferLump(ZipFile* zip, const zipfile_lumprecord_t* lumpRecord, - uint8_t* buffer) -{ - assert(zip && lumpRecord && buffer); - - DFile_Seek(zip->base._file, lumpRecord->baseOffset, SEEK_SET); - - if(lumpRecord->info.compressedSize != lumpRecord->info.size) - { - boolean result; - uint8_t* compressedData = (uint8_t*)malloc(lumpRecord->info.compressedSize); - if(!compressedData) - Con_Error("ZipFile::BufferLump: Failed on allocation of %lu bytes for decompression buffer.", lumpRecord->info.compressedSize); - - // Read the compressed data into a temporary buffer for decompression. - DFile_Read(zip->base._file, compressedData, lumpRecord->info.compressedSize); - - // Uncompress into the buffer provided by the caller. - result = ZipFile_UncompressRaw(compressedData, lumpRecord->info.compressedSize, - buffer, lumpRecord->info.size); - - free(compressedData); - if(!result) return 0; // Inflate failed. - } - else - { - // Read the uncompressed data directly to the buffer provided by the caller. - DFile_Read(zip->base._file, buffer, lumpRecord->info.size); - } - return lumpRecord->info.size; + return CPP_BOOL( zipfile_s::uncompressRaw(in, inSize, out, outSize) ); } size_t ZipFile_ReadLumpSection2(ZipFile* zip, int lumpIdx, uint8_t* buffer, size_t startOffset, size_t length, boolean tryCache) { - const zipfile_lumprecord_t* lumpRecord = ZipFile_LumpRecord(zip, lumpIdx); - size_t readBytes; - - if(!lumpRecord) return 0; - - VERBOSE2( - AutoStr* path = ZipFile_ComposeLumpPath(zip, lumpIdx, '/'); - Con_Printf("ZipFile::ReadLumpSection: \"%s:%s\" (%lu bytes%s) [%lu +%lu]", - F_PrettyPath(Str_Text(AbstractFile_Path((abstractfile_t*)zip))), - F_PrettyPath(Str_Text(path)), (unsigned long) lumpRecord->info.size, - (lumpRecord->info.compressedSize != lumpRecord->info.size? ", compressed" : ""), - (unsigned long) startOffset, (unsigned long)length); - ) - - // Try to avoid a file system read by checking for a cached copy. - if(tryCache && zip->lumpCache) - { - boolean isCached; - void** cachePtr; - - if(ZipFile_LumpCount(zip) > 1) - { - const uint cacheIdx = lumpIdx; - cachePtr = &zip->lumpCache[cacheIdx]; - isCached = (NULL != zip->lumpCache[cacheIdx]); - } - else - { - cachePtr = (void**)&zip->lumpCache; - isCached = (NULL != zip->lumpCache); - } - - if(isCached) - { - VERBOSE2( Con_Printf(" from cache\n") ) - readBytes = MIN_OF(lumpRecord->info.size, length); - memcpy(buffer, (uint8_t*)*cachePtr + startOffset, readBytes); - return readBytes; - } - } - VERBOSE2( Con_Printf("\n") ) - - if(!startOffset && length == lumpRecord->info.size) - { - // Read it straight to the caller's data buffer. - readBytes = ZipFile_BufferLump(zip, lumpRecord, buffer); - } - else - { - // Allocate a temporary buffer and read the whole lump into it(!). - uint8_t* lumpData = (uint8_t*)malloc(lumpRecord->info.size); - if(!lumpData) - Con_Error("ZipFile::ReadLumpSection: Failed on allocation of %lu bytes for work buffer.", lumpRecord->info.size); - - if(ZipFile_BufferLump(zip, lumpRecord, lumpData)) - { - readBytes = MIN_OF(lumpRecord->info.size, length); - memcpy(buffer, lumpData + startOffset, readBytes); - } - else - { - readBytes = 0; - } - free(lumpData); - } - - if(readBytes < MIN_OF(lumpRecord->info.size, length)) - { - /// @todo Do not do this here. - Con_Error("ZipFile::ReadLumpSection: Only read %lu of %lu bytes of lump #%i.", - (unsigned long) readBytes, (unsigned long) length, lumpIdx); - } - - return readBytes; + SELF(zip); + return self->readLumpSection(lumpIdx, buffer, startOffset, length, tryCache); } size_t ZipFile_ReadLumpSection(ZipFile* zip, int lumpIdx, uint8_t* buffer, size_t startOffset, size_t length) { - return ZipFile_ReadLumpSection2(zip, lumpIdx, buffer, startOffset, length, true); + SELF(zip); + return self->readLumpSection(lumpIdx, buffer, startOffset, length); } size_t ZipFile_ReadLump2(ZipFile* zip, int lumpIdx, uint8_t* buffer, boolean tryCache) { - const LumpInfo* info = ZipFile_LumpInfo(zip, lumpIdx); - if(!info) return 0; - return ZipFile_ReadLumpSection2(zip, lumpIdx, buffer, 0, info->size, tryCache); + SELF(zip); + return self->readLump(lumpIdx, buffer, tryCache); } size_t ZipFile_ReadLump(ZipFile* zip, int lumpIdx, uint8_t* buffer) { - return ZipFile_ReadLump2(zip, lumpIdx, buffer, true); + SELF(zip); + return self->readLump(lumpIdx, buffer); } -const uint8_t* ZipFile_CacheLump(ZipFile* zip, int lumpIdx, int tag) +uint8_t const* ZipFile_CacheLump(ZipFile* zip, int lumpIdx, int tag) { - const LumpInfo* info = ZipFile_LumpInfo(zip, lumpIdx); - const uint cacheIdx = lumpIdx; - boolean isCached; - void** cachePtr; - - VERBOSE2( - AutoStr* path = ZipFile_ComposeLumpPath(zip, lumpIdx, '/'); - Con_Printf("ZipFile::CacheLump: \"%s:%s\" (%lu bytes%s)", - F_PrettyPath(Str_Text(AbstractFile_Path((abstractfile_t*)zip))), - F_PrettyPath(Str_Text(path)), (unsigned long) info->size, - (info->compressedSize != info->size? ", compressed" : "")); - ) - - if(ZipFile_LumpCount(zip) > 1) - { - // Time to allocate the cache ptr table? - if(!zip->lumpCache) - zip->lumpCache = (void**)calloc(ZipFile_LumpCount(zip), sizeof(*zip->lumpCache)); - cachePtr = &zip->lumpCache[cacheIdx]; - isCached = (NULL != zip->lumpCache[cacheIdx]); - } - else - { - cachePtr = (void**)&zip->lumpCache; - isCached = (NULL != zip->lumpCache); - } - - VERBOSE2( Con_Printf(" %s\n", isCached? "hit":"miss") ) - - if(!isCached) - { - uint8_t* ptr = (uint8_t*)Z_Malloc(info->size, tag, cachePtr); - if(NULL == ptr) - Con_Error("ZipFile::CacheLump: Failed on allocation of %lu bytes for " - "cache copy of lump #%i.", (unsigned long) info->size, lumpIdx); - ZipFile_ReadLump2(zip, lumpIdx, ptr, false); - } - else - { - Z_ChangeTag(*cachePtr, tag); - } - - return (uint8_t*)(*cachePtr); + SELF(zip); + return self->cacheLump(lumpIdx, tag); } void ZipFile_ChangeLumpCacheTag(ZipFile* zip, int lumpIdx, int tag) { - boolean isCached; - void** cachePtr; - assert(zip); - - if(ZipFile_LumpCount(zip) > 1) - { - const uint cacheIdx = lumpIdx; - if(!zip->lumpCache) return; // Obviously not cached. - - cachePtr = &zip->lumpCache[cacheIdx]; - isCached = (NULL != zip->lumpCache[cacheIdx]); - } - else - { - cachePtr = (void**)&zip->lumpCache; - isCached = (NULL != zip->lumpCache); - } - - if(isCached) - { - VERBOSE2( - AutoStr* path = ZipFile_ComposeLumpPath(zip, lumpIdx, '/'); - Con_Printf("ZipFile::ChangeLumpCacheTag: \"%s:%s\" tag=%i\n", - F_PrettyPath(Str_Text(AbstractFile_Path((abstractfile_t*)zip))), - F_PrettyPath(Str_Text(path)), tag); - ) - - Z_ChangeTag2(*cachePtr, tag); - } + SELF(zip); + self->changeLumpCacheTag(lumpIdx, tag); } int ZipFile_LumpCount(ZipFile* zip) { - assert(zip); - return zip->lumpDirectory? PathDirectory_Size(zip->lumpDirectory) : 0; + SELF(zip); + return self->lumpCount(); } boolean ZipFile_Recognise(DFile* file) { - boolean knownFormat = false; - localfileheader_t hdr; - size_t readBytes, initPos = DFile_Tell(file); - DFile_Seek(file, 0, SEEK_SET); - readBytes = DFile_Read(file, (uint8_t*)&hdr, sizeof(hdr)); - if(!(readBytes < sizeof(hdr))) - { - // Seek to the start of the signature. - if(ULONG(hdr.signature) == SIG_LOCAL_FILE_HEADER) - { - knownFormat = true; - } - } - // Reposition the stream in case another handler needs to process this file. - DFile_Seek(file, initPos, SEEK_SET); - return knownFormat; + if(!file) return false; + return CPP_BOOL( zipfile_s::recognise(*file) ); }