lib/Gto/zhacks.cpp lib/Gto/zhacks.h @@ -1,4 +1,9 @@ +* Version 3.5.3 + + * Sped up random access in gzipped binary files + + * Version 3.5.2 * Minor patches to GtoContainer C++ lib NEWS @@ -1,2 +1,2 @@ -This is release version 3.5.2 +This is release version 3.5.3 RELEASE_VERSION @@ -170,14 +170,19 @@ Reader::descriptionComplete() { if (big) { - if (isSwapped()) cout << "little endian binary\n"; - else cout << "big endian binary\n"; + if (isSwapped()) cout << "little endian binary"; + else cout << "big endian binary"; } else { - if (isSwapped()) cout << "big endian binary\n"; - else cout << "little endian binary\n"; + if (isSwapped()) cout << "big endian binary"; + else cout << "little endian binary"; } + if (hasIndex()) + { + cout << ", indexed"; + } + cout << std::endl; } for (Gto::Reader::Properties::const_iterator p = properties().begin(); bin/gtoinfo/main.cpp @@ -41,7 +41,7 @@ fi dnl ********************** Processing starts here... -AC_INIT(Gto, 3.5.2) +AC_INIT(Gto, 3.5.3) dnl AC_PREREQ(2.57) AC_CONFIG_AUX_DIR(config) configure.in @@ -10,9 +10,9 @@ SUBDIRS = test lib_LTLIBRARIES = libGto.la libGto_la_SOURCES = FlexLexer.cpp Parser.cpp Writer.cpp Reader.cpp \ -RawData.cpp Utilities.cpp +RawData.cpp Utilities.cpp zhacks.cpp -noinst_HEADERS = Parser.h FlexLexer.h +noinst_HEADERS = Parser.h FlexLexer.h zhacks.h libGto_la_LIBS = @LIBS@ lib/Gto/Makefile.am @@ -17,6 +17,7 @@ // USA // +#include "zhacks.h" #include "Reader.h" #include "Utilities.h" #include <fstream> @@ -274,38 +275,6 @@ Reader::property(const string& name, return property(name, p); } -static void -swapWords(void *data, size_t size) -{ - struct bytes { char c[4]; }; - - bytes* ip = reinterpret_cast<bytes*>(data); - - for (size_t i=0; i<size; i++) - { - bytes temp = ip[i]; - ip[i].c[0] = temp.c[3]; - ip[i].c[1] = temp.c[2]; - ip[i].c[2] = temp.c[1]; - ip[i].c[3] = temp.c[0]; - } -} - -static void -swapShorts(void *data, size_t size) -{ - struct bytes { char c[2]; }; - - bytes* ip = reinterpret_cast<bytes*>(data); - - for (size_t i=0; i<size; i++) - { - bytes temp = ip[i]; - ip[i].c[0] = temp.c[1]; - ip[i].c[1] = temp.c[0]; - } -} - void Reader::readMagicNumber() { @@ -491,6 +460,7 @@ Reader::readComponents() void Reader::readProperties() { + size_t index = 0; for (Components::iterator i = m_components.begin(); i != m_components.end(); ++i) @@ -501,6 +471,8 @@ Reader::readProperties() { PropertyInfo p; + p.index = index++; + if (m_header.version == 2) { read((char*)&p, sizeof(PropertyHeader_v2)); @@ -554,7 +526,7 @@ Reader::accessProperty(PropertyInfo& p) if (p.requested) { - seekTo(p.offset); + seekTo(p); readProperty(p); } @@ -627,6 +599,7 @@ Reader::readTextGTO() bool Reader::readBinaryGTO() { + readIndexTable(); // OK to continue if this fails readHeader(); if (m_error) return false; readStringTable(); if (m_error) return false; readObjects(); if (m_error) return false; @@ -695,11 +668,18 @@ Reader::readProperty(PropertyInfo& prop) { if ((buffer = (char*)data(prop, bytes))) { - // Do we need to be somewhere else? - if(m_currentReadOffset != tell()) + if(prop.index < m_dataOffsets.size()) { - // If so, move the actual file pointer there. - seekForward(m_currentReadOffset - tell()); + seekTo(prop); + } + else + { + // Do we need to be somewhere else? + if(m_currentReadOffset != tell()) + { + // If so, move the actual file pointer there. + seekForward(m_currentReadOffset - tell()); + } } read(buffer, bytes); readok = true; @@ -811,7 +791,6 @@ Reader::read(char *buffer, size_t size) { char *buffer_pos = buffer; size_t remaining = size; - //while (true) while (remaining != 0) { int retval = gzread(m_gzfile, buffer_pos, remaining); @@ -906,8 +885,9 @@ void Reader::seekForward(size_t bytes) #endif } -void Reader::seekTo(size_t bytes) +void Reader::seekTo(const PropertyInfo &p) { + size_t bytes = p.offset; if (m_inRAM) { if (bytes > m_inRAMSize) @@ -927,7 +907,14 @@ void Reader::seekTo(size_t bytes) #ifdef GTO_SUPPORT_ZIP else { - gzseek(m_gzfile, bytes, SEEK_SET); + if(p.index < m_dataOffsets.size()) + { + gzseek_raw(m_gzfile, m_dataOffsets[p.index]); + } + else + { + gzseek(m_gzfile, p.offset, SEEK_SET); + } } #endif @@ -1194,4 +1181,88 @@ void Reader::endFile() m_header.numStrings = m_strings.size(); } + +void Reader::readIndexTable() +{ + // See zhacks.h for details + + m_dataOffsets.clear(); + + FILE *file = fopen(m_inName.c_str(), "rb"); + + // + // Read the FLG header field + // + unsigned char flags = 0; + fseek(file, 3L, SEEK_SET); + fread(&flags, sizeof(unsigned char), 1, file); + if ( ! (flags & EXTRA_FIELD) ) + { + // No offsets in this file + fclose(file); + return; + } + + // According to RFC 1952, gzip header byte order is always little-endian + + // + // Read the XLEN extra field length + // + unsigned char xlen_bytes[2]; + fseek(file, 10L, SEEK_SET); + fread(xlen_bytes, sizeof(unsigned char), 2, file); + unsigned short xlen = xlen_bytes[0] + (xlen_bytes[1] << 8); + + if( xlen != (2 * sizeof(unsigned int)) ) + { + std::cerr << "Warning: ignoring malformed index table." << std::endl; + fclose(file); + return; + } + + // + // Read the index table offset and size + // + unsigned int indexTableOffset = 0; + unsigned int indexTableSize = 0; + fread(&indexTableOffset, sizeof(unsigned int), 1, file); + fread(&indexTableSize, sizeof(unsigned int), 1, file); + fclose(file); + + unsigned short needsSwap = 0xFF00; + if( *(unsigned char*)(&needsSwap) ) + { + swapWords(&indexTableOffset, 1); + swapWords(&indexTableSize, 1); + } + + // + // Read the offset table + // + m_dataOffsets.resize(indexTableSize); + unsigned int restore_gz_pos = gztell(m_gzfile); + gzseek_raw(m_gzfile, indexTableOffset); + int r = gzread(m_gzfile, &m_dataOffsets.front(), + indexTableSize * sizeof(unsigned int)); + + // + // Offsets are always stored as little-endian + // + if( *(unsigned char*)(&needsSwap) ) + { + swapWords(&m_dataOffsets.front(), m_dataOffsets.size()); + } + + // + // Rewind and seek past the GTO magic number + // + // Note: I can't figure out how to reset the gzip stream + // to 'normal' after a gzseek_raw, so I'm closing and re- + // opening it. + // + gzclose(m_gzfile); + m_gzfile = gzopen(m_inName.c_str(), "rb"); + gzseek(m_gzfile, restore_gz_pos, SEEK_SET); +} + } // Gto lib/Gto/Reader.cpp @@ -84,12 +84,13 @@ public: struct PropertyInfo : PropertyHeader { void* propertyData; - int offset; // file offset + unsigned int offset; // file offset const ComponentInfo* component; private: bool requested; + size_t index; friend class Reader; }; @@ -99,6 +100,7 @@ public: typedef std::vector<std::string> StringTable; typedef std::vector<unsigned char> ByteArray; typedef std::map<std::string,int> StringMap; + typedef std::vector<unsigned int> DataOffsets; // @@ -167,6 +169,7 @@ public: const StringTable& stringTable() { return m_strings; } bool isSwapped() const { return m_swapped; } + bool hasIndex() const { return !m_dataOffsets.empty(); } unsigned int readMode() const { return m_mode; } const std::string& infileName() const { return m_inName; } @@ -351,13 +354,14 @@ private: void readObjects(); void readComponents(); void readProperties(); + void readIndexTable(); void read(char *, size_t); void get(char &); bool notEOF(); void seekForward(size_t); int tell(); - void seekTo(size_t); + void seekTo(const PropertyInfo &p); private: Header m_header; @@ -383,6 +387,7 @@ private: ByteArray m_buffer; TypeSpec m_currentType; size_t m_currentReadOffset; + DataOffsets m_dataOffsets; }; template <typename T> lib/Gto/Reader.h @@ -143,4 +143,37 @@ bool isGTOFile(const char* infile) header.magic == GTO_MAGIC_TEXTl; } +void +swapWords(void *data, size_t size) +{ + struct bytes { char c[4]; }; + + bytes* ip = reinterpret_cast<bytes*>(data); + + for (size_t i=0; i<size; i++) + { + bytes temp = ip[i]; + ip[i].c[0] = temp.c[3]; + ip[i].c[1] = temp.c[2]; + ip[i].c[2] = temp.c[1]; + ip[i].c[3] = temp.c[0]; + } +} + +void +swapShorts(void *data, size_t size) +{ + struct bytes { char c[2]; }; + + bytes* ip = reinterpret_cast<bytes*>(data); + + for (size_t i=0; i<size; i++) + { + bytes temp = ip[i]; + ip[i].c[0] = temp.c[1]; + ip[i].c[1] = temp.c[0]; + } +} + + } // Gto lib/Gto/Utilities.cpp @@ -52,6 +52,9 @@ Number asNumber(void*, Gto::DataType); bool isGTOFile(const char*); +void swapWords(void *data, size_t size); +void swapShorts(void *data, size_t size); + } // Gto lib/Gto/Utilities.h @@ -1,5 +1,5 @@ //****************************************************************************** -// Copyright (c) 2003 Tweak Films. +// Copyright (c) 2003 Tweak Films. // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public License @@ -21,6 +21,7 @@ #define _GNU_SOURCE #endif +#include "zhacks.h" #include "Writer.h" #include "Utilities.h" #include <fstream> @@ -31,6 +32,7 @@ #define GTO_DEBUG 0 #ifdef GTO_SUPPORT_ZIP +#include <fcntl.h> #include <zlib.h> #endif @@ -42,10 +44,11 @@ namespace Gto { using namespace std; -Writer::Writer() - : m_out(0), +Writer::Writer() + : m_out(0), m_gzfile(0), - m_needsClosing(false), + m_gzRawFd(-1), + m_needsClosing(false), m_error(false), m_tableFinished(false), m_currentProperty(0), @@ -58,11 +61,12 @@ Writer::Writer() init(0); } -Writer::Writer(ostream &o) - : m_out(0), +Writer::Writer(ostream &o) + : m_out(0), m_gzfile(0), - m_needsClosing(true), - m_error(false), + m_gzRawFd(-1), + m_needsClosing(true), + m_error(false), m_tableFinished(false), m_currentProperty(0), m_type(CompressedGTO), @@ -86,12 +90,13 @@ Writer::init(ostream *o) } bool -Writer::open(const char* filename, FileType type) +Writer::open(const char* filename, FileType type, bool writeIndex) { if (m_out || m_gzfile) return false; - m_outName = filename; - m_type = type; + m_outName = filename; + m_type = type; + m_writeIndexTable = writeIndex; #ifndef GTO_SUPPORT_ZIP if (type == CompressedGTO) type = BinaryGTO; @@ -118,11 +123,12 @@ Writer::open(const char* filename, FileType type) #ifdef GTO_SUPPORT_ZIP else if (type == CompressedGTO) { - m_gzfile = gzopen(filename, "wb"); + m_gzRawFd = ::open(filename, O_WRONLY|O_CREAT|O_TRUNC, 0666); - if (!m_gzfile) + if (m_gzRawFd < 0) { m_gzfile = 0; + m_gzRawFd = -1; m_error = true; return false; } @@ -137,7 +143,7 @@ Writer::open(const char* filename, FileType type) void Writer::close() { - if (m_beginDataCalled && !m_endDataCalled) + if (m_beginDataCalled && !m_endDataCalled) { // // Should this be an error? We can gracefully finish like @@ -155,12 +161,13 @@ Writer::close() delete m_out; } #ifdef GTO_SUPPORT_ZIP - else if (m_gzfile && m_needsClosing) + else if ((m_gzRawFd > 0) && m_needsClosing) { - gzclose(m_gzfile); + ::close(m_gzRawFd); } m_gzfile = 0; + m_gzRawFd = -1; #endif m_out = 0; } @@ -184,6 +191,11 @@ Writer::beginData(const std::string *orderedStrings, int num) } else { + if(m_type == CompressedGTO) + { + m_gzfile = gzdopen(dup(m_gzRawFd), "w"); + prepIndexTable(); + } writeHead(); } @@ -197,6 +209,10 @@ Writer::endData() { writeText(" }\n}\n"); } + else if (m_type == CompressedGTO) + { + writeIndexTable(); + } m_endDataCalled = true; } @@ -277,8 +293,8 @@ Writer::lookup(int n) const } void -Writer::beginObject(const char* name, - const char *protocol, +Writer::beginObject(const char* name, + const char *protocol, unsigned int version) { if (m_objectActive) @@ -310,37 +326,11 @@ Writer::beginObject(const char* name, void Writer::beginComponent(const char* name, unsigned int flags) { - if (!m_objectActive) - { - throw std::runtime_error("ERROR: Gto::Writer::beginComponent() -- " - "you forgot to call beginObject()"); - } - - if (m_componentActive) - { - throw std::runtime_error("ERROR: Gto::Writer::beginComponent() -- " - "you forgot to call endComponent()"); - } - - m_names.push_back(name); - m_objects.back().numComponents++; - ComponentHeader header; - memset(&header, 0, sizeof(ComponentHeader)); - - header.numProperties = 0; - header.flags = flags; - header.name = m_names.size() - 1; - header.pad = 0; - - m_names.push_back(""); - header.interpretation = m_names.size() - 1; - - m_components.push_back(header); - m_componentActive = true; + beginComponent(name, "", flags); } void -Writer::beginComponent(const char* name, +Writer::beginComponent(const char* name, const char* interp, unsigned int flags) { @@ -360,7 +350,7 @@ Writer::beginComponent(const char* name, m_objects.back().numComponents++; ComponentHeader header; memset(&header, 0, sizeof(ComponentHeader)); - + header.numProperties = 0; header.flags = flags; header.name = m_names.size() - 1; @@ -428,21 +418,21 @@ Writer::property(const char* name, void Writer::constructStringTable(const std::string *orderedStrings, int num) { - if (num < 0) + if (num < 0) { throw std::runtime_error("Gto::Writer::constructStringTable(): " "ordered string list length was negative"); } - - if (num > 0 && orderedStrings == 0) + + if (num > 0 && orderedStrings == 0) { throw std::runtime_error("Gto::Writer::constructStringTable(): " "ordered string list length was positive " "but string list was null"); } - - intern( "(Gto::Writer compiled " __DATE__ - " " __TIME__ + + intern( "(Gto::Writer compiled " __DATE__ + " " __TIME__ ", $Id: Writer.cpp,v 1.37 2008/04/11 23:25:24 src Exp $)" ); for (size_t i=0; i < m_names.size(); i++) @@ -481,7 +471,7 @@ Writer::constructStringTable(const std::string *orderedStrings, int num) } for (StringMap::iterator i = m_strings.begin(); - i != m_strings.end(); + i != m_strings.end(); ++i) { if (i->second == -1) i->second = count++; @@ -563,7 +553,7 @@ Writer::writeMaybeQuotedString(const string& str) return; } } - + if (gto_isalnum(str)) { writeText(str); @@ -582,42 +572,42 @@ Writer::writeQuotedString(const string& str) for (size_t i=0; i < str.size(); i++) { - char c = str[i]; + char c = str[i]; - if (c == 0) - { + if (c == 0) + { write(""); - } - else if (iscntrl(c)) - { - switch (c) - { - case '\n': writeFormatted("\\n"); break; - case '\b': writeFormatted("\\b"); break; - case '\r': writeFormatted("\\r"); break; - case '\t': writeFormatted("\\t"); break; - default: + } + else if (iscntrl(c)) + { + switch (c) + { + case '\n': writeFormatted("\\n"); break; + case '\b': writeFormatted("\\b"); break; + case '\r': writeFormatted("\\r"); break; + case '\t': writeFormatted("\\t"); break; + default: { char temp[41]; temp[40] = 0; snprintf(temp, 40, "\\%o", int(c)); writeFormatted(temp); } - break; - } - } - else if (c == delim) - { + break; + } + } + else if (c == delim) + { writeFormatted("\\%c", delim); - } + } else if (c & 0x80) { // UTF-8 } - else - { + else + { write(&c, sizeof(char)); - } + } } writeFormatted("\""); @@ -764,14 +754,14 @@ Writer::propertySanityCheck(const char *propertyName, int size, int width) m_error = true; return false; } - + return true; } -void +void Writer::propertyDataRaw(const void* data, - const char *propertyName, - int size, + const char *propertyName, + int size, int width) { if (!m_beginDataCalled) beginData(); @@ -846,7 +836,7 @@ Writer::propertyDataRaw(const void* data, if (isNumber(DataType(info.type))) { - Number num = asNumber(bdata + (i * ds), + Number num = asNumber(bdata + (i * ds), DataType(info.type)); if (num.type == Int) @@ -873,12 +863,100 @@ Writer::propertyDataRaw(const void* data, } else { +#ifdef GTO_SUPPORT_ZIP + if( (m_type == CompressedGTO) && m_writeIndexTable ) + { + gzflush(m_gzfile, Z_FULL_FLUSH); + m_dataOffsets.push_back(lseek(m_gzRawFd, 0, SEEK_CUR)); + } +#endif write(data, dataSize(m_properties[p].type) * n); } } } +void +Writer::prepIndexTable() +{ + // See zhacks.h for details + + if( ! m_writeIndexTable ) return; + gz_stream *s = (gz_stream*)m_gzfile; + + // + // Write the FLG header field + // + fseek(s->file, 3L, SEEK_SET); + fprintf(s->file, "%c", EXTRA_FIELD); + + // + // Write the XLEN extra field length + // + unsigned short xlen = 2 * sizeof(unsigned int); + fseek(s->file, 10L, SEEK_SET); + fwrite(&xlen, 1, sizeof(unsigned short), s->file); + + // + // Leave space for the index table offset and size + // + unsigned int placeholders[2] = {0, 0}; + fwrite(placeholders, sizeof(unsigned int), 2, s->file); + + gzflush(m_gzfile, Z_FULL_FLUSH); + s->start = ftell(s->file); +} + + +void +Writer::writeIndexTable() +{ + // See zhacks.h for details + + if( ! m_writeIndexTable ) return; + assert(m_dataOffsets.size() == m_properties.size()); + + // + // Lay down a gzip marker and remember where it is + // + gzflush(m_gzfile, Z_FULL_FLUSH); + unsigned int indexTableOffset = lseek(m_gzRawFd, 0, SEEK_CUR); + unsigned int indexTableSize = m_dataOffsets.size(); + + // + // Always store offsets as little-endian + // + unsigned short needsSwap = 0xFF00; + if( *(unsigned char*)(&needsSwap) ) + { + swapWords(&m_dataOffsets.front(), m_dataOffsets.size()); + swapWords(&indexTableOffset, 1); + swapWords(&indexTableSize, 1); + } + + // + // Write the index table into the gzip stream + // + int w = gzwrite(m_gzfile, &m_dataOffsets.front(), + m_dataOffsets.size() * sizeof(unsigned int)); + + // + // Close the gzip file. Must be done before lseeking back + // to the header in the next step. I believe this writes + // the CRC32 and ISIZE fields at the end of the file. + // + gzclose(m_gzfile); + + // + // Write the raw offset to the index table into the + // 'extra' gzip header field. + // + lseek(m_gzRawFd, 12L, SEEK_SET); + ::write(m_gzRawFd, &indexTableOffset, sizeof(unsigned int)); + ::write(m_gzRawFd, &indexTableSize, sizeof(unsigned int)); +} + + } // Gto #ifdef _MSC_VER lib/Gto/Writer.cpp @@ -54,6 +54,7 @@ public: typedef std::vector<PropertyHeader> Properties; typedef std::vector<ObjectHeader> Objects; typedef std::map<size_t, PropertyPath> PropertyMap; + typedef std::vector<unsigned int> DataOffsets; enum FileType { @@ -71,7 +72,8 @@ public: // bool open(const char* filename, - FileType mode = CompressedGTO); + FileType mode = CompressedGTO, + bool writeIndex=true); // // Deprecated open API @@ -211,11 +213,15 @@ private: void writeMaybeQuotedString(const std::string&); void flush(); + void prepIndexTable(); + void writeIndexTable(); + bool propertySanityCheck(const char*, int, int); private: std::ostream* m_out; void* m_gzfile; + int m_gzRawFd; Objects m_objects; Components m_components; Properties m_properties; @@ -232,6 +238,9 @@ private: bool m_beginDataCalled : 1; bool m_objectActive : 1; bool m_componentActive : 1; + bool m_writeIndexTable : 1; + size_t m_bytesWritten; + DataOffsets m_dataOffsets; }; template<typename T> lib/Gto/Writer.h b1ba195f3672a26fdb8803c119fb5ab8e5d16744 Mike Root mike-gto@redpointmaine.com http://github.com/jimhourihan/gto/commit/c1f1f9673d57b6218593ea8b2fda5ea676d001f0 c1f1f9673d57b6218593ea8b2fda5ea676d001f0 2009-05-30T12:18:15-07:00 2009-05-30T12:18:15-07:00 Directly seek to properties in gzipped files 5e9eaae65955292ed986d96e3415dd38eb2e671e Mike Root mike-gto@redpointmaine.com