From 6f0dac7003b506b3e234b621fe74b786e28bf6cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaakko=20Kera=CC=88nen?= Date: Sun, 12 Feb 2017 21:27:29 +0200 Subject: [PATCH] libcore|FS: Added MetadataBank for caching ZIP archive metadata File::metaId() generates identifiers suitable for caching. ZipArchive can be initialized either from a cached central directory or from the source file. --- doomsday/sdk/libcore/include/de/MetadataBank | 1 + doomsday/sdk/libcore/include/de/core/app.h | 6 + doomsday/sdk/libcore/include/de/data/bank.h | 12 +- .../sdk/libcore/include/de/data/ziparchive.h | 8 +- .../include/de/filesys/archiveentryfile.h | 14 +- .../sdk/libcore/include/de/filesys/file.h | 6 + .../libcore/include/de/filesys/metadatabank.h | 63 ++++ .../libcore/include/de/filesys/nativefile.h | 1 + doomsday/sdk/libcore/src/core/app.cpp | 21 +- doomsday/sdk/libcore/src/data/bank.cpp | 6 +- doomsday/sdk/libcore/src/data/ziparchive.cpp | 316 +++++++++++------- .../libcore/src/filesys/archiveentryfile.cpp | 10 + .../sdk/libcore/src/filesys/archivefeed.cpp | 6 +- doomsday/sdk/libcore/src/filesys/file.cpp | 10 + .../sdk/libcore/src/filesys/metadatabank.cpp | 107 ++++++ .../sdk/libcore/src/filesys/nativefile.cpp | 5 + .../libcore/src/scriptsys/bindings_core.cpp | 8 +- .../sdk/libgui/src/graphics/imagebank.cpp | 2 +- 18 files changed, 454 insertions(+), 148 deletions(-) create mode 100644 doomsday/sdk/libcore/include/de/MetadataBank create mode 100644 doomsday/sdk/libcore/include/de/filesys/metadatabank.h create mode 100644 doomsday/sdk/libcore/src/filesys/metadatabank.cpp diff --git a/doomsday/sdk/libcore/include/de/MetadataBank b/doomsday/sdk/libcore/include/de/MetadataBank new file mode 100644 index 0000000000..f0ef15eca9 --- /dev/null +++ b/doomsday/sdk/libcore/include/de/MetadataBank @@ -0,0 +1 @@ +#include "filesys/metadatabank.h" diff --git a/doomsday/sdk/libcore/include/de/core/app.h b/doomsday/sdk/libcore/include/de/core/app.h index 56b7cb701e..a2f82f7cee 100644 --- a/doomsday/sdk/libcore/include/de/core/app.h +++ b/doomsday/sdk/libcore/include/de/core/app.h @@ -43,6 +43,7 @@ class FileSystem; class Folder; class LogBuffer; class LogFilter; +class MetadataBank; class Module; class Path; class NativePath; @@ -274,6 +275,11 @@ class DENG2_PUBLIC App : DENG2_OBSERVES(Clock, TimeChange) */ static FileSystem &fileSystem(); + /** + * Returns the application's metadata cache. + */ + static MetadataBank &metadataBank(); + /** * Returns the root folder of the file system. */ diff --git a/doomsday/sdk/libcore/include/de/data/bank.h b/doomsday/sdk/libcore/include/de/data/bank.h index ff1a5de74e..123d018165 100644 --- a/doomsday/sdk/libcore/include/de/data/bank.h +++ b/doomsday/sdk/libcore/include/de/data/bank.h @@ -82,12 +82,16 @@ class DENG2_PUBLIC Bank enum Flag { + SingleThread = 0, + /** * Separate thread used for managing the bank's data (loading, caching * data). Requires data items and sources to be thread-safe. */ BackgroundThread = 0x1, + EnableHotStorage = 0, + /** * Do not use the hot storage to keep serialized copies of data items. * This is useful if recreating the data from source is trivial. @@ -102,7 +106,7 @@ class DENG2_PUBLIC Bank */ ClearHotStorageWhenBankDestroyed = 0x4, - DefaultFlags = DisableHotStorage + DefaultFlags = SingleThread | DisableHotStorage }; Q_DECLARE_FLAGS(Flags, Flag) @@ -166,9 +170,11 @@ class DENG2_PUBLIC Bank public: virtual ~IData() {} + enum SerialMode { Serializing, Deserializing }; + /// Returns an ISerializable pointer to the object. Required /// for putting the data in hot storage. - virtual ISerializable *asSerializable() { return 0; } + virtual ISerializable *asSerializable(SerialMode) { return 0; } /// Returns the size of the data that it occupies in memory. virtual duint sizeInMemory() const { return 0; } @@ -370,7 +376,7 @@ class DENG2_PUBLIC Bank /** * Construct a new concrete instance of the data item. Called before - * deserialization. Default implementation just returns NULL (seriliazation + * deserialization. Default implementation just returns NULL (serialization * not supported). * * @return IData instance. Ownership given to caller. diff --git a/doomsday/sdk/libcore/include/de/data/ziparchive.h b/doomsday/sdk/libcore/include/de/data/ziparchive.h index e21614320d..c0730f20dc 100644 --- a/doomsday/sdk/libcore/include/de/data/ziparchive.h +++ b/doomsday/sdk/libcore/include/de/data/ziparchive.h @@ -73,10 +73,9 @@ class DENG2_PUBLIC ZipArchive : public Archive * data is made, so the caller must make sure the * byte array remains in existence for the lifetime * of the Archive instance. + * @param dirCacheId ID of cached ZIP directory data. */ - ZipArchive(IByteArray const &data); - - virtual ~ZipArchive(); + ZipArchive(IByteArray const &data, Block const &dirCacheId = Block()); void operator >> (Writer &to) const; @@ -122,6 +121,9 @@ class DENG2_PUBLIC ZipArchive : public Archive typedef PathTreeT Index; Index const &index() const; + +private: + DENG2_PRIVATE(d) }; } // namespace de diff --git a/doomsday/sdk/libcore/include/de/filesys/archiveentryfile.h b/doomsday/sdk/libcore/include/de/filesys/archiveentryfile.h index b58b035a61..855057a95e 100644 --- a/doomsday/sdk/libcore/include/de/filesys/archiveentryfile.h +++ b/doomsday/sdk/libcore/include/de/filesys/archiveentryfile.h @@ -46,10 +46,10 @@ class ArchiveEntryFile : public ByteArrayFile ~ArchiveEntryFile(); - String describe() const; + String describe() const override; String entryPath() const; - void clear(); + void clear() override; /** * Flushes the entire archive that this file is part of into its source @@ -63,7 +63,9 @@ class ArchiveEntryFile : public ByteArrayFile * manual flushing this occurs automatically when the root ArchiveFeed * instance is deleted. */ - void flush(); + void flush() override; + + Block metaId() const override; /// Returns the archive of the file. Archive &archive(); @@ -74,8 +76,8 @@ class ArchiveEntryFile : public ByteArrayFile void uncache() const; // Implements IByteArray. - Size size() const; - void get(Offset at, Byte *values, Size count) const; + Size size() const override; + void get(Offset at, Byte *values, Size count) const override; /** * Modifies the content of an archive entry. Changes are made instantly @@ -86,7 +88,7 @@ class ArchiveEntryFile : public ByteArrayFile * @param values Data. * @param count Length of data. */ - void set(Offset at, Byte const *values, Size count); + void set(Offset at, Byte const *values, Size count) override; private: DENG2_PRIVATE(d) diff --git a/doomsday/sdk/libcore/include/de/filesys/file.h b/doomsday/sdk/libcore/include/de/filesys/file.h index 2234dbfba9..0fbe4afdc5 100644 --- a/doomsday/sdk/libcore/include/de/filesys/file.h +++ b/doomsday/sdk/libcore/include/de/filesys/file.h @@ -273,6 +273,12 @@ class DENG2_PUBLIC File : public filesys::Node, public IIOStream, public IObject */ dsize size() const; + /** + * Generates a unique identifier generated based on the metadata of the file. This is + * suitable for use in caches to identify a particular instance of a file. + */ + virtual Block metaId() const; + /** * Returns the mode of the file. */ diff --git a/doomsday/sdk/libcore/include/de/filesys/metadatabank.h b/doomsday/sdk/libcore/include/de/filesys/metadatabank.h new file mode 100644 index 0000000000..380a50af4b --- /dev/null +++ b/doomsday/sdk/libcore/include/de/filesys/metadatabank.h @@ -0,0 +1,63 @@ +/** @file metadatabank.h File metadata cache. + * + * @authors Copyright (c) 2017 Jaakko Keränen + * + * @par License + * LGPL: http://www.gnu.org/licenses/lgpl.html + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or (at your + * option) any later version. This program is distributed in the hope that it + * will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser + * General Public License for more details. You should have received a copy of + * the GNU Lesser General Public License along with this program; if not, see: + * http://www.gnu.org/licenses + */ + +#ifndef LIBDENG2_METADATABANK_H +#define LIBDENG2_METADATABANK_H + +#include + +namespace de { + +/** + * File metadata cache. + * + * @ingroup fs + */ +class DENG2_PUBLIC MetadataBank : public Bank +{ +public: + MetadataBank(); + + static MetadataBank &get(); + + /** + * Adds a new metadata entry into the bank. + * + * @param id Meta ID. + * + * @return The cached metadata, if available. This will be an empty Block if no + * metadata has yet been cached. + */ + Block check(Block const &id); + + void setMetadata(Block const &id, Block const &metadata); + + Block metadata(Block const &id) const; + +protected: + IData *loadFromSource(ISource &source) override; + + IData *newData() override; + +private: + DENG2_PRIVATE(d) +}; + +} // namespace de + +#endif // LIBDENG2_METADATABANK_H diff --git a/doomsday/sdk/libcore/include/de/filesys/nativefile.h b/doomsday/sdk/libcore/include/de/filesys/nativefile.h index 8ac0529fc3..c27146a696 100644 --- a/doomsday/sdk/libcore/include/de/filesys/nativefile.h +++ b/doomsday/sdk/libcore/include/de/filesys/nativefile.h @@ -50,6 +50,7 @@ class DENG2_PUBLIC NativeFile : public ByteArrayFile virtual ~NativeFile(); String describe() const; + Block metaId() const; void clear(); void flush(); diff --git a/doomsday/sdk/libcore/src/core/app.cpp b/doomsday/sdk/libcore/src/core/app.cpp index fe7441c024..1459a5a7a5 100644 --- a/doomsday/sdk/libcore/src/core/app.cpp +++ b/doomsday/sdk/libcore/src/core/app.cpp @@ -35,6 +35,7 @@ #include "de/LogBuffer" #include "de/LogFilter" #include "de/math.h" +#include "de/MetadataBank" #include "de/Module" #include "de/NativeFile" #include "de/NumberValue" @@ -83,14 +84,15 @@ DENG2_PIMPL(App) ScriptSystem scriptSys; FileSystem fs; - QScopedPointer basePackFile; + std::unique_ptr metaBank; + std::unique_ptr basePackFile; Record appModule; /// Archive where persistent data should be stored. Written to /home/persist.pack. /// The archive is owned by the file system. Archive *persistentData; - QScopedPointer unixInfo; + std::unique_ptr unixInfo; /// The configuration. Path configPath; @@ -102,7 +104,7 @@ DENG2_PIMPL(App) void (*terminateFunc)(char const *); /// Optional sink for warnings and errors (set with "-errors"). - QScopedPointer errorSink; + std::unique_ptr errorSink; Impl(Public *a, QStringList args) : Base(a) @@ -140,7 +142,9 @@ DENG2_PIMPL(App) ~Impl() { - if (!errorSink.isNull()) + metaBank->unloadAll(Bank::InHotStorage); + + if (errorSink) { logBuffer.removeSink(*errorSink); } @@ -227,6 +231,9 @@ DENG2_PIMPL(App) // Internal data (for runtime use only, thus writable). fs.makeFolder("/sys").setMode(File::Write); + // Metadata for files. + metaBank.reset(new MetadataBank); + // Populate the file system. fs.refresh(); Folder::waitForPopulation(); @@ -758,6 +765,12 @@ FileSystem &App::fileSystem() return DENG2_APP->d->fs; } +MetadataBank &App::metadataBank() +{ + DENG2_ASSERT(DENG2_APP->d->metaBank); + return *DENG2_APP->d->metaBank; +} + PackageLoader &App::packageLoader() { return DENG2_APP->d->packageLoader; diff --git a/doomsday/sdk/libcore/src/data/bank.cpp b/doomsday/sdk/libcore/src/data/bank.cpp index fdc2017a5c..80f95927cc 100644 --- a/doomsday/sdk/libcore/src/data/bank.cpp +++ b/doomsday/sdk/libcore/src/data/bank.cpp @@ -223,7 +223,7 @@ DENG2_PIMPL(Bank) if (isValidSerialTime(timestamp)) { QScopedPointer blank(bank->newData()); - reader >> *blank->asSerializable(); + reader >> *blank->asSerializable(IData::Deserializing); setData(blank.take()); LOG_RES_XVERBOSE("Deserialized \"%s\" in %.2f seconds", path(bank->d->sepChar) << startedAt.since()); @@ -259,7 +259,7 @@ DENG2_PIMPL(Bank) loadFromSource(); } - DENG2_ASSERT(data->asSerializable() != 0); + DENG2_ASSERT(data->asSerializable(IData::Serializing) != 0); try { @@ -275,7 +275,7 @@ DENG2_PIMPL(Bank) Writer(*serial).withHeader() << source->modifiedAt() - << *data->asSerializable(); + << *data->asSerializable(IData::Serializing); } catch (...) { diff --git a/doomsday/sdk/libcore/src/data/ziparchive.cpp b/doomsday/sdk/libcore/src/data/ziparchive.cpp index 63405cc56f..b25a244a56 100644 --- a/doomsday/sdk/libcore/src/data/ziparchive.cpp +++ b/doomsday/sdk/libcore/src/data/ziparchive.cpp @@ -27,6 +27,7 @@ #include "de/ISerializable" #include "de/LittleEndianByteOrder" #include "de/LogBuffer" +#include "de/MetadataBank" #include "de/Reader" #include "de/Writer" #include "de/Zeroed" @@ -260,126 +261,227 @@ struct CentralEnd : public ISerializable { using namespace internal; -ZipArchive::ZipArchive() : Archive() +DENG2_PIMPL(ZipArchive) { - setIndex(new Index); -} - -ZipArchive::ZipArchive(IByteArray const &archive) : Archive(archive) -{ - setIndex(new Index); + Block directoryCacheId; - Reader reader(archive, littleEndianByteOrder); + Impl(Public *i) : Base(i) {} - // Locate the central directory. Start from the earliest location where - // the signature might be. - duint centralEndPos = 0; - for (duint pos = CENTRAL_END_SIZE; pos < MAXIMUM_COMMENT_SIZE; pos++) + void readCentralDirectory(Reader &reader, bool updateFromLocalHeaders) { - reader.setOffset(archive.size() - pos); - duint32 signature; - reader >> signature; - if (signature == SIG_END_OF_CENTRAL_DIR) + // The central directory end record is at the signature we found. + CentralEnd summary; + reader >> summary; + + duint const entryCount = summary.totalEntryCount; + + // The ZIP must have only one part, all entries in the same archive. + if (entryCount != summary.diskEntryCount) { - // This is it! - centralEndPos = archive.size() - pos; - break; + /// @throw MultiPartError ZIP archives in more than one part are not supported + /// by the implementation. + throw MultiPartError("ZipArchive::readCentralDirectory", "Multipart archives are not supported"); } - } - if (!centralEndPos) - { - /// @throw MissingCentralDirectoryError The ZIP central directory was not found - /// in the end of the source data. - throw MissingCentralDirectoryError("ZipArchive::Archive", - "Could not locate the central directory of the archive"); - } - // The central directory end record is at the signature we found. - CentralEnd summary; - reader >> summary; + // Read all the entries of the central directory. + reader.setOffset(summary.offset); + for (duint index = 0; index < entryCount; ++index) + { + CentralFileHeader header; + reader >> header; - duint const entryCount = summary.totalEntryCount; + // Check the signature. + if (header.signature != SIG_CENTRAL_FILE_HEADER) + { + /// @throw FormatError Invalid signature in a central directory entry. + throw FormatError("ZipArchive::readCentralDirectory", "Corrupt central directory"); + } - // The ZIP must have only one part, all entries in the same archive. - if (entryCount != summary.diskEntryCount) - { - /// @throw MultiPartError ZIP archives in more than one part are not supported - /// by the implementation. - throw MultiPartError("ZipArchive::Archive", "Multipart archives are not supported"); + Block latin1Name; + reader.readBytes(header.fileNameSize, latin1Name); + String const fileName = String::fromLatin1(latin1Name); /// @todo UTF-8? + + // Advance the cursor past the variable sized fields. + reader.seek(header.extraFieldSize + header.commentSize); + + // Skip folders. + if (fileName.endsWith("/") && !header.size) + { + continue; + } + + // Check for unsupported features. + if (header.compression != NO_COMPRESSION && header.compression != DEFLATED) + { + /// @throw UnknownCompressionError Deflation is the only compression + /// algorithm supported by the implementation. + throw UnknownCompressionError("ZipArchive::readCentralDirectory", + "Entry '" + fileName + "' uses an unsupported compression algorithm"); + } + if (header.flags & ZFH_ENCRYPTED) + { + /// @throw EncryptionError Archive is encrypted, which is not supported + /// by the implementation. + throw EncryptionError("ZipArchive::readCentralDirectory", + "Entry '" + fileName + "' is encrypted and thus cannot be read"); + } + + LocalFileHeader localHeader; + if (updateFromLocalHeaders) + { + // Read the local file header, which contains the correct extra + // field size (Info-ZIP!). + reader.mark(); + reader.setOffset(header.relOffset); + reader >> localHeader; + } + + // Make an index entry for this. + ZipEntry &entry = static_cast(self().insertEntry(fileName)); + + entry.size = header.size; + entry.sizeInArchive = header.compressedSize; + entry.compression = header.compression; + entry.crc32 = header.crc32; + entry.localHeaderOffset = header.relOffset; + + // Unpack the last modified time from the ZIP entry header. + DOSDate lastModDate(header.lastModDate); + DOSTime lastModTime(header.lastModTime); + entry.modifiedAt = QDateTime(QDate(lastModDate.year + 1980, lastModDate.month, lastModDate.dayOfMonth), + QTime(lastModTime.hours, lastModTime.minutes, lastModTime.seconds)); + + if (updateFromLocalHeaders) + { + entry.offset = reader.offset() + header.fileNameSize + localHeader.extraFieldSize; + + // Back to the central directory. + reader.rewind(); + } + } } - // Read all the entries of the central directory. - reader.setOffset(summary.offset); - for (duint index = 0; index < entryCount; ++index) + void writeCentralDirectory(Writer &writer) { - CentralFileHeader header; - reader >> header; + CentralEnd summary; + summary.diskEntryCount = summary.totalEntryCount = self().index().leafNodes().size(); - // Check the signature. - if (header.signature != SIG_CENTRAL_FILE_HEADER) + // This is where the central directory begins. + summary.offset = writer.offset(); + + // Write the central directory. + for (PathTreeIterator iter(self().index().leafNodes()); iter.hasNext(); ) { - /// @throw FormatError Invalid signature in a central directory entry. - throw FormatError("ZipArchive::Archive", "Corrupt central directory"); + ZipEntry const &entry = iter.next(); + String const fullPath = entry.path(); + + CentralFileHeader header; + header.signature = SIG_CENTRAL_FILE_HEADER; + header.version = 20; + header.requiredVersion = 20; + Date at(entry.modifiedAt); + header.lastModTime = DOSTime(at.hours(), at.minutes(), at.seconds()); + header.lastModDate = DOSDate(at.year() - 1980, at.month(), at.dayOfMonth()); + header.compression = entry.compression; + header.crc32 = entry.crc32; + header.compressedSize = entry.sizeInArchive; + header.size = entry.size; + header.fileNameSize = fullPath.size(); + header.relOffset = entry.localHeaderOffset; + + writer << header << FixedByteArray(fullPath.toLatin1()); } - String fileName = String::fromLatin1(ByteSubArray(archive, reader.offset(), header.fileNameSize)); + // Size of the central directory. + summary.size = writer.offset() - summary.offset; - // Advance the cursor past the variable sized fields. - reader.seek(header.fileNameSize + header.extraFieldSize + header.commentSize); + // End of central directory. + writer << duint32(SIG_END_OF_CENTRAL_DIR) << summary; - // Skip folders. - if (fileName.endsWith("/") && !header.size) + writer << duint32(SIG_DIGITAL_SIGNATURE) + << duint16(0); // No signature data. + } + + void updateCachedDirectory() + { + if (!directoryCacheId.isEmpty()) { - continue; + Block meta; + Writer writer(meta); + writeCentralDirectory(writer); + MetadataBank::get().setMetadata(directoryCacheId, meta); } + } - // Check for unsupported features. - if (header.compression != NO_COMPRESSION && header.compression != DEFLATED) + bool restoreFromCache() + { + if (directoryCacheId.isEmpty()) return false; + + auto &bank = MetadataBank::get(); + + try { - /// @throw UnknownCompressionError Deflation is the only compression - /// algorithm supported by the implementation. - throw UnknownCompressionError("ZipArchive::Archive", - "Entry '" + fileName + "' uses an unsupported compression algorithm"); + Block const meta = bank.check(directoryCacheId); + if (meta.isEmpty()) return false; + qDebug() << "restoring from cache" << directoryCacheId; + Reader reader(meta); + readCentralDirectory(reader, false); + return true; } - if (header.flags & ZFH_ENCRYPTED) + catch (Error const &) { - /// @throw EncryptionError Archive is encrypted, which is not supported - /// by the implementation. - throw EncryptionError("ZipArchive::Archive", - "Entry '" + fileName + "' is encrypted and thus cannot be read"); + return false; } + } +}; - // Read the local file header, which contains the correct extra - // field size (Info-ZIP!). - reader.mark(); - reader.setOffset(header.relOffset); - LocalFileHeader localHeader; - reader >> localHeader; +ZipArchive::ZipArchive() : d(new Impl(this)) +{ + setIndex(new Index); +} - // Make an index entry for this. - ZipEntry &entry = static_cast(insertEntry(fileName)); +ZipArchive::ZipArchive(IByteArray const &archive, Block const &dirCacheId) + : Archive(archive) + , d(new Impl(this)) +{ + setIndex(new Index); - entry.size = header.size; - entry.sizeInArchive = header.compressedSize; - entry.compression = header.compression; - entry.crc32 = header.crc32; - entry.localHeaderOffset = header.relOffset; + d->directoryCacheId = dirCacheId; - // Unpack the last modified time from the ZIP entry header. - DOSDate lastModDate(header.lastModDate); - DOSTime lastModTime(header.lastModTime); - entry.modifiedAt = QDateTime(QDate(lastModDate.year + 1980, lastModDate.month, lastModDate.dayOfMonth), - QTime(lastModTime.hours, lastModTime.minutes, lastModTime.seconds)); + if (d->restoreFromCache()) + { + // No need to check the file itself. + return; + } - entry.offset = reader.offset() + header.fileNameSize + localHeader.extraFieldSize; + Reader reader(archive, littleEndianByteOrder); - // Back to the central directory. - reader.rewind(); + // Locate the central directory. Start from the earliest location where + // the signature might be. + duint centralEndPos = 0; + for (duint pos = CENTRAL_END_SIZE; pos < MAXIMUM_COMMENT_SIZE; pos++) + { + reader.setOffset(archive.size() - pos); + duint32 signature; + reader >> signature; + if (signature == SIG_END_OF_CENTRAL_DIR) + { + // This is it! + centralEndPos = archive.size() - pos; + break; + } + } + if (!centralEndPos) + { + /// @throw MissingCentralDirectoryError The ZIP central directory was not found + /// in the end of the source data. + throw MissingCentralDirectoryError("ZipArchive::Archive", + "Could not locate the central directory of the archive"); } -} -ZipArchive::~ZipArchive() -{} + d->readCentralDirectory(reader, true); + d->updateCachedDirectory(); +} void ZipArchive::readFromSource(Entry const &e, Path const &, IBlock &uncompressedData) const { @@ -470,7 +572,7 @@ void ZipArchive::operator >> (Writer &to) const */ Writer writer(to, littleEndianByteOrder); - // First write the local headers. + // First write the local headers and entry contents. for (PathTreeIterator iter(index().leafNodes()); iter.hasNext(); ) { // We will be updating relevant members of the entry. @@ -568,43 +670,7 @@ void ZipArchive::operator >> (Writer &to) const } } - CentralEnd summary; - summary.diskEntryCount = summary.totalEntryCount = index().leafNodes().size(); - - // This is where the central directory begins. - summary.offset = writer.offset(); - - // Write the central directory. - for (PathTreeIterator iter(index().leafNodes()); iter.hasNext(); ) - { - ZipEntry const &entry = iter.next(); - String const fullPath = entry.path(); - - CentralFileHeader header; - header.signature = SIG_CENTRAL_FILE_HEADER; - header.version = 20; - header.requiredVersion = 20; - Date at(entry.modifiedAt); - header.lastModTime = DOSTime(at.hours(), at.minutes(), at.seconds()); - header.lastModDate = DOSDate(at.year() - 1980, at.month(), at.dayOfMonth()); - header.compression = entry.compression; - header.crc32 = entry.crc32; - header.compressedSize = entry.sizeInArchive; - header.size = entry.size; - header.fileNameSize = fullPath.size(); - header.relOffset = entry.localHeaderOffset; - - writer << header << FixedByteArray(fullPath.toLatin1()); - } - - // Size of the central directory. - summary.size = writer.offset() - summary.offset; - - // End of central directory. - writer << duint32(SIG_END_OF_CENTRAL_DIR) << summary; - - // No signature data. - writer << duint32(SIG_DIGITAL_SIGNATURE) << duint16(0); + d->writeCentralDirectory(writer); // Since we used our own writer, seek the writer that was given to us // by the amount of data we wrote. diff --git a/doomsday/sdk/libcore/src/filesys/archiveentryfile.cpp b/doomsday/sdk/libcore/src/filesys/archiveentryfile.cpp index 32ba9f430f..0f4558df06 100644 --- a/doomsday/sdk/libcore/src/filesys/archiveentryfile.cpp +++ b/doomsday/sdk/libcore/src/filesys/archiveentryfile.cpp @@ -101,6 +101,16 @@ void ArchiveEntryFile::flush() } } +Block ArchiveEntryFile::metaId() const +{ + Block data = File::metaId() + d->entryPath.toUtf8(); + if (auto const *archFeed = originFeed()->maybeAs()) + { + data += archFeed->archiveSourceFile().metaId(); + } + return data.md5Hash(); +} + Archive &ArchiveEntryFile::archive() { return *d->archive; diff --git a/doomsday/sdk/libcore/src/filesys/archivefeed.cpp b/doomsday/sdk/libcore/src/filesys/archivefeed.cpp index 5ae566bd1e..b36e90ce91 100644 --- a/doomsday/sdk/libcore/src/filesys/archivefeed.cpp +++ b/doomsday/sdk/libcore/src/filesys/archivefeed.cpp @@ -67,7 +67,9 @@ DENG2_PIMPL(ArchiveFeed) { LOG_RES_XVERBOSE("Source %s is a byte array", f.description()); - arch = new ZipArchive(*bytes); + qDebug() << "loading" << f.description() << f.metaId(); + + arch = new ZipArchive(*bytes, f.metaId()); } else { @@ -322,7 +324,7 @@ File const &ArchiveFeed::archiveSourceFile() const { return *d->file; } - throw InvalidSourceError("ArchiveFeed::archiveSourceFile", "Archive source file is gone"); + throw InvalidSourceError("ArchiveFeed::archiveSourceFile", "Archive source file is missing"); } void ArchiveFeed::rewriteFile() diff --git a/doomsday/sdk/libcore/src/filesys/file.cpp b/doomsday/sdk/libcore/src/filesys/file.cpp index 42d72a1975..0ec1eaaa8b 100644 --- a/doomsday/sdk/libcore/src/filesys/file.cpp +++ b/doomsday/sdk/libcore/src/filesys/file.cpp @@ -444,4 +444,14 @@ dsize File::size() const return status().size; } +Block File::metaId() const +{ + auto const &st = target().status(); + + Block data; + Writer writer(data); + writer << path() << duint64(st.size) << st.modifiedAt; + return data.md5Hash(); +} + } // namespace de diff --git a/doomsday/sdk/libcore/src/filesys/metadatabank.cpp b/doomsday/sdk/libcore/src/filesys/metadatabank.cpp new file mode 100644 index 0000000000..de2fd5fca1 --- /dev/null +++ b/doomsday/sdk/libcore/src/filesys/metadatabank.cpp @@ -0,0 +1,107 @@ +/** @file metadatabank.cpp Cache for file metadata. + * + * @authors Copyright (c) 2017 Jaakko Keränen + * + * @par License + * LGPL: http://www.gnu.org/licenses/lgpl.html + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or (at your + * option) any later version. This program is distributed in the hope that it + * will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser + * General Public License for more details. You should have received a copy of + * the GNU Lesser General Public License along with this program; if not, see: + * http://www.gnu.org/licenses + */ + +#include "de/MetadataBank" +#include "de/App" + +namespace de { + +DENG2_PIMPL(MetadataBank), public Lockable +{ + struct Source : public ISource + { + Block metaId; + + Source(Block const &id) : metaId(id) {} + }; + + struct Data : public IData + { + Block metadata; + + ISerializable *asSerializable(SerialMode mode) override { + if (mode == Serializing && metadata.isEmpty()) { + // Never serialize empty metadata, since it hasn't been set. + return nullptr; + } + return &metadata; + } + virtual duint sizeInMemory() const override { + return duint(metadata.size()); + } + }; + + Impl(Public *i) : Base(i) {} + + static DotPath pathFromId(Block const &id) { return id.asHexadecimalText(); } +}; + +MetadataBank::MetadataBank() + : Bank("MetadataBank", SingleThread | EnableHotStorage, "/home/cache/metadata") + , d(new Impl(this)) +{} + +MetadataBank &MetadataBank::get() // static +{ + return App::metadataBank(); +} + +Block MetadataBank::check(Block const &id) +{ + DENG2_GUARD(d); + + DotPath const path = Impl::pathFromId(id); + if (!has(path)) + { + Bank::add(path, new Impl::Source(id)); + } + return data(path).as().metadata; +} + +void MetadataBank::setMetadata(Block const &id, Block const &metadata) +{ + DENG2_GUARD(d); + + DotPath const path = Impl::pathFromId(id); + if (!has(path)) + { + Bank::add(path, new Impl::Source(id)); + } + data(path).as().metadata = metadata; +} + +Block MetadataBank::metadata(Block const &id) const +{ + DENG2_GUARD(d); + + return data(Impl::pathFromId(id)).as().metadata; +} + +Bank::IData *MetadataBank::loadFromSource(ISource &) +{ + // The cached metadata can only be deserialized from hot storage or replaced. + // Loading from source is not possible. + return newData(); +} + +Bank::IData *MetadataBank::newData() +{ + return new Impl::Data; +} + +} // namespace de diff --git a/doomsday/sdk/libcore/src/filesys/nativefile.cpp b/doomsday/sdk/libcore/src/filesys/nativefile.cpp index d22143a521..7e644ceba0 100644 --- a/doomsday/sdk/libcore/src/filesys/nativefile.cpp +++ b/doomsday/sdk/libcore/src/filesys/nativefile.cpp @@ -147,6 +147,11 @@ String NativeFile::describe() const return String("\"%1\"").arg(d->nativePath.pretty()); } +Block NativeFile::metaId() const +{ + return Block(File::metaId() + d->nativePath.toUtf8()).md5Hash(); +} + void NativeFile::close() { DENG2_GUARD(this); diff --git a/doomsday/sdk/libcore/src/scriptsys/bindings_core.cpp b/doomsday/sdk/libcore/src/scriptsys/bindings_core.cpp index cb595c4f0b..bd90b26ced 100644 --- a/doomsday/sdk/libcore/src/scriptsys/bindings_core.cpp +++ b/doomsday/sdk/libcore/src/scriptsys/bindings_core.cpp @@ -147,6 +147,11 @@ static Value *Function_File_ReadUtf8(Context &ctx, Function::ArgumentValues cons return new TextValue(String::fromUtf8(raw)); } +static Value *Function_File_MetaId(Context &ctx, Function::ArgumentValues const &) +{ + return new TextValue(fileInstance(ctx).metaId().asHexadecimalText()); +} + static Value *Function_Folder_List(Context &ctx, Function::ArgumentValues const &) { Folder const &folder = fileInstance(ctx).as(); @@ -252,7 +257,8 @@ void initCoreModule(Binder &binder, Record &coreModule) << DENG2_FUNC_NOARG(File_ModifiedAt, "modifiedAt") << DENG2_FUNC (File_Locate, "locate", "relativePath") << DENG2_FUNC_NOARG(File_Read, "read") - << DENG2_FUNC_NOARG(File_ReadUtf8, "readUtf8"); + << DENG2_FUNC_NOARG(File_ReadUtf8, "readUtf8") + << DENG2_FUNC_NOARG(File_MetaId, "metaId"); } // Folder diff --git a/doomsday/sdk/libgui/src/graphics/imagebank.cpp b/doomsday/sdk/libgui/src/graphics/imagebank.cpp index 1dacc4d077..a4219b2b73 100644 --- a/doomsday/sdk/libgui/src/graphics/imagebank.cpp +++ b/doomsday/sdk/libgui/src/graphics/imagebank.cpp @@ -52,7 +52,7 @@ DENG2_PIMPL_NOREF(ImageBank) ImageData() {} ImageData(Image const &img) : image(img) {} - ISerializable *asSerializable() + ISerializable *asSerializable(SerialMode) { return ℑ }