Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
libcore|FS: Added MetadataBank for caching ZIP archive metadata
File::metaId() generates identifiers suitable for caching.

ZipArchive can be initialized either from a cached central directory
or from the source file.
  • Loading branch information
skyjake committed Feb 12, 2017
1 parent c01c6e0 commit 6f0dac7
Show file tree
Hide file tree
Showing 18 changed files with 454 additions and 148 deletions.
1 change: 1 addition & 0 deletions doomsday/sdk/libcore/include/de/MetadataBank
@@ -0,0 +1 @@
#include "filesys/metadatabank.h"
6 changes: 6 additions & 0 deletions doomsday/sdk/libcore/include/de/core/app.h
Expand Up @@ -43,6 +43,7 @@ class FileSystem;
class Folder;
class LogBuffer;
class LogFilter;
class MetadataBank;
class Module;
class Path;
class NativePath;
Expand Down Expand Up @@ -274,6 +275,11 @@ class DENG2_PUBLIC App : DENG2_OBSERVES(Clock, TimeChange)
*/
static FileSystem &fileSystem();

/**
* Returns the application's metadata cache.
*/
static MetadataBank &metadataBank();

/**
* Returns the root folder of the file system.
*/
Expand Down
12 changes: 9 additions & 3 deletions doomsday/sdk/libcore/include/de/data/bank.h
Expand Up @@ -82,12 +82,16 @@ class DENG2_PUBLIC Bank

enum Flag
{
SingleThread = 0,

/**
* Separate thread used for managing the bank's data (loading, caching
* data). Requires data items and sources to be thread-safe.
*/
BackgroundThread = 0x1,

EnableHotStorage = 0,

/**
* Do not use the hot storage to keep serialized copies of data items.
* This is useful if recreating the data from source is trivial.
Expand All @@ -102,7 +106,7 @@ class DENG2_PUBLIC Bank
*/
ClearHotStorageWhenBankDestroyed = 0x4,

DefaultFlags = DisableHotStorage
DefaultFlags = SingleThread | DisableHotStorage
};
Q_DECLARE_FLAGS(Flags, Flag)

Expand Down Expand Up @@ -166,9 +170,11 @@ class DENG2_PUBLIC Bank
public:
virtual ~IData() {}

enum SerialMode { Serializing, Deserializing };

/// Returns an ISerializable pointer to the object. Required
/// for putting the data in hot storage.
virtual ISerializable *asSerializable() { return 0; }
virtual ISerializable *asSerializable(SerialMode) { return 0; }

/// Returns the size of the data that it occupies in memory.
virtual duint sizeInMemory() const { return 0; }
Expand Down Expand Up @@ -370,7 +376,7 @@ class DENG2_PUBLIC Bank

/**
* Construct a new concrete instance of the data item. Called before
* deserialization. Default implementation just returns NULL (seriliazation
* deserialization. Default implementation just returns NULL (serialization
* not supported).
*
* @return IData instance. Ownership given to caller.
Expand Down
8 changes: 5 additions & 3 deletions doomsday/sdk/libcore/include/de/data/ziparchive.h
Expand Up @@ -73,10 +73,9 @@ class DENG2_PUBLIC ZipArchive : public Archive
* data is made, so the caller must make sure the
* byte array remains in existence for the lifetime
* of the Archive instance.
* @param dirCacheId ID of cached ZIP directory data.
*/
ZipArchive(IByteArray const &data);

virtual ~ZipArchive();
ZipArchive(IByteArray const &data, Block const &dirCacheId = Block());

void operator >> (Writer &to) const;

Expand Down Expand Up @@ -122,6 +121,9 @@ class DENG2_PUBLIC ZipArchive : public Archive
typedef PathTreeT<ZipEntry> Index;

Index const &index() const;

private:
DENG2_PRIVATE(d)
};

} // namespace de
Expand Down
14 changes: 8 additions & 6 deletions doomsday/sdk/libcore/include/de/filesys/archiveentryfile.h
Expand Up @@ -46,10 +46,10 @@ class ArchiveEntryFile : public ByteArrayFile

~ArchiveEntryFile();

String describe() const;
String describe() const override;
String entryPath() const;

void clear();
void clear() override;

/**
* Flushes the entire archive that this file is part of into its source
Expand All @@ -63,7 +63,9 @@ class ArchiveEntryFile : public ByteArrayFile
* manual flushing this occurs automatically when the root ArchiveFeed
* instance is deleted.
*/
void flush();
void flush() override;

Block metaId() const override;

/// Returns the archive of the file.
Archive &archive();
Expand All @@ -74,8 +76,8 @@ class ArchiveEntryFile : public ByteArrayFile
void uncache() const;

// Implements IByteArray.
Size size() const;
void get(Offset at, Byte *values, Size count) const;
Size size() const override;
void get(Offset at, Byte *values, Size count) const override;

/**
* Modifies the content of an archive entry. Changes are made instantly
Expand All @@ -86,7 +88,7 @@ class ArchiveEntryFile : public ByteArrayFile
* @param values Data.
* @param count Length of data.
*/
void set(Offset at, Byte const *values, Size count);
void set(Offset at, Byte const *values, Size count) override;

private:
DENG2_PRIVATE(d)
Expand Down
6 changes: 6 additions & 0 deletions doomsday/sdk/libcore/include/de/filesys/file.h
Expand Up @@ -273,6 +273,12 @@ class DENG2_PUBLIC File : public filesys::Node, public IIOStream, public IObject
*/
dsize size() const;

/**
* Generates a unique identifier generated based on the metadata of the file. This is
* suitable for use in caches to identify a particular instance of a file.
*/
virtual Block metaId() const;

/**
* Returns the mode of the file.
*/
Expand Down
63 changes: 63 additions & 0 deletions doomsday/sdk/libcore/include/de/filesys/metadatabank.h
@@ -0,0 +1,63 @@
/** @file metadatabank.h File metadata cache.
*
* @authors Copyright (c) 2017 Jaakko Keränen <jaakko.keranen@iki.fi>
*
* @par License
* LGPL: http://www.gnu.org/licenses/lgpl.html
*
* <small>This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation; either version 3 of the License, or (at your
* option) any later version. This program is distributed in the hope that it
* will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
* General Public License for more details. You should have received a copy of
* the GNU Lesser General Public License along with this program; if not, see:
* http://www.gnu.org/licenses</small>
*/

#ifndef LIBDENG2_METADATABANK_H
#define LIBDENG2_METADATABANK_H

#include <de/Bank>

namespace de {

/**
* File metadata cache.
*
* @ingroup fs
*/
class DENG2_PUBLIC MetadataBank : public Bank
{
public:
MetadataBank();

static MetadataBank &get();

/**
* Adds a new metadata entry into the bank.
*
* @param id Meta ID.
*
* @return The cached metadata, if available. This will be an empty Block if no
* metadata has yet been cached.
*/
Block check(Block const &id);

void setMetadata(Block const &id, Block const &metadata);

Block metadata(Block const &id) const;

protected:
IData *loadFromSource(ISource &source) override;

IData *newData() override;

private:
DENG2_PRIVATE(d)
};

} // namespace de

#endif // LIBDENG2_METADATABANK_H
1 change: 1 addition & 0 deletions doomsday/sdk/libcore/include/de/filesys/nativefile.h
Expand Up @@ -50,6 +50,7 @@ class DENG2_PUBLIC NativeFile : public ByteArrayFile
virtual ~NativeFile();

String describe() const;
Block metaId() const;

void clear();
void flush();
Expand Down
21 changes: 17 additions & 4 deletions doomsday/sdk/libcore/src/core/app.cpp
Expand Up @@ -35,6 +35,7 @@
#include "de/LogBuffer"
#include "de/LogFilter"
#include "de/math.h"
#include "de/MetadataBank"
#include "de/Module"
#include "de/NativeFile"
#include "de/NumberValue"
Expand Down Expand Up @@ -83,14 +84,15 @@ DENG2_PIMPL(App)

ScriptSystem scriptSys;
FileSystem fs;
QScopedPointer<NativeFile> basePackFile;
std::unique_ptr<MetadataBank> metaBank;
std::unique_ptr<NativeFile> basePackFile;
Record appModule;

/// Archive where persistent data should be stored. Written to /home/persist.pack.
/// The archive is owned by the file system.
Archive *persistentData;

QScopedPointer<UnixInfo> unixInfo;
std::unique_ptr<UnixInfo> unixInfo;

/// The configuration.
Path configPath;
Expand All @@ -102,7 +104,7 @@ DENG2_PIMPL(App)
void (*terminateFunc)(char const *);

/// Optional sink for warnings and errors (set with "-errors").
QScopedPointer<FileLogSink> errorSink;
std::unique_ptr<FileLogSink> errorSink;

Impl(Public *a, QStringList args)
: Base(a)
Expand Down Expand Up @@ -140,7 +142,9 @@ DENG2_PIMPL(App)

~Impl()
{
if (!errorSink.isNull())
metaBank->unloadAll(Bank::InHotStorage);

if (errorSink)
{
logBuffer.removeSink(*errorSink);
}
Expand Down Expand Up @@ -227,6 +231,9 @@ DENG2_PIMPL(App)
// Internal data (for runtime use only, thus writable).
fs.makeFolder("/sys").setMode(File::Write);

// Metadata for files.
metaBank.reset(new MetadataBank);

// Populate the file system.
fs.refresh();
Folder::waitForPopulation();
Expand Down Expand Up @@ -758,6 +765,12 @@ FileSystem &App::fileSystem()
return DENG2_APP->d->fs;
}

MetadataBank &App::metadataBank()
{
DENG2_ASSERT(DENG2_APP->d->metaBank);
return *DENG2_APP->d->metaBank;
}

PackageLoader &App::packageLoader()
{
return DENG2_APP->d->packageLoader;
Expand Down
6 changes: 3 additions & 3 deletions doomsday/sdk/libcore/src/data/bank.cpp
Expand Up @@ -223,7 +223,7 @@ DENG2_PIMPL(Bank)
if (isValidSerialTime(timestamp))
{
QScopedPointer<IData> blank(bank->newData());
reader >> *blank->asSerializable();
reader >> *blank->asSerializable(IData::Deserializing);
setData(blank.take());
LOG_RES_XVERBOSE("Deserialized \"%s\" in %.2f seconds",
path(bank->d->sepChar) << startedAt.since());
Expand Down Expand Up @@ -259,7 +259,7 @@ DENG2_PIMPL(Bank)
loadFromSource();
}

DENG2_ASSERT(data->asSerializable() != 0);
DENG2_ASSERT(data->asSerializable(IData::Serializing) != 0);

try
{
Expand All @@ -275,7 +275,7 @@ DENG2_PIMPL(Bank)

Writer(*serial).withHeader()
<< source->modifiedAt()
<< *data->asSerializable();
<< *data->asSerializable(IData::Serializing);
}
catch (...)
{
Expand Down

0 comments on commit 6f0dac7

Please sign in to comment.