From cecbdf1e58917efcaf5f0b62217ea9cb4a212166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaakko=20Kera=CC=88nen?= Date: Mon, 23 Oct 2017 09:48:43 +0300 Subject: [PATCH] FS: Experimenting with a web-hosted file repository This needs some refactoring: a repository should present a collection of packages, not raw files. Also, libcore should not be aware of an idgames-style file archive (move to a specialized class in libdoomsday). --- doomsday/apps/libdoomsday/src/doomsdayapp.cpp | 16 +- .../libdoomsday/src/resource/databundle.cpp | 6 + .../include/de/filesys/remotefeedrelay.h | 6 + .../sdk/libcore/src/filesys/remotefeed.cpp | 32 +- .../libcore/src/filesys/remotefeedrelay.cpp | 363 ++++++++++++++++-- 5 files changed, 385 insertions(+), 38 deletions(-) diff --git a/doomsday/apps/libdoomsday/src/doomsdayapp.cpp b/doomsday/apps/libdoomsday/src/doomsdayapp.cpp index 1a99548ffb..b9ea312751 100644 --- a/doomsday/apps/libdoomsday/src/doomsdayapp.cpp +++ b/doomsday/apps/libdoomsday/src/doomsdayapp.cpp @@ -48,6 +48,7 @@ #include #include #include +#include #include #include #include @@ -348,7 +349,7 @@ DENG2_PIMPL(DoomsdayApp) void initPackageFolders() { - Folder &packs = FileSystem::get().makeFolder(PATH_LOCAL_PACKS, FS::DontInheritFeeds); + Folder &packs = FS::get().makeFolder(PATH_LOCAL_PACKS, FS::DontInheritFeeds); packs.clear(); packs.clearFeeds(); @@ -385,6 +386,13 @@ DENG2_PIMPL(DoomsdayApp) packs.populate(Folder::PopulateAsyncFullTree); } + void initRemoteRepositories() + { + FS::get().makeFolderWithFeed("/remote/idgames", + RemoteFeedRelay::get().addRepository("http://www.gamers.org/pub/idgames/"), + Folder::PopulateAsyncFullTree); + } + void folderPopulationFinished() override { if (initialized) @@ -480,11 +488,15 @@ void DoomsdayApp::initialize() d->initWadFolders(); d->initPackageFolders(); - Folder::waitForPopulation(); + // We need to access the local file system to complete initialization. + Folder::waitForPopulation(Folder::BlockingMainThread); d->dataBundles.identify(); d->gameProfiles.deserialize(); + // Register some remote repositories. + d->initRemoteRepositories(); + d->initialized = true; } diff --git a/doomsday/apps/libdoomsday/src/resource/databundle.cpp b/doomsday/apps/libdoomsday/src/resource/databundle.cpp index 0eb1ee2f46..3c446e1f26 100644 --- a/doomsday/apps/libdoomsday/src/resource/databundle.cpp +++ b/doomsday/apps/libdoomsday/src/resource/databundle.cpp @@ -1248,6 +1248,12 @@ String DataBundle::guessCompatibleGame() const File *DataBundle::Interpreter::interpretFile(File *sourceData) const { + // Broken links cannot be interpreted. + if (LinkFile *link = maybeAs(sourceData)) + { + if (link->isBroken()) return nullptr; + } + // Naive check using the file extension. static struct { String str; Format format; } formats[] = { { ".pk3.zip", Pk3 }, diff --git a/doomsday/sdk/libcore/include/de/filesys/remotefeedrelay.h b/doomsday/sdk/libcore/include/de/filesys/remotefeedrelay.h index 2ab5ca20fa..4e5d546e0f 100644 --- a/doomsday/sdk/libcore/include/de/filesys/remotefeedrelay.h +++ b/doomsday/sdk/libcore/include/de/filesys/remotefeedrelay.h @@ -54,6 +54,8 @@ class DENG2_PUBLIC RemoteFeedRelay StringList repositories() const; + bool isConnected(String const &address) const; + FileListRequest fetchFileList(String const &repository, String folderPath, FileListFunc result); @@ -62,6 +64,10 @@ class DENG2_PUBLIC RemoteFeedRelay String filePath, DataReceivedFunc dataReceived); +public: + enum Status { Disconnected, Connected }; + DENG2_DEFINE_AUDIENCE2(Status, void remoteRepositoryStatusChanged(String const &address, Status)) + private: DENG2_PRIVATE(d) }; diff --git a/doomsday/sdk/libcore/src/filesys/remotefeed.cpp b/doomsday/sdk/libcore/src/filesys/remotefeed.cpp index cd59d89884..7067947fab 100644 --- a/doomsday/sdk/libcore/src/filesys/remotefeed.cpp +++ b/doomsday/sdk/libcore/src/filesys/remotefeed.cpp @@ -35,10 +35,12 @@ namespace de { static TimeDelta const POPULATE_TIMEOUT = 15.0; DENG2_PIMPL(RemoteFeed) +, DENG2_OBSERVES(RemoteFeedRelay, Status) { String repository; Path remotePath; std::unique_ptr fileList; + SafePtr pendingPopulation; Impl(Public *i) : Base(i) {} @@ -82,6 +84,20 @@ DENG2_PIMPL(RemoteFeed) } return populated; } + + void remoteRepositoryStatusChanged(String const &address, RemoteFeedRelay::Status status) override + { + if (repository == address && status == RemoteFeedRelay::Connected) + { + if (pendingPopulation) + { + // There is a pending population request, let's do it now. + pendingPopulation->populate(Folder::PopulateAsyncFullTree); + pendingPopulation.reset(); + } + RemoteFeedRelay::get().audienceForStatus() -= this; + } + } }; RemoteFeed::RemoteFeed(String const &repository, String const &remotePath) @@ -105,15 +121,23 @@ String RemoteFeed::repository() const String RemoteFeed::description() const { - return String("remote repository \"%1%2\"") - .arg(d->repository) - .arg(d->remotePath); + return String("remote repository \"%1\"") + .arg(d->repository / d->remotePath); } Feed::PopulatedFiles RemoteFeed::populate(Folder const &folder) { + LOG_AS("RemoteFeed"); + auto &relay = RemoteFeedRelay::get(); PopulatedFiles files; - auto request = RemoteFeedRelay::get().fetchFileList + if (!relay.isConnected(d->repository)) + { + //qDebug() << "Population deferred:" << folder.path(); + d->pendingPopulation.reset(const_cast(&folder)); + relay.audienceForStatus() += d; + return files; + } + auto request = relay.fetchFileList (d->repository, d->remotePath, [this, &folder, &files] diff --git a/doomsday/sdk/libcore/src/filesys/remotefeedrelay.cpp b/doomsday/sdk/libcore/src/filesys/remotefeedrelay.cpp index dc34e82433..e31a7bf108 100644 --- a/doomsday/sdk/libcore/src/filesys/remotefeedrelay.cpp +++ b/doomsday/sdk/libcore/src/filesys/remotefeedrelay.cpp @@ -19,11 +19,27 @@ #include "de/RemoteFeedRelay" #include "de/App" +#include "de/Async" +#include "de/Date" +#include "de/DictionaryValue" #include "de/Loop" #include "de/Message" +#include "de/PathTree" +#include "de/RecordValue" #include "de/RemoteFeedProtocol" #include "de/Socket" +#include "de/TextValue" +#include "de/Version" #include "de/charsymbols.h" +#include "de/data/gzip.h" + +#include +#include +#include +#include +#include + +#include namespace de { @@ -33,7 +49,7 @@ DENG2_PIMPL(RemoteFeedRelay) * Active connection to a remote repository. One link is shared by all * RemoteFeed instances accessing the same repository. */ - struct RepositoryLink + struct RepositoryLink : protected AsyncScope { RemoteFeedRelay::Impl *d; @@ -81,36 +97,52 @@ DENG2_PIMPL(RemoteFeedRelay) virtual ~RepositoryLink() { - // All queries will be cancelled. - for (auto i = deferredQueries.begin(); i != deferredQueries.end(); ++i) - { - i->cancel(); - } - for (auto i = pendingQueries.begin(); i != pendingQueries.end(); ++i) - { - i.value().cancel(); - } + cancelAllQueries(); } virtual void wasConnected() { + DENG2_ASSERT_IN_MAIN_THREAD(); state = Ready; sendDeferredQueries(); + DENG2_FOR_EACH_OBSERVER(StatusAudience, i, d->audienceForStatus) + { + i->remoteRepositoryStatusChanged(address, Connected); + } } virtual void wasDisconnected() { state = Deinitialized; + cancelAllQueries(); + cleanup(); + DENG2_FOR_EACH_OBSERVER(StatusAudience, i, d->audienceForStatus) + { + i->remoteRepositoryStatusChanged(address, Disconnected); + } } virtual void handleError(QString errorMessage) { LOG_NET_ERROR("Error accessing remote file repository \"%s\": %s " DENG2_CHAR_MDASH - "files from repository may not be available") + " files from repository may not be available") << address << errorMessage; } + void cancelAllQueries() + { + // All queries will be cancelled. + for (auto i = deferredQueries.begin(); i != deferredQueries.end(); ++i) + { + i->cancel(); + } + for (auto i = pendingQueries.begin(); i != pendingQueries.end(); ++i) + { + i.value().cancel(); + } + } + void sendQuery(Query query) { try @@ -147,50 +179,56 @@ DENG2_PIMPL(RemoteFeedRelay) deferredQueries.clear(); } - void metadataReceived(Query::Id id, DictionaryValue const &metadata) + Query *findQuery(Query::Id id) { auto found = pendingQueries.find(id); if (found != pendingQueries.end()) { - auto &query = found.value(); - if (query.fileList) + return &found.value(); + } + return nullptr; + } + + void metadataReceived(Query::Id id, DictionaryValue const &metadata) + { + if (auto *query = findQuery(id)) + { + if (query->fileList) { - query.fileList->call(metadata); + query->fileList->call(metadata); } - pendingQueries.erase(found); + pendingQueries.remove(id); } } - void chunkReceived(Query::Id id, duint64 startOffset, Block const &chunk, duint64 fileSize) + void chunkReceived(Query::Id id, duint64 startOffset, Block const &chunk,duint64 fileSize) { - auto found = pendingQueries.find(id); - if (found != pendingQueries.end()) + if (auto *query = findQuery(id)) { - auto &query = found.value(); - // Get rid of cancelled queries. - if (!query.isValid()) + if (!query->isValid()) { - pendingQueries.erase(found); + pendingQueries.remove(id); return; } // Before the first chunk, notify about the total size. - if (!query.fileSize) + if (!query->fileSize) { - query.fileContents->call(0, Block(), fileSize); + query->fileContents->call(0, Block(), fileSize); } - query.fileSize = fileSize; - query.receivedBytes += chunk.size(); + query->fileSize = fileSize; + query->receivedBytes += chunk.size(); // Notify about progress and provide the data chunk to the requestor. - query.fileContents->call(startOffset, chunk, fileSize - query.receivedBytes); + query->fileContents->call(startOffset, chunk, + fileSize - query->receivedBytes); - if (fileSize == query.receivedBytes) + if (fileSize == query->receivedBytes) { // Transfer complete. - pendingQueries.erase(found); + pendingQueries.remove(id); } } } @@ -294,18 +332,267 @@ DENG2_PIMPL(RemoteFeedRelay) } }; + /** + * Repository of files hosted on a web server as a file tree. Assumed to come + * with a Unix-style "ls-laR.gz" directory tree index (e.g., an idgames mirror). + */ + struct WebRepositoryLink : public RepositoryLink + { + QSet pendingRequests; + + struct FileEntry : public PathTree::Node + { + duint64 size = 0; + Time modTime; + + FileEntry(PathTree::NodeArgs const &args) : Node(args) {} + FileEntry() = delete; + }; + using FileTree = PathTreeT; + LockableT> fileTree; + + WebRepositoryLink(RemoteFeedRelay::Impl *d, String const &address) + : RepositoryLink(d, address) + { + // Fetch the repository index. + { + QNetworkRequest req(QUrl(address / "ls-laR.gz")); + req.setRawHeader("User-Agent", Version::currentBuild().userAgent().toLatin1()); + + QNetworkReply *reply = d->network->get(req); + QObject::connect(reply, &QNetworkReply::finished, [this, reply] () + { + reply->deleteLater(); + if (reply->error() == QNetworkReply::NoError) + { + parseUnixDirectoryListing(reply->readAll()); + } + else + { + handleError(reply->errorString()); + wasDisconnected(); + } + }); + } + } + + void parseUnixDirectoryListing(QByteArray data) + { + // This may be a long list, so let's do it in a background thread. + // The link will be marked connected only after the data has been parsed. + + *this += async([this, data] () -> String + { + Block const listing = gDecompress(data); + QTextStream is(listing, QIODevice::ReadOnly); + is.setCodec("UTF-8"); + QRegularExpression const reDir("^\\.?(.*):$"); + QRegularExpression const reTotal("^total\\s+\\d+$"); + QRegularExpression const reFile("^(-|d)[-rwxs]+\\s+\\d+\\s+\\w+\\s+\\w+\\s+" + "(\\d+)\\s+(\\w+\\s+\\d+\\s+[0-9:]+)\\s+(.*)$", + QRegularExpression::CaseInsensitiveOption); + String currentPath; + bool ignore = false; + QRegularExpression const reIncludedPaths("^/(levels|music|sounds|themes)"); + + std::shared_ptr tree(new FileTree); + while (!is.atEnd()) + { + if (String const line = is.readLine().trimmed()) + { + if (!currentPath) + { + // This should be a directory path. + auto match = reDir.match(line); + if (match.hasMatch()) + { + currentPath = match.captured(1); + qDebug() << "[WebRepositoryLink] Parsing path:" << currentPath; + + ignore = !reIncludedPaths.match(currentPath).hasMatch(); + } + } + else if (!ignore && reTotal.match(line).hasMatch()) + { + // Ignore directory sizes. + } + else if (!ignore) + { + auto match = reFile.match(line); + if (match.hasMatch()) + { + bool const isFolder = (match.captured(1) == QStringLiteral("d")); + if (!isFolder) + { + String const name = match.captured(4); + if (name.startsWith(QChar('.')) || name.contains(" -> ")) + continue; + + auto &entry = tree->insert(currentPath / name); + entry.size = match.captured(2).toULongLong(nullptr, 10);; + entry.modTime = Time::fromText(match.captured(3), Time::UnixLsStyleDateTime); + } + } + } + } + else + { + currentPath.clear(); + } + } + qDebug() << "file tree contains" << tree->size() << "entries"; + { + // It is now ready for use. + DENG2_GUARD(fileTree); + fileTree.value = tree; + } + return String(); + }, + [this] (String const &errorMessage) + { + if (!errorMessage) + { + wasConnected(); + } + else + { + handleError("Failed to parse directory listing: " + errorMessage); + wasDisconnected(); + } + }); + } + + ~WebRepositoryLink() + { + foreach (auto *reply, pendingRequests) + { + reply->deleteLater(); + } + } + + void transmit(Query const &query) override + { + // We can answer population queries instantly. + if (query.fileList) + { + handleFileListQueryAsync(query); + return; + } + + String url = address; + QNetworkRequest req(url); + req.setRawHeader("User-Agent", Version::currentBuild().userAgent().toLatin1()); + + // TODO: Configure the request. + + QNetworkReply *reply = d->network->get(req); + pendingRequests.insert(reply); + + auto const id = query.id; + QObject::connect(reply, &QNetworkReply::finished, [this, id, reply] () + { + handleReply(id, reply); + }); + } + + Block metaIdForFileEntry(FileEntry const &entry) const + { + if (entry.isBranch()) return Block(); // not applicable + + Block data; + Writer writer(data); + writer << address << entry.path() << entry.size << entry.modTime; + return data.md5Hash(); + } + + void handleFileListQueryAsync(Query query) + { + Query::Id id = query.id; + String const queryPath = query.path; + *this += async([this, queryPath] () -> std::shared_ptr + { + DENG2_GUARD(fileTree); + if (auto const *dir = fileTree.value->tryFind + (queryPath, FileTree::MatchFull | FileTree::NoLeaf)) + { + std::shared_ptr list(new DictionaryValue); + + static String const VAR_TYPE("type"); + static String const VAR_MODIFIED_AT("modifiedAt"); + static String const VAR_SIZE("size"); + static String const VAR_META_ID("metaId"); + + auto addMeta = [this] + (DictionaryValue &list, PathTree::Nodes const &nodes) + { + for (auto i = nodes.begin(); i != nodes.end(); ++i) + { + auto const &entry = i.value()->as(); + list.add(new TextValue(entry.name()), + RecordValue::takeRecord( + Record::withMembers( + VAR_TYPE, entry.isLeaf()? 0 : 1, + VAR_SIZE, entry.size, + VAR_MODIFIED_AT, entry.modTime, + VAR_META_ID, metaIdForFileEntry(entry) + ))); + } + }; + + addMeta(*list.get(), dir->children().branches); + addMeta(*list.get(), dir->children().leaves); + + return list; + } + return nullptr; + }, + [this, id] (std::shared_ptr list) + { + metadataReceived(id, list? *list : DictionaryValue()); + }); + } + + void handleReply(Query::Id id, QNetworkReply *reply) + { + reply->deleteLater(); + if (reply->error() == QNetworkReply::NoError) + { + QByteArray const data = reply->readAll(); + + } + else + { + LOG_NET_WARNING(reply->errorString()); + } + } + + }; + RemoteFeedProtocol protocol; QHash repositories; // owned + std::unique_ptr network; Impl(Public *i) : Base(i) - {} + { + network.reset(new QNetworkAccessManager); + + auto *cache = new QNetworkDiskCache; + String const dir = NativePath(QStandardPaths::writableLocation(QStandardPaths::CacheLocation)) + / "RemoteFiles"; + cache->setCacheDirectory(dir); + network->setCache(cache); + } ~Impl() { qDeleteAll(repositories.values()); } + + DENG2_PIMPL_AUDIENCE(Status) }; +DENG2_AUDIENCE_METHOD(RemoteFeedRelay, Status) + RemoteFeedRelay &RemoteFeedRelay::get() { return App::remoteFeedRelay(); @@ -324,7 +611,9 @@ RemoteFeed *RemoteFeedRelay::addServerRepository(String const &serverAddress, St RemoteFeed *RemoteFeedRelay::addRepository(String const &address) { - return nullptr; + auto *repo = new Impl::WebRepositoryLink(d, address); + d->repositories.insert(address, repo); + return new RemoteFeed(address); } void RemoteFeedRelay::removeRepository(const de::String &address) @@ -345,6 +634,16 @@ StringList RemoteFeedRelay::repositories() const return repos; } +bool RemoteFeedRelay::isConnected(String const &address) const +{ + auto found = d->repositories.constFind(address); + if (found != d->repositories.constEnd()) + { + return found.value()->state == Impl::RepositoryLink::Ready; + } + return false; +} + RemoteFeedRelay::FileListRequest RemoteFeedRelay::fetchFileList(String const &repository, String folderPath, FileListFunc result) {