From c754ddaa76db956df339148e3643f3575b5b159f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaakko=20Kera=CC=88nen?= Date: Tue, 31 Oct 2017 20:23:51 +0200 Subject: [PATCH] FS|libcore: Downloading files from a web-hosted package repository --- .../include/de/filesys/remote/remotefile.h | 5 ++- .../include/de/filesys/remote/webhostedlink.h | 2 + doomsday/sdk/libcore/src/core/app.cpp | 42 ++++++++++++++++++- .../libcore/src/filesys/remote/nativelink.cpp | 2 +- .../libcore/src/filesys/remote/remotefile.cpp | 28 +++++++++---- .../src/filesys/remote/webhostedlink.cpp | 41 ++++++++++++------ 6 files changed, 97 insertions(+), 23 deletions(-) diff --git a/doomsday/sdk/libcore/include/de/filesys/remote/remotefile.h b/doomsday/sdk/libcore/include/de/filesys/remote/remotefile.h index 5d94e9c064..e3b8eee5ed 100644 --- a/doomsday/sdk/libcore/include/de/filesys/remote/remotefile.h +++ b/doomsday/sdk/libcore/include/de/filesys/remote/remotefile.h @@ -37,8 +37,11 @@ class DENG2_PUBLIC RemoteFile : public LinkFile, public Asset, public IDownloada /// Data of the file has not yet been fetched. @ingroup errors DENG2_ERROR(UnfetchedError); + static String const CACHE_PATH; + public: - RemoteFile(String const &name, String const &remotePath, Block const &remoteMetaId); + RemoteFile(String const &name, String const &remotePath, Block const &remoteMetaId, + String const &repositoryAddress = String()); String describe() const override; Block metaId() const override; diff --git a/doomsday/sdk/libcore/include/de/filesys/remote/webhostedlink.h b/doomsday/sdk/libcore/include/de/filesys/remote/webhostedlink.h index a241378f48..8913794ba4 100644 --- a/doomsday/sdk/libcore/include/de/filesys/remote/webhostedlink.h +++ b/doomsday/sdk/libcore/include/de/filesys/remote/webhostedlink.h @@ -55,6 +55,8 @@ class WebHostedLink : public Link FileTree const &fileTree() const; + FileEntry const *findFile(Path const &path) const; + virtual void parseRepositoryIndex(QByteArray data) = 0; virtual String findPackagePath(String const &packageId) const = 0; diff --git a/doomsday/sdk/libcore/src/core/app.cpp b/doomsday/sdk/libcore/src/core/app.cpp index 866f60c798..f83fdd5da8 100644 --- a/doomsday/sdk/libcore/src/core/app.cpp +++ b/doomsday/sdk/libcore/src/core/app.cpp @@ -46,6 +46,7 @@ #include "de/RemoteFeedRelay" #include "de/ScriptSystem" #include "de/StaticLibraryFeed" +#include "de/TextValue" #include "de/UnixInfo" #include "de/Version" #include "de/Writer" @@ -62,6 +63,37 @@ namespace de { static App *singletonApp; +static Value *Function_App_Locate(Context &, Function::ArgumentValues const &args) +{ + std::unique_ptr result(new DictionaryValue); + + String const packageId = args.first()->asText(); + + // Local packages. + if (File const *pkg = PackageLoader::get().select(packageId)) + { + result->add(new TextValue(packageId), RecordValue::takeRecord( + Record::withMembers( + "localPath", pkg->path(), + "description", pkg->description() + ))); + } + + // Remote packages. + auto found = App::remoteFeedRelay().locatePackages(StringList({ packageId })); + for (auto i = found.begin(); i != found.end(); ++i) + { + result->add(new TextValue(i.key()), RecordValue::takeRecord( + Record::withMembers( + "repository", i->link->address(), + "localPath", i->localPath, + "remotePath", i->remotePath + ))); + } + + return result.release(); +} + DENG2_PIMPL(App) , DENG2_OBSERVES(PackageLoader, Activity) { @@ -90,10 +122,12 @@ DENG2_PIMPL(App) QList systems; ScriptSystem scriptSys; + Record appModule; + Binder binder; + FileSystem fs; std::unique_ptr metaBank; std::unique_ptr basePackFile; - Record appModule; /// Archive where persistent data should be stored. Written to /home/persist.pack. /// The archive is owned by the file system. @@ -157,7 +191,11 @@ DENG2_PIMPL(App) fs.addInterpreter(intrpZipArchive); // Native App module. - scriptSys.addNativeModule("App", appModule); + { + scriptSys.addNativeModule("App", appModule); + binder.init(appModule) + << DENG2_FUNC(App_Locate, "locate", "packageId"); + } } ~Impl() diff --git a/doomsday/sdk/libcore/src/filesys/remote/nativelink.cpp b/doomsday/sdk/libcore/src/filesys/remote/nativelink.cpp index 7926c7b767..f9815e9b8e 100644 --- a/doomsday/sdk/libcore/src/filesys/remote/nativelink.cpp +++ b/doomsday/sdk/libcore/src/filesys/remote/nativelink.cpp @@ -31,7 +31,7 @@ namespace filesys { String const NativeLink::URL_SCHEME("doomsday:"); -static String const PATH_SERVER_REPOSITORY_ROOT("/sys/server/files"); // serverside folder +static String const PATH_SERVER_REPOSITORY_ROOT("/sys/server/public"); // serverside folder DENG2_PIMPL(NativeLink) { diff --git a/doomsday/sdk/libcore/src/filesys/remote/remotefile.cpp b/doomsday/sdk/libcore/src/filesys/remote/remotefile.cpp index f657e49220..3a97c1b796 100644 --- a/doomsday/sdk/libcore/src/filesys/remote/remotefile.cpp +++ b/doomsday/sdk/libcore/src/filesys/remote/remotefile.cpp @@ -31,12 +31,13 @@ namespace de { using namespace de::filesys; -static String const CACHE_PATH = "/home/cache/remote"; +String const RemoteFile::CACHE_PATH = "/home/cache/remote"; DENG2_PIMPL(RemoteFile) { String remotePath; Block remoteMetaId; + String repositoryAddress; // If empty, use feed's repository. Block buffer; Request fetching; @@ -103,22 +104,35 @@ DENG2_PIMPL(RemoteFile) } return false; } + + String repository() const + { + if (repositoryAddress) + { + return repositoryAddress; + } + DENG2_ASSERT(is(self().originFeed())); + return self().originFeed()->as().repository(); + } }; -RemoteFile::RemoteFile(String const &name, String const &remotePath, Block const &remoteMetaId) +RemoteFile::RemoteFile(String const &name, String const &remotePath, Block const &remoteMetaId, + String const &repositoryAddress) : LinkFile(name) , d(new Impl(this)) { objectNamespace().addSuperRecord(ScriptSystem::builtInClass(QStringLiteral("RemoteFile"))); - d->remotePath = remotePath; - d->remoteMetaId = remoteMetaId; + d->repositoryAddress = repositoryAddress; + d->remotePath = remotePath; + d->remoteMetaId = remoteMetaId; + + qDebug() << "RemoteFile remotePath:" << remotePath; + setState(NotReady); } void RemoteFile::download() { - DENG2_ASSERT(is(originFeed())); - if (state() != NotReady) return; setState(Recovering); @@ -136,7 +150,7 @@ void RemoteFile::download() LOG_NET_MSG("Requesting download of \"%s\"") << name(); d->fetching = filesys::RemoteFeedRelay::get().fetchFileContents - (originFeed()->as().repository(), + (d->repository(), d->remotePath, [this] (duint64 startOffset, Block const &chunk, duint64 remainingBytes) { diff --git a/doomsday/sdk/libcore/src/filesys/remote/webhostedlink.cpp b/doomsday/sdk/libcore/src/filesys/remote/webhostedlink.cpp index a925164fc3..8089f9afb8 100644 --- a/doomsday/sdk/libcore/src/filesys/remote/webhostedlink.cpp +++ b/doomsday/sdk/libcore/src/filesys/remote/webhostedlink.cpp @@ -41,12 +41,7 @@ DENG2_PIMPL(WebHostedLink), public Lockable {} ~Impl() - { - foreach (auto *reply, pendingRequests) - { - reply->deleteLater(); - } - } + {} Block metaIdForFileEntry(FileEntry const &entry) const { @@ -101,17 +96,27 @@ DENG2_PIMPL(WebHostedLink), public Lockable }); } - void handleReply(QueryId id, QNetworkReply *reply) + void receiveFileContents(QueryId id, QNetworkReply *reply) { - reply->deleteLater(); if (reply->error() == QNetworkReply::NoError) { + //qDebug() << "Content-Length:" << reply->header(QNetworkRequest::ContentLengthHeader); + dsize const contentLength = reply->header(QNetworkRequest::ContentLengthHeader).toULongLong(); + + //qDebug() << "pos:" << pos << contentLength << reply->url(); + + // Ths is the complete downloaded file. QByteArray const data = reply->readAll(); + Query const *query = self().findQuery(id); + self().chunkReceived(id, query->receivedBytes, data, + contentLength? contentLength : dsize(data.size())); } else { LOG_NET_WARNING(reply->errorString()); + + /// @todo Abort query with error. } } }; @@ -153,6 +158,12 @@ WebHostedLink::FileTree const &WebHostedLink::fileTree() const return *d->fileTree; } +WebHostedLink::FileEntry const *WebHostedLink::findFile(Path const &path) const +{ + DENG2_GUARD(d); + return d->fileTree->tryFind(path, PathTree::MatchFull); +} + filesys::PackagePaths WebHostedLink::locatePackages(StringList const &packageIds) const { PackagePaths remotePaths; @@ -177,19 +188,25 @@ void WebHostedLink::transmit(Query const &query) return; } + DENG2_ASSERT(query.fileContents); + String url = address(); - QNetworkRequest req(url); + QNetworkRequest req(url.concatenateRelativePath(query.path)); + qDebug() << req.url().toString(); req.setRawHeader("User-Agent", Version::currentBuild().userAgent().toLatin1()); - // TODO: Configure the request. - QNetworkReply *reply = RemoteFeedRelay::get().network().get(req); d->pendingRequests.insert(reply); auto const id = query.id; + QObject::connect(reply, &QNetworkReply::readyRead, [this, id, reply] () + { + d->receiveFileContents(id, reply); + }); QObject::connect(reply, &QNetworkReply::finished, [this, id, reply] () { - d->handleReply(id, reply); + d->pendingRequests.remove(reply); + reply->deleteLater(); }); }