FS|libcore: Downloading files from a web-hosted package repository

doomsday/sdk/libcore/include/de/filesys/remote/remotefile.h

@@ -37,8 +37,11 @@ class DENG2_PUBLIC RemoteFile : public LinkFile, public Asset, public IDownloada
     /// Data of the file has not yet been fetched. @ingroup errors
     DENG2_ERROR(UnfetchedError);
 
+    static String const CACHE_PATH;
+
 public:
-    RemoteFile(String const &name, String const &remotePath, Block const &remoteMetaId);
+    RemoteFile(String const &name, String const &remotePath, Block const &remoteMetaId,
+               String const &repositoryAddress = String());
 
     String describe() const override;
     Block  metaId()   const override;

doomsday/sdk/libcore/include/de/filesys/remote/webhostedlink.h

@@ -55,6 +55,8 @@ class WebHostedLink : public Link
 
     FileTree const &fileTree() const;
 
+    FileEntry const *findFile(Path const &path) const;
+
     virtual void parseRepositoryIndex(QByteArray data) = 0;
 
     virtual String findPackagePath(String const &packageId) const = 0;

doomsday/sdk/libcore/src/core/app.cpp

@@ -46,6 +46,7 @@
 #include "de/RemoteFeedRelay"
 #include "de/ScriptSystem"
 #include "de/StaticLibraryFeed"
+#include "de/TextValue"
 #include "de/UnixInfo"
 #include "de/Version"
 #include "de/Writer"
@@ -62,6 +63,37 @@ namespace de {
 
 static App *singletonApp;
 
+static Value *Function_App_Locate(Context &, Function::ArgumentValues const &args)
+{
+    std::unique_ptr<DictionaryValue> result(new DictionaryValue);
+
+    String const packageId = args.first()->asText();
+
+    // Local packages.
+    if (File const *pkg = PackageLoader::get().select(packageId))
+    {
+        result->add(new TextValue(packageId), RecordValue::takeRecord(
+                        Record::withMembers(
+                            "localPath",   pkg->path(),
+                            "description", pkg->description()
+                        )));
+    }
+
+    // Remote packages.
+    auto found = App::remoteFeedRelay().locatePackages(StringList({ packageId }));
+    for (auto i = found.begin(); i != found.end(); ++i)
+    {
+        result->add(new TextValue(i.key()), RecordValue::takeRecord(
+                        Record::withMembers(
+                            "repository", i->link->address(),
+                            "localPath",  i->localPath,
+                            "remotePath", i->remotePath
+                        )));
+    }
+
+    return result.release();
+}
+
 DENG2_PIMPL(App)
 , DENG2_OBSERVES(PackageLoader, Activity)
 {
@@ -90,10 +122,12 @@ DENG2_PIMPL(App)
     QList<System *> systems;
 
     ScriptSystem scriptSys;
+    Record appModule;
+    Binder binder;
+
     FileSystem fs;
     std::unique_ptr<MetadataBank> metaBank;
     std::unique_ptr<NativeFile> basePackFile;
-    Record appModule;
 
     /// Archive where persistent data should be stored. Written to /home/persist.pack.
     /// The archive is owned by the file system.
@@ -157,7 +191,11 @@ DENG2_PIMPL(App)
         fs.addInterpreter(intrpZipArchive);
 
         // Native App module.
-        scriptSys.addNativeModule("App", appModule);
+        {
+            scriptSys.addNativeModule("App", appModule);
+            binder.init(appModule)
+                    << DENG2_FUNC(App_Locate, "locate", "packageId");
+        }
     }
 
     ~Impl()

doomsday/sdk/libcore/src/filesys/remote/nativelink.cpp

@@ -31,7 +31,7 @@ namespace filesys {
 
 String const NativeLink::URL_SCHEME("doomsday:");
 
-static String const PATH_SERVER_REPOSITORY_ROOT("/sys/server/files"); // serverside folder
+static String const PATH_SERVER_REPOSITORY_ROOT("/sys/server/public"); // serverside folder
 
 DENG2_PIMPL(NativeLink)
 {

doomsday/sdk/libcore/src/filesys/remote/remotefile.cpp

@@ -31,12 +31,13 @@ namespace de {
 
 using namespace de::filesys;
 
-static String const CACHE_PATH = "/home/cache/remote";
+String const RemoteFile::CACHE_PATH = "/home/cache/remote";
 
 DENG2_PIMPL(RemoteFile)
 {
     String remotePath;
     Block remoteMetaId;
+    String repositoryAddress; // If empty, use feed's repository.
     Block buffer;
     Request<FileContents> fetching;
 
@@ -103,22 +104,35 @@ DENG2_PIMPL(RemoteFile)
         }
         return false;
     }
+
+    String repository() const
+    {
+        if (repositoryAddress)
+        {
+            return repositoryAddress;
+        }
+        DENG2_ASSERT(is<RemoteFeed>(self().originFeed()));
+        return self().originFeed()->as<RemoteFeed>().repository();
+    }
 };
 
-RemoteFile::RemoteFile(String const &name, String const &remotePath, Block const &remoteMetaId)
+RemoteFile::RemoteFile(String const &name, String const &remotePath, Block const &remoteMetaId,
+                       String const &repositoryAddress)
     : LinkFile(name)
     , d(new Impl(this))
 {
     objectNamespace().addSuperRecord(ScriptSystem::builtInClass(QStringLiteral("RemoteFile")));
-    d->remotePath   = remotePath;
-    d->remoteMetaId = remoteMetaId;
+    d->repositoryAddress = repositoryAddress;
+    d->remotePath        = remotePath;
+    d->remoteMetaId      = remoteMetaId;
+
+    qDebug() << "RemoteFile remotePath:" << remotePath;
+
     setState(NotReady);
 }
 
 void RemoteFile::download()
 {
-    DENG2_ASSERT(is<RemoteFeed>(originFeed()));
-
     if (state() != NotReady) return;
 
     setState(Recovering);
@@ -136,7 +150,7 @@ void RemoteFile::download()
     LOG_NET_MSG("Requesting download of \"%s\"") << name();
 
     d->fetching = filesys::RemoteFeedRelay::get().fetchFileContents
-            (originFeed()->as<RemoteFeed>().repository(),
+            (d->repository(),
              d->remotePath,
              [this] (duint64 startOffset, Block const &chunk, duint64 remainingBytes)
     {

doomsday/sdk/libcore/src/filesys/remote/webhostedlink.cpp

@@ -41,12 +41,7 @@ DENG2_PIMPL(WebHostedLink), public Lockable
     {}
 
     ~Impl()
-    {
-        foreach (auto *reply, pendingRequests)
-        {
-            reply->deleteLater();
-        }
-    }
+    {}
 
     Block metaIdForFileEntry(FileEntry const &entry) const
     {
@@ -101,17 +96,27 @@ DENG2_PIMPL(WebHostedLink), public Lockable
         });
     }
 
-    void handleReply(QueryId id, QNetworkReply *reply)
+    void receiveFileContents(QueryId id, QNetworkReply *reply)
     {
-        reply->deleteLater();
         if (reply->error() == QNetworkReply::NoError)
         {
+            //qDebug() << "Content-Length:" << reply->header(QNetworkRequest::ContentLengthHeader);
+            dsize const contentLength = reply->header(QNetworkRequest::ContentLengthHeader).toULongLong();
+
+            //qDebug() << "pos:" << pos << contentLength << reply->url();
+
+            // Ths is the complete downloaded file.
             QByteArray const data = reply->readAll();
 
+            Query const *query = self().findQuery(id);
+            self().chunkReceived(id, query->receivedBytes, data,
+                                 contentLength? contentLength : dsize(data.size()));
         }
         else
         {
             LOG_NET_WARNING(reply->errorString());
+
+            /// @todo Abort query with error.
         }
     }
 };
@@ -153,6 +158,12 @@ WebHostedLink::FileTree const &WebHostedLink::fileTree() const
     return *d->fileTree;
 }
 
+WebHostedLink::FileEntry const *WebHostedLink::findFile(Path const &path) const
+{
+    DENG2_GUARD(d);
+    return d->fileTree->tryFind(path, PathTree::MatchFull);
+}
+
 filesys::PackagePaths WebHostedLink::locatePackages(StringList const &packageIds) const
 {
     PackagePaths remotePaths;
@@ -177,19 +188,25 @@ void WebHostedLink::transmit(Query const &query)
         return;
     }
 
+    DENG2_ASSERT(query.fileContents);
+
     String url = address();
-    QNetworkRequest req(url);
+    QNetworkRequest req(url.concatenateRelativePath(query.path));
+    qDebug() << req.url().toString();
     req.setRawHeader("User-Agent", Version::currentBuild().userAgent().toLatin1());
 
-    // TODO: Configure the request.
-
     QNetworkReply *reply = RemoteFeedRelay::get().network().get(req);
     d->pendingRequests.insert(reply);
 
     auto const id = query.id;
+    QObject::connect(reply, &QNetworkReply::readyRead, [this, id, reply] ()
+    {
+        d->receiveFileContents(id, reply);
+    });
     QObject::connect(reply, &QNetworkReply::finished, [this, id, reply] ()
     {
-        d->handleReply(id, reply);
+        d->pendingRequests.remove(reply);
+        reply->deleteLater();
     });
 }