FS: Experimenting with a web-hosted file repository

This needs some refactoring: a repository should present a collection of packages, not raw files. Also, libcore should not be aware of an idgames-style file archive (move to a specialized class in libdoomsday).
skyjake · Oct 23, 2017 · cecbdf1 · cecbdf1
1 parent 3c2cc7b
commit cecbdf1
Show file tree

Hide file tree

Showing 5 changed files with 385 additions and 38 deletions.
diff --git a/doomsday/apps/libdoomsday/src/doomsdayapp.cpp b/doomsday/apps/libdoomsday/src/doomsdayapp.cpp
@@ -48,6 +48,7 @@
 #include <de/MetadataBank>
 #include <de/NativeFile>
 #include <de/PackageLoader>
+#include <de/RemoteFeedRelay>
 #include <de/ScriptSystem>
 #include <de/TextValue>
 #include <de/c_wrapper.h>
@@ -348,7 +349,7 @@ DENG2_PIMPL(DoomsdayApp)
 
     void initPackageFolders()
     {
-        Folder &packs = FileSystem::get().makeFolder(PATH_LOCAL_PACKS, FS::DontInheritFeeds);
+        Folder &packs = FS::get().makeFolder(PATH_LOCAL_PACKS, FS::DontInheritFeeds);
         packs.clear();
         packs.clearFeeds();
 
@@ -385,6 +386,13 @@ DENG2_PIMPL(DoomsdayApp)
         packs.populate(Folder::PopulateAsyncFullTree);
     }
 
+    void initRemoteRepositories()
+    {
+        FS::get().makeFolderWithFeed("/remote/idgames",
+                RemoteFeedRelay::get().addRepository("http://www.gamers.org/pub/idgames/"),
+                Folder::PopulateAsyncFullTree);
+    }
+
     void folderPopulationFinished() override
     {
         if (initialized)
@@ -480,11 +488,15 @@ void DoomsdayApp::initialize()
     d->initWadFolders();
     d->initPackageFolders();
 
-    Folder::waitForPopulation();
+    // We need to access the local file system to complete initialization.
+    Folder::waitForPopulation(Folder::BlockingMainThread);
 
     d->dataBundles.identify();
     d->gameProfiles.deserialize();
 
+    // Register some remote repositories.
+    d->initRemoteRepositories();
+
     d->initialized = true;
 }
 

diff --git a/doomsday/apps/libdoomsday/src/resource/databundle.cpp b/doomsday/apps/libdoomsday/src/resource/databundle.cpp
@@ -1248,6 +1248,12 @@ String DataBundle::guessCompatibleGame() const
 
 File *DataBundle::Interpreter::interpretFile(File *sourceData) const
 {
+    // Broken links cannot be interpreted.
+    if (LinkFile *link = maybeAs<LinkFile>(sourceData))
+    {
+        if (link->isBroken()) return nullptr;
+    }
+
     // Naive check using the file extension.
     static struct { String str; Format format; } formats[] = {
         { ".pk3.zip", Pk3 },

diff --git a/doomsday/sdk/libcore/include/de/filesys/remotefeedrelay.h b/doomsday/sdk/libcore/include/de/filesys/remotefeedrelay.h
@@ -54,6 +54,8 @@ class DENG2_PUBLIC RemoteFeedRelay
 
     StringList repositories() const;
 
+    bool isConnected(String const &address) const;
+
     FileListRequest fetchFileList(String const &repository,
                                   String folderPath,
                                   FileListFunc result);
@@ -62,6 +64,10 @@ class DENG2_PUBLIC RemoteFeedRelay
                                           String filePath,
                                           DataReceivedFunc dataReceived);
 
+public:
+    enum Status { Disconnected, Connected };
+    DENG2_DEFINE_AUDIENCE2(Status, void remoteRepositoryStatusChanged(String const &address, Status))
+
 private:
     DENG2_PRIVATE(d)
 };

diff --git a/doomsday/sdk/libcore/src/filesys/remotefeed.cpp b/doomsday/sdk/libcore/src/filesys/remotefeed.cpp
@@ -35,10 +35,12 @@ namespace de {
 static TimeDelta const POPULATE_TIMEOUT = 15.0;
 
 DENG2_PIMPL(RemoteFeed)
+, DENG2_OBSERVES(RemoteFeedRelay, Status)
 {
     String repository;
     Path remotePath;
     std::unique_ptr<RemoteFeedRelay::FileList> fileList;
+    SafePtr<Folder> pendingPopulation;
 
     Impl(Public *i) : Base(i)
     {}
@@ -82,6 +84,20 @@ DENG2_PIMPL(RemoteFeed)
         }
         return populated;
     }
+
+    void remoteRepositoryStatusChanged(String const &address, RemoteFeedRelay::Status status) override
+    {
+        if (repository == address && status == RemoteFeedRelay::Connected)
+        {
+            if (pendingPopulation)
+            {
+                // There is a pending population request, let's do it now.
+                pendingPopulation->populate(Folder::PopulateAsyncFullTree);
+                pendingPopulation.reset();
+            }
+            RemoteFeedRelay::get().audienceForStatus() -= this;
+        }
+    }
 };
 
 RemoteFeed::RemoteFeed(String const &repository, String const &remotePath)
@@ -105,15 +121,23 @@ String RemoteFeed::repository() const
 
 String RemoteFeed::description() const
 {
-    return String("remote repository \"%1%2\"")
-            .arg(d->repository)
-            .arg(d->remotePath);
+    return String("remote repository \"%1\"")
+            .arg(d->repository / d->remotePath);
 }
 
 Feed::PopulatedFiles RemoteFeed::populate(Folder const &folder)
 {
+    LOG_AS("RemoteFeed");
+    auto &relay = RemoteFeedRelay::get();
     PopulatedFiles files;
-    auto request = RemoteFeedRelay::get().fetchFileList
+    if (!relay.isConnected(d->repository))
+    {
+        //qDebug() << "Population deferred:" << folder.path();
+        d->pendingPopulation.reset(const_cast<Folder *>(&folder));
+        relay.audienceForStatus() += d;
+        return files;
+    }
+    auto request = relay.fetchFileList
             (d->repository,
              d->remotePath,
              [this, &folder, &files]