FS|Network: Improving remote file access

RemoteFile now stores the actual file in the cache using the
original name, and is derived from LinkFile to provide access to it.
This is similar to how local data files are linked as packages.

The special path "/sys/server/files" is where servers are expected
to keep files for remote access.

doomsday/sdk/libcore/include/de/filesys/linkfile.h

@@ -49,6 +49,8 @@ class DENG2_PUBLIC LinkFile : public File
      */
     void setTarget(File const &file);
 
+    void setTarget(File const *fileOrNull);
+
     /**
      * Returns the file's target. This is used for indirection when descending into
      * subfolders, to implement symbolic links.
@@ -75,6 +77,9 @@ class DENG2_PUBLIC LinkFile : public File
 
     String describe() const;
 
+    // Stream access:
+    IIStream const &operator >> (IByteArray &bytes) const override;
+
     // filesys::Node overrides:
     Node const *tryFollowPath(PathRef const &path) const;
     Node const *tryGetChild(String const &name) const;

doomsday/sdk/libcore/include/de/filesys/packagefeed.h

@@ -23,6 +23,7 @@
 
 namespace de {
 
+class Package;
 class PackageLoader;
 
 /**
@@ -33,7 +34,14 @@ class PackageLoader;
 class PackageFeed : public Feed
 {
 public:
-    PackageFeed(PackageLoader &loader);
+    enum LinkMode { LinkPackages, LinkSourceFiles };
+
+    typedef std::function<bool (Package const &)> Filter;
+
+public:
+    PackageFeed(PackageLoader &loader, LinkMode linkMode = LinkPackages);
+
+    void setFilter(Filter filter);
 
     PackageLoader &loader();
 

doomsday/sdk/libcore/include/de/filesys/remotefile.h

@@ -19,7 +19,7 @@
 #ifndef LIBDENG2_REMOTEFILE_H
 #define LIBDENG2_REMOTEFILE_H
 
-#include "../ByteArrayFile"
+#include "../LinkFile"
 #include "../Asset"
 
 namespace de {
@@ -30,7 +30,7 @@ namespace de {
  *
  * RemoteFile provides status information as an Asset.
  */
-class RemoteFile : public ByteArrayFile, public Asset
+class DENG2_PUBLIC RemoteFile : public LinkFile, public Asset
 {
 public:
     /// Data of the file has not yet been fetched. @ingroup errors
@@ -48,8 +48,8 @@ class RemoteFile : public ByteArrayFile, public Asset
     void fetchContents();
 
     // Implements IByteArray.
-    void get(Offset at, Byte *values, Size count) const override;
-    void set(Offset at, Byte const *values, Size count) override;
+    //void get(Offset at, Byte *values, Size count) const override;
+    //void set(Offset at, Byte const *values, Size count) override;
 
     // File streaming.
     IIStream const &operator >> (IByteArray &bytes) const override;

doomsday/sdk/libcore/src/filesys/linkfile.cpp

@@ -84,6 +84,13 @@ void LinkFile::setTarget(File const &file)
     d->target.reset(&file);
 }
 
+void LinkFile::setTarget(File const *fileOrNull)
+{
+    DENG2_GUARD(this);
+
+    d->target.reset(fileOrNull);
+}
+
 bool LinkFile::isBroken() const
 {
     return &target() == this;
@@ -101,6 +108,19 @@ String LinkFile::describe() const
     return "broken link";
 }
 
+IIStream const &LinkFile::operator >> (IByteArray &bytes) const
+{
+    if (!isBroken())
+    {
+        target() >> bytes;
+        return *this;
+    }
+    else
+    {
+        return File::operator >> (bytes);
+    }
+}
+
 filesys::Node const *LinkFile::tryFollowPath(PathRef const &path) const
 {
     if (Folder const *folder = targetFolder())

doomsday/sdk/libcore/src/filesys/packagefeed.cpp

@@ -24,12 +24,16 @@
 
 namespace de {
 
+static String const VAR_LINK_PACKAGE_ID("link.package");
+
 DENG2_PIMPL(PackageFeed)
 {
     PackageLoader &loader;
+    LinkMode linkMode;
+    Filter filter;
 
-    Impl(Public *i, PackageLoader &ldr)
-        : Base(i), loader(ldr)
+    Impl(Public *i, PackageLoader &ldr, LinkMode lm)
+        : Base(i), loader(ldr), linkMode(lm)
     {}
 
     File *linkToPackage(Package &pkg, String const &linkName, Folder const &folder)
@@ -38,12 +42,36 @@ DENG2_PIMPL(PackageFeed)
 
         if (folder.has(linkName)) return nullptr; // Already there, keep the existing link.
 
+        // Packages can be optionally filtered from the feed.
+        if (filter && !filter(pkg)) return nullptr;
+
         // Create a link to the loaded package's file.
-        LinkFile *link = LinkFile::newLinkToFile(pkg.file(), linkName);
+        File const *target;
+        String name;
+        if (linkMode == LinkPackages)
+        {
+            target = &pkg.file();
+            name = linkName;
+        }
+        else
+        {
+            target = &pkg.file();
+            //name = String("%1.%2").arg(target->metaId().asHexadecimalText())
+             //                     .arg(target->name());
+            name = Package::versionedIdentifierForFile(pkg.file());
+        }
+        LinkFile *link = LinkFile::newLinkToFile(*target, name);
 
         // We will decide on pruning this.
         link->setOriginFeed(thisPublic);
 
+        link->objectNamespace().addText(VAR_LINK_PACKAGE_ID, pkg.identifier());
+//        if (linkMode == LinkSourceFiles)
+//        {
+//            link->objectNamespace().addText(QStringLiteral("link.sourceName"),
+//                                            pkg.sourceFile().name());
+//        }
+
         return link;
     }
 
@@ -72,9 +100,15 @@ DENG2_PIMPL(PackageFeed)
     }
 };
 
-PackageFeed::PackageFeed(PackageLoader &loader) : d(new Impl(this, loader))
+PackageFeed::PackageFeed(PackageLoader &loader, LinkMode linkMode)
+    : d(new Impl(this, loader, linkMode))
 {}
 
+void PackageFeed::setFilter(Filter filter)
+{
+    d->filter = filter;
+}
+
 PackageLoader &PackageFeed::loader()
 {
     return d->loader;
@@ -94,13 +128,20 @@ bool PackageFeed::prune(File &file) const
 {
     if (LinkFile const *link = maybeAs<LinkFile>(file))
     {
-        if (Folder const *pkg = maybeAs<Folder>(link->target()))
+        // Links to unloaded packages should be pruned.
+        if (!d->loader.isLoaded(link->objectNamespace().gets(VAR_LINK_PACKAGE_ID)))
+            return true;
+
+        //if (Folder const *pkg = maybeAs<Folder>(link->target()))
         {
             // Links to unloaded packages should be pruned.
-            if (!d->loader.isLoaded(*pkg)) return true;
+            //if (!d->loader.isLoaded(*pkg)) return true;
+
+//            qDebug() << "Link:" << link->description() << link->status().modifiedAt.asText();
+//            qDebug() << " tgt:" << link->target().description() << link->target().status().modifiedAt.asText();
 
             // Package has been modified, should be pruned.
-            if (link->status() != pkg->status()) return true;
+            if (link->status() != link->target().status()) return true;
         }
     }
     return false; // Don't prune.

doomsday/sdk/libcore/src/filesys/remotefeed.cpp

@@ -73,6 +73,9 @@ DENG2_PIMPL(RemoteFeed)
                 if (md.has("package"))
                 {
                     file->objectNamespace().add("package", new Record(md.subrecord("package")));
+
+                    //qDebug() << "Package metadata for" << file->path();
+                    //qDebug() << file->objectNamespace().getr("package").asText();
                 }
                 file->setStatus(File::Status(fileType, fileSize, modTime));
                 file->setOriginFeed(thisPublic);

doomsday/sdk/libcore/src/filesys/remotefeedprotocol.cpp

@@ -80,8 +80,8 @@ RemoteFeedMetadataPacket::RemoteFeedMetadataPacket()
 
 void RemoteFeedMetadataPacket::addFile(File const &file, String const &prefix)
 {
-    auto const &ns = file.objectNamespace();
-    auto const status = file.status();
+    auto const &ns = file.target().objectNamespace();
+    auto const status = file.target().status();
 
     std::unique_ptr<Record> fileMeta(new Record);
 
@@ -98,9 +98,13 @@ void RemoteFeedMetadataPacket::addFile(File const &file, String const &prefix)
     }
     if (ns.hasSubrecord("package"))
     {
-        fileMeta->add("package", new Record(ns["package"].valueAsRecord(),
+        fileMeta->add("package", new Record(ns.getr("package").dereference(),
                       Record::IgnoreDoubleUnderscoreMembers));
     }
+//    if (ns.hasSubrecord("link"))
+//    {
+//        fileMeta->add("link", new Record(ns.getr("link").dereference()));
+//    }
 
     _metadata.add(new TextValue(prefix / file.name()),
                   new RecordValue(fileMeta.release(), RecordValue::OwnsRecord));

doomsday/sdk/libcore/src/filesys/remotefeedrelay.cpp

@@ -181,13 +181,15 @@ DENG2_PIMPL(RemoteFeedRelay)
                 if (!query.fileSize)
                 {
                     // Before the first chunk, notify about the total size.
+                    qDebug() << "notifying full file size:" << fileSize;
                     query.fileContents->call(0, Block(), fileSize);
                 }
                 //query.receivedData.set(startOffset, chunk.data(), chunk.size());
                 query.fileSize = fileSize;
                 query.receivedBytes += chunk.size();
 
                 // Notify about progress.
+                qDebug() << "notifying chunk with" << chunk.size() << "bytes";
                 query.fileContents->call(startOffset, chunk, fileSize - query.receivedBytes);
 
                 if (fileSize == query.receivedBytes)
@@ -325,7 +327,7 @@ RemoteFeed *RemoteFeedRelay::addServerRepository(String const &serverAddress)
 {
     auto *repo = new Impl::NativeRepositoryLink(d, serverAddress);
     d->repositories.insert(serverAddress, repo);
-    return new RemoteFeed(serverAddress, "/local");
+    return new RemoteFeed(serverAddress, "/sys/server/files");
 }
 
 RemoteFeed *RemoteFeedRelay::addRepository(String const &address)