diff --git a/doomsday/sdk/libcore/include/de/filesys/linkfile.h b/doomsday/sdk/libcore/include/de/filesys/linkfile.h index c8ce7b5acd..ef2bf7855a 100644 --- a/doomsday/sdk/libcore/include/de/filesys/linkfile.h +++ b/doomsday/sdk/libcore/include/de/filesys/linkfile.h @@ -49,6 +49,8 @@ class DENG2_PUBLIC LinkFile : public File */ void setTarget(File const &file); + void setTarget(File const *fileOrNull); + /** * Returns the file's target. This is used for indirection when descending into * subfolders, to implement symbolic links. @@ -75,6 +77,9 @@ class DENG2_PUBLIC LinkFile : public File String describe() const; + // Stream access: + IIStream const &operator >> (IByteArray &bytes) const override; + // filesys::Node overrides: Node const *tryFollowPath(PathRef const &path) const; Node const *tryGetChild(String const &name) const; diff --git a/doomsday/sdk/libcore/include/de/filesys/packagefeed.h b/doomsday/sdk/libcore/include/de/filesys/packagefeed.h index 7febb1b1c1..8e85b92682 100644 --- a/doomsday/sdk/libcore/include/de/filesys/packagefeed.h +++ b/doomsday/sdk/libcore/include/de/filesys/packagefeed.h @@ -23,6 +23,7 @@ namespace de { +class Package; class PackageLoader; /** @@ -33,7 +34,14 @@ class PackageLoader; class PackageFeed : public Feed { public: - PackageFeed(PackageLoader &loader); + enum LinkMode { LinkPackages, LinkSourceFiles }; + + typedef std::function Filter; + +public: + PackageFeed(PackageLoader &loader, LinkMode linkMode = LinkPackages); + + void setFilter(Filter filter); PackageLoader &loader(); diff --git a/doomsday/sdk/libcore/include/de/filesys/remotefile.h b/doomsday/sdk/libcore/include/de/filesys/remotefile.h index 25812bb48d..9650f4db52 100644 --- a/doomsday/sdk/libcore/include/de/filesys/remotefile.h +++ b/doomsday/sdk/libcore/include/de/filesys/remotefile.h @@ -19,7 +19,7 @@ #ifndef LIBDENG2_REMOTEFILE_H #define LIBDENG2_REMOTEFILE_H -#include "../ByteArrayFile" +#include "../LinkFile" #include "../Asset" namespace de { @@ -30,7 +30,7 @@ namespace de { * * RemoteFile provides status information as an Asset. */ -class RemoteFile : public ByteArrayFile, public Asset +class DENG2_PUBLIC RemoteFile : public LinkFile, public Asset { public: /// Data of the file has not yet been fetched. @ingroup errors @@ -48,8 +48,8 @@ class RemoteFile : public ByteArrayFile, public Asset void fetchContents(); // Implements IByteArray. - void get(Offset at, Byte *values, Size count) const override; - void set(Offset at, Byte const *values, Size count) override; + //void get(Offset at, Byte *values, Size count) const override; + //void set(Offset at, Byte const *values, Size count) override; // File streaming. IIStream const &operator >> (IByteArray &bytes) const override; diff --git a/doomsday/sdk/libcore/src/filesys/linkfile.cpp b/doomsday/sdk/libcore/src/filesys/linkfile.cpp index b399d18230..6b9327aa4c 100644 --- a/doomsday/sdk/libcore/src/filesys/linkfile.cpp +++ b/doomsday/sdk/libcore/src/filesys/linkfile.cpp @@ -84,6 +84,13 @@ void LinkFile::setTarget(File const &file) d->target.reset(&file); } +void LinkFile::setTarget(File const *fileOrNull) +{ + DENG2_GUARD(this); + + d->target.reset(fileOrNull); +} + bool LinkFile::isBroken() const { return &target() == this; @@ -101,6 +108,19 @@ String LinkFile::describe() const return "broken link"; } +IIStream const &LinkFile::operator >> (IByteArray &bytes) const +{ + if (!isBroken()) + { + target() >> bytes; + return *this; + } + else + { + return File::operator >> (bytes); + } +} + filesys::Node const *LinkFile::tryFollowPath(PathRef const &path) const { if (Folder const *folder = targetFolder()) diff --git a/doomsday/sdk/libcore/src/filesys/packagefeed.cpp b/doomsday/sdk/libcore/src/filesys/packagefeed.cpp index ff5d3f0f67..ed7f485045 100644 --- a/doomsday/sdk/libcore/src/filesys/packagefeed.cpp +++ b/doomsday/sdk/libcore/src/filesys/packagefeed.cpp @@ -24,12 +24,16 @@ namespace de { +static String const VAR_LINK_PACKAGE_ID("link.package"); + DENG2_PIMPL(PackageFeed) { PackageLoader &loader; + LinkMode linkMode; + Filter filter; - Impl(Public *i, PackageLoader &ldr) - : Base(i), loader(ldr) + Impl(Public *i, PackageLoader &ldr, LinkMode lm) + : Base(i), loader(ldr), linkMode(lm) {} File *linkToPackage(Package &pkg, String const &linkName, Folder const &folder) @@ -38,12 +42,36 @@ DENG2_PIMPL(PackageFeed) if (folder.has(linkName)) return nullptr; // Already there, keep the existing link. + // Packages can be optionally filtered from the feed. + if (filter && !filter(pkg)) return nullptr; + // Create a link to the loaded package's file. - LinkFile *link = LinkFile::newLinkToFile(pkg.file(), linkName); + File const *target; + String name; + if (linkMode == LinkPackages) + { + target = &pkg.file(); + name = linkName; + } + else + { + target = &pkg.file(); + //name = String("%1.%2").arg(target->metaId().asHexadecimalText()) + // .arg(target->name()); + name = Package::versionedIdentifierForFile(pkg.file()); + } + LinkFile *link = LinkFile::newLinkToFile(*target, name); // We will decide on pruning this. link->setOriginFeed(thisPublic); + link->objectNamespace().addText(VAR_LINK_PACKAGE_ID, pkg.identifier()); +// if (linkMode == LinkSourceFiles) +// { +// link->objectNamespace().addText(QStringLiteral("link.sourceName"), +// pkg.sourceFile().name()); +// } + return link; } @@ -72,9 +100,15 @@ DENG2_PIMPL(PackageFeed) } }; -PackageFeed::PackageFeed(PackageLoader &loader) : d(new Impl(this, loader)) +PackageFeed::PackageFeed(PackageLoader &loader, LinkMode linkMode) + : d(new Impl(this, loader, linkMode)) {} +void PackageFeed::setFilter(Filter filter) +{ + d->filter = filter; +} + PackageLoader &PackageFeed::loader() { return d->loader; @@ -94,13 +128,20 @@ bool PackageFeed::prune(File &file) const { if (LinkFile const *link = maybeAs(file)) { - if (Folder const *pkg = maybeAs(link->target())) + // Links to unloaded packages should be pruned. + if (!d->loader.isLoaded(link->objectNamespace().gets(VAR_LINK_PACKAGE_ID))) + return true; + + //if (Folder const *pkg = maybeAs(link->target())) { // Links to unloaded packages should be pruned. - if (!d->loader.isLoaded(*pkg)) return true; + //if (!d->loader.isLoaded(*pkg)) return true; + +// qDebug() << "Link:" << link->description() << link->status().modifiedAt.asText(); +// qDebug() << " tgt:" << link->target().description() << link->target().status().modifiedAt.asText(); // Package has been modified, should be pruned. - if (link->status() != pkg->status()) return true; + if (link->status() != link->target().status()) return true; } } return false; // Don't prune. diff --git a/doomsday/sdk/libcore/src/filesys/remotefeed.cpp b/doomsday/sdk/libcore/src/filesys/remotefeed.cpp index c2a74aed80..69697d40ce 100644 --- a/doomsday/sdk/libcore/src/filesys/remotefeed.cpp +++ b/doomsday/sdk/libcore/src/filesys/remotefeed.cpp @@ -73,6 +73,9 @@ DENG2_PIMPL(RemoteFeed) if (md.has("package")) { file->objectNamespace().add("package", new Record(md.subrecord("package"))); + + //qDebug() << "Package metadata for" << file->path(); + //qDebug() << file->objectNamespace().getr("package").asText(); } file->setStatus(File::Status(fileType, fileSize, modTime)); file->setOriginFeed(thisPublic); diff --git a/doomsday/sdk/libcore/src/filesys/remotefeedprotocol.cpp b/doomsday/sdk/libcore/src/filesys/remotefeedprotocol.cpp index e76dd0c50f..a45e4cee4b 100644 --- a/doomsday/sdk/libcore/src/filesys/remotefeedprotocol.cpp +++ b/doomsday/sdk/libcore/src/filesys/remotefeedprotocol.cpp @@ -80,8 +80,8 @@ RemoteFeedMetadataPacket::RemoteFeedMetadataPacket() void RemoteFeedMetadataPacket::addFile(File const &file, String const &prefix) { - auto const &ns = file.objectNamespace(); - auto const status = file.status(); + auto const &ns = file.target().objectNamespace(); + auto const status = file.target().status(); std::unique_ptr fileMeta(new Record); @@ -98,9 +98,13 @@ void RemoteFeedMetadataPacket::addFile(File const &file, String const &prefix) } if (ns.hasSubrecord("package")) { - fileMeta->add("package", new Record(ns["package"].valueAsRecord(), + fileMeta->add("package", new Record(ns.getr("package").dereference(), Record::IgnoreDoubleUnderscoreMembers)); } +// if (ns.hasSubrecord("link")) +// { +// fileMeta->add("link", new Record(ns.getr("link").dereference())); +// } _metadata.add(new TextValue(prefix / file.name()), new RecordValue(fileMeta.release(), RecordValue::OwnsRecord)); diff --git a/doomsday/sdk/libcore/src/filesys/remotefeedrelay.cpp b/doomsday/sdk/libcore/src/filesys/remotefeedrelay.cpp index af1f6c9942..b7ee927701 100644 --- a/doomsday/sdk/libcore/src/filesys/remotefeedrelay.cpp +++ b/doomsday/sdk/libcore/src/filesys/remotefeedrelay.cpp @@ -181,6 +181,7 @@ DENG2_PIMPL(RemoteFeedRelay) if (!query.fileSize) { // Before the first chunk, notify about the total size. + qDebug() << "notifying full file size:" << fileSize; query.fileContents->call(0, Block(), fileSize); } //query.receivedData.set(startOffset, chunk.data(), chunk.size()); @@ -188,6 +189,7 @@ DENG2_PIMPL(RemoteFeedRelay) query.receivedBytes += chunk.size(); // Notify about progress. + qDebug() << "notifying chunk with" << chunk.size() << "bytes"; query.fileContents->call(startOffset, chunk, fileSize - query.receivedBytes); if (fileSize == query.receivedBytes) @@ -325,7 +327,7 @@ RemoteFeed *RemoteFeedRelay::addServerRepository(String const &serverAddress) { auto *repo = new Impl::NativeRepositoryLink(d, serverAddress); d->repositories.insert(serverAddress, repo); - return new RemoteFeed(serverAddress, "/local"); + return new RemoteFeed(serverAddress, "/sys/server/files"); } RemoteFeed *RemoteFeedRelay::addRepository(String const &address) diff --git a/doomsday/sdk/libcore/src/filesys/remotefile.cpp b/doomsday/sdk/libcore/src/filesys/remotefile.cpp index 64435b0244..92cc01cd60 100644 --- a/doomsday/sdk/libcore/src/filesys/remotefile.cpp +++ b/doomsday/sdk/libcore/src/filesys/remotefile.cpp @@ -17,10 +17,13 @@ */ #include "de/RemoteFile" -#include "de/RemoteFeedRelay" + #include "de/App" #include "de/FileSystem" +#include "de/RecordValue" +#include "de/RemoteFeedRelay" #include "de/ScriptSystem" +#include "de/TextValue" namespace de { @@ -32,7 +35,7 @@ DENG2_PIMPL(RemoteFile) Block remoteMetaId; Block buffer; RemoteFeedRelay::FileContentsRequest fetching; - SafePtr cachedFile; + //SafePtr cachedFile; Impl(Public *i) : Base(i) {} @@ -47,35 +50,53 @@ DENG2_PIMPL(RemoteFile) String cachePath() const { String const hex = remoteMetaId.asHexadecimalText(); - return CACHE_PATH / String(hex.last()) / hex; + String path = CACHE_PATH / hex.right(3); + String original = self().objectNamespace().gets("package.path", remotePath); + return path / original.fileName(); } void findCachedFile(bool requireExists = true) { - if (cachedFile) return; + if (!self().isBroken()) return; + if (self().state() == NotReady) { throw UnfetchedError("RemoteFile::operator >>", self().description() + " not downloaded"); } - if (!cachedFile) + if (self().isBroken()) { - cachedFile.reset(FS::tryLocate(cachePath())); + self().setTarget(FS::tryLocate(cachePath())); } - if (requireExists && !cachedFile) + if (requireExists && self().isBroken()) { throw InputError("RemoteFile::operator >>", self().description() + " has no locally cached data"); } } + + void prepareForUse() + { +// File *interp = self().reinterpret(); +// if (interp != thisPublic) +// { +// // Make sure the remote package metadata is visible. +// interp->objectNamespace().add +// ("package", new Record(self().objectNamespace().subrecord("package"))); + +// qDebug() << "Interpreted as" << interp->description(); +// qDebug() << interp->objectNamespace().asText(); +// } +// self().setState(Ready); + } }; RemoteFile::RemoteFile(String const &name, String const &remotePath, Block const &remoteMetaId) - : ByteArrayFile(name) + : LinkFile(name) , d(new Impl(this)) { objectNamespace().addSuperRecord(ScriptSystem::builtInClass(QStringLiteral("RemoteFile"))); - d->remotePath = remotePath; + d->remotePath = remotePath; d->remoteMetaId = remoteMetaId; setState(NotReady); } @@ -88,14 +109,17 @@ void RemoteFile::fetchContents() setState(Recovering); +#if 0 d->findCachedFile(false /* doesn't have to exist */); - if (d->cachedFile) + if (!isBroken()) { // There is a cached copy already. - setState(Ready); - reinterpret(); + d->prepareForUse(); + //setState(Ready); + //reinterpret(); return; } +#endif d->fetching = RemoteFeedRelay::get().fetchFileContents (originFeed()->as().repository(), @@ -112,6 +136,7 @@ void RemoteFile::fetchContents() { d->buffer.resize(remainingBytes); } + d->buffer.set(startOffset, chunk.data(), chunk.size()); // When fully transferred, the file can be cached locally and interpreted. @@ -124,19 +149,31 @@ void RemoteFile::fetchContents() Folder &cacheFolder = FS::get().makeFolder(fn.fileNamePath()); File &data = cacheFolder.replaceFile(fn); data << d->buffer; + d->buffer.clear(); + data.flush(); - d->buffer.clear(); - d->cachedFile.reset(&data); + setTarget(data.reinterpret()); + + // d->cachedFile.reset(&data); + + //d->prepareForUse(); + + if (objectNamespace().has("package.path")) + { + objectNamespace()["package.path"] = target().path(); + } + qDebug() << "RemoteFile metadata:\n" << objectNamespace().asText().toUtf8().constData(); + setState(Ready); // Now this RemoteFile can become the source of an interpreted file, // which replaces the RemoteFile within the parent folder. - reinterpret(); } }); } +/* void RemoteFile::get(Offset at, Byte *values, Size count) const { d->findCachedFile(); @@ -154,13 +191,16 @@ void RemoteFile::set(Offset, Byte const *, Size) { verifyWriteAccess(); // intended to throw exception } +*/ IIStream const &RemoteFile::operator >> (IByteArray &bytes) const { - d->findCachedFile(); - DENG2_ASSERT(d->cachedFile); - *d->cachedFile >> bytes; - return *this; + //d->findCachedFile(); + DENG2_ASSERT(!isBroken()); + return LinkFile::operator >> (bytes); + + //*d->cachedFile >> bytes; + //return *this; } String RemoteFile::describe() const @@ -169,11 +209,17 @@ String RemoteFile::describe() const { return String("\"%1\"").arg(name()); } + String targetDesc; + if (!isBroken()) + { + targetDesc = " cached in " + target().description(); + } return String("remote file \"%1\" (%2)") .arg(name()) .arg( state() == NotReady ? "not ready" : state() == Recovering ? "downloading" - : "ready"); + : "ready") + + targetDesc; } Block RemoteFile::metaId() const