libcore|FS: Improved FS finding API and Package loading

FileSystem now has methods for finding multiple types of files and
a method that accepts a custom predicate for excluding matches.

Package metadata is now parsed and stored into its File's info record.
Metadata may come from two places: a file named "Info" at the root
of the package, or the "__init__.de" script.

Packages are now required to specify a title, version number, license,
and tags in their metadata.

Fixed PackageFeed to not attempt duplicate links.

PackageLoader allows packages to be loaded from plain folders or
zipped folders, if they use the ".pack" file extension.

doomsday/libcore/include/de/filesys/filesystem.h

@@ -174,15 +174,24 @@ class DENG2_PUBLIC FileSystem : public System
      * using the file system's index; no recursive descent into folders is
      * done.
      *
-     * @param path   Path or file name to look for.
-     * @param found  Set of files that match the result.
+     * @param partialPath   Partial path or file name to look for.
+     * @param found         Set of files that match the result.
      *
      * @return  Number of files found.
      */
-    int findAll(String const &path, FoundFiles &found) const;
+    int findAll(String const &partialPath, FoundFiles &found) const;
+
+    template <typename Predicate>
+    int findAll(Predicate exclusion, String const &partialPath, FoundFiles &found) const {
+        findAll(partialPath, found);
+        found.remove_if(exclusion);
+        return int(found.size());
+    }
 
     int findAllOfType(String const &typeIdentifier, String const &path, FoundFiles &found) const;
 
+    int findAllOfTypes(StringList const &typeIdentifiers, String const &path, FoundFiles &found) const;
+
     /**
      * Finds a single file matching a full or partial path. The search is
      * done using the file system's index; no recursive descent into
@@ -208,9 +217,8 @@ class DENG2_PUBLIC FileSystem : public System
     template <typename Type>
     Type &find(String const &path) const {
         FoundFiles found;
-        findAll(path, found);
         // Filter out the wrong types.
-        found.remove_if(internal::cannotCastFileTo<Type>);
+        findAll(internal::cannotCastFileTo<Type>, path, found);
         if(found.size() > 1) {
             /// @throw AmbiguousError  More than one file matches the conditions.
             throw AmbiguousError("FS::find", "More than one file found matching '" + path + "'");

doomsday/libcore/include/de/filesys/package.h

@@ -82,7 +82,7 @@ class Package
      * Returns the package's root folder, if it has one. Returns @c NULL if the package
      * is "flat" and comes with no folder structure.
      */
-    Folder const *root() const;
+    Folder const &root() const;
 
     Record &info();
 
@@ -115,6 +115,13 @@ class Package
     virtual void aboutToUnload();
 
 public:
+    /**
+     * Parse the embedded metadata found in a package file.
+     *
+     * @param packageFile  File containing a package.
+     */
+    static void parseMetadata(File &packageFile);
+
     /**
      * Checks that all the metadata seems legit. An IncompleteMetadataError or
      * another exception is thrown if the package is not deemed valid.

doomsday/libcore/include/de/filesys/packageloader.h

@@ -58,7 +58,7 @@ class DENG2_PUBLIC PackageLoader
 public:
     PackageLoader();
 
-    void load(String const &packageId);
+    Package const &load(String const &packageId);
 
     void unload(String const &packageId);
 
@@ -71,6 +71,16 @@ class DENG2_PUBLIC PackageLoader
      */
     LoadedPackages const &loadedPackages() const;
 
+    /**
+     * Retrieves a specific loaded package. The package must already be loaded
+     * using load().
+     *
+     * @param packageId
+     *
+     * @return Package.
+     */
+    Package const &package(String const &packageId) const;
+
 private:
     DENG2_PRIVATE(d)
 };

doomsday/libcore/src/core/app.cpp

@@ -25,6 +25,7 @@
 #include "de/DictionaryValue"
 #include "de/DirectoryFeed"
 #include "de/FileLogSink"
+#include "de/PackageFeed"
 #include "de/Log"
 #include "de/LogBuffer"
 #include "de/LogFilter"
@@ -49,6 +50,7 @@ namespace de {
 static App *singletonApp;
 
 DENG2_PIMPL(App)
+, DENG2_OBSERVES(PackageLoader, Activity)
 {
     QThread *mainThread;
 
@@ -147,6 +149,8 @@ DENG2_PIMPL(App)
 
     ~Instance()
     {
+        packageLoader.audienceForActivity() -= this;
+
         if(!errorSink.isNull())
         {
             logBuffer.removeSink(*errorSink);
@@ -214,8 +218,12 @@ DENG2_PIMPL(App)
         fs.makeFolder("/home", FS::DontInheritFeeds).attach(new DirectoryFeed(self.nativeHomePath(),
                 DirectoryFeed::AllowWrite | DirectoryFeed::CreateIfMissing));
 
+        fs.makeFolder("/packs").attach(new PackageFeed(packageLoader));
+
         // Populate the file system.
         fs.refresh();
+
+        packageLoader.audienceForActivity() += this;
     }
 
     void setLogLevelAccordingToOptions()
@@ -277,6 +285,12 @@ DENG2_PIMPL(App)
         }
     }
 
+    void setOfLoadedPackagesChanged()
+    {
+        // Make sure the package links are up to date.
+        fs.root().locate<Folder>("/packs").populate();
+    }
+
     DENG2_PIMPL_AUDIENCE(StartupComplete)
     DENG2_PIMPL_AUDIENCE(GameUnload)
     DENG2_PIMPL_AUDIENCE(GameChange)

doomsday/libcore/src/filesys/filesystem.cpp

@@ -48,9 +48,9 @@ DENG2_PIMPL_NOREF(FileSystem), public ReadWriteLockable
 
     void findInIndex(Index const &idx, String const &path, FoundFiles &found) const
     {
-        found.clear();
         String baseName = path.fileName().lower();
-        String dir = path.fileNamePath().lower();
+        String dir      = path.fileNamePath().lower();
+
         if(!dir.empty() && !dir.beginsWith("/"))
         {
             // Always begin with a slash. We don't want to match partial folder names.
@@ -232,6 +232,7 @@ int FileSystem::findAll(String const &path, FoundFiles &found) const
 {
     LOG_AS("FS::findAll");
 
+    found.clear();
     d->findInIndex(d->index, path, found);
     return int(found.size());
 }
@@ -240,7 +241,18 @@ int FileSystem::findAllOfType(String const &typeIdentifier, String const &path,
 {
     LOG_AS("FS::findAllOfType");
 
-    d->findInIndex(indexFor(typeIdentifier), path, found);
+    return findAllOfTypes(StringList() << typeIdentifier, path, found);
+}
+
+int FileSystem::findAllOfTypes(StringList const &typeIdentifiers, String const &path, FoundFiles &found) const
+{
+    LOG_AS("FS::findAllOfTypes");
+
+    found.clear();
+    foreach(String const &id, typeIdentifiers)
+    {
+        d->findInIndex(indexFor(id), path, found);
+    }
     return int(found.size());
 }
 

doomsday/libcore/src/filesys/package.cpp

@@ -20,6 +20,8 @@
 #include "de/PackageLoader"
 #include "de/Process"
 #include "de/Script"
+#include "de/ScriptedInfo"
+#include "de/TimeValue"
 #include "de/App"
 
 namespace de {
@@ -67,25 +69,10 @@ File const &Package::file() const
     return *d->file;
 }
 
-void Package::validateMetadata(Record const &packageInfo)
-{
-    if(!packageInfo.has("name"))
-    {
-        throw IncompleteMetadataError("Package::validateMetadata",
-                                      "Package does not have a name");
-    }
-
-    if(!packageInfo.has("version"))
-    {
-        throw IncompleteMetadataError("Package::validateMetadata",
-                                      "Package '" + packageInfo.gets("name") +
-                                      "' does not have a version");
-    }
-}
-
-Folder const *Package::root() const
+Folder const &Package::root() const
 {
-    return d->file->maybeAs<Folder>();
+    d->verifyFile();
+    return d->file->as<Folder>();
 }
 
 Record &Package::info()
@@ -121,12 +108,91 @@ bool Package::executeFunction(String const &name)
 
 void Package::didLoad()
 {
-    executeFunction(L"onLoad");
+    executeFunction("onLoad");
 }
 
 void Package::aboutToUnload()
 {
-    executeFunction(L"onUnload");
+    executeFunction("onUnload");
+}
+
+void Package::parseMetadata(File &packageFile)
+{
+    static String const TIMESTAMP("__timestamp__");
+
+    if(Folder *folder = packageFile.maybeAs<Folder>())
+    {
+        Record &metadata        = packageFile.info();
+        File *metadataInfo      = folder->tryLocateFile("Info");
+        File *initializerScript = folder->tryLocateFile("__init__.de");
+        Time parsedAt           = Time::invalidTime();
+        bool needParse          = true;
+
+        if(!metadataInfo && !initializerScript) return; // Nothing to do.
+
+        if(metadata.has(TIMESTAMP))
+        {
+            // Already parsed.
+            needParse = false;
+
+            // Only parse if the source has changed.
+            if(TimeValue const *time = metadata.get(TIMESTAMP).maybeAs<TimeValue>())
+            {
+                needParse =
+                        (metadataInfo      && metadataInfo->status().modifiedAt      > time->time()) ||
+                        (initializerScript && initializerScript->status().modifiedAt > time->time());
+            }
+        }
+
+        if(!needParse) return;
+
+        // Check for a ScriptedInfo source.
+        if(metadataInfo)
+        {
+            ScriptedInfo script(&metadata);
+            script.parse(*metadataInfo);
+
+            parsedAt = metadataInfo->status().modifiedAt;
+        }
+
+        // Check for an initialization script.
+        if(initializerScript)
+        {
+            Script script(*initializerScript);
+            Process proc(&metadata);
+            proc.run(script);
+            proc.execute();
+
+            if(parsedAt.isValid() && initializerScript->status().modifiedAt > parsedAt)
+            {
+                parsedAt = initializerScript->status().modifiedAt;
+            }
+        }
+
+        metadata.addTime(TIMESTAMP, parsedAt);
+
+        LOG_MSG("Parsed metadata of '%s':\n")
+                << identifierForFile(packageFile)
+                << metadata.asText();
+    }
+}
+
+void Package::validateMetadata(Record const &packageInfo)
+{
+    char const *required[] = {
+        "title", "version", "license", "tags", 0
+    };
+
+    for(int i = 0; required[i]; ++i)
+    {
+        if(!packageInfo.has(required[i]))
+        {
+            throw IncompleteMetadataError("Package::validateMetadata",
+                                          QString("Package \"%1\" does not have '%2' in its metadata")
+                                          .arg(packageInfo.gets("path"))
+                                          .arg(required[i]));
+        }
+    }
 }
 
 String Package::identifierForFile(File const &file)

doomsday/libcore/src/filesys/packagefeed.cpp

@@ -52,11 +52,12 @@ void PackageFeed::populate(Folder &folder)
 {
     DENG2_FOR_EACH_CONST(PackageLoader::LoadedPackages, i, d->loader.loadedPackages())
     {
-        String const id = i.value()->identifier();
-        if(folder.has(id)) continue; // Already there.
+        Package *pkg = i.value();
+
+        if(folder.has(pkg->file().name())) continue; // Already there.
 
         // Create a link to the loaded package's file.
-        LinkFile &link = folder.add(LinkFile::newLinkToFile(i.value()->file()));
+        LinkFile &link = folder.add(LinkFile::newLinkToFile(pkg->file()));
 
         // We will decide on pruning this.
         link.setOriginFeed(this);

doomsday/libcore/src/filesys/packageloader.cpp

@@ -80,7 +80,10 @@ DENG2_PIMPL(PackageLoader)
         LOG_AS("selectPackage");
 
         FS::FoundFiles found;
-        if(!App::fileSystem().findAllOfType(DENG2_TYPE_NAME(ArchiveFolder), packageId + ".pack", found))
+        if(!App::fileSystem().findAllOfTypes(StringList()
+                                             << DENG2_TYPE_NAME(Folder)
+                                             << DENG2_TYPE_NAME(ArchiveFolder),
+                                             packageId + ".pack", found))
         {
             // None found.
             return 0;
@@ -89,11 +92,15 @@ DENG2_PIMPL(PackageLoader)
         // Each must have a version specified.
         DENG2_FOR_EACH_CONST(FS::FoundFiles, i, found)
         {
-            Package::validateMetadata((*i)->info());
+            File *pkg = *i;
+            Package::parseMetadata(*pkg);
+            Package::validateMetadata(pkg->info());
         }
 
         found.sort(ascendingPackagesByLatest);
 
+        LOG_RES_VERBOSE("Selected '%s': %s") << packageId << found.back()->description();
+
         return found.back();
     }
 
@@ -132,7 +139,7 @@ DENG2_AUDIENCE_METHOD(PackageLoader, Activity)
 PackageLoader::PackageLoader() : d(new Instance(this))
 {}
 
-void PackageLoader::load(String const &packageId)
+Package const &PackageLoader::load(String const &packageId)
 {
     LOG_AS("PackageLoader");
 
@@ -149,6 +156,8 @@ void PackageLoader::load(String const &packageId)
     {
         i->setOfLoadedPackagesChanged();
     }
+
+    return package(packageId);
 }
 
 void PackageLoader::unload(String const &packageId)
@@ -177,4 +186,13 @@ PackageLoader::LoadedPackages const &PackageLoader::loadedPackages() const
     return d->loaded;
 }
 
+Package const &PackageLoader::package(String const &packageId) const
+{
+    if(!isLoaded(packageId))
+    {
+        throw NotFoundError("PackageLoader::package", "Package '" + packageId + "' is not loaded");
+    }
+    return *d->loaded[packageId];
+}
+
 } // namespace de