Refactor: Organized core resources into packages

Added new packages:

- net.dengine.base: contains files shared by the client and server
- net.dengine.client: files for the client only
- net.dengine.stdlib: Doomsday Script standard library
- net.dengine.stdlib.gui: Optional GUI package for the std lib

These packages are loaded as "init" packages, meaning they will be
loaded by App immediately after the file system has been first
populated. This makes them available when the application configuration
is parsed (appconfig imported from Config).

The "deployPackages" macro can be used to easily deploy packages via
the app's .pro file.

With these new packages, it is no longer necessary to deploy the
individual .de scripts, help strings, and other files.

Note that the client's old "data/graphics" and "data/fonts" folders are
still untouched and accessed solely via FS1.

As a new feature, a package may now specify additional DS module
import folders using the 'importPath' metadata variable. The import
path is naturally effective only when the package is loaded.

The main DED file (doomsday.ded) is now loaded via FS2, explicitly
from the net.dengine.base package. Similarly, the help strings are
loaded from net.dengine.base by libdoomsday.

doomsday/build/scripts/packres.py

@@ -68,27 +68,28 @@ def process_dir(path, dest_path):
 # Directory contents added recursively.
 p = Pack()
 p.add_files(
-    [ ('client/defs', 'defs'),
-      ('client/data', 'data') ] )
+    [ ('client/data', 'data') ] )
 p.create('doomsday.pk3')
 
-# net.dengine.pack
+# net.dengine.base.pack
 p = Pack()
-p.add_files(
-    [ ('client/data/net.dengine.pack', '') ] )
-p.create('net.dengine.pack')
+p.add_files( [ ('net.dengine.base.pack', '') ] )
+p.create('net.dengine.base.pack')
+
+# net.dengine.stdlib.pack
+p = Pack()
+p.add_files( [ ('libcore/net.dengine.stdlib.pack', '') ] )
+p.create('net.dengine.stdlib.pack')
 
-# # renderer.pack
-# p = Pack()
-# p.add_files(
-#     [ ('client/data/renderer.pack', '') ] )
-# p.create('renderer.pack')
-#
-# # lensflares.pack
-# p = Pack()
-# p.add_files(
-#     [ ('client/data/lensflares.pack', '') ] )
-# p.create('lensflares.pack')
+# net.dengine.stdlib.gui.pack
+p = Pack()
+p.add_files( [ ('libgui/net.dengine.stdlib.gui.pack', '') ] )
+p.create('net.dengine.stdlib.gui.pack')
+
+# net.dengine.pack (client stuff)
+p = Pack()
+p.add_files( [ ('client/net.dengine.pack', '') ] )
+p.create('net.dengine.pack')
 
 # libdoom.pk3
 p = Pack()

doomsday/client/client.pro

@@ -772,14 +772,7 @@ SOURCES += \
     SOURCES += src/audio/sys_audiod_sdlmixer.cpp
 }
 
-DOOMSDAY_SCRIPTS += \
-    modules/appconfig.de \
-    modules/bootstrap.de \
-    modules/Updater.de
-
 OTHER_FILES += \
-    $$DOOMSDAY_SCRIPTS \
-    data/cphelp.txt \
     include/template.h.template \
     src/template.c.template \
     client-mac.doxy \
@@ -789,13 +782,6 @@ OTHER_FILES += \
 
 data.files = $$OUT_PWD/../doomsday.pk3
 
-mod.files = \
-    $$DOOMSDAY_SCRIPTS \
-    $$DENG_MODULES_DIR/Config.de \
-    $$DENG_MODULES_DIR/Log.de \
-    $$DENG_MODULES_DIR/recutil.de \
-    $$DENG_MODULES_DIR/../../libgui/modules/gui.de
-
 # These fonts may be needed during the initial startup busy mode.
 startupfonts.files = \
     data/fonts/console11.dfn \
@@ -818,10 +804,9 @@ macx {
         res/macx/deng.icns
 
     data.path         = $${res.path}
-    mod.path          = $${res.path}/modules
     startupfonts.path = $${res.path}/data/fonts
 
-    QMAKE_BUNDLE_DATA += mod res data startupfonts
+    QMAKE_BUNDLE_DATA += res data startupfonts
 
     QMAKE_INFO_PLIST = res/macx/Info.plist
 
@@ -858,14 +843,17 @@ macx {
 
 # Installation ---------------------------------------------------------------
 
+DENG_PACKAGES += net.dengine.pack
+
+deployPackages($$DENG_PACKAGES, $$OUT_PWD/..)
+
 !macx {
     # Common (non-Mac) parts of the installation.
-    INSTALLS += target data startupfonts mod
+    INSTALLS += target data startupfonts
 
     target.path       = $$DENG_BIN_DIR
     data.path         = $$DENG_DATA_DIR
     startupfonts.path = $$DENG_DATA_DIR/fonts
-    mod.path          = $$DENG_BASE_DIR/modules
 
     win32 {
         # Windows-specific installation.

doomsday/client/net.dengine.pack/client.pack/Info

@@ -0,0 +1,8 @@
+# Client's primary resources
+
+title: Doomsday Client
+version: 1.15.0
+license: GPL 3+
+tags: core client
+
+importPath <modules>

doomsday/client/src/clientapp.cpp

@@ -43,6 +43,7 @@
 #include "dd_def.h"
 #include "dd_loop.h"
 #include "de_audio.h"
+#include "def_main.h"
 #include "sys_system.h"
 #include "audio/s_main.h"
 #include "gl/gl_main.h"
@@ -330,6 +331,11 @@ void ClientApp::initialize()
     // subsystems and Config.
     DisplayMode_Init();
 
+    // Initialize definitions before the files are indexed.
+    Def_Init();
+
+    addInitPackage("net.dengine.base");
+    addInitPackage("net.dengine.client");
     initSubsystems(); // loads Config
 
     // Set up the log alerts (observes Config variables).

doomsday/client/src/dd_main.cpp

@@ -2120,9 +2120,6 @@ static int DD_StartupWorker(void * /*context*/)
     initPathMappings();
     App_FileSystem().resetAllSchemes();
 
-    // Initialize the definition databases.
-    Def_Init();
-
     Con_SetProgress(40);
 
     Net_Init();

doomsday/client/src/def_main.cpp

@@ -585,11 +585,17 @@ static void readAllDefinitions()
     /*
      * Start with engine's own top-level definition file.
      */
+
+    /*
     String foundPath = App_FileSystem().findPath(de::Uri("doomsday.ded", RC_DEFINITION),
                                                   RLF_DEFAULT, App_ResourceClass(RC_DEFINITION));
     foundPath = App_BasePath() / foundPath; // Ensure the path is absolute.
 
     readDefinitionFile(foundPath);
+    */
+
+    readDefinitionFile(App::packageLoader().package("net.dengine.base").root()
+                       .locate<File const>("defs/doomsday.ded").path());
 
     /*
      * Now any definition files required by the game on load.

doomsday/dep_core.pri

@@ -74,3 +74,5 @@ macx {
         linkBinaryToBundledLibcore($${1}.dylib)
     }
 }
+
+DENG_PACKAGES += net.dengine.stdlib.pack

doomsday/dep_doomsday.pri

@@ -18,3 +18,5 @@ macx {
         fixInstallName($${1}.bundle/$$1, libdeng_doomsday.1.dylib, ..)
     }
 }
+
+DENG_PACKAGES += net.dengine.base.pack

doomsday/dep_gui.pri

@@ -25,3 +25,4 @@ INCLUDEPATH += $$PWD/libgui/include
     }
 }
 
+DENG_PACKAGES += net.dengine.stdlib.gui.pack

doomsday/libcore/include/de/core/app.h

@@ -108,6 +108,14 @@ class DENG2_PUBLIC App : DENG2_OBSERVES(Clock, TimeChange)
     virtual void setMetadata(String const &orgName, String const &orgDomain,
                              String const &appName, String const &appVersion) = 0;
 
+    /**
+     * Add a new package to be loaded at initialization time. Call this before
+     * initSubsystems().
+     *
+     * @param identifier  Package identifier.
+     */
+    void addInitPackage(String const &identifier);
+
     /**
      * Sets the path of the configuration script that will be automatically run if needed
      * during application launch. The script governs the contents of the special

doomsday/libcore/include/de/filesys/fileindex.h

@@ -34,15 +34,15 @@ class File;
  *
  * @ingroup fs
  */
-class FileIndex
+class DENG2_PUBLIC FileIndex
 {
 public:
     typedef std::multimap<String, File *> Index;
     typedef std::pair<Index::iterator, Index::iterator> IndexRange;
     typedef std::pair<Index::const_iterator, Index::const_iterator> ConstIndexRange;
     typedef std::list<File *> FoundFiles;
 
-    class IPredicate
+    class DENG2_PUBLIC IPredicate
     {
     public:
         virtual ~IPredicate() {}

doomsday/libcore/include/de/filesys/package.h

@@ -26,23 +26,6 @@ namespace de {
 
 class Package;
 
-#if 0
-/**
- * Interface for anything that can contain a Package instance.
- */
-class IPackageContainer
-{
-public:
-    virtual ~IPackageContainer() {}
-
-    virtual Package &package() = 0;
-
-    virtual Package const &package() const {
-        return const_cast<IPackageContainer *>(this)->package();
-    }
-};
-#endif
-
 /**
  * Container package with metadata, data, and/or files.
  * @ingroup fs
@@ -132,6 +115,8 @@ class Package
 
     static String identifierForFile(File const &file);
 
+    static String identifierForContainerOfFile(File const &file);
+
 private:
     DENG2_PRIVATE(d)
 };

doomsday/libcore/include/de/scriptsys/scriptsystem.h

@@ -42,6 +42,19 @@ class DENG2_PUBLIC ScriptSystem : public System
 
     ~ScriptSystem();
 
+    /**
+     * Specifies an additional path where modules may be imported from.
+     * "Config.importPath" is checked before any of the paths specified using this
+     * method.
+     *
+     * Paths specified using this method are not saved persistently in Config.
+     *
+     * @param path  Additional module import path.
+     */
+    void addModuleImportPath(Path const &path);
+
+    void removeModuleImportPath(Path const &path);
+
     /**
      * Adds a native module to the set of modules that can be imported in
      * scripts.

doomsday/libcore/net.dengine.stdlib.pack/Info

@@ -0,0 +1,6 @@
+title: Doomsday Script Standard Library
+version: 1.15.0
+license: GPL 3+
+tags: core script
+
+importPath <modules>

doomsday/libcore/src/core/app.cpp

@@ -93,6 +93,7 @@ DENG2_PIMPL(App)
 
     game::Game *currentGame;
 
+    StringList packagesToLoadAtInit;
     PackageLoader packageLoader;
 
     void (*terminateFunc)(char const *);
@@ -123,11 +124,13 @@ DENG2_PIMPL(App)
         , cmdLine(args)
         , unixHomeFolder(".doomsday")
         , persistentData(0)
-        , configPath("/modules/Config.de")
+        , configPath("/packs/net.dengine.stdlib/modules/Config.de")
         , config(0)
         , currentGame(0)
         , terminateFunc(0)
     {
+        packagesToLoadAtInit << "net.dengine.stdlib";
+
         singletonApp = a;
         mainThread = QThread::currentThread();
 
@@ -171,6 +174,16 @@ DENG2_PIMPL(App)
         Clock::setAppClock(0);
     }
 
+    NativePath defaultNativeModulePath() const
+    {
+#ifdef WIN32
+        NativePath appDir = appPath.fileNamePath();
+        return appDir / "..\\modules";
+#else
+        return self.nativeBasePath() / "modules";
+#endif
+    }
+
     void initFileSystem(bool allowPlugins)
     {
         Folder &binFolder = fs.makeFolder("/bin");
@@ -194,19 +207,17 @@ DENG2_PIMPL(App)
             NativePath appDir = appPath.fileNamePath();
             binFolder.attach(new DirectoryFeed(appDir));
             fs.makeFolder("/data").attach(new DirectoryFeed(self.nativeBasePath()));
-            if((self.nativeBasePath() / "modules").exists())
-            {
-                fs.makeFolder("/modules").attach(new DirectoryFeed(self.nativeBasePath() / "modules"));
-            }
 #elif WIN32
             NativePath appDir = appPath.fileNamePath();
             fs.makeFolder("/data").attach(new DirectoryFeed(appDir / "..\\data"));
-            fs.makeFolder("/modules").attach(new DirectoryFeed(appDir / "..\\modules"));
 
 #else // UNIX
             fs.makeFolder("/data").attach(new DirectoryFeed(self.nativeBasePath() / "data"));
-            fs.makeFolder("/modules").attach(new DirectoryFeed(self.nativeBasePath() / "modules"));
 #endif
+            if(defaultNativeModulePath().exists())
+            {
+                fs.makeFolder("/modules").attach(new DirectoryFeed(defaultNativeModulePath()));
+            }
         }
 
         if(allowPlugins)
@@ -349,6 +360,11 @@ App::~App()
     singletonApp = 0;
 }
 
+void App::addInitPackage(String const &identifier)
+{
+    d->packagesToLoadAtInit << identifier;
+}
+
 void App::setConfigScript(Path const &path)
 {
     d->configPath = path;
@@ -543,18 +559,25 @@ void App::initSubsystems(SubsystemInitFlags flags)
 
     d->initFileSystem(allowPlugins);
 
+    // Load the init packages.
+    foreach(String pkg, d->packagesToLoadAtInit)
+    {
+        d->packageLoader.load(pkg);
+    }
+
     if(!flags.testFlag(DisablePersistentData))
     {
+        // Recreate the persistent state data package, if necessary.
         if(!homeFolder().has("persist.pack") || commandLine().has("-reset"))
         {
-            // Recreate the persistent state data package.
             ZipArchive arch;
             arch.add("Info", String(QString("# Package for %1's persistent state.\n").arg(d->appName)).toUtf8());
             Writer(homeFolder().replaceFile("persist.pack")) << arch;
 
             homeFolder().populate(Folder::PopulateOnlyThisFolder);
         }
 
+        // Load the persistent data.
         d->persistentData = &homeFolder().locate<ArchiveFolder>("persist.pack").archive();
     }
 
@@ -623,7 +646,7 @@ void App::initSubsystems(SubsystemInitFlags flags)
     d->clock.setTime(Time::currentHighPerformanceTime());
 
     // Now we can start observing progress of time.
-    d->clock.audienceForTimeChange() += this;
+    d->clock.audienceForTimeChange() += this;    
 
     if(allowPlugins)
     {