Adminotech - AssetAPI asset bundle support + ArchivePlugin for zip bundle implementation

CMakeBuildConfigTemplate.txt

@@ -156,3 +156,7 @@ endif ()
 
 #AddProject(Application AssetInterestPlugin)    # Options to only keep assets below certain distance threshold in memory. Can also unload all non used assets from memory. Exposed to scripts so scenes can set the behaviour.
 AddProject(Application CanvasPlugin)           # Component that draws a graphics scene with any number of widgets into a mesh and provides 3D mouse input.
+
+if (WIN32 OR APPLE) # Enable on linux once the needed zziplib dependency is satisfied in the build scripts.
+    AddProject(Application ArchivePlugin)          # Provides archived asset bundle capabilities. Enables example sub asset referencing into eg. zip files.
+endif ()

bin/plugins.xml

@@ -10,6 +10,7 @@
         dynamic objects (Framework::RegisterDynamicObject) in OnScriptEngineCreated(QScriptEngine*), must be loaded before JavascriptModule. -->
   <plugin path="JavascriptModule" />            <!-- JavascriptModule depends on TundraProtocolModule --> 
   <plugin path="AssetModule" />                 <!-- AssetModule depends on TundraProtocolModule -->
+  <plugin path="ArchivePlugin" />               <!-- Does not depend on any other module -->
   <plugin path="AvatarModule" />                <!-- AvatarModule depends on AssetModule and OgreRenderingModule -->
   <plugin path="SkyXHydrax" />                  <!-- SkyXHydrax depends on OgreRenderingModule -->
   <plugin path="OgreAssetEditorModule" />       <!-- OgreAssetEditorModule depends on OgreRenderingModule -->

bin/viewer-browser.xml

@@ -10,6 +10,7 @@
         dynamic objects (Framework::RegisterDynamicObject) in OnScriptEngineCreated(QScriptEngine*), must be loaded before JavascriptModule. -->
   <plugin path="JavascriptModule" />            <!-- JavascriptModule depends on TundraProtocolModule --> 
   <plugin path="AssetModule" />                 <!-- AssetModule depends on TundraProtocolModule -->
+  <plugin path="ArchivePlugin" />               <!-- Does not depend on any other module -->
   <plugin path="AvatarModule" />                <!-- AvatarModule depends on AssetModule and OgreRenderingModule -->
   <plugin path="SkyXHydrax" />                  <!-- SkyXHydrax depends on OgreRenderingModule -->
   <plugin path="OgreAssetEditorModule" />       <!-- OgreAssetEditorModule depends on OgreRenderingModule -->

bin/viewer.xml

@@ -10,6 +10,7 @@
         dynamic objects (Framework::RegisterDynamicObject) in OnScriptEngineCreated(QScriptEngine*), must be loaded before JavascriptModule. -->
   <plugin path="JavascriptModule" />            <!-- JavascriptModule depends on TundraProtocolModule --> 
   <plugin path="AssetModule" />                 <!-- AssetModule depends on TundraProtocolModule -->
+  <plugin path="ArchivePlugin" />               <!-- Does not depend on any other module -->
   <plugin path="AvatarModule" />                <!-- AvatarModule depends on AssetModule and OgreRenderingModule -->
   <plugin path="SkyXHydrax" />                  <!-- SkyXHydrax depends on OgreRenderingModule -->
   <plugin path="OgreAssetEditorModule" />       <!-- OgreAssetEditorModule depends on OgreRenderingModule -->

bin/viewerautomove.xml

@@ -11,6 +11,7 @@
         dynamic objects (Framework::RegisterDynamicObject) in OnScriptEngineCreated(QScriptEngine*), must be loaded before JavascriptModule. -->
   <plugin path="JavascriptModule" />            <!-- JavascriptModule depends on TundraProtocolModule --> 
   <plugin path="AssetModule" />                 <!-- AssetModule depends on TundraProtocolModule -->
+  <plugin path="ArchivePlugin" />               <!-- Does not depend on any other module -->
   <plugin path="AvatarModule" />                <!-- AvatarModule depends on AssetModule and OgreRenderingModule -->
   <plugin path="DebugStatsModule" />            <!-- DebugStatsModule depends on OgreRenderingModule, EnvironmentModule and AssetModule -->
   <plugin path="SkyXHydrax" />                  <!-- SkyXHydrax depends on OgreRenderingModule -->

src/Application/ArchivePlugin/ArchiveBundleFactory.cpp

@@ -0,0 +1,26 @@
+// For conditions of distribution and use, see copyright notice in LICENSE
+
+#include "ArchiveBundleFactory.h"
+#include "ZipAssetBundle.h"
+
+ArchiveBundleFactory::ArchiveBundleFactory()
+{
+    typesExtensions_ << ".zip";
+}
+
+QString ArchiveBundleFactory::Type() const
+{
+    return "Archive";
+}
+
+QStringList ArchiveBundleFactory::TypeExtensions() const
+{
+    return typesExtensions_;
+}
+
+AssetBundlePtr ArchiveBundleFactory::CreateEmptyAssetBundle(AssetAPI *owner, const QString &name)
+{
+    if (name.endsWith(".zip", Qt::CaseInsensitive))
+        return AssetBundlePtr(new ZipAssetBundle(owner, Type(), name));
+    return AssetBundlePtr();
+}

src/Application/ArchivePlugin/ArchiveBundleFactory.h

@@ -0,0 +1,21 @@
+// For conditions of distribution and use, see copyright notice in LICENSE
+
+#pragma once
+
+#include "IAssetBundleTypeFactory.h"
+
+/// This class provides a factory for archived bundle assets like zip files.
+class ArchiveBundleFactory : public IAssetBundleTypeFactory
+{
+Q_OBJECT
+
+public:
+    ArchiveBundleFactory();
+
+    virtual QString Type() const;
+    virtual QStringList TypeExtensions() const;
+    virtual AssetBundlePtr CreateEmptyAssetBundle(AssetAPI *owner, const QString &name);
+
+private:
+    QStringList typesExtensions_;
+};

src/Application/ArchivePlugin/ArchivePlugin.cpp

@@ -0,0 +1,16 @@
+// For conditions of distribution and use, see copyright notice in LICENSE
+
+#include "ArchiveBundleFactory.h"
+
+#include "Framework.h"
+#include "CoreDefines.h"
+#include "AssetAPI.h"
+
+extern "C"
+{
+    DLLEXPORT void TundraPluginMain(Framework *framework)
+    {
+        Framework::SetInstance(framework);
+        framework->Asset()->RegisterAssetBundleTypeFactory(AssetBundleTypeFactoryPtr(new ArchiveBundleFactory()));
+    }
+}

src/Application/ArchivePlugin/CMakeLists.txt

@@ -0,0 +1,48 @@
+
+macro (configure_zziplib)
+    if ("${ZZIPLIB_ROOT}" STREQUAL "")
+        file (TO_CMAKE_PATH "$ENV{ZZIPLIB_ROOT}" ZZIPLIB_ROOT)
+    endif()
+    if ("${ZZIPLIB_ROOT}" STREQUAL "")
+        SET(ZZIPLIB_ROOT ${ENV_TUNDRA_DEP_PATH}/zziplib)
+    endif()
+
+    message("${ZZIPLIB_ROOT}")
+    sagase_configure_package(ZZIPLIB
+        NAMES zziplib zzip # zzip find headers. Important in NAMES so it will pick up /include instead of /include/zzip
+        COMPONENTS zziplib zzip
+        PREFIXES ${ZZIPLIB_ROOT} ${ENV_TUNDRA_DEP_PATH})
+
+    sagase_configure_report (ZZIPLIB)
+endmacro (configure_zziplib)
+
+configure_zziplib ()
+
+#################################################################################
+
+# Define target name and output directory
+init_target (ArchivePlugin OUTPUT plugins)
+
+# Define source files
+file (GLOB CPP_FILES *.cpp)
+file (GLOB H_FILES *.h)
+
+MocFolder ()
+UiFolder ()
+
+QT4_WRAP_CPP (MOC_SRCS ${H_FILES})
+
+set (SOURCE_FILES ${CPP_FILES} ${H_FILES} ${MOC_SRCS})
+
+use_package (ZZIPLIB)
+use_core_modules (Framework Asset)
+
+build_library (${TARGET_NAME} SHARED ${SOURCE_FILES})
+
+link_modules (Framework Asset)
+link_package (ZZIPLIB)
+
+SetupCompileFlags()
+
+final_target ()
+

src/Application/ArchivePlugin/ZipAssetBundle.cpp

@@ -0,0 +1,193 @@
+// For conditions of distribution and use, see copyright notice in LICENSE
+
+#include "ZipAssetBundle.h"
+#include "ZipHelpers.h"
+
+#include "CoreDefines.h"
+#include "AssetCache.h"
+#include "LoggingFunctions.h"
+
+#include "zzip/zzip.h"
+#include <QDir>
+#include <QDateTime>
+
+ZipAssetBundle::ZipAssetBundle(AssetAPI *owner, const QString &type, const QString &name) :
+    IAssetBundle(owner, type, name),
+    archive_(0),
+    worker_(0)
+{
+}
+
+ZipAssetBundle::~ZipAssetBundle()
+{
+    Unload();
+}
+
+void ZipAssetBundle::DoUnload()
+{
+    Close();
+    CloseWorker();
+}
+
+bool ZipAssetBundle::DeserializeFromDiskSource()
+{
+    if (DiskSource().isEmpty())
+    {
+        LogError("ZipAssetBundle::DeserializeFromDiskSource: Cannot process archive, no disk source for " + Name());
+        return false;
+    }
+
+    /* We want to detect if the extracted files are already up to date to save time.
+       If the last modified date for the sub asset is the same as the parent zip file, 
+       we don't extract it. If the zip is re-downloaded from source everything will get unpacked even
+       if only one file would have changed inside it. We could do uncompressed size comparisons
+       but that is not a absolute guarantee that the file has not changed. We'll be on the safe side
+       to unpack the whole zip file. Zip files are meant for deploying the scene and should be touched
+       rather rarely. Note that local:// refs are unpacked to cache but the zips disk source is not in the
+       cache. Meaning that local:// zip files will always be extracted fully even if the disk source
+       was not changed, we don't have a mechanism to get the last modified date properly except from
+       the asset cache. For local scenes this should be fine as there is no real need to
+       zip the scene up as you already have the disk sources right there in the storage.
+       The last modified query will fail if the file is open with zziplib, do it first. */
+    QDateTime zipLastModified = assetAPI_->GetAssetCache()->LastModified(Name());
+
+    zzip_error_t error = ZZIP_NO_ERROR;
+    archive_ = zzip_dir_open(QDir::fromNativeSeparators(DiskSource()).toStdString().c_str(), &error);
+    if (CheckAndLogZzipError(error) || CheckAndLogArchiveError(archive_) || !archive_)
+    {
+        archive_ = 0;
+        return false;
+    }
+
+    ZZIP_DIRENT archiveEntry;
+    while(zzip_dir_read(archive_, &archiveEntry))
+    {
+        QString relativePath = QDir::fromNativeSeparators(archiveEntry.d_name);
+        if (!relativePath.endsWith("/"))
+        {
+            QString subAssetRef = GetFullAssetReference(relativePath);
+
+            ZipArchiveFile file;
+            file.relativePath = relativePath;
+            file.cachePath = assetAPI_->GetAssetCache()->GetDiskSourceByRef(subAssetRef);
+            file.lastModified = assetAPI_->GetAssetCache()->LastModified(subAssetRef);
+            file.compressedSize = archiveEntry.d_csize;
+            file.uncompressedSize = archiveEntry.st_size;
+
+            /* Mark this file for extraction. If both cache files have valid dates
+               and they differ extract. If they have the same date stamp skip extraction.
+               Note that file.lastModified will be non-valid for non cached files so we 
+               will cover also missing files. */
+            file.doExtract = (zipLastModified.isValid() && file.lastModified.isValid()) ? (zipLastModified != file.lastModified) : true;
+
+            files_ << file;
+        }
+    }
+
+    // Close the zzip directory ptr
+    Close();
+
+    // If the zip file was empty we don't want IsLoaded to fail on the files_ check.
+    // The bundle loaded fine but there was no content, log a warning.
+    if (files_.isEmpty())
+    {
+        LogWarning("ZipAssetBundle: Bundle loaded but does not contain any files " + Name());
+        files_ << ZipArchiveFile();
+
+        // In this case there is no need to spin up the worker.
+        emit Loaded(this);
+        return true;
+    }
+
+    // Now that the file info has been read, continue in a worker thread.
+    LogDebug("ZipAssetBundle: File information read for " + Name() + ". File count: " + QString::number(files_.size()) + ". Starting worker thread.");
+
+    CloseWorker();
+    worker_ = new ZipWorker(DiskSource(), files_);
+    worker_->moveToThread(worker_);
+
+    connect(worker_, SIGNAL(AsynchLoadCompleted(bool)), this, SLOT(OnAsynchLoadCompleted(bool)), Qt::QueuedConnection);
+    worker_->start(QThread::HighPriority);
+
+    return true;
+}
+
+bool ZipAssetBundle::DeserializeFromData(const u8 *data, size_t numBytes)
+{
+    /** @note At this point it seems zzip needs a disk source to do processing
+        so we require disk source for the archive. This might change in the future by changing the lib. */
+    return false;
+}
+
+std::vector<u8> ZipAssetBundle::GetSubAssetData(const QString &subAssetName)
+{
+    /* Makes no sense to keep the whole zip file contents in memory as only
+       few files could be wanted from a 100mb bundle. Additionally all asset would take 2x the memory.
+       We could make this function also open the zip file and uncompress the data for every sub asset request. 
+       But that would be rather pointless, not to mention slower, as we already have the unpacked individual 
+       assets on disk. If the unpacking to disk changes we might need to rethink this. */
+
+    QString filePath = GetSubAssetDiskSource(subAssetName);
+    if (filePath.isEmpty())
+        return std::vector<u8>();
+
+    std::vector<u8> data;
+    return LoadFileToVector(filePath, data) ? data : std::vector<u8>();
+}
+
+QString ZipAssetBundle::GetSubAssetDiskSource(const QString &subAssetName)
+{
+    return assetAPI_->GetAssetCache()->FindInCache(GetFullAssetReference(subAssetName));
+}
+
+QString ZipAssetBundle::GetFullAssetReference(const QString &subAssetName)
+{
+    return Name() + "#" + subAssetName;
+}
+
+bool ZipAssetBundle::IsLoaded() const
+{
+    return (archive_ != 0 || !files_.isEmpty());
+}
+
+void ZipAssetBundle::OnAsynchLoadCompleted(bool successful)
+{
+    CloseWorker();
+
+    // Write new timestamps for extracted files. Cannot be done (?!) in the worker
+    // thread as it would need to access Framework, AssetAPI and AssetCache ptrs 
+    // and they might not be safe to access from outside the main thread.
+    QDateTime zipLastModified = assetAPI_->GetAssetCache()->LastModified(Name());
+    if (zipLastModified.isValid())
+    {
+        foreach(ZipArchiveFile file, files_)
+            if (file.doExtract)
+                assetAPI_->GetAssetCache()->SetLastModified(GetFullAssetReference(file.relativePath), zipLastModified);
+    }
+
+    LogDebug("ZipAssetBundle: Zip file extracted " + Name());
+
+    if (successful)
+        emit Loaded(this);
+    else
+        emit Failed(this);
+}
+
+void ZipAssetBundle::Close()
+{
+    if (archive_)
+    {
+        zzip_dir_close(archive_);
+        archive_ = 0;
+    }
+}
+
+void ZipAssetBundle::CloseWorker()
+{
+    if (worker_ && worker_->isRunning())
+    {
+        worker_->exit();
+        worker_->wait();
+    }
+    SAFE_DELETE(worker_);
+}