Optimize|FS|libcore: Memory optimizations in the file system

ArchiveFeed was caching all uncompressed entries in memory. This is
problematic in the case where archives contain other archives,
because that means the sub-archives were kept uncompressed in memory
even when nobody needed them (populating the file system requires
reading the sub-archives’ directories, too).

Added a new uncaching feature that allows releasing all these cached
entries if they are unmodified.

Archive also has a method for manually caching or uncaching the
archive contents.

doomsday/sdk/libcore/include/de/data/archive.h

@@ -111,20 +111,28 @@ class DENG2_PUBLIC Archive : public IWritable
      */
     IByteArray const *source() const;
 
-    enum CacheAttachment {
-        RemainAttachedToSource = 0,
-        DetachFromSource = 1
+    enum CacheOperation {
+        CacheAndRemainAttachedToSource = 0,
+        CacheAndDetachFromSource = 1,
+        UncacheUnmodifiedEntries = 2,
     };
 
     /**
-     * Loads a copy of the serialized data into memory for all the entries that
-     * don't already have deserialized data stored.
+     * Caches or uncaches archive entries.
      *
-     * @param attach  If DetachFromSource, the archive becomes a standalone
-     *                archive that no longer needs the source byte array to
-     *                remain in existence.
+     * The archive must have a source for any of the caching operations to be possible.
+     *
+     * @param operation  One of the following:
+     *      - CacheAndRemainAttachedToSource: Loads a copy of the serialized data into
+     *          memory for all the entries that don't already have deserialized data stored.
+     *      - CacheAndDetachFromSource: Loads a copy of all the serialized data into memory,
+     *          and the archive becomes a standalone archive that no longer needs the
+     *          source byte array to remain in existence.
+     *      - UncacheUnmodifiedEntries: All entries that have not been modified will be
+     *          released from memory. Accessing them in the future will require reloading
+     *          the entry contents into memory.
      */
-    void cache(CacheAttachment attach = DetachFromSource);
+    void cache(CacheOperation operation = CacheAndDetachFromSource);
 
     /**
      * Determines whether the archive contains an entry (not a folder).
@@ -201,6 +209,12 @@ class DENG2_PUBLIC Archive : public IWritable
      */
     Block &entryBlock(Path const &path);
 
+    /**
+     * Release all cached data of a block. Unmodified blocks cannot be uncached.
+     * The archive must have a source for uncaching to be possible.
+     */
+    void uncacheBlock(Path const &path) const;
+
     /**
      * Adds an entry to the archive. The entry will not be committed to the
      * source, but instead remains as-is in memory.
@@ -256,11 +270,11 @@ class DENG2_PUBLIC Archive : public IWritable
         Time modifiedAt;        ///< Latest modification timestamp.
         bool maybeChanged;      ///< @c true, if the data must be re-serialized when writing.
 
-        /// Deserialized data. Can be @c NULL. Entry has ownership.
-        Block *data;
+        /// Deserialized data. Can be @c nullptr. Entry has ownership.
+        std::unique_ptr<Block> data;
 
-        /// Cached copy of the serialized data. Can be @c NULL. Entry has ownership.
-        Block mutable *dataInArchive;
+        /// Cached copy of the serialized data. Can be @c nullptr. Entry has ownership.
+        mutable std::unique_ptr<Block> dataInArchive;
 
         Entry(PathTree::NodeArgs const &args);
         virtual ~Entry();

doomsday/sdk/libcore/include/de/filesys/archiveentryfile.h

@@ -71,6 +71,8 @@ class ArchiveEntryFile : public ByteArrayFile
     /// Returns the archive of the file (non-modifiable).
     Archive const &archive() const;
 
+    void uncache() const;
+
     // Implements IByteArray.
     Size size() const;
     void get(Offset at, Byte *values, Size count) const;

doomsday/sdk/libcore/include/de/filesys/archivefeed.h

@@ -98,6 +98,15 @@ class DENG2_PUBLIC ArchiveFeed : public Feed
      */
     void rewriteFile();
 
+    void uncache();
+
+    /**
+     * Uncaches all unmodified indexed archive entries from memory. Doing this at
+     * specific suitable points in the application's lifetime is good so that unnecessary
+     * data is not kept cached.
+     */
+    static void uncacheAllEntries(StringList const &folderTypes);
+
 private:
     DENG2_PRIVATE(d)
 };

doomsday/sdk/libcore/include/de/filesys/archivefolder.h

@@ -13,7 +13,7 @@
  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
  * General Public License for more details. You should have received a copy of
  * the GNU Lesser General Public License along with this program; if not, see:
- * http://www.gnu.org/licenses</small> 
+ * http://www.gnu.org/licenses</small>
  */
 
 #ifndef LIBDENG2_ARCHIVEFOLDER_H

doomsday/sdk/libcore/src/data/archive.cpp

@@ -78,7 +78,7 @@ IByteArray const *Archive::source() const
     return d->source;
 }
 
-void Archive::cache(CacheAttachment attach)
+void Archive::cache(CacheOperation operation)
 {
     if (!d->source)
     {
@@ -89,14 +89,28 @@ void Archive::cache(CacheAttachment attach)
     while (iter.hasNext())
     {
         Entry &entry = static_cast<Entry &>(iter.next());
-        if (!entry.data && !entry.dataInArchive)
+        switch (operation)
         {
-            entry.dataInArchive = new Block(*d->source, entry.offset, entry.sizeInArchive);
+        case CacheAndDetachFromSource:
+        case CacheAndRemainAttachedToSource:
+            if (!entry.data && !entry.dataInArchive)
+            {
+                entry.dataInArchive.reset(new Block(*d->source, entry.offset, entry.sizeInArchive));
+            }
+            break;
+
+        case UncacheUnmodifiedEntries:
+            if (!entry.maybeChanged)
+            {
+                entry.data.reset();
+                entry.dataInArchive.reset();
+            }
+            break;
         }
     }
-    if (attach == DetachFromSource)
+    if (operation == CacheAndDetachFromSource)
     {
-        d->source = 0;
+        d->source = nullptr;
     }
 }
 
@@ -159,21 +173,20 @@ Block const &Archive::entryBlock(Path const &path) const
 {
     DENG2_ASSERT(d->index != 0);
 
-    try
+    // The entry contents will be cached in memory.
+    if (Entry *entry = static_cast<Entry *>(d->index->tryFind(path, PathTree::MatchFull | PathTree::NoBranch)))
     {
-        // We'll need to modify the entry.
-        Entry &entry = static_cast<Entry &>(d->index->find(path, PathTree::MatchFull | PathTree::NoBranch));
-        if (entry.data)
+        if (entry->data)
         {
-            // Got it.
-            return *entry.data;
+            // Already got it.
+            return *entry->data;
         }
         std::unique_ptr<Block> cached(new Block);
         d->readEntry(path, *cached.get());
-        entry.data = cached.release();
-        return *entry.data;
+        entry->data.reset(cached.release());
+        return *entry->data;
     }
-    catch (PathTree::NotFoundError const &)
+    else
     {
         /// @throw NotFoundError Entry with @a path was not found.
         throw NotFoundError("Archive::entryBlock", String("'%1' not found").arg(path));
@@ -199,6 +212,29 @@ Block &Archive::entryBlock(Path const &path)
     return const_cast<Block &>(block);
 }
 
+void Archive::uncacheBlock(Path const &path) const
+{
+    if (!d->source) return; // Wouldn't be able to re-cache the data.
+
+    if (Entry *entry = static_cast<Entry *>(d->index->tryFind(path, PathTree::MatchFull | PathTree::NoBranch)))
+    {
+        if (!entry->data && !entry->dataInArchive) return;
+
+        qDebug() << "Archive:" << path << "uncached by archive" << this;
+
+        if (!entry->maybeChanged)
+        {
+            entry->data.reset();
+        }
+        entry->dataInArchive.reset();
+    }
+    else
+    {
+        /// @throw NotFoundError Entry with @a path was not found.
+        throw NotFoundError("Archive::uncacheBlock", String("'%1' not found").arg(path));
+    }
+}
+
 void Archive::add(Path const &path, IByteArray const &data)
 {
     if (path.isEmpty())
@@ -214,7 +250,7 @@ void Archive::add(Path const &path, IByteArray const &data)
     DENG2_ASSERT(d->index != 0);
 
     Entry &entry = static_cast<Entry &>(d->index->insert(path));
-    entry.data         = new Block(data);
+    entry.data.reset(new Block(data));
     entry.modifiedAt   = Time::currentHighPerformanceTime();
     entry.maybeChanged = true;
 
@@ -276,15 +312,9 @@ Archive::Entry::Entry(PathTree::NodeArgs const &args)
     , sizeInArchive(0)
     , modifiedAt(Time::invalidTime())
     , maybeChanged(false)
-    , data(0)
-    , dataInArchive(0)
 {}
 
 Archive::Entry::~Entry()
-{
-    // Entry has ownership of the cached data.
-    delete data;
-    delete dataInArchive;
-}
+{}
 
 } // namespace de

doomsday/sdk/libcore/src/data/path.cpp

@@ -29,7 +29,7 @@
 namespace de {
 
 /// Size of the fixed-size portion of the segment buffer.
-static int const SEGMENT_BUFFER_SIZE = 24;
+static int const SEGMENT_BUFFER_SIZE = 8;
 
 static String emptyPath;
 
@@ -90,6 +90,8 @@ String Path::Segment::toString() const
     return range.string()->mid(range.position(), range.size());
 }
 
+//---------------------------------------------------------------------------------------
+
 struct Path::Impl
 {
     DENG2_NO_ASSIGN(Impl)
@@ -127,7 +129,7 @@ struct Path::Impl
      * List of the extra segments that don't fit in segments, in reverse
      * order.
      */
-    QList<Path::Segment *> extraSegments;
+    QList<Path::Segment> extraSegments;
 
     Impl() : separator('/'), segmentCount(0)
     {}
@@ -147,10 +149,7 @@ struct Path::Impl
      */
     void clearSegments()
     {
-        while (!extraSegments.isEmpty())
-        {
-            delete extraSegments.takeFirst();
-        }
+        extraSegments.clear();
         zap(segments);
         segmentCount = 0;
     }
@@ -171,8 +170,8 @@ struct Path::Impl
         else
         {
             // Allocate an "extra" node.
-            segment = new Path::Segment;
-            extraSegments.append(segment);
+            extraSegments.append(Path::Segment());
+            segment = &extraSegments.last();
         }
 
         zapPtr(segment);
@@ -319,7 +318,7 @@ Path::Segment const &Path::reverseSegment(int reverseIndex) const
     }
 
     // No - an extra segment.
-    return *d->extraSegments[reverseIndex - SEGMENT_BUFFER_SIZE];
+    return d->extraSegments[reverseIndex - SEGMENT_BUFFER_SIZE];
 }
 
 Path Path::subPath(Rangei const &range) const

doomsday/sdk/libcore/src/data/pointerset.cpp

@@ -108,6 +108,7 @@ void PointerSet::insert(Pointer ptr)
         if (_range.size() == _size)
         {
             DENG2_ASSERT(_size < POINTERSET_MAX_SIZE);
+            if (_size == POINTERSET_MAX_SIZE) return; // Can't do it.
 
             Pointer *oldBase = _pointers;
             duint const oldSize = _size;

doomsday/sdk/libcore/src/data/ziparchive.cpp

@@ -407,7 +407,7 @@ void ZipArchive::readFromSource(Entry const &e, Path const &, IBlock &uncompress
         if (!entry.dataInArchive)
         {
             DENG2_ASSERT(source() != NULL);
-            entry.dataInArchive = new Block(*source(), entry.offset, entry.sizeInArchive);
+            entry.dataInArchive.reset(new Block(*source(), entry.offset, entry.sizeInArchive));
         }
 
         z_stream stream;
@@ -453,6 +453,7 @@ void ZipArchive::readFromSource(Entry const &e, Path const &, IBlock &uncompress
 
         // We're done.
         inflateEnd(&stream);
+        entry.dataInArchive.reset(); // Now have the decompressed version.
     }
 }
 

doomsday/sdk/libcore/src/filesys/archiveentryfile.cpp

@@ -111,11 +111,22 @@ Archive const &ArchiveEntryFile::archive() const
     return *d->archive;
 }
 
+void ArchiveEntryFile::uncache() const
+{
+    DENG2_GUARD(this);
+
+    if (d->readBlock)
+    {
+        archive().uncacheBlock(d->entryPath);
+        d->readBlock = nullptr;
+    }
+}
+
 IByteArray::Size ArchiveEntryFile::size() const
 {
     DENG2_GUARD(this);
 
-    return d->entryData().size();
+    return archive().entryStatus(d->entryPath).size;
 }
 
 void ArchiveEntryFile::get(Offset at, Byte *values, Size count) const