QResource: add support for resources compressed with Zstandard

See next commit for details on why this is a good idea.

[ChangeLog][Important Behavior Changes] The Qt resource system now
supports compressing content using the Zstandard (https://zstd.net)
algorithm. Compared to zlib, it compresses better for the same CPU time,
so this algorithm is the default. QResource::isCompressed() returns true
for either compression algorithm. Use QResource::compressionAlgorithm()
to find out which algorithm to decompress. QFile will automatically
decompress using the correct algorithm.

Change-Id: I343f2beed55440a7ac0bfffd1562e9a8f94933a7
Reviewed-by: Oswald Buddenhagen <oswald.buddenhagen@qt.io>
Reviewed-by: Samuel Gaist <samuel.gaist@idiap.ch>
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

configure.json

@@ -136,7 +136,8 @@
             "Werror": { "type": "boolean", "name": "warnings_are_errors" },
             "widgets": "boolean",
             "xplatform": "string",
-            "zlib": { "type": "enum", "name": "system-zlib", "values": { "system": "yes", "qt": "no" } }
+            "zlib": { "type": "enum", "name": "system-zlib", "values": { "system": "yes", "qt": "no" } },
+            "zstd": "boolean"
         },
         "prefix": {
             "D": "defines",
@@ -167,6 +168,21 @@
                 { "libs": "-s USE_ZLIB=1", "condition": "config.wasm" }
             ]
         },
+        "zstd": {
+            "label": "Zstandard",
+            "test": {
+                "include": "zstd.h",
+                "main": [
+                    "(void) ZSTD_compress(NULL, 0, NULL, 0, 1);",
+                    "unsigned long long n = ZSTD_getFrameContentSize(NULL, 0);",
+                    "(void) ZSTD_decompress(NULL, 0, NULL, n);"
+                ]
+            },
+            "sources": [
+                { "type": "pkgConfig", "args": "libzstd >= 1.3" },
+                "-lzstd"
+            ]
+        },
         "dbus": {
             "label": "D-Bus >= 1.2",
             "test": {
@@ -1126,6 +1142,11 @@
             "condition": "libs.zlib",
             "output": [ "privateFeature" ]
         },
+        "zstd": {
+            "label": "Zstandard support",
+            "condition": "libs.zstd",
+            "output": [ "privateFeature" ]
+        },
         "thread": {
             "label": "Thread support",
             "purpose": "Provides QThread and related classes.",
@@ -1452,7 +1473,8 @@ Configure with '-qreal float' to create a build that is binary-compatible with 5
             "entries": [
                 "pkg-config",
                 "libudev",
-                "system-zlib"
+                "system-zlib",
+                "zstd"
             ]
         }
     ]

src/corelib/global/qconfig-bootstrapped.h

@@ -121,6 +121,7 @@
 #define QT_FEATURE_topleveldomain -1
 #define QT_NO_TRANSLATION
 #define QT_FEATURE_translation -1
+#define QT_FEATURE_zstd -1
 
 #ifdef QT_BUILD_QMAKE
 #define QT_FEATURE_commandlineparser -1

src/corelib/io/io.pri

@@ -79,6 +79,8 @@ SOURCES += \
         io/qloggingcategory.cpp \
         io/qloggingregistry.cpp
 
+qtConfig(zstd): QMAKE_USE_PRIVATE += zstd
+
 qtConfig(filesystemwatcher) {
     HEADERS += \
         io/qfilesystemwatcher.h \

src/corelib/io/qresource.cpp

@@ -1,6 +1,7 @@
 /****************************************************************************
 **
-** Copyright (C) 2016 The Qt Company Ltd.
+** Copyright (C) 2018 The Qt Company Ltd.
+** Copyright (C) 2018 Intel Corporation.
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of the QtCore module of the Qt Toolkit.
@@ -56,8 +57,13 @@
 #include "private/qabstractfileengine_p.h"
 #include "private/qnumeric_p.h"
 #include "private/qsimd_p.h"
+#include "private/qtools_p.h"
 #include "private/qsystemerror_p.h"
 
+#if QT_CONFIG(zstd)
+#  include <zstd.h>
+#endif
+
 #ifdef Q_OS_UNIX
 # include "private/qcore_unix_p.h"
 #endif
@@ -100,8 +106,10 @@ class QResourceRoot
 {
     enum Flags
     {
+        // must match rcc.h
         Compressed = 0x01,
-        Directory = 0x02
+        Directory = 0x02,
+        CompressedZstd = 0x04
     };
     const uchar *tree, *names, *payloads;
     int version;
@@ -117,7 +125,15 @@ class QResourceRoot
     virtual ~QResourceRoot() { }
     int findNode(const QString &path, const QLocale &locale=QLocale()) const;
     inline bool isContainer(int node) const { return flags(node) & Directory; }
-    inline bool isCompressed(int node) const { return flags(node) & Compressed; }
+    QResource::Compression compressionAlgo(int node)
+    {
+        uint compressionFlags = flags(node) & (Compressed | CompressedZstd);
+        if (compressionFlags == Compressed)
+            return QResource::ZlibCompression;
+        if (compressionFlags == CompressedZstd)
+            return QResource::ZstdCompression;
+        return QResource::NoCompression;
+    }
     const uchar *data(int node, qint64 *size) const;
     quint64 lastModified(int node) const;
     QStringList children(int node) const;
@@ -229,6 +245,23 @@ static inline QStringList *resourceSearchPaths()
     \sa {The Qt Resource System}, QFile, QDir, QFileInfo
 */
 
+/*!
+    \enum QResource::Compression
+    \since 5.13
+
+    This enum is used by compressionAlgorithm() to indicate which algorithm the
+    RCC tool used to compress the payload.
+
+    \value NoCompression    Contents are not compressed (isCompressed() is false).
+    \value ZlibCompression  Contents are compressed using \l{zlib}{https://zlib.net} and can
+                            be decompressed using the qUncompress() function.
+    \value ZstdCompression  Contents are compressed using \l{zstd}{https://zstd.net}. To
+                            decompress, use the \c{ZSTD_decompress} function from the zstd
+                            library.
+
+    \sa compressionAlgorithm(), isCopressed()
+*/
+
 class QResourcePrivate {
 public:
     inline QResourcePrivate(QResource *_q) : q_ptr(_q) { clear(); }
@@ -243,12 +276,13 @@ class QResourcePrivate {
     QLocale locale;
     QString fileName, absoluteFilePath;
     QList<QResourceRoot*> related;
-    uint container : 1;
-    mutable uint compressed : 1;
     mutable qint64 size;
+    mutable quint64 lastModified;
     mutable const uchar *data;
     mutable QStringList children;
-    mutable quint64 lastModified;
+    mutable quint8 compressionAlgo;
+    bool container;
+    /* 2 or 6 padding bytes */
 
     QResource *q_ptr;
     Q_DECLARE_PUBLIC(QResource)
@@ -258,7 +292,7 @@ void
 QResourcePrivate::clear()
 {
     absoluteFilePath.clear();
-    compressed = 0;
+    compressionAlgo = QResource::NoCompression;
     data = 0;
     size = 0;
     children.clear();
@@ -287,11 +321,11 @@ QResourcePrivate::load(const QString &file)
                 container = res->isContainer(node);
                 if(!container) {
                     data = res->data(node, &size);
-                    compressed = res->isCompressed(node);
+                    compressionAlgo = res->compressionAlgo(node);
                 } else {
-                    data = 0;
+                    data = nullptr;
                     size = 0;
-                    compressed = 0;
+                    compressionAlgo = QResource::NoCompression;
                 }
                 lastModified = res->lastModified(node);
             } else if(res->isContainer(node) != container) {
@@ -301,9 +335,9 @@ QResourcePrivate::load(const QString &file)
             related.append(res);
         } else if(res->mappingRootSubdir(file)) {
             container = true;
-            data = 0;
+            data = nullptr;
             size = 0;
-            compressed = 0;
+            compressionAlgo = QResource::NoCompression;
             lastModified = 0;
             res->ref.ref();
             related.append(res);
@@ -493,16 +527,41 @@ bool QResource::isValid() const
 
 /*!
     Returns \c true if the resource represents a file and the data backing it
-    is in a compressed format, false otherwise.
+    is in a compressed format, false otherwise. If the data is compressed,
+    check compressionAlgorithm() to verify what algorithm to use to decompress
+    the data.
 
-    \sa data(), isFile()
+    \sa data(), compressionType(), isFile()
 */
 
 bool QResource::isCompressed() const
+{
+    return compressionAlgorithm() != NoCompression;
+}
+
+/*!
+    \since 5.13
+
+    Returns the compression type that this resource is compressed with, if any.
+    If it is not compressed, this function returns QResource::NoCompression.
+
+    If this function returns QResource::ZlibCompression, you may decompress the
+    data using the qUncompress() function. Up until Qt 5.13, this was the only
+    possible compression algorithm.
+
+    If this function returns QResource::ZstdCompression, you need to use the
+    Zstandard library functios (\c{<zstd.h> header). Qt does not provide a
+    wrapper.
+
+    See \l{http://facebook.github.io/zstd/zstd_manual.html}{Zstandard manual}.
+
+    \sa isCompressed(), data(), isFile()
+*/
+QResource::Compression QResource::compressionAlgorithm() const
 {
     Q_D(const QResource);
     d->ensureInitialized();
-    return d->compressed;
+    return Compression(d->compressionAlgo);
 }
 
 /*!
@@ -1531,12 +1590,40 @@ bool QResourceFileEnginePrivate::unmap(uchar *ptr)
 
 void QResourceFileEnginePrivate::uncompress() const
 {
-    if (resource.isCompressed() && uncompressed.isEmpty() && resource.size()) {
+    if (uncompressed.isEmpty() && resource.size()) {
+        quint64 size;
+        switch (resource.compressionAlgorithm()) {
+        case QResource::NoCompression:
+            return;     // nothing to do
+
+        case QResource::ZlibCompression:
 #ifndef QT_NO_COMPRESS
-        uncompressed = qUncompress(resource.data(), resource.size());
+            uncompressed = qUncompress(resource.data(), resource.size());
+#else
+            Q_ASSERT(!"QResourceFileEngine::open: Qt built without support for Zlib compression");
+#endif
+            break;
+
+        case QResource::ZstdCompression:
+#if QT_CONFIG(zstd)
+            size = ZSTD_getFrameContentSize(resource.data(), resource.size());
+            if (!ZSTD_isError(size)) {
+                if (size >= MaxAllocSize) {
+                    qWarning("QResourceFileEngine::open: content bigger than memory (size %lld)", size);
+                } else {
+                    uncompressed = QByteArray(size, Qt::Uninitialized);
+                    size = ZSTD_decompress(const_cast<char *>(uncompressed.data()), size,
+                                           resource.data(), resource.size());
+                }
+            }
+            if (ZSTD_isError(size))
+                qWarning("QResourceFileEngine::open: error decoding: %s", ZSTD_getErrorName(size));
 #else
-        Q_ASSERT(!"QResourceFileEngine::open: Qt built without support for compression");
+            Q_UNUSED(size);
+            Q_ASSERT(!"QResourceFileEngine::open: Qt built without support for Zstd compression");
 #endif
+            break;
+        }
     }
 }
 

src/corelib/io/qresource.h

@@ -54,6 +54,12 @@ class QResourcePrivate;
 class Q_CORE_EXPORT QResource
 {
 public:
+    enum Compression {
+        NoCompression,
+        ZlibCompression,
+        ZstdCompression
+    };
+
     QResource(const QString &file=QString(), const QLocale &locale=QLocale());
     ~QResource();
 
@@ -67,6 +73,7 @@ class Q_CORE_EXPORT QResource
     bool isValid() const;
 
     bool isCompressed() const;
+    Compression compressionAlgorithm() const;
     qint64 size() const;
     const uchar *data() const;
     QDateTime lastModified() const;

src/tools/rcc/rcc.cpp

@@ -94,6 +94,7 @@ class RCCFileInfo
 public:
     enum Flags
     {
+        // must match qresource.cpp
         NoFlags = 0x00,
         Compressed = 0x01,
         Directory = 0x02