Initial commit of new HTTP and Websocket code

Note: the following functionality is incomplete and/or missing:-
 - authentication (easy enough)
 - WSDL (started - but will take time a little effort to complete)
 - actual handling of websocket functionality (i.e. remote procedure
calls)
 - no existing service APIs are converted in this commit - and each will
be require an update
 - no conversion of UPnP end points

- the principle aims here are to remove the dependency on QScript (which
is end of life), invert the dependency between HTTP and UPnP (i.e. UPnP
should require HTTP and not the other way around), extend functionality,
improve performance and memory footprint and ease maintenance.
- this is entirely standalone and can be run alongside the existing
server implementation (port settings are currently ignored and the
server will listen on 8081 and 8091 (SSL))
- the intention is that the new code behaves in exactly the same way as
the old code - with the exception of extending functionality.
- websocket handling is currently embedded and is fully compliant with
the autobahn test suite
- the server code is entirely lock free, event driven and thread safe -
but there will be instances where services require locking to process
requests

mythtv/libs/libmythbase/http/mythhttpcache.cpp

@@ -0,0 +1,192 @@
+// Qt
+#include <QCryptographicHash>
+
+// MythTV
+#include "mythlogging.h"
+#include "mythdate.h"
+#include "http/mythhttpdata.h"
+#include "http/mythhttpfile.h"
+#include "http/mythhttpcache.h"
+
+/*! \brief Process precondition checks
+ *
+ * This should be the first step in processing an HTTP request, typically only
+ * when there is meaningful content to be returned (i.e. it is not used for error
+ * responses, even though they may contain content).
+ *
+ * The status of the response may be updated and the ETag for the content may
+ * be set.
+ *
+ * 'Last-Modified' and 'ETag' outgoing headers are supported, dependant on the cache
+ * setting for the content. This defaults to HTTPIgnoreCache for errors etc,
+ * HTTPLastModified for files and HTTPNoCache for in-memory data responses. It
+ * is expected that the latter will be configured as required by the handler's
+ * implementation.
+ *
+ * 'If-Modified-Since', 'If-None-Match' and 'If-Range' headers are supported in
+ * incoming headers (i.e. last modified and ETag validators).
+ *
+ * There is currently no suport for 'If-Match' and 'If-Unmodified-Since' headers.
+ *
+ * We only ever use one validator and it is assumed the validator is used consistently.
+ * i.e. we use ETag or Last-Modified across invocations and not both. This ensures
+ * the client sends the correct validator in response.
+ *
+ * \note Last-Modified is used for files as they have valid last modified information, last
+ * modified data gives reasonable validation (see below) and hashing the file contents
+ * (or in memory data for compressed content) for the ETag is potentially expensive.
+ *
+ * \note Last-Modified is considered a weak validator as it only has second accuracy.
+ * This should however be sufficient for our purposes when sending files. Likewise
+ * our hashing for file responses when using ETag should also be considered weak
+ * (as it does not guarantee byte-for-byte accuracy). Again however it should
+ * be considered accurate enough for our needs - and more accurate than Last-Modified
+ * as we use millisecond accuracy for our hash generation.
+ *
+ * \note It is recommended that Etags vary depending on the content encoding (e.g. gzipped).
+ * We do not currently support this as our decision to compress depends on content
+ * size, type, and the client range and encoding requests. It should not however
+ * be an issue (!)
+ */
+void MythHTTPCache::PreConditionCheck(HTTPResponse Response)
+{
+    // Retrieve content
+    auto data = std::get_if<HTTPData>(&Response->m_response);
+    auto file = std::get_if<HTTPFile>(&Response->m_response);
+    if (!(file || data))
+        return;
+
+    int cache = data ? (*data)->m_cacheType : (*file)->m_cacheType;
+
+    // Explicitly ignore caching (error responses etc)
+    if (cache == HTTPIgnoreCache)
+        return;
+
+    // Explicitly request no caching
+    if ((cache & HTTPNoCache) == HTTPNoCache)
+        return;
+
+    // We use the last modified data for both the last modified header and potentially
+    // hashing for the etag
+    auto & lastmodified = data ? (*data)->m_lastModified : (*file)->m_lastModified;
+    if (!lastmodified.isValid())
+        lastmodified = QDateTime::currentDateTime();
+
+    // If-Range precondition can contain either a last-modified or ETag validator
+    QString ifrange = MythHTTP::GetHeader(Response->m_requestHeaders, "if-range");
+    bool checkifrange = !ifrange.isEmpty();
+    bool removeranges = false;
+
+    if ((cache & HTTPETag) == HTTPETag)
+    {
+        QByteArray& etag = data ? (*data)->m_etag : (*file)->m_etag;
+        if (file)
+        {
+            QByteArray hashdata = ((*file)->fileName() + lastmodified.toString("ddMMyyyyhhmmsszzz")).toLocal8Bit().constData();
+            etag = QCryptographicHash::hash(hashdata, QCryptographicHash::Sha224).toHex();
+        }
+        else
+        {
+            etag = QCryptographicHash::hash((*data)->constData(), QCryptographicHash::Sha224).toHex();
+        }
+
+        // This assumes only one or other is present...
+        if (checkifrange)
+        {
+            removeranges = !ifrange.contains(etag);
+        }
+        else
+        {
+            QString nonematch = MythHTTP::GetHeader(Response->m_requestHeaders, "if-none-match");
+            if (!nonematch.isEmpty() && (nonematch.contains(etag)))
+                Response->m_status = HTTPNotModified;
+        }
+    }
+    else if (((cache & HTTPLastModified) == HTTPLastModified))
+    {
+        auto RemoveMilliseconds = [](QDateTime& DateTime)
+        {
+            auto residual = DateTime.time().msec();
+            DateTime = DateTime.addMSecs(-residual);
+        };
+
+        auto ParseModified = [](const QString& Modified)
+        {
+            QDateTime time = QDateTime::fromString(Modified, Qt::RFC2822Date);
+            time.setTimeSpec(Qt::OffsetFromUTC);
+            return time;
+        };
+
+        if (checkifrange)
+        {
+            auto time = ParseModified(ifrange);
+            if (time.isValid())
+            {
+                RemoveMilliseconds(lastmodified);
+                removeranges = lastmodified > time;
+            }
+        }
+        else if (Response->m_requestType == HTTPGet || Response->m_requestType == HTTPHead)
+        {
+            QString modified = MythHTTP::GetHeader(Response->m_requestHeaders, "if-modified-since");
+            if (!modified.isEmpty())
+            {
+                auto time = ParseModified(modified);
+                if (time.isValid())
+                {
+                    RemoveMilliseconds(lastmodified);
+                    if (lastmodified <= time)
+                        Response->m_status = HTTPNotModified;
+                }
+            }
+        }
+    }
+
+    // If the If-Range precondition fails, then we need to send back the complete
+    // contents i.e. ignore any ranges. So search for the range headers and remove them.
+    if (removeranges && Response->m_requestHeaders && Response->m_requestHeaders->contains("range"))
+        Response->m_requestHeaders->replace("range", "");
+}
+
+/*! \brief Add precondition (cache) headers to the response.
+ *
+ * Must be after a call to PreConditionCheck.
+*/
+void MythHTTPCache::PreConditionHeaders(HTTPResponse Response)
+{
+    // Retrieve content
+    auto data = std::get_if<HTTPData>(&Response->m_response);
+    auto file = std::get_if<HTTPFile>(&Response->m_response);
+    if (!(file || data))
+        return;
+
+    int cache = data ? (*data)->m_cacheType : (*file)->m_cacheType;
+
+    // Explicitly ignore caching (error responses etc)
+    if (cache == HTTPIgnoreCache)
+        return;
+
+    // Explicitly request no caching
+    if ((cache & HTTPNoCache) == HTTPNoCache)
+    {
+        Response->AddHeader("Cache-Control", "no-store, max-age=0");
+        return;
+    }
+
+    // Add the default cache control header
+    QString duration = ((cache & HTTPLongLife) == HTTPLongLife)   ? "31536000" : // 1 Year
+                       ((cache & HTTPLongLife) == HTTPMediumLife) ? "7200" :     // 2 Hours
+                                                                    "0";         // ??
+    Response->AddHeader("Cache-Control", "no-cache=\"Ext\",max-age=" + duration);
+
+    if ((cache & HTTPETag) == HTTPETag)
+    {
+        Response->AddHeader("ETag", "\"" + (data ? (*data)->m_etag : (*file)->m_etag) + "\"");
+    }
+    else if ((cache & HTTPLastModified) == HTTPLastModified)
+    {
+        auto & lastmodified = data ? (*data)->m_lastModified : (*file)->m_lastModified;
+        auto last = MythDate::toString(lastmodified, MythDate::kOverrideUTC | MythDate::kRFC822);
+        Response->AddHeader("Last-Modified", last);
+    }
+}

mythtv/libs/libmythbase/http/mythhttpcache.h

@@ -0,0 +1,14 @@
+#ifndef MYTHHTTPCACHE_H
+#define MYTHHTTPCACHE_H
+
+// MythTV
+#include "http/mythhttpresponse.h"
+
+class MythHTTPCache
+{
+  public:
+    static void PreConditionCheck   (HTTPResponse Response);
+    static void PreConditionHeaders (HTTPResponse Response);
+};
+
+#endif

mythtv/libs/libmythbase/http/mythhttpcommon.cpp

@@ -0,0 +1,78 @@
+// Qt
+#include <QStringList>
+
+// MythTV
+#include "http/mythhttpcommon.h"
+
+DataPayload MythSharedData::CreatePayload(size_t Size)
+{
+    return std::shared_ptr<MythSharedData>(new MythSharedData(Size));
+}
+
+DataPayload MythSharedData::CreatePayload(const QString& Text)
+{
+    return std::shared_ptr<MythSharedData>(new MythSharedData(Text));
+}
+
+DataPayload MythSharedData::CreatePayload(const QByteArray &Other)
+{
+    return std::shared_ptr<MythSharedData>(new MythSharedData(Other));
+}
+
+MythSharedData::MythSharedData(uint64_t Size)
+  : QByteArray(static_cast<int>(Size), Qt::Uninitialized)
+{
+}
+
+MythSharedData::MythSharedData(const QString& Text)
+  : QByteArray(Text.toUtf8())
+{
+}
+
+MythSharedData::MythSharedData(const QByteArray& Other)
+  : QByteArray(Other)
+{
+}
+
+StringPayload MythSharedString::CreateString()
+{
+    return std::shared_ptr<MythSharedString>(new MythSharedString());
+}
+
+QString MythHTTPWS::ProtocolToString(MythSocketProtocol Protocol)
+{
+    switch (Protocol)
+    {
+        case ProtFrame:    return "Default";
+        case ProtJSONRPC:  return JSONRPC;
+        case ProtXMLRPC:   return XMLRPC;
+        case ProtPLISTRPC: return PLISTRPC;
+        case ProtCBORRPC:  return CBORRPC;
+        default: break;
+    }
+    return "noprotocol";
+}
+
+MythSocketProtocol MythHTTPWS::ProtocolFromString(const QString &Protocols)
+{
+    auto ParseProtocol = [](const QString& Protocol)
+    {
+        if (Protocol.contains(JSONRPC))  return ProtJSONRPC;
+        if (Protocol.contains(XMLRPC))   return ProtXMLRPC;
+        if (Protocol.contains(PLISTRPC)) return ProtPLISTRPC;
+        if (Protocol.contains(CBORRPC))  return ProtCBORRPC;
+        return ProtFrame;
+    };
+
+#if QT_VERSION < QT_VERSION_CHECK(5,14,0)
+    auto protocols = Protocols.trimmed().toLower().split(",", QString::SkipEmptyParts);
+#else
+    auto protocols = Protocols.trimmed().toLower().split(",", Qt::SkipEmptyParts);
+#endif
+
+    for (const auto & protocol : protocols)
+        if (auto valid = ParseProtocol(protocol); valid != ProtFrame)
+            return valid;
+
+    return ProtFrame;
+}

mythtv/libs/libmythbase/http/mythhttpcommon.h

@@ -0,0 +1,90 @@
+#ifndef MYTHHTTPCOMMON_H
+#define MYTHHTTPCOMMON_H
+
+// Qt
+#include <QString>
+#include <QMetaType>
+
+// MythTV
+#include "mythbaseexp.h"
+
+// Std
+#include <memory>
+
+/// A group of functions shared between HTTP and WebSocket code.
+
+#define HTTP_CHUNKSIZE 65536L // 64k
+
+#define JSONRPC  QStringLiteral("jsonrpc")
+#define XMLRPC   QStringLiteral("xmlrpc")
+#define PLISTRPC QStringLiteral("plistrpc")
+#define CBORRPC  QStringLiteral("cborrpc")
+
+enum MythSocketProtocol
+{
+    ProtHTTP = 0, // Socket has not been upgraded
+    ProtFrame,    // Default WebSocket text and binary frame transmission
+    ProtJSONRPC,
+    ProtXMLRPC,
+    ProtPLISTRPC,
+    ProtCBORRPC
+};
+
+class MythSharedData;
+using DataPayload  = std::shared_ptr<MythSharedData>;
+using DataPayloads = std::vector<DataPayload>;
+
+class MythSharedData : public QByteArray
+{
+  public:
+    static DataPayload CreatePayload(size_t Size);
+    static DataPayload CreatePayload(const QString& Text);
+    static DataPayload CreatePayload(const QByteArray& Other);
+
+    int64_t m_readPosition  { 0 };
+    int64_t m_writePosition { 0 };
+
+  protected:
+    explicit MythSharedData(uint64_t Size);
+    explicit MythSharedData(const QString& Text);
+    explicit MythSharedData(const QByteArray& Other);
+};
+
+class MythSharedString;
+using StringPayload = std::shared_ptr<MythSharedString>;
+
+class MythSharedString : public QString
+{
+  public:
+    static StringPayload CreateString();
+
+  protected:
+    MythSharedString() = default;
+};
+
+Q_DECLARE_METATYPE(DataPayload)
+Q_DECLARE_METATYPE(DataPayloads)
+Q_DECLARE_METATYPE(StringPayload)
+
+class MythHTTPWS
+{
+  public:
+    static QString ProtocolToString(MythSocketProtocol Protocol);
+    static MythSocketProtocol ProtocolFromString(const QString& Protocols);
+    static inline QString BitrateToString(uint64_t Rate)
+    {
+        if (Rate < 1)
+            return "-";
+        else if (Rate > (1073741824L * 1024))
+            return ">1TBps";
+        else if (Rate >= 1073741824)
+            return QStringLiteral("%1GBps").arg(Rate / 1073741824.0);
+        else if (Rate >= 1048576)
+            return QStringLiteral("%1MBps").arg(Rate / 1048576.0);
+        else if (Rate >= 1024)
+            return QStringLiteral("%1kBps").arg(Rate / 1024.0);
+        return QStringLiteral("%1Bps").arg(Rate);
+    }
+};
+
+#endif