Delay subsequent requests to the same host

src/base/net/downloadmanager.cpp

@@ -40,6 +40,7 @@
 #include <QNetworkReply>
 #include <QNetworkRequest>
 #include <QSslError>
+#include <QTimer>
 #include <QUrl>
 
 #include "base/global.h"
@@ -181,11 +182,17 @@ Net::DownloadHandler *Net::DownloadManager::download(const DownloadRequest &down
     return downloadHandler;
 }
 
-void Net::DownloadManager::registerSequentialService(const Net::ServiceID &serviceID)
+void Net::DownloadManager::registerSequentialService(const Net::ServiceID &serviceID, std::chrono::seconds delay)
 {
+    setSequentialServiceDelay(serviceID ,delay);
     m_sequentialServices.insert(serviceID);
 }
 
+void Net::DownloadManager::setSequentialServiceDelay(const ServiceID &serviceID, std::chrono::seconds delay)
+{
+    m_serviceDelay[serviceID] = delay;
+}
+
 QList<QNetworkCookie> Net::DownloadManager::cookiesForUrl(const QUrl &url) const
 {
     return m_networkCookieJar->cookiesForUrl(url);
@@ -259,9 +266,8 @@ void Net::DownloadManager::applyProxySettings()
         m_proxy.setCapabilities(m_proxy.capabilities() & ~QNetworkProxy::HostNameLookupCapability);
 }
 
-void Net::DownloadManager::handleDownloadFinished(DownloadHandlerImpl *finishedHandler)
+void Net::DownloadManager::handleDownloadFinished(const ServiceID &id)
 {
-    const ServiceID id = ServiceID::fromURL(finishedHandler->url());
     const auto waitingJobsIter = m_waitingJobs.find(id);
     if ((waitingJobsIter == m_waitingJobs.end()) || waitingJobsIter.value().isEmpty())
     {
@@ -304,7 +310,10 @@ void Net::DownloadManager::processRequest(DownloadHandlerImpl *downloadHandler)
     QNetworkReply *reply = m_networkManager->get(request);
     connect(reply, &QNetworkReply::finished, this, [this, downloadHandler]
     {
-        handleDownloadFinished(downloadHandler);
+        const ServiceID id = ServiceID::fromURL(downloadHandler->url());
+        QTimer::singleShot(m_serviceDelay[id] , this, [this, id](){
+            handleDownloadFinished(id);
+        });
     });
     downloadHandler->assignNetworkReply(reply);
 }

src/base/net/downloadmanager.h

@@ -29,6 +29,8 @@
 
 #pragma once
 
+#include <chrono>
+
 #include <QtTypes>
 #include <QHash>
 #include <QNetworkProxy>
@@ -137,7 +139,8 @@ namespace Net
         template <typename Context, typename Func>
         void download(const DownloadRequest &downloadRequest, bool useProxy, Context context, Func &&slot);
 
-        void registerSequentialService(const ServiceID &serviceID);
+        void registerSequentialService(const ServiceID &serviceID, std::chrono::seconds delay = std::chrono::seconds(0));
+        void setSequentialServiceDelay(const ServiceID &serviceID, std::chrono::seconds delay);
 
         QList<QNetworkCookie> cookiesForUrl(const QUrl &url) const;
         bool setCookiesFromUrl(const QList<QNetworkCookie> &cookieList, const QUrl &url);
@@ -153,7 +156,7 @@ namespace Net
         explicit DownloadManager(QObject *parent = nullptr);
 
         void applyProxySettings();
-        void handleDownloadFinished(DownloadHandlerImpl *finishedHandler);
+        void handleDownloadFinished(const ServiceID &id);
         void processRequest(DownloadHandlerImpl *downloadHandler);
 
         static DownloadManager *m_instance;
@@ -164,6 +167,7 @@ namespace Net
         QSet<ServiceID> m_sequentialServices;
         QSet<ServiceID> m_busyServices;
         QHash<ServiceID, QQueue<DownloadHandlerImpl *>> m_waitingJobs;
+        QHash<ServiceID, std::chrono::seconds> m_serviceDelay;
     };
 
     template <typename Context, typename Func>

src/base/rss/rss_feed.cpp

@@ -98,7 +98,7 @@ Feed::Feed(const QUuid &uid, const QString &url, const QString &path, Session *s
     else
         connect(m_session, &Session::processingStateChanged, this, &Feed::handleSessionProcessingEnabledChanged);
 
-    Net::DownloadManager::instance()->registerSequentialService(Net::ServiceID::fromURL(m_url));
+    Net::DownloadManager::instance()->registerSequentialService(Net::ServiceID::fromURL(m_url), m_session->fetchDelay());
 
     load();
 }
@@ -159,6 +159,11 @@ void Feed::refresh()
     emit stateChanged(this);
 }
 
+void Feed::updateFetchDelay()
+{
+    Net::DownloadManager::instance()->setSequentialServiceDelay(Net::ServiceID::fromURL(m_url), m_session->fetchDelay());
+}
+
 QUuid Feed::uid() const
 {
     return m_uid;

src/base/rss/rss_feed.h

@@ -75,6 +75,7 @@ namespace RSS
         int unreadCount() const override;
         void markAsRead() override;
         void refresh() override;
+        void updateFetchDelay() override;
 
         QUuid uid() const;
         QString url() const;

src/base/rss/rss_folder.cpp

@@ -91,6 +91,12 @@ void Folder::refresh()
         item->refresh();
 }
 
+void Folder::updateFetchDelay()
+{
+    for (Item *item : asConst(items()))
+        item->updateFetchDelay();
+}
+
 QList<Item *> Folder::items() const
 {
     return m_items;

src/base/rss/rss_folder.h

@@ -54,6 +54,7 @@ namespace RSS
         int unreadCount() const override;
         void markAsRead() override;
         void refresh() override;
+        void updateFetchDelay() override;
 
         QList<Item *> items() const;
 

src/base/rss/rss_item.h

@@ -52,6 +52,7 @@ namespace RSS
         virtual int unreadCount() const = 0;
         virtual void markAsRead() = 0;
         virtual void refresh() = 0;
+        virtual void updateFetchDelay() = 0;
 
         QString path() const;
         QString name() const;

src/base/rss/rss_session.cpp

@@ -56,12 +56,14 @@ const QString DATA_FOLDER_NAME = u"rss/articles"_s;
 const QString FEEDS_FILE_NAME = u"feeds.json"_s;
 
 using namespace RSS;
+using namespace std::chrono_literals;
 
 QPointer<Session> Session::m_instance = nullptr;
 
 Session::Session()
     : m_storeProcessingEnabled(u"RSS/Session/EnableProcessing"_s)
     , m_storeRefreshInterval(u"RSS/Session/RefreshInterval"_s, 30)
+    , m_storeFetchDelay(u"RSS/Session/FetchDelay"_s, 2s)
     , m_storeMaxArticlesPerFeed(u"RSS/Session/MaxArticlesPerFeed"_s, 50)
     , m_workingThread(new QThread)
 {
@@ -525,6 +527,19 @@ void Session::setRefreshInterval(const int refreshInterval)
     }
 }
 
+std::chrono::seconds Session::fetchDelay() const
+{
+    return m_storeFetchDelay;
+}
+
+void Session::setFetchDelay(const std::chrono::seconds delay)
+{
+    if (delay == fetchDelay())
+        return;
+    m_storeFetchDelay = delay;
+    rootFolder()->updateFetchDelay();
+}
+
 QThread *Session::workingThread() const
 {
     return m_workingThread.get();

src/base/rss/rss_session.h

@@ -68,6 +68,8 @@
  * 3.   Feed is JSON object (keys are property names, values are property values; 'uid' and 'url' are required)
  */
 
+#include <chrono>
+
 #include <QHash>
 #include <QObject>
 #include <QPointer>
@@ -114,6 +116,9 @@ namespace RSS
         int refreshInterval() const;
         void setRefreshInterval(int refreshInterval);
 
+        std::chrono::seconds fetchDelay() const;
+        void setFetchDelay(std::chrono::seconds delay);
+
         nonstd::expected<void, QString> addFolder(const QString &path);
         nonstd::expected<void, QString> addFeed(const QString &url, const QString &path);
         nonstd::expected<void, QString> setFeedURL(const QString &path, const QString &url);
@@ -161,6 +166,7 @@ namespace RSS
 
         CachedSettingValue<bool> m_storeProcessingEnabled;
         CachedSettingValue<int> m_storeRefreshInterval;
+        CachedSettingValue<std::chrono::seconds> m_storeFetchDelay;
         CachedSettingValue<int> m_storeMaxArticlesPerFeed;
         Utils::Thread::UniquePtr m_workingThread;
         AsyncFileStorage *m_confFileStorage = nullptr;

src/base/settingsstorage.h

@@ -30,11 +30,13 @@
 
 #pragma once
 
+#include <chrono>
 #include <type_traits>
 
 #include <QObject>
 #include <QReadWriteLock>
 #include <QTimer>
+#include <qtypes.h>
 #include <QVariant>
 #include <QVariantHash>
 
@@ -84,6 +86,11 @@ class SettingsStorage final : public QObject
             const typename T::Int value = loadValue(key, static_cast<typename T::Int>(defaultValue));
             return T {value};
         }
+        else if constexpr (std::same_as<T, std::chrono::seconds>)
+        {
+            const qlonglong value = loadValue(key, static_cast<qlonglong>(defaultValue.count()));
+            return std::chrono::seconds(value);
+        }
         else
         {
             const QVariant value = loadValueImpl(key);
@@ -103,6 +110,8 @@ class SettingsStorage final : public QObject
             storeValueImpl(key, Utils::String::fromEnum(value));
         else if constexpr (IsQFlags<T>)
             storeValueImpl(key, static_cast<typename T::Int>(value));
+        else if constexpr (std::same_as<T, std::chrono::seconds>)
+            storeValueImpl(key, static_cast<qlonglong>(value.count()));
         else
             storeValueImpl(key, QVariant::fromValue(value));
     }

src/gui/optionsdialog.cpp

@@ -29,6 +29,7 @@
 
 #include "optionsdialog.h"
 
+#include <chrono>
 #include <cstdlib>
 #include <limits>
 
@@ -1193,6 +1194,7 @@ void OptionsDialog::loadRSSTabOptions()
 
     m_ui->checkRSSEnable->setChecked(rssSession->isProcessingEnabled());
     m_ui->spinRSSRefreshInterval->setValue(rssSession->refreshInterval());
+    m_ui->spinRSSFetchDelay->setValue(rssSession->fetchDelay().count());
     m_ui->spinRSSMaxArticlesPerFeed->setValue(rssSession->maxArticlesPerFeed());
     m_ui->checkRSSAutoDownloaderEnable->setChecked(autoDownloader->isProcessingEnabled());
     m_ui->textSmartEpisodeFilters->setPlainText(autoDownloader->smartEpisodeFilters().join(u'\n'));
@@ -1209,6 +1211,7 @@ void OptionsDialog::loadRSSTabOptions()
     connect(m_ui->textSmartEpisodeFilters, &QPlainTextEdit::textChanged, this, &OptionsDialog::enableApplyButton);
     connect(m_ui->checkSmartFilterDownloadRepacks, &QCheckBox::toggled, this, &OptionsDialog::enableApplyButton);
     connect(m_ui->spinRSSRefreshInterval, qSpinBoxValueChanged, this, &OptionsDialog::enableApplyButton);
+    connect(m_ui->spinRSSFetchDelay, qSpinBoxValueChanged, this, &OptionsDialog::enableApplyButton);
     connect(m_ui->spinRSSMaxArticlesPerFeed, qSpinBoxValueChanged, this, &OptionsDialog::enableApplyButton);
 }
 
@@ -1219,6 +1222,7 @@ void OptionsDialog::saveRSSTabOptions() const
 
     rssSession->setProcessingEnabled(m_ui->checkRSSEnable->isChecked());
     rssSession->setRefreshInterval(m_ui->spinRSSRefreshInterval->value());
+    rssSession->setFetchDelay(std::chrono::seconds(m_ui->spinRSSFetchDelay->value()));
     rssSession->setMaxArticlesPerFeed(m_ui->spinRSSMaxArticlesPerFeed->value());
     autoDownloader->setProcessingEnabled(m_ui->checkRSSAutoDownloaderEnable->isChecked());
     autoDownloader->setSmartEpisodeFilters(m_ui->textSmartEpisodeFilters->toPlainText().split(u'\n', Qt::SkipEmptyParts));

src/gui/optionsdialog.ui

@@ -3127,8 +3127,8 @@ Disable encryption: Only connect to peers without protocol encryption</string>
             <rect>
              <x>0</x>
              <y>0</y>
-             <width>336</width>
-             <height>391</height>
+             <width>541</width>
+             <height>539</height>
             </rect>
            </property>
            <layout class="QVBoxLayout" name="verticalLayout_27">
@@ -3147,14 +3147,7 @@ Disable encryption: Only connect to peers without protocol encryption</string>
                </item>
                <item>
                 <layout class="QGridLayout" name="gridLayout_5">
-                 <item row="0" column="0">
-                  <widget class="QLabel" name="label_111">
-                   <property name="text">
-                    <string>Feeds refresh interval:</string>
-                   </property>
-                  </widget>
-                 </item>
-                 <item row="1" column="1">
+                 <item row="2" column="1">
                   <widget class="QSpinBox" name="spinRSSMaxArticlesPerFeed">
                    <property name="maximum">
                     <number>2147483646</number>
@@ -3164,12 +3157,18 @@ Disable encryption: Only connect to peers without protocol encryption</string>
                    </property>
                   </widget>
                  </item>
-                 <item row="1" column="0">
-                  <widget class="QLabel" name="label_12">
-                   <property name="text">
-                    <string>Maximum number of articles per feed:</string>
+                 <item row="0" column="2">
+                  <spacer name="horizontalSpacer_6">
+                   <property name="orientation">
+                    <enum>Qt::Horizontal</enum>
                    </property>
-                  </widget>
+                   <property name="sizeHint" stdset="0">
+                    <size>
+                     <width>40</width>
+                     <height>20</height>
+                    </size>
+                   </property>
+                  </spacer>
                  </item>
                  <item row="0" column="1">
                   <widget class="QSpinBox" name="spinRSSRefreshInterval">
@@ -3187,18 +3186,39 @@ Disable encryption: Only connect to peers without protocol encryption</string>
                    </property>
                   </widget>
                  </item>
-                 <item row="0" column="2">
-                  <spacer name="horizontalSpacer_6">
-                   <property name="orientation">
-                    <enum>Qt::Horizontal</enum>
+                 <item row="2" column="0">
+                  <widget class="QLabel" name="label_12">
+                   <property name="text">
+                    <string>Maximum number of articles per feed:</string>
                    </property>
-                   <property name="sizeHint" stdset="0">
-                    <size>
-                     <width>40</width>
-                     <height>20</height>
-                    </size>
+                  </widget>
+                 </item>
+                 <item row="0" column="0">
+                  <widget class="QLabel" name="label_111">
+                   <property name="text">
+                    <string>Feeds refresh interval:</string>
                    </property>
-                  </spacer>
+                  </widget>
+                 </item>
+                 <item row="1" column="0">
+                  <widget class="QLabel" name="label_rss_fetch_delay">
+                   <property name="text">
+                    <string>Delay between requests:</string>
+                   </property>
+                  </widget>
+                 </item>
+                 <item row="1" column="1">
+                  <widget class="QSpinBox" name="spinRSSFetchDelay">
+                   <property name="suffix">
+                    <string> sec</string>
+                   </property>
+                   <property name="maximum">
+                    <number>999999</number>
+                   </property>
+                   <property name="value">
+                    <number>2</number>
+                   </property>
+                  </widget>
                  </item>
                 </layout>
                </item>

src/webui/api/appcontroller.cpp

@@ -319,6 +319,7 @@ void AppController::preferencesAction()
 
     // RSS settings
     data[u"rss_refresh_interval"_s] = RSS::Session::instance()->refreshInterval();
+    data[u"rss_fetch_delay"_s] = static_cast<int>(RSS::Session::instance()->fetchDelay().count());
     data[u"rss_max_articles_per_feed"_s] = RSS::Session::instance()->maxArticlesPerFeed();
     data[u"rss_processing_enabled"_s] = RSS::Session::instance()->isProcessingEnabled();
     data[u"rss_auto_downloading_enabled"_s] = RSS::AutoDownloader::instance()->isProcessingEnabled();
@@ -865,6 +866,8 @@ void AppController::setPreferencesAction()
 
     if (hasKey(u"rss_refresh_interval"_s))
         RSS::Session::instance()->setRefreshInterval(it.value().toInt());
+    if (hasKey(u"rss_fetch_delay"_s))
+        RSS::Session::instance()->setFetchDelay(std::chrono::seconds(it.value().toInt()));
     if (hasKey(u"rss_max_articles_per_feed"_s))
         RSS::Session::instance()->setMaxArticlesPerFeed(it.value().toInt());
     if (hasKey(u"rss_processing_enabled"_s))