Merge #1012: [Performance][Wallet][QT][Model] TransactionTableModel m…

…ulti-thread initialization + tx filter sort speed improved.

dbf26ee9cbbcc8d39decb372538556cd14171757 [Build] QtConcurrent include path added. (furszy)
5cd44b639995b51b7b172d87175bf091b2fe4af9 [Build] QtConcurrent checks. (furszy)
a24a2511b49ac5953996af93a9743dffaaa1603a [Build] QtConcurrent feature added to config options. (furszy)
35f63f2cfbe62e959196b50a1ae0bbf1b5a79389 [QT] Dashboard chart, do not refresh the chart when a refresh is being processed. (furszy)
70abbcc0e44e4889658b7e9507e90d7752f6bffc [Build] QtConcurrent included into the qt libs. (furszy)
3d58d00a56aa2fe63a0549ca4b56b9584d01ea4d [Wallet][Model][QT] Multi-thread transactionTableModel startup corrections + worker object finish signal broadcasted when the thread task fails too. (furszy)
9b4facc0b7feabc232f09f84f22185aadd042021 [Performance][Wallet][QT][Model] * Parallelization of the transactionTableModel initial loop (for-each loop over every wallet transaction to generate the TransactionRecord list that is used in the UI).  * QSortFilterProxyModel stakeFilter in dashboard widget and txFilter in the privacyWidget sort speed drastically improved, only done over the editRole and pre sort filter type flag set. (furszy)

Pull request description:

  Rationale

  When the splash screen finishes loading and the wallet is big enough (wallet.dat with lot of transactions), the UI gets frozen for a large period of time processing data in the main thread.

  To solve it, two improvements were done:

  1)  Parallelization of the transactionTableModel initial loop (for-each loop over every wallet transaction to generate the TransactionRecord list that is used in the UI).

  2) stakeFilter in dashboard widget and txFilter in the privacyWidget sort speed drastically improved, done after the editRole and filter type flag set to not do it over all of the transactions and not do it twice in any of the filter flag set list invalidation.

  Test environment:

  - Wallet with:
      - 4,400,00 tPIV (4.4kk).
      - 40,150 ztPIV (40k)
      -  104,000 transactions (104k).

  - Computer macbook pro i7 16gb ram.

  Test results:

  	Current master:
  		- Startup time: 17:15 minutes.
  		- UI frozen for 13 minutes after backend finish loading.

  	PR:
  		- Startup time:  1:59 minutes.
  		- UI frozen for 30 seconds after backend finish loading.

ACKs for top commit:
  Fuzzbawls:
    ACK dbf26ee9cbbcc8d39decb372538556cd14171757
  Warrows:
    ACK dbf26ee
  random-zebra:
    ACK dbf26ee9cbbcc8d39decb372538556cd14171757

Tree-SHA512: 3dec43324ced36d40c29191b71774764303f507ca0bb37ee7ad55569aede912492bdab5c37ea42e6fe5ecc1914c272748ce4841d361789d3020affd9d582ca21

build-aux/m4/bitcoin_qt.m4

@@ -471,7 +471,7 @@ AC_DEFUN([_BITCOIN_QT_FIND_LIBS_WITHOUT_PKGCONFIG],[
   TEMP_LIBS="$LIBS"
   BITCOIN_QT_CHECK([
     if test "x$qt_include_path" != x; then
-      QT_INCLUDES="-I$qt_include_path -I$qt_include_path/QtCore -I$qt_include_path/QtGui -I$qt_include_path/QtWidgets -I$qt_include_path/QtNetwork -I$qt_include_path/QtTest -I$qt_include_path/QtDBus -I$qt_include_path/QtSvg -I$qt_include_path/QtCharts"
+      QT_INCLUDES="-I$qt_include_path -I$qt_include_path/QtCore -I$qt_include_path/QtGui -I$qt_include_path/QtWidgets -I$qt_include_path/QtNetwork -I$qt_include_path/QtTest -I$qt_include_path/QtDBus -I$qt_include_path/QtConcurrent -I$qt_include_path/QtSvg -I$qt_include_path/QtCharts"
       CPPFLAGS="$QT_INCLUDES $CPPFLAGS"
     fi
   ])

depends/packages/qt.mk

@@ -7,7 +7,7 @@ $(package)_sha256_hash=36dd9574f006eaa1e5af780e4b33d11fe39d09fd7c12f3b9d83294174
 $(package)_dependencies=openssl zlib
 $(package)_linux_dependencies=freetype fontconfig libxcb libX11 xproto libXext
 $(package)_build_subdir=qtbase
-$(package)_qt_libs=corelib network widgets gui plugins testlib
+$(package)_qt_libs=corelib network widgets gui plugins testlib concurrent
 $(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch fix_riscv64_arch.patch fix_s390x_powerpc_mips_mipsel_architectures.patch xkb-default.patch
 
 $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix)
@@ -84,7 +84,7 @@ $(package)_config_opts += -no-feature-lcdnumber
 $(package)_config_opts += -no-feature-pdf
 $(package)_config_opts += -no-feature-printer
 $(package)_config_opts += -no-feature-printdialog
-$(package)_config_opts += -no-feature-concurrent
+$(package)_config_opts += -feature-concurrent
 $(package)_config_opts += -no-feature-sql
 $(package)_config_opts += -no-feature-statemachine
 $(package)_config_opts += -no-feature-syntaxhighlighter

src/qt/askpassphrasedialog.cpp

@@ -358,7 +358,7 @@ void AskPassphraseDialog::run(int type){
         }
     }
 }
-void AskPassphraseDialog::onError(int type, QString error){
+void AskPassphraseDialog::onError(QString error, int type){
     newpassCache = "";
 }
 

src/qt/askpassphrasedialog.h

@@ -67,7 +67,7 @@ class AskPassphraseDialog : public QDialog, public Runnable
     SecureString newpassCache = "";
 
     void run(int type) override;
-    void onError(int type, QString error) override;
+    void onError(QString error, int type) override;
     QCheckBox *btnWatch;
 
     void initWatch(QWidget *parent);

src/qt/transactiontablemodel.cpp

@@ -25,6 +25,10 @@
 #include <QDebug>
 #include <QIcon>
 #include <QList>
+#include <QtConcurrent/QtConcurrent>
+#include <QFuture>
+
+#define SINGLE_THREAD_MAX_TXES_SIZE 4000
 
 // Amount column is right-aligned it contains numbers
 static int column_alignments[] = {
@@ -71,25 +75,79 @@ class TransactionTablePriv
     QList<TransactionRecord> cachedWallet;
     bool hasZcTxes = false;
 
+
     /* Query entire wallet anew from core.
      */
     void refreshWallet()
     {
         qDebug() << "TransactionTablePriv::refreshWallet";
         cachedWallet.clear();
-        {
-            LOCK2(cs_main, wallet->cs_wallet);
-            for (auto it = wallet->mapWallet.begin(); it != wallet->mapWallet.end(); ++it) {
-                if (TransactionRecord::showTransaction(it->second)) {
-                    QList<TransactionRecord> records = TransactionRecord::decomposeTransaction(wallet, it->second);
-                    for (const TransactionRecord& record : records) {
-                        updateHasZcTxesIfNeeded(record);
+
+        std::vector<CWalletTx> walletTxes = wallet->getWalletTxs();
+
+        // Divide the work between multiple threads to speedup the process if the vector is larger than 4k txes
+        std::size_t txesSize = walletTxes.size();
+        if (txesSize > SINGLE_THREAD_MAX_TXES_SIZE) {
+
+            // Simple way to get the processors count
+            std::size_t threadsCount = (QThreadPool::globalInstance()->maxThreadCount() / 2 ) + 1;
+
+            // Size of the tx subsets
+            std::size_t const subsetSize = txesSize / (threadsCount + 1);
+            std::size_t totalSumSize = 0;
+            QList<QFuture<QList<TransactionRecord>>> tasks;
+
+            // Subsets + run task
+            for (std::size_t i = 0; i < threadsCount; ++i) {
+                tasks.append(
+                        QtConcurrent::run(
+                                convertTxToRecords,
+                                this,
+                                wallet,
+                                std::vector<CWalletTx>(walletTxes.begin() + totalSumSize, walletTxes.begin() + totalSumSize + subsetSize)
+                        )
+                 );
+                totalSumSize += subsetSize;
+            }
+
+            // Now take the remaining ones and do the work here
+            std::size_t const remainingSize = txesSize - totalSumSize;
+            cachedWallet.append(convertTxToRecords(this, wallet,
+                                                   std::vector<CWalletTx>(walletTxes.end() - remainingSize, walletTxes.end())
+                                                   ));
+
+            for (QFuture<QList<TransactionRecord>> &future : tasks) {
+                future.waitForFinished();
+                cachedWallet.append(future.result());
+            }
+        } else {
+            // Single thread flow
+            cachedWallet.append(convertTxToRecords(this, wallet, walletTxes));
+        }
+    }
+
+    static QList<TransactionRecord> convertTxToRecords(TransactionTablePriv* tablePriv, const CWallet* wallet, const std::vector<CWalletTx>& walletTxes) {
+        QList<TransactionRecord> cachedWallet;
+        bool hasZcTxes = tablePriv->hasZcTxes;
+        for (const auto &tx : walletTxes) {
+            if (TransactionRecord::showTransaction(tx)) {
+                QList<TransactionRecord> records = TransactionRecord::decomposeTransaction(wallet, tx);
+
+                if (!hasZcTxes) {
+                    for (const TransactionRecord &record : records) {
+                        hasZcTxes = HasZcTxesIfNeeded(record);
                         if (hasZcTxes) break;
                     }
-                    cachedWallet.append(records);
                 }
+
+                cachedWallet.append(records);
             }
         }
+
+        if (hasZcTxes) // Only update it if it's true, multi-thread operation.
+            tablePriv->hasZcTxes = true;
+
+        return cachedWallet;
     }
 
     void updateHasZcTxesIfNeeded(const TransactionRecord& record) {
@@ -154,7 +212,7 @@ class TransactionTablePriv
                     int insert_idx = lowerIndex;
                     foreach (const TransactionRecord& rec, toInsert) {
                         cachedWallet.insert(insert_idx, rec);
-                        updateHasZcTxesIfNeeded(rec);
+                        if (!hasZcTxes) hasZcTxes = HasZcTxesIfNeeded(rec);
                         insert_idx += 1;
                     }
                     parent->endInsertRows();

src/qt/transcendence.cpp

@@ -377,10 +377,8 @@ void BitcoinApplication::createSplashScreen(const NetworkStyle* networkStyle)
 bool BitcoinApplication::createTutorialScreen()
 {
     WelcomeContentWidget* widget = new WelcomeContentWidget();
-    //widget->setOptionsModel(optionsModel);
 
     connect(widget, &WelcomeContentWidget::onLanguageSelected, [this](){
-        std::cout << "updating translations.." << std::endl;
         updateTranslation();
     });
 

src/qt/transcendence/dashboardwidget.cpp

@@ -155,6 +155,7 @@ DashboardWidget::DashboardWidget(TELOSGUI* parent) :
 bool hasCharts = false;
 #ifdef USE_QTCHARTS
     hasCharts = true;
+    isLoading = false;
     setChartShow(YEAR);
     connect(ui->pushButtonYear, &QPushButton::clicked, [this](){setChartShow(YEAR);});
     connect(ui->pushButtonMonth, &QPushButton::clicked, [this](){setChartShow(MONTH);});
@@ -218,9 +219,13 @@ void DashboardWidget::loadWalletModel(){
 #ifdef USE_QTCHARTS
         // chart filter
         stakesFilter = new TransactionFilterProxy();
+        stakesFilter->setDynamicSortFilter(true);
+        stakesFilter->setSortCaseSensitivity(Qt::CaseInsensitive);
+        stakesFilter->setFilterCaseSensitivity(Qt::CaseInsensitive);
+        stakesFilter->setSortRole(Qt::EditRole);
+        stakesFilter->setOnlyStakes(true);
         stakesFilter->setSourceModel(txModel);
         stakesFilter->sort(TransactionTableModel::Date, Qt::AscendingOrder);
-        stakesFilter->setOnlyStakes(true);
         loadChart();
 #endif
     }
@@ -366,11 +371,18 @@ void DashboardWidget::loadChart(){
 
 void DashboardWidget::showHideEmptyChart(bool showEmpty, bool loading, bool forceView) {
     if (stakesFilter->rowCount() > SHOW_EMPTY_CHART_VIEW_THRESHOLD || forceView) {
-        if (!ui->layoutChart->isVisible()) {
+        if (ui->emptyContainerChart->isVisible() != showEmpty) {
             ui->layoutChart->setVisible(!showEmpty);
             ui->emptyContainerChart->setVisible(showEmpty);
         }
     }
+    // Enable/Disable sort buttons
+    bool invLoading = !loading;
+    ui->comboBoxMonths->setEnabled(invLoading);
+    ui->comboBoxYears->setEnabled(invLoading);
+    ui->pushButtonMonth->setEnabled(invLoading);
+    ui->pushButtonAll->setEnabled(invLoading);
+    ui->pushButtonYear->setEnabled(invLoading);
     ui->labelEmptyChart->setText(loading ? tr("Loading chart..") : tr("You have no staking rewards"));
 }
 
@@ -428,8 +440,7 @@ void DashboardWidget::changeChartColors(){
     if (set1) set1->setBorderColor(gridLineColorX);
 }
 
-// pair TELOS, zTELOS
-QMap<int, std::pair<qint64, qint64>> DashboardWidget::getAmountBy() {
+void DashboardWidget::updateStakeFilter() {
     if (chartShow != ALL) {
         bool filterByMonth = false;
         if (monthFilter != 0 && chartShow == MONTH) {
@@ -462,6 +473,11 @@ QMap<int, std::pair<qint64, qint64>> DashboardWidget::getAmountBy() {
     } else {
         stakesFilter->clearDateRange();
     }
+}
+
+// pair TELOS, zTELOS
+QMap<int, std::pair<qint64, qint64>> DashboardWidget::getAmountBy() {
+    updateStakeFilter();
     int size = stakesFilter->rowCount();
     QMap<int, std::pair<qint64, qint64>> amountBy;
     // Get all of the stakes
@@ -570,6 +586,8 @@ void DashboardWidget::onChartMonthChanged(const QString& monthStr) {
 }
 
 bool DashboardWidget::refreshChart(){
+    if (isLoading) return false;
+    isLoading = true;
     isChartMin = width() < 1300;
     isChartInitialized = false;
     showHideEmptyChart(true, true);
@@ -672,6 +690,7 @@ void DashboardWidget::onChartRefreshed() {
     // back to normal
     isChartInitialized = true;
     showHideEmptyChart(false, false, true);
+    isLoading = false;
 }
 
 std::pair<int, int> DashboardWidget::getChartRange(QMap<int, std::pair<qint64, qint64>> amountsBy) {
@@ -756,7 +775,7 @@ void DashboardWidget::run(int type) {
     }
 #endif
 }
-void DashboardWidget::onError(int type, QString error) {
+void DashboardWidget::onError(QString error, int type) {
     inform(tr("Error loading chart: %1").arg(error));
 }
 

src/qt/transcendence/dashboardwidget.h

@@ -102,7 +102,7 @@ class DashboardWidget : public PWidget
     void loadChart();
 
     void run(int type) override;
-    void onError(int type, QString error) override;
+    void onError(QString error, int type) override;
 
 public slots:
     void walletSynced(bool isSync);
@@ -143,6 +143,7 @@ private slots:
 #ifdef USE_QTCHARTS
 
     int64_t lastRefreshTime = 0;
+    std::atomic<bool> isLoading;
 
     // Chart
     TransactionFilterProxy* stakesFilter = nullptr;
@@ -169,6 +170,7 @@ private slots:
     void showHideEmptyChart(bool show, bool loading, bool forceView = false);
     bool refreshChart();
     void tryChartRefresh();
+    void updateStakeFilter();
     QMap<int, std::pair<qint64, qint64>> getAmountBy();
     void loadChartData(bool withMonthNames);
     void updateAxisX(const QStringList *arg = nullptr);

src/qt/transcendence/loadingdialog.cpp

@@ -10,11 +10,13 @@ void Worker::process(){
     try {
         if (runnable)
             runnable->run(type);
-        emit finished();
     } catch (std::exception& e) {
-        std::cout << e.what() << std::endl;
-        emit error(QString::fromStdString(e.what()));
+        QString errorStr = QString::fromStdString(e.what());
+        runnable->onError(errorStr, type);
+        emit error(errorStr, type);
     }
+    emit finished();
+
 };
 
 LoadingDialog::LoadingDialog(QWidget *parent) :
@@ -45,7 +47,6 @@ void LoadingDialog::execute(Runnable *runnable, int type){
     QThread* thread = new QThread;
     Worker* worker = new Worker(runnable, type);
     worker->moveToThread(thread);
-    connect(worker, SIGNAL (error(QString)), this, SLOT (errorString(QString)));
     connect(thread, SIGNAL (started()), worker, SLOT (process()));
     connect(worker, SIGNAL (finished()), thread, SLOT (quit()));
     connect(worker, SIGNAL (finished()), worker, SLOT (deleteLater()));

src/qt/transcendence/loadingdialog.h

@@ -24,7 +24,7 @@ public slots:
     void process();
 signals:
     void finished();
-    void error(QString err);
+    void error(QString err,int type);
 
 private:
     Runnable* runnable;

src/qt/transcendence/prunnable.h

@@ -8,7 +8,7 @@
 class Runnable {
 public:
     virtual void run(int type) = 0;
-    virtual void onError(int type, QString error) = 0;
+    virtual void onError(QString error, int type) = 0;
 };
 
 #endif //PIVX_CORE_NEW_GUI_PRUNNABLE_H

src/qt/transcendence/pwidget.cpp

@@ -78,7 +78,7 @@ class WorkerTask : public QRunnable {
 
 bool PWidget::execute(int type){
     Worker* worker = new Worker(this, type);
-    connect(worker, SIGNAL (error(QString&)), this, SLOT (errorString(QString)));
+    connect(worker, SIGNAL (error(QString, int)), this, SLOT (errorString(QString, int)));
     connect(worker, SIGNAL (finished()), worker, SLOT (deleteLater()));
 
     WorkerTask* task = new WorkerTask(worker);
@@ -95,6 +95,10 @@ bool PWidget::verifyWalletUnlocked(){
     return true;
 }
 
+void PWidget::errorString(QString error, int type) {
+    onError(error, type);
+}
+
 ////////////////////////////////////////////////////////////////
 //////////////////Override methods//////////////////////////////
 ////////////////////////////////////////////////////////////////

src/qt/transcendence/pwidget.h

@@ -31,7 +31,7 @@ class PWidget : public QWidget, public Runnable
     TELOSGUI* getWindow() { return this->window; }
 
     void run(int type) override;
-    void onError(int type, QString error) override;
+    void onError(QString error, int type) override;
 
 signals:
     void message(const QString& title, const QString& body, unsigned int style, bool* ret = nullptr);
@@ -62,6 +62,8 @@ protected slots:
 
 private:
     void init();
+private slots:
+    void errorString(QString, int);
 };
 
 #endif // PWIDGET_H

src/qt/transcendence/res/css/style_dark.css

@@ -1740,6 +1740,15 @@ QComboBox[cssClass="btn-combo-chart-selected"]::down-arrow {
     padding-right: 10px;
     height: 24px;
 }
+QComboBox[cssClass="btn-combo-chart-selected"]:disabled {
+    background-color:#2e2e19;
+    padding:6px 12px 6px 6px;
+    font-size:16px;
+    border:1px solid #7d694b;
+    color: #FFFFFF;
+    text-align: right;
+    border-radius:2px;
+}
 
 QComboBox[cssClass="btn-combo-chart-selected"]:on {
     padding-top: 10px;
@@ -1771,6 +1780,15 @@ QPushButton[cssClass="btn-check-time"]:checked {
     color: #FFFFFF;
 }
 
+QPushButton[cssClass="btn-check-time"]:disabled {
+    background-color:#2e2e19;
+    border-radius:2px;
+    font-size:14px;
+    padding:4px;
+    color: #FFFFFF;
+    border:1px solid #7d694b;
+}
+
 QPushButton[cssClass="btn-check-time"]:hover {
     background-color:#B37d744b;
     border-radius:2px;