Squashed commit of the following:

commit 277e0851c12596eb09de77b465e608ec85359c27 Author: zekses <enmarantispam@gmail.com> Date: Sun Nov 11 04:55:33 2018 +0300 server load - ok commit 55e6313 Author: zekses <enmarantispam@gmail.com> Date: Sun Nov 11 02:12:44 2018 +0300 somewhat tolerable commit 0f79f91 Author: zekses <enmarantispam@gmail.com> Date: Sun Nov 11 01:34:36 2018 +0300 complete bullshit commit 7dfc628 Author: zekses <enmarantispam@gmail.com> Date: Sun Nov 11 01:01:04 2018 +0300 hm commit 29f1be8 Author: zekses <enmarantispam@gmail.com> Date: Sat Nov 10 23:56:32 2018 +0300 managed to get the bastard to compile# Please enter the commit message for your changes. Lines starting commit 6b7f315 Author: zekses <enmarantispam@gmail.com> Date: Sat Nov 10 22:29:50 2018 +0300 rip msvc commit 85d5e0c Author: zekses <enmarantispam@gmail.com> Date: Thu Nov 8 22:05:41 2018 +0300 hm hm commit 15ccb97 Author: zekses <enmarantispam@gmail.com> Date: Thu Nov 8 21:01:13 2018 +0300 mhmm commit a3f4eb7 Author: zekses <enmarantispam@gmail.com> Date: Thu Nov 8 19:33:46 2018 +0300 testing commit 799c426 Author: zekses <enmarantispam@gmail.com> Date: Thu Nov 8 17:20:32 2018 +0300 me stupid
Zeks · Nov 11, 2018 · 65a967d · 65a967d
1 parent eab8823
commit 65a967d
Show file tree

Hide file tree

Showing 18 changed files with 20,450 additions and 271 deletions.
diff --git a/Run/settings_server.ini b/Run/settings_server.ini
@@ -16,6 +16,7 @@ serverPort=3055
 thinClient=true
 motdRequired=false
 motd="Have fun!"
+usestoreddata=true
 
 [Logging]
 loglevel=0

diff --git a/UI/servitorwindow.ui b/UI/servitorwindow.ui
@@ -185,6 +185,13 @@
         </property>
        </widget>
       </item>
+      <item>
+       <widget class="QSpinBox" name="sbFicCount">
+        <property name="maximum">
+         <number>1000000</number>
+        </property>
+       </widget>
+      </item>
       <item>
        <widget class="QPushButton" name="pbCalcWeights">
         <property name="text">

diff --git a/feed_server.qbs b/feed_server.qbs
@@ -28,16 +28,21 @@ App{
     cpp.defines: base.concat(["L_LOGGER_LIBRARY"])
     cpp.includePaths: [
         sourceDirectory,
-        sourceDirectory + "/../",
+        //sourceDirectory + "/../",
         sourceDirectory + "/include",
         sourceDirectory + "/libs",
         sourceDirectory + "/third_party/zlib",
         sourceDirectory + "/libs/Logger/include",
     ]
 
     files: [
+        "include/calc_data_holder.h",
         "include/grpc/grpc_source.h",
         "include/Interfaces/data_source.h",
+        "include/threaded_data/common_traits.h",
+        "include/threaded_data/threaded_load.h",
+        "include/threaded_data/threaded_save.h",
+        "src/calc_data_holder.cpp",
         "src/grpc/grpc_source.cpp",
         "src/Interfaces/data_source.cpp",
         "include/Interfaces/base.h",
@@ -67,6 +72,8 @@ App{
         "include/core/section.h",
         "include/storyfilter.h",
         "include/url_utils.h",
+        "src/threaded_data/threaded_load.cpp",
+        "src/threaded_data/threaded_save.cpp",
         "third_party/sqlite3/sqlite3.c",
         "third_party/sqlite3/sqlite3.h",
         "src/sqlcontext.cpp",
@@ -99,6 +106,10 @@ App{
         "src/servers/database_context.cpp",
         "include/servers/database_context.h",
     ]
+    cpp.systemIncludePaths: [
+        sourceDirectory +"/proto",
+        sourceDirectory + "/third_party",
+        sourceDirectory + "/../"]
 
     cpp.staticLibraries: {
         //var libs = ["UniversalModels", "logger", "quazip"]

diff --git a/include/calc_data_holder.h b/include/calc_data_holder.h
@@ -6,16 +6,18 @@
 #include <QDataStream>
 #include <array>
 #include "include/core/section.h"
-
+struct ListWithIdentifier{
+    int id;
+    QSet<int>  favourites;
+};
 struct CalcDataHolder{
     QVector<core::FicWeightPtr> fics;
     QList<core::AuthorPtr>  authors;
 
     QHash<int, QSet<int> > favourites;
-    QHash<int, QSet<int> > favouritesReduced;
     QHash<int, std::array<double, 22> > genreData;
     QHash<int, core::AuthorFavFandomStatsPtr> fandomLists;
-
+    QVector<ListWithIdentifier> filteredFavourites;
     QFile data;
     QDataStream out;
     int fileCounter = 0;
@@ -30,3 +32,4 @@ struct CalcDataHolder{
     void SaveFicsData();
     void LoadFicsData();
 };
+
diff --git a/include/favholder.h b/include/favholder.h
@@ -21,6 +21,10 @@ class FavHolder
 {
 public:
     void LoadFavourites(QSharedPointer<interfaces::Authors> authorInterface);
+    void CreateTempDataDir();
+    void LoadDataFromDatabase(QSharedPointer<interfaces::Authors> authorInterface);
+    void LoadStoredData();
+    void SaveData();
     RecommendationListResult GetMatchedFicsForFavList(QSet<int> sourceFics,  QSharedPointer<core::RecommendationList> params);
     QHash<int, QSet<int>> favourites;
 };

diff --git a/include/servitorwindow.h b/include/servitorwindow.h
@@ -3,6 +3,7 @@
 
 #include <QMainWindow>
 #include "environment.h"
+#include "third_party/roaring/roaring.hh"
 #include "include/calc_data_holder.h"
 
 namespace Ui {
@@ -25,6 +26,13 @@ class ServitorWindow : public QMainWindow
 
     void DetectGenres(int minAuthorRecs, int minFoundLists);
     void LoadDataForCalculation(CalcDataHolder& data);
+    void ProcessCDHData(CalcDataHolder& data);
+    void CalcConstantMemory();
+    QHash<uint32_t, core::FicWeightPtr> ficData;
+    QHash<uint32_t, QSet<uint32_t>> ficsForFandoms;
+    QHash<uint32_t, Roaring> ficsToFavLists;
+    QList<uint32_t> keys;
+
     QSharedPointer<database::IDBWrapper> dbInterface;
     CoreEnvironment env;
 

diff --git a/include/threaded_data/common_traits.h b/include/threaded_data/common_traits.h
@@ -0,0 +1,60 @@
+#pragma once 
+#include <QSharedPointer>
+#include <QVector>
+#include <QSet>
+#include <QList>
+#include <array>
+template <typename T>
+struct is_shared_ptr {
+  static const bool value = false;
+};
+
+template <typename Key>
+struct is_shared_ptr<QSharedPointer<Key> > {
+  static const bool value = true;
+};
+
+template <typename T>
+struct is_hash {
+  static const bool value = false;
+};
+
+template <typename Key,typename Value>
+struct is_hash<QHash<Key,Value> > {
+  static const bool value = true;
+};
+
+template <typename T>
+struct is_vector{
+  static const bool value = false;
+};
+
+template <typename Key>
+struct is_vector<QVector<Key> > {
+  static const bool value = true;
+};
+template <typename Key>
+struct is_vector<QList<Key> > {
+  static const bool value = true;
+};
+
+namespace detail{
+  template<class> struct sfinae_true : std::true_type{};
+
+  template<class T, class A0>
+  static auto test_serialize(int)
+    -> sfinae_true<decltype(std::declval<T>().Serialize(std::declval<A0>()))>;
+  template<class, class A0>
+  static auto test_serialize(long) -> std::false_type;
+} // detail::
+
+template<class T, class Arg>
+struct has_serialize: decltype(detail::test_serialize<T, Arg>(0)){};
+
+template <typename T>
+constexpr bool is_serializable_pointer (){
+    if constexpr(!is_shared_ptr<T>::value)
+        return false;
+
+    return has_serialize<typename T::value_type, QDataStream>();
+};
diff --git a/include/threaded_data/threaded_load.h b/include/threaded_data/threaded_load.h
@@ -0,0 +1,16 @@
+#pragma once
+#include <QVector>
+#include <QList>
+#include <QHash>
+#include <QFile>
+#include <QDataStream>
+
+#include "include/core/section.h"
+
+namespace thread_boost{
+void LoadFicWeightCalcData(QString storage, QVector<core::FicWeightPtr>& fics);
+void LoadAuthorsData(QString storage,QList<core::AuthorPtr> &authors);
+void LoadFavouritesData(QString storage, QHash<int, QSet<int>>& favourites);
+void LoadGenreDataForFavLists(QString storage, QHash<int, std::array<double, 22> >& genreData);
+void LoadFandomDataForFavLists(QString storage, QHash<int, core::AuthorFavFandomStatsPtr>& fandomLists);
+}
diff --git a/include/threaded_data/threaded_save.h b/include/threaded_data/threaded_save.h
@@ -0,0 +1,16 @@
+#pragma once
+#include <QVector>
+#include <QList>
+#include <QHash>
+#include <QFile>
+#include <QDataStream>
+#include <array>
+#include "include/core/section.h"
+
+namespace thread_boost{
+void SaveFicWeightCalcData(QString storageFolder,QVector<core::FicWeightPtr>& fics);
+void SaveAuthorsData(QString storageFolder,QList<core::AuthorPtr>& authors);
+void SaveFavouritesData(QString storageFolder, QHash<int, QSet<int>>& favourites);
+void SaveGenreDataForFavLists(QString storageFolder, QHash<int, std::array<double, 22> > &genreData);
+void SaveFandomDataForFavLists(QString storageFolder, QHash<int, core::AuthorFavFandomStatsPtr>& fandomLists);
+}
diff --git a/servitor.qbs b/servitor.qbs
@@ -41,6 +41,9 @@ App{
         "include/servitorwindow.h",
         "include/tasks/author_cache_reprocessor.h",
         "include/tasks/slash_task_processor.h",
+        "include/threaded_data/common_traits.h",
+        "include/threaded_data/threaded_load.h",
+        "include/threaded_data/threaded_save.h",
         "src/calc_data_holder.cpp",
         "src/main_servitor.cpp",
         "src/servitorwindow.cpp",
@@ -75,6 +78,11 @@ App{
         "include/url_utils.h",
         "src/tasks/author_cache_reprocessor.cpp",
         "src/tasks/slash_task_processor.cpp",
+        "src/threaded_data/threaded_load.cpp",
+        "src/threaded_data/threaded_save.cpp",
+        "third_party/roaring/roaring.c",
+        "third_party/roaring/roaring.h",
+        "third_party/roaring/roaring.hh",
         "third_party/sqlite3/sqlite3.c",
         "third_party/sqlite3/sqlite3.h",
         "src/sqlcontext.cpp",