fixing slash stuff

UI/servitorwindow.ui

@@ -185,7 +185,14 @@
     </item>
     <item row="0" column="0">
      <layout class="QGridLayout" name="gridLayout">
-      <item row="1" column="1">
+      <item row="0" column="3">
+       <widget class="QPushButton" name="pbPCRescue">
+        <property name="text">
+         <string>Rescue PageCache</string>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="2">
        <widget class="QLineEdit" name="leReprocessFics"/>
       </item>
       <item row="1" column="0">
@@ -202,13 +209,20 @@
         </property>
        </widget>
       </item>
-      <item row="0" column="1">
+      <item row="1" column="1">
+       <widget class="QPushButton" name="pbSlashCalc">
+        <property name="text">
+         <string>Recalc Slash</string>
+        </property>
+       </widget>
+      </item>
+      <item row="0" column="1" colspan="2">
        <widget class="QLineEdit" name="leFicUrl"/>
       </item>
-      <item row="0" column="2">
-       <widget class="QPushButton" name="pbPCRescue">
+      <item row="1" column="3">
+       <widget class="QPushButton" name="pbFindSlashSummary">
         <property name="text">
-         <string>Rescue PageCache</string>
+         <string>Check Summary</string>
         </property>
        </widget>
       </item>

flipper.qbs

@@ -48,7 +48,6 @@ App{
         "include/Interfaces/fanfics.h",
         "include/Interfaces/ffn/ffn_fanfics.h",
         "include/Interfaces/ffn/ffn_authors.h",
-
         "include/Interfaces/genres.h",
         "include/Interfaces/interface_sqlite.h",
         "include/Interfaces/pagetask_interface.h",
@@ -64,6 +63,7 @@ App{
         "include/parsers/ffn/ffnparserbase.h",
         "include/parsers/ffn/ficparser.h",
         "include/parsers/ffn/fandomindexparser.h",
+        "include/tasks/author_cache_reprocessor.h",
         "include/tasks/fandom_task_processor.h",
         "include/tasks/author_task_processor.h",
         "include/timeutils.h",
@@ -87,6 +87,7 @@ App{
         "include/url_utils.h",
         "qml_ficmodel.cpp",
         "qml_ficmodel.h",
+        "src/tasks/author_cache_reprocessor.cpp",
         "src/tasks/fandom_task_processor.cpp",
         "src/tasks/author_task_processor.cpp",
         "src/pageconsumer.cpp",
@@ -99,7 +100,6 @@ App{
         "src/Interfaces/fanfics.cpp",
         "src/Interfaces/ffn/ffn_fanfics.cpp",
         "src/Interfaces/ffn/ffn_authors.cpp",
-
         "src/Interfaces/genres.cpp",
         "src/Interfaces/interface_sqlite.cpp",
         "src/Interfaces/pagetask_interface.cpp",

include/servitorwindow.h

@@ -60,6 +60,10 @@ private slots:
 
     void on_pbPCRescue_clicked();
 
+    void on_pbSlashCalc_clicked();
+
+    void on_pbFindSlashSummary_clicked();
+
 private:
     Ui::servitorWindow *ui;
 };

include/tasks/slash_task_processor.h

@@ -42,7 +42,6 @@ Q_OBJECT
     SlashProcessor(QSqlDatabase db,
                         QSharedPointer<interfaces::Fanfics> fanficInterface,
                         QSharedPointer<interfaces::Fandoms> fandomsInterface,
-                        QSharedPointer<interfaces::PageTask> pageInterface,
                         QSharedPointer<interfaces::Authors> authorsInterface,
                         QSharedPointer<interfaces::RecommendationLists> recsInterface,
                         QSharedPointer<database::IDBWrapper> dbInterface,

servitor.qbs

@@ -39,6 +39,7 @@ App{
         "include/core/fic_genre_data.h",
         "include/servitorwindow.h",
         "include/tasks/author_cache_reprocessor.h",
+        "include/tasks/slash_task_processor.h",
         "src/main_servitor.cpp",
         "src/servitorwindow.cpp",
         "src/Interfaces/data_source.cpp",
@@ -71,6 +72,7 @@ App{
         "include/storyfilter.h",
         "include/url_utils.h",
         "src/tasks/author_cache_reprocessor.cpp",
+        "src/tasks/slash_task_processor.cpp",
         "third_party/sqlite3/sqlite3.c",
         "third_party/sqlite3/sqlite3.h",
         "src/sqlcontext.cpp",

src/grpc/grpc_source.cpp

@@ -127,6 +127,7 @@ ProtoSpace::Filter StoryFilterIntoProto(const core::StoryFilter& filter,
     slashFilter->set_use_slash_filter(filter.slashFilter.slashFilterEnabled);
     slashFilter->set_exclude_slash(filter.slashFilter.excludeSlash);
     slashFilter->set_include_slash(filter.slashFilter.includeSlash);
+    slashFilter->set_slash_filter_level(filter.slashFilter.slashFilterLevel);
     slashFilter->set_enable_exceptions(filter.slashFilter.enableFandomExceptions);
     for(auto exception : filter.slashFilter.fandomExceptions)
         slashFilter->add_fandom_exceptions(exception);
@@ -232,6 +233,7 @@ core::StoryFilter ProtoIntoStoryFilter(const ProtoSpace::Filter& filter, const P
     result.slashFilter.excludeSlash = filter.slash_filter().exclude_slash();
     result.slashFilter.includeSlash = filter.slash_filter().include_slash();
     result.slashFilter.enableFandomExceptions = filter.slash_filter().enable_exceptions();
+    result.slashFilter.slashFilterLevel = filter.slash_filter().slash_filter_level();
 
     for(int i = 0; i < filter.slash_filter().fandom_exceptions_size(); i++)
         result.slashFilter.fandomExceptions.push_back(filter.slash_filter().fandom_exceptions(i));

src/regex_utils.cpp

@@ -185,7 +185,7 @@ void CommonRegex::Init()
     fixedNamesOnly = names + characterSeparator + names;
     slashRegexPerFandom["Naruto"] = fixedNamesOnly;
 
-    names = "((ichi(go){0,1})|(ren(ji){0,1})|(zara(ki){0,1})|(hi(tsugaya){0,1})|(ikka(ku){0,1})|(byaku(ya){0,1})|(chad))";
+    names = "((ichi(go){0,1})|(ren(ji){0,1})|(zara(ki){0,1})|(hitsugaya)|(ikka(ku){0,1})|(byaku(ya){0,1})|(chad))";
     fixedNamesOnly = names + characterSeparator + names;
     slashRegexPerFandom["Bleach"] = fixedNamesOnly;
     slashRegexPerFandom["Death Note"] = "(L/Light)|(Light/L)";

src/servitorwindow.cpp

@@ -9,6 +9,7 @@
 #include "include/Interfaces/ffn/ffn_authors.h"
 #include "include/url_utils.h"
 #include "include/favholder.h"
+#include "include/tasks/slash_task_processor.h"
 #include <QTextCodec>
 #include <QSettings>
 #include <QSqlRecord>
@@ -650,3 +651,41 @@ void ServitorWindow::on_pbPCRescue_clicked()
 
 
 }
+
+void ServitorWindow::on_pbSlashCalc_clicked()
+{
+    auto db = QSqlDatabase::database();
+    auto authorInterface = QSharedPointer<interfaces::Authors> (new interfaces::FFNAuthors());
+    authorInterface->db = db;
+    authorInterface->portableDBInterface = dbInterface;
+
+    auto authors = authorInterface->GetAllAuthors("ffn");
+
+
+    auto fandomInterface = QSharedPointer<interfaces::Fandoms> (new interfaces::Fandoms());
+    fandomInterface->db = db;
+    fandomInterface->portableDBInterface = dbInterface;
+
+    auto fanficsInterface = QSharedPointer<interfaces::Fanfics> (new interfaces::FFNFanfics());
+    fanficsInterface->db = db;
+    fanficsInterface->authorInterface = authorInterface;
+    fanficsInterface->fandomInterface = fandomInterface;
+
+    auto recsInterface = QSharedPointer<interfaces::RecommendationLists> (new interfaces::RecommendationLists());
+    recsInterface->db = db;
+    recsInterface->portableDBInterface = dbInterface;
+    recsInterface->authorInterface = authorInterface;
+
+    SlashProcessor slash(db,fanficsInterface, fandomInterface, authorInterface, recsInterface, dbInterface);
+    slash.DoFullCycle(db, 2);
+
+}
+
+void ServitorWindow::on_pbFindSlashSummary_clicked()
+{
+    CommonRegex rx;
+    rx.Init();
+    auto result = rx.ContainsSlash("A year after his mother's death, Ichigo finds himself in a world that he knows doesn't exist and met four spirits. Deciding to know what they truly are, he goes to his father who takes him to a shop. Take a different road IchiHime fans. Dual Wield Zanpakuto! Resurreccion! Quincy powers! OOC. New chapters every week or two. HIATUS",
+                                   "[Ichigo K., Yoruichi S.] Rukia K., T. Harribel",
+                                   "Bleach");
+}

src/tasks/author_stats_processor.cpp

@@ -54,7 +54,7 @@ void AuthorStatsProcessor::ReprocessAllAuthorsStats(ECacheMode cacheMode)
         auto authors = this->authorsInterface->GetAllAuthors("ffn", true);
 
         statistics_utils::UserPageSource source;
-        statistics_utils::UserPageSink sink;
+        statistics_utils::UserPageSink<core::FicSectionStatsTemporaryToken> sink;
 
         QHash<int, QList<WebPage>>pages;
         int counter = 0;

src/tasks/humor_task_processor.cpp

@@ -86,8 +86,8 @@ void HumorProcessor::CreateListOfHumorCandidates(QList<core::AuthorPtr > authors
     QSqlDatabase db = QSqlDatabase::database();
     database::Transaction transaction(db);
 
-    QHash<int, std::array<double, 21>> authorGenreHash;
-    QHash<int, std::array<double, 21>> ficGenreHash;
+    QHash<int, std::array<double, 22>> authorGenreHash;
+    QHash<int, std::array<double, 22>> ficGenreHash;
     QHash<int, double> reviewRatios;
     TimedAction getReviewRatios("get review ratios", [&](){
         reviewRatios = fanficsInterface->GetDoubleValueHashForFics("reviewstofavourites");
@@ -217,7 +217,7 @@ void HumorProcessor::CreateRecListOfHumorProfiles(QList<core::AuthorPtr> authors
 {
     database::Transaction transaction(db);
 
-    QHash<int, std::array<double, 21>> authorGenreHash;
+    QHash<int, std::array<double, 22>> authorGenreHash;
 
     TimedAction getGenres("getGenres", [&](){
         authorGenreHash = authorsInterface->GetListGenreData();

src/tasks/slash_task_processor.cpp

@@ -33,15 +33,13 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 SlashProcessor::SlashProcessor(QSqlDatabase db,
                                QSharedPointer<interfaces::Fanfics> fanficInterface,
                                QSharedPointer<interfaces::Fandoms> fandomsInterface,
-                               QSharedPointer<interfaces::PageTask> pageInterface,
                                QSharedPointer<interfaces::Authors> authorsInterface,
                                QSharedPointer<interfaces::RecommendationLists> recsInterface,
                                QSharedPointer<database::IDBWrapper> dbInterface, QObject *obj) : QObject(obj)
 
 {
     this->fanficsInterface = fanficInterface;
     this->fandomsInterface = fandomsInterface;
-    this->pageInterface = pageInterface;
     this->dbInterface = dbInterface;
     this->authorsInterface = authorsInterface;
     this->recsInterface = recsInterface;
@@ -131,8 +129,6 @@ void SlashProcessor::CreateListOfSlashCandidates(double neededNotslashMatchesCoe
     processNotSlash.run();
     QList<int> intersection;
     intersection.reserve(50000);
-    //    CommonRegex rx;
-    //    rx.Init();
 
     // sufficient matches depends on if a fic is present in lists 0 and 1
     // 0 - 2 matches or more requires 7 non slash
@@ -151,7 +147,7 @@ void SlashProcessor::CreateListOfSlashCandidates(double neededNotslashMatchesCoe
                 sufficientMatchesCount = 7;
             else if(slashFics[1][fic] >= 5)
                 sufficientMatchesCount = 5;
-            //auto ficPtr = env.interfaces.fanfics->GetFicById(fic);
+
             bool soleTMatch = (TRepo.contains(fic) && slashFics[2][fic]==1);
             bool cantTellReliably = slashFics[2][fic]==1 && slashFics[1][fic] == 0;
             bool sufficientMatches = notSlashFics[fic] >= static_cast<double>(sufficientMatchesCount)/neededNotslashMatchesCoeff;
@@ -211,18 +207,25 @@ void SlashProcessor::DoFullCycle(QSqlDatabase db, int passCount)
     auto authors = authorsInterface->GetAllAuthors("ffn", true);
     {
         database::Transaction transaction(db);
+        qDebug() << "Assigning metainformation for first pass";
         AssignSlashKeywordsMetaInfomation(db);
+        qDebug() << "Calculating statistics for first pass";
         authorsInterface->CalculateSlashStatisticsPercentages("keywords_pass_result");
+        qDebug() << "Calculated statistics for first pass";
         transaction.finalize();
     }
-
+    qDebug() << "Starting iterations";
     for(int i = 1; i < passCount+1; i++)
     {
+        qDebug() << "Iteration: " << i;
         {
             database::Transaction transaction(db);
             auto authors = authorsInterface->GetAllAuthors("ffn", true);
+            qDebug() << "Calculating statistics for pass";
             CreateListOfSlashCandidates(i, authors);
+            qDebug() << "Assigning iteration";
             fanficsInterface->AssignIterationOfSlash("pass_" + QString::number(i));
+            qDebug() << "Calculating in database";
             authorsInterface->CalculateSlashStatisticsPercentages("pass_" + QString::number(i));
             transaction.finalize();
             lastI = i;