tuning bm25 parameters, setting sort

openzim · Feb 13, 2021 · 95fe273 · 95fe273
1 parent 86c62d4
commit 95fe273
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 0 deletions.
diff --git a/src/search.cpp b/src/search.cpp
@@ -353,6 +353,7 @@ Search::iterator Search::begin() const {
     }
     Xapian::Query query;
     try {
+      // flags = Xapian::QueryParser::FLAG_PHRASE;
       query = queryParser->parse_query(this->query, flags, prefix);
     } catch (Xapian::QueryParserError& e) {
       estimated_matches_number = 0;
@@ -379,6 +380,9 @@ Search::iterator Search::begin() const {
         }
     }
 
+    // Tune BM25 to reduce wdf parameter and set sort order
+    enquire.set_weighting_scheme(Xapian::BM25Weight(0.001,0,1,1,0.5));
+    enquire.set_sort_by_relevance_then_value(valuesmap["titles"], true);
     enquire.set_query(query);
 
 #if WITH_LEV

diff --git a/src/writer/creator.cpp b/src/writer/creator.cpp
@@ -170,7 +170,23 @@ namespace zim
 
       auto dirent = data->createItemDirent(item.get());
       data->addItemData(dirent, item->getContentProvider(), compressContent);
+<<<<<<< HEAD
       data->handle(dirent, item);
+=======
+
+#if defined(ENABLE_XAPIAN)
+      if (item->getMimeType() == "text/html" && !item->getTitle().empty()) {
+        data->nbIndexItems++;
+        std::string ss = item->getTitle();
+        ss.insert(0, "zzzz ");
+        data->titleIndexer.indexTitle(item->getPath(), ss);
+        std::cout<<"indexing.: "<<ss<<std::endl;
+        if(m_withIndex) {
+          data->taskList.pushToQueue(new IndexTask(item));
+        }
+      }
+#endif
+>>>>>>> tuning bm25 parameters, setting sort
 
       if (data->dirents.size()%1000 == 0) {
         TPROGRESS();
@@ -199,8 +215,18 @@ namespace zim
         TPROGRESS();
       }
 
+<<<<<<< HEAD
       data->handle(dirent);
     }
+=======
+#if defined(ENABLE_XAPIAN)
+      if (!title.empty()) {
+        data->titleIndexer.indexTitle(path, title);
+        std::cout<<"indexing: "<<title<<std::endl;
+      }
+#endif
+     }
+>>>>>>> tuning bm25 parameters, setting sort
 
     void Creator::finishZimCreation()
     {