Fixed two unicode-related bugs in the text index.

src/index/Index.Text.cpp

@@ -167,6 +167,8 @@ void Index::passContextFileIntoVector(const string& contextFile,
             LOG(WARN) << "There are more entities not in the KB..."
                       << " suppressing further warnings...\n";
           }
+        } else {
+          entityNotFoundErrorMsgCount++;
         }
       }
     } else {
@@ -189,6 +191,11 @@ void Index::passContextFileIntoVector(const string& contextFile,
       LOG(INFO) << "Lines processed: " << i << '\n';
     }
   }
+  LOG(WARN)
+      << "Number of entity mentions where the entity is not part of the KB: "
+      << entityNotFoundErrorMsgCount << std::endl;
+  LOG(WARN) << "Number of total entity mentions: " << nofEntityPostings
+            << std::endl;
   ++nofContexts;
   addContextToVector(writer, currentContext, wordsInContext, entitiesInContext);
   _textMeta.setNofTextRecords(nofContexts);

src/index/StringSortComparator.h

@@ -336,7 +336,7 @@ class SimpleStringComparator {
     if (cmpRes != 0 || level != Level::TOTAL) {
       return cmpRes;
     }
-    return a.compare(b) < 0;
+    return a.compare(b);
   }
 
   /**
@@ -409,7 +409,7 @@ class TripleComponentComparator {
  public:
   using Level = LocaleManager::Level;
 
-  /**
+  /* mentions*
    * @param lang The language of the locale, e.g. "en" or "de"
    * @param country The country of the locale, e.g. "US" or "CA"
    * @param ignorePunctuationAtFirstLevel If true then spaces/punctuation etc.

src/index/Vocabulary.cpp

@@ -153,7 +153,10 @@ void Vocabulary<S, C>::createFromSet(const ad_utility::HashSet<S>& set) {
   _words.reserve(set.size());
   _words.insert(begin(_words), begin(set), end(set));
   LOG(INFO) << "... sorting ...\n";
-  std::sort(begin(_words), end(_words), _caseComparator);
+  auto totalComparison = [this](const auto& a, const auto& b) {
+    return _caseComparator(a, b, SortLevel::TOTAL);
+  };
+  std::sort(begin(_words), end(_words), totalComparison);
   LOG(INFO) << "Done creating vocabulary.\n";
 }
 
@@ -407,6 +410,9 @@ template <typename S, typename C>
 bool Vocabulary<S, C>::getId(const string& word, Id* id) const {
   if (!shouldBeExternalized(word)) {
     // need the TOTAL level because we want the unique word.
+    if (ad_utility::startsWith(word, "krivokapic")) {
+      LOG(INFO) << word << '\n';
+    }
     *id = lower_bound(word, SortLevel::TOTAL);
     // works for the case insensitive version because
     // of the strict ordering.