Fix externalization bug in text index (#534)

Words in the text index should never be externalized. The bug was that they were considered as externalizable at query time, when they looked like a literal, for example "...@...

src/index/Index.Text.cpp

@@ -95,6 +95,10 @@ void Index::addTextFromOnDiskIndex() {
 
 // _____________________________________________________________________________
 size_t Index::passContextFileForVocabulary(string const& contextFile) {
+  // We have to have a configuration by now. Rereading it is necessary,
+  // because we might only add an text index to the already existing KB index.
+  // In this case we also need the correct locale settings etc.
+  readConfiguration();
   LOG(INFO) << "Making pass over ContextFile " << contextFile
             << " for vocabulary." << std::endl;
   ContextFileParser::Line line;
@@ -174,16 +178,12 @@ void Index::passContextFileIntoVector(const string& contextFile,
     } else {
       ++nofWordPostings;
       Id wid;
-#ifndef NDEBUG
       bool ret = _textVocab.getId(line._word, &wid);
       if (!ret) {
-        LOG(INFO) << "ERROR: word " << line._word
-                  << "not found in textVocab. Terminating\n";
+        LOG(ERROR) << "ERROR: word \"" << line._word << "\" "
+                   << "not found in textVocab. Terminating\n";
+        AD_CHECK(false);
       }
-      assert(ret);
-#else
-      _textVocab.getId(line._word, &wid);
-#endif
       wordsInContext[wid] += line._score;
     }
     ++i;

src/index/Vocabulary.cpp

@@ -206,10 +206,19 @@ bool Vocabulary<S, C>::isExternalizedLiteral(const string& word) {
 // _____________________________________________________________________________
 template <class S, class C>
 bool Vocabulary<S, C>::shouldBeExternalized(const string& word) const {
-  if (!isLiteral(word)) {
-    return shouldEntityBeExternalized(word);
+  // TODO<joka921> Completely refactor the Vocabulary on the different
+  // Types, it is a mess.
+
+  // If the string is not compressed, this means that this is a text vocabulary
+  // and thus doesn't support externalization.
+  if constexpr (std::is_same_v<S, CompressedString>) {
+    if (!isLiteral(word)) {
+      return shouldEntityBeExternalized(word);
+    } else {
+      return shouldLiteralBeExternalized(word);
+    }
   } else {
-    return shouldLiteralBeExternalized(word);
+    return false;
   }
 }
 

test/StringSortComparatorTest.cpp

@@ -154,7 +154,7 @@ TEST(StringSortComparatorTest, TripleComponentComparatorTotal) {
 
 // ______________________________________________________________________________________________
 TEST(StringSortComparatorTest, SimpleStringComparator) {
-  SimpleStringComparator comp("en", "US", false);
+  SimpleStringComparator comp("en", "US", true);
 
   // strange casings must not affect order
   ASSERT_TRUE(comp("ALPHA", "beta"));
@@ -177,4 +177,7 @@ TEST(StringSortComparatorTest, SimpleStringComparator) {
 
   // something is not smaller thant itself
   ASSERT_FALSE(comp("beta", "beta"));
+
+  ASSERT_TRUE(comp("\"@u2", "@u2"));
+  ASSERT_FALSE(comp("@u2", "\"@u2"));
 }