Two improvements for the IndexBuilding

src/index/ExternalVocabulary.cpp

@@ -8,6 +8,7 @@
 
 #include "../parser/RdfEscaping.h"
 #include "../parser/Tokenizer.h"
+#include "../util/BufferedVector.h"
 #include "../util/Log.h"
 
 // _____________________________________________________________________________
@@ -71,7 +72,8 @@ void ExternalVocabulary<Comp>::buildFromTextFile(const string& textFileName,
   _file.open(outFileName.c_str(), "w");
   std::ifstream infile(textFileName);
   AD_CHECK(infile.is_open());
-  vector<off_t> offsets;
+  ad_utility::BufferedVector<off_t> offsets(
+      1'000'000'000, textFileName + ".offset.tmp.buffer");
   off_t currentOffset = 0;
   _size = 0;
   std::string word;

src/index/Index.cpp

@@ -252,6 +252,7 @@ VocabularyData Index::passFileForVocabulary(const string& filename,
     LOG(INFO) << "Merging temporary vocabulary for prefix compression";
     {
       VocabularyMerger m;
+      m._ignoreExternalVocabulary = true;
       m.mergeVocabulary(_onDiskBase + TMP_BASENAME_COMPRESSION, numFiles,
                         std::less<>());
       LOG(INFO) << "Finished merging additional Vocabulary.";

src/index/VocabularyGenerator.h

@@ -28,6 +28,7 @@ using TripleVec = stxxl::vector<array<Id, 3>>;
  */
 class VocabularyMerger {
  public:
+  bool _ignoreExternalVocabulary = false;
   // result of a call to mergeVocabulary
   struct VocMergeRes {
     size_t _numWordsTotal;   // that many distinct words were found (size of the

src/index/VocabularyGeneratorImpl.h

@@ -76,6 +76,13 @@ VocabularyMerger::VocMergeRes VocabularyMerger::mergeVocabulary(
 
   // start k-way merge
   while (!queue.empty()) {
+    // for the prefix compression vocabulary, we don't need the external
+    // vocabulary
+    if (_ignoreExternalVocabulary &&
+        queue.top()._value >= EXTERNALIZED_LITERALS_PREFIX) {
+      break;
+    }
+
     // accumulated the globally ordered queue words in a buffer.
     sortedBuffer.push_back(std::move(queue.top()));
     queue.pop();