Made the IdTable class use the LimitedAllocator

- Integrated the memory Limit into the complete project.
- Required minor adaptations for a lot of Unit tests.

src/ServerMain.cpp

@@ -27,6 +27,7 @@ using std::vector;
 struct option options[] = {{"help", no_argument, NULL, 'h'},
                            {"index", required_argument, NULL, 'i'},
                            {"worker-threads", required_argument, NULL, 'j'},
+                           {"memory-for-queries", required_argument, NULL, 'm'},
                            {"on-disk-literals", no_argument, NULL, 'l'},
                            {"port", required_argument, NULL, 'p'},
                            {"no-patterns", no_argument, NULL, 'P'},
@@ -48,6 +49,11 @@ void printUsage(char* execName) {
        << "The location of the index files." << endl;
   cout << "  " << std::setw(20) << "p, port" << std::setw(1) << "    "
        << "The port on which to run the web interface." << endl;
+  cout << "  " << std::setw(20) << "m, memory-for-queries" << std::setw(1)
+       << "    "
+       << "The number of GB that may be used by query (intermediate) results, "
+          "including the cache"
+       << endl;
   cout << "  " << std::setw(20) << "no-patterns" << std::setw(1) << "    "
        << "Disable the use of patterns. This disables ql:has-predicate."
        << endl;
@@ -80,10 +86,12 @@ int main(int argc, char** argv) {
   bool usePatterns = true;
   bool enablePatternTrick = true;
 
+  size_t memLimit = MAX_MEM_FOR_QUERIES_IN_GB;
+
   optind = 1;
   // Process command line arguments.
   while (true) {
-    int c = getopt_long(argc, argv, "i:p:j:tauhmlT", options, NULL);
+    int c = getopt_long(argc, argv, "i:p:j:tauhm:lT", options, NULL);
     if (c == -1) break;
     switch (c) {
       case 'i':
@@ -104,6 +112,9 @@ int main(int argc, char** argv) {
       case 'j':
         numThreads = atoi(optarg);
         break;
+      case 'm':
+        memLimit = atoi(optarg);
+        break;
       case 'h':
         printUsage(argv[0]);
         exit(0);
@@ -142,7 +153,7 @@ int main(int argc, char** argv) {
   cout << "Set locale LC_CTYPE to: " << locale << endl;
 
   try {
-    Server server(port, numThreads);
+    Server server(port, numThreads, memLimit * 1 << 30u);
     server.initialize(index, text, usePatterns, enablePatternTrick);
     server.run();
   } catch (const std::exception& e) {

src/SparqlEngineMain.cpp

@@ -149,7 +149,10 @@ int main(int argc, char** argv) {
       index.addTextFromOnDiskIndex();
     }
 
-    QueryExecutionContext qec(index, engine, &cache, &pinnedSizes);
+    ad_utility::LimitedAllocator<Id> allocator{
+        ad_utility::makeAllocationState(MAX_MEM_FOR_QUERIES_IN_GB)};
+
+    QueryExecutionContext qec(index, engine, &cache, &pinnedSizes, allocator);
     if (costFactosFileName.size() > 0) {
       qec.readCostFactorsFromTSVFile(costFactosFileName);
     }

src/WriteIndexListsMain.cpp

@@ -89,7 +89,9 @@ int main(int argc, char** argv) {
     Engine engine;
     SubtreeCache cache(NOF_SUBTREES_TO_CACHE);
     PinnedSizes pinnedSizes;
-    QueryExecutionContext qec(index, engine, &cache, &pinnedSizes);
+    ad_utility::LimitedAllocator<Id> allocator{
+        ad_utility::makeAllocationState(MAX_MEM_FOR_QUERIES_IN_GB)};
+    QueryExecutionContext qec(index, engine, &cache, &pinnedSizes, allocator);
     ParsedQuery q;
     if (!freebase) {
       q = SparqlParser("SELECT ?x WHERE {?x <is-a> <Scientist>}").parse();

src/engine/CountAvailablePredicates.cpp

@@ -152,7 +152,7 @@ void CountAvailablePredicates::computeResult(ResultTable* result) {
     // If the entity exists return the all predicates for that entitity,
     // otherwise return an empty result.
     if (getIndex().getVocab().getId(_subjectEntityName.value(), &entityId)) {
-      IdTable input(1);
+      IdTable input(1, _executionContext->getAllocator());
       input.push_back({entityId});
       int width = input.cols();
       CALL_FIXED_SIZE_1(width, CountAvailablePredicates::computePatternTrick,

src/engine/Filter.cpp

@@ -268,7 +268,7 @@ void Filter::computeResult(ResultTable* result) {
 template <ResultTable::ResultType T, int WIDTH, bool INVERSE>
 void Filter::computeFilterRange(IdTableStatic<WIDTH>* res, size_t lhs,
                                 Id rhs_lower, Id rhs_upper,
-                                const IdTableStatic<WIDTH>& input,
+                                const IdTableView<WIDTH>& input,
                                 shared_ptr<const ResultTable> subRes) const {
   bool lhs_is_sorted =
       subRes->_sortedBy.size() > 0 && subRes->_sortedBy[0] == lhs;

src/engine/Filter.h

@@ -144,7 +144,7 @@ class Filter : public Operation {
    */
   template <ResultTable::ResultType T, int WIDTH, bool INVERSE = false>
   void computeFilterRange(IdTableStatic<WIDTH>* res, size_t lhs, Id rhs_lower,
-                          Id rhs_upper, const IdTableStatic<WIDTH>& input,
+                          Id rhs_upper, const IdTableView<WIDTH>& input,
                           shared_ptr<const ResultTable> subRes) const;
 
   template <int WIDTH>

src/engine/IdTable.h

@@ -10,25 +10,28 @@
 #include <cstring>
 #include <initializer_list>
 #include <ostream>
+
 #include "../global/Id.h"
+#include "../util/LimitedAllocator.h"
 #include "../util/Log.h"
 
 namespace detail {
 // The actual data storage of the Id Tables, basically a wrapper around a
 // std::vector<Id>
-template<typename Alloc>
+template <typename Alloc>
 struct IdTableVectorWrapper {
   static constexpr bool ManagesStorage = true;  // is able to grow/allocate
   std::vector<Id, Alloc> _data;
 
-  IdTableVectorWrapper(Alloc a) : _data{a}{};
+  IdTableVectorWrapper(Alloc a) : _data{a} {};
 
   // construct from c-style array by copying the content
-  explicit IdTableVectorWrapper(const Id* const ptr, size_t sz, Alloc a) : _data{a}{
+  explicit IdTableVectorWrapper(const Id* const ptr, size_t sz, Alloc a)
+      : _data{a} {
     _data.assign(ptr, ptr + sz);
   }
 
-  Alloc getAllocator() const { return _data.get_allocator();}
+  Alloc getAllocator() const { return _data.get_allocator(); }
 
   // unified interface to the data
   Id* data() noexcept { return _data.data(); }
@@ -40,7 +43,7 @@ struct IdTableVectorWrapper {
 // similar interface to the IdTableVectorWrapper but doesn't own storage, used
 // for cheap to copy const views into IdTables
 // We still need the allocator information to be able to copy to a owning vector
-template<typename Alloc>
+template <typename Alloc>
 struct IdTableViewWrapper {
   static constexpr bool ManagesStorage =
       false;  // is not able to grow and allocate
@@ -54,7 +57,9 @@ struct IdTableViewWrapper {
 
   // construct as a view into an owning VectorWrapper
   explicit IdTableViewWrapper(const IdTableVectorWrapper<Alloc>& rhs) noexcept
-      : _data(rhs.data()), _size(rhs._data.size()) ,_alloc{rhs.getAllocator()} {}
+      : _data(rhs.data()),
+        _size(rhs._data.size()),
+        _alloc{rhs.getAllocator()} {}
 
   // convert to an owning VectorWrapper by making a copy. Explicit since
   // expensive
@@ -244,11 +249,11 @@ class IdTableImpl {
   size_t _size = 0;
   size_t _capacity = 0;
   DATA _data;  // something that lets us access a (const) Id*, might or might
-               // not own its storage
+  // not own its storage
 
-  template <int INCOLS, typename INDATA>
+  template <int INCOLS, typename INDATA, typename>
   friend class IdTableImpl;  // make the conversions work from static to dynamic
-                             // etc.
+  // etc.
 
   // these constructors are protected so they can only be used by the
   // moveToStatic and moveToStaticView functions of the IdTableStatic class
@@ -269,7 +274,7 @@ class IdTableImpl {
   }
 
  public:
-  IdTableImpl() = default;
+  IdTableImpl(Alloc al) : _data{al} {};
 
   using const_row_type = const std::array<Id, COLS>;
   using row_type = const std::array<Id, COLS>;
@@ -347,8 +352,8 @@ class ConstRow final {
  * This provides access to a single row of a Table. The class can optionally
  * manage its own data, allowing for it to be swappable (otherwise swapping
  * two rows during e.g. a std::sort would lead to bad behaviour).
- * Rows currently do not pass down the allocator information, but there are only few
- * of them typically.
+ * Rows currently do not pass down the allocator information, but there are only
+ *few of them typically.
  **/
 class Row {
   Id* _data;
@@ -635,7 +640,7 @@ class IdTableDynamicIterator {
 // at runtime) IdTable
 template <typename DATA, typename Alloc>
 class IdTableImpl<0, DATA, Alloc> {
-  template <int INCOLS, typename INDATA>
+  template <int INCOLS, typename INDATA, typename INALLOC>
   friend class IdTableImpl;
 
  protected:
@@ -645,7 +650,7 @@ class IdTableImpl<0, DATA, Alloc> {
   size_t _cols = 0;
 
  public:
-  IdTableImpl() = default;
+  IdTableImpl(Alloc al) : _data{al} {};
 
  protected:
   template <int INCOLS, typename INDATA>
@@ -704,7 +709,7 @@ class IdTableImpl<0, DATA, Alloc> {
 template <int COLS, typename DATA, typename Alloc>
 class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
   // Make all other instantiations of this template friends of this.
-  template <int, typename>
+  template <int, typename, typename>
   friend class IdTableTemplated;
 
  protected:
@@ -721,7 +726,6 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
   using typename Base::row_reference;
   static constexpr float GROWTH_FACTOR = 1.5;
   // the allocator
-  Alloc _allocator;
 
  public:
   using typename Base::const_iterator;
@@ -740,7 +744,9 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
 
   IdTableTemplated(Alloc al = Alloc{}) : Base{al} {};
 
-  IdTableTemplated(size_t cols) { setCols(cols); }
+  IdTableTemplated(size_t cols, Alloc al = Alloc{}) : Base{al} {
+    setCols(cols);
+  }
 
   virtual ~IdTableTemplated() = default;
 
@@ -753,6 +759,8 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
   // move assignment
   IdTableTemplated& operator=(IdTableTemplated&& other) noexcept = default;
 
+  Alloc getAllocator() const { return _data.getAllocator(); }
+
  protected:
   // internally convert between "dynamic" and "static" mode and possibly
   // convert from Owning to non owning storage, is used in the conversion
@@ -762,7 +770,8 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
       : Base(o) {}
 
   template <int INCOLS, typename INDATA>
-  explicit IdTableTemplated(IdTableTemplated<INCOLS, INDATA, Alloc>&& o) noexcept
+  explicit IdTableTemplated(
+      IdTableTemplated<INCOLS, INDATA, Alloc>&& o) noexcept
       : Base(std::move(o)) {}
 
  public:
@@ -892,7 +901,8 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
 
   template <typename INDATA, bool ManagesStorage = DATA::ManagesStorage,
             typename = std::enable_if_t<ManagesStorage>>
-  void push_back(const IdTableTemplated<COLS, INDATA, Alloc>& init, size_t row) {
+  void push_back(const IdTableTemplated<COLS, INDATA, Alloc>& init,
+                 size_t row) {
     assert(init._cols == _cols);
     if (_size + 1 >= _capacity) {
       grow();
@@ -1027,7 +1037,8 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
    * columns this table already has or an assertion will fail
    */
   template <int NEW_COLS>
-  const IdTableTemplated<NEW_COLS, IdTableViewWrapper<Alloc>, Alloc> asStaticView() const {
+  const IdTableTemplated<NEW_COLS, IdTableViewWrapper<Alloc>, Alloc>
+  asStaticView() const {
     return IdTableTemplated<NEW_COLS, IdTableViewWrapper<Alloc>, Alloc>(
         *static_cast<const IdTableTemplated<COLS, DATA, Alloc>*>(this));
   };
@@ -1041,7 +1052,8 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
    */
   template <bool ManagesStorage = DATA::ManagesStorage,
             typename = std::enable_if_t<!ManagesStorage>>
-  const IdTableTemplated<COLS, IdTableVectorWrapper<Alloc>, Alloc> clone() const {
+  const IdTableTemplated<COLS, IdTableVectorWrapper<Alloc>, Alloc> clone()
+      const {
     return IdTableTemplated<COLS, IdTableVectorWrapper<Alloc>, Alloc>(*this);
   };
 
@@ -1071,8 +1083,8 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
   }
 
   // Support for ostreams
-  friend std::ostream& operator<<(std::ostream& out,
-                                  const IdTableTemplated<COLS, DATA, Alloc>& table) {
+  friend std::ostream& operator<<(
+      std::ostream& out, const IdTableTemplated<COLS, DATA, Alloc>& table) {
     out << "IdTable(" << ((void*)table.data()) << ") with " << table.size()
         << " rows and " << table.cols() << " columns" << std::endl;
     for (size_t row = 0; row < table.size(); row++) {
@@ -1090,14 +1102,15 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
 /// The general IdTable class. Can be modified and owns its data. If COLS > 0,
 /// COLS specifies the compile-time number of columns COLS == 0 means "runtime
 /// number of cols"
-template <int COLS, typename Alloc = std::allocator<Id>>
+template <int COLS, typename Alloc = ad_utility::LimitedAllocator<Id>>
 using IdTableStatic =
     detail::IdTableTemplated<COLS, detail::IdTableVectorWrapper<Alloc>, Alloc>;
 
 // the "runtime number of cols" variant
-template <typename Alloc = std::allocator<Id>>
-using IdTable = IdTableStatic<0, Alloc>;
+// template <typename Alloc = ad_utility::LimitedAllocator<Id>>
+using IdTable = IdTableStatic<0, ad_utility::LimitedAllocator<Id>>;
 
 /// A constant view into an IdTable that does not own its data
-template <int COLS, typename Alloc = std::allocator<Id>>
-using IdTableView = detail::IdTableTemplated<COLS, detail::IdTableViewWrapper<Alloc>, Alloc>;
+template <int COLS, typename Alloc = ad_utility::LimitedAllocator<Id>>
+using IdTableView =
+    detail::IdTableTemplated<COLS, detail::IdTableViewWrapper<Alloc>, Alloc>;

src/engine/Join.cpp

@@ -335,7 +335,7 @@ void Join::doComputeJoinWithFullScanDummyLeft(const IdTable& ndr,
     } else {
       // Do a scan.
       LOG(TRACE) << "Inner scan with ID: " << currentJoinId << endl;
-      IdTable jr(2);
+      IdTable jr(2, _executionContext->getAllocator());
       scan(currentJoinId, &jr);
       LOG(TRACE) << "Got #items: " << jr.size() << endl;
       // Build the cross product.
@@ -348,7 +348,7 @@ void Join::doComputeJoinWithFullScanDummyLeft(const IdTable& ndr,
   }
   // Do the scan for the final element.
   LOG(TRACE) << "Inner scan with ID: " << currentJoinId << endl;
-  IdTable jr(2);
+  IdTable jr(2, _executionContext->getAllocator());
   scan(currentJoinId, &jr);
   LOG(TRACE) << "Got #items: " << jr.size() << endl;
   // Build the cross product.
@@ -376,7 +376,7 @@ void Join::doComputeJoinWithFullScanDummyRight(const IdTable& ndr,
     } else {
       // Do a scan.
       LOG(TRACE) << "Inner scan with ID: " << currentJoinId << endl;
-      IdTable jr(2);
+      IdTable jr(2, _executionContext->getAllocator());
       scan(currentJoinId, &jr);
       LOG(TRACE) << "Got #items: " << jr.size() << endl;
       // Build the cross product.
@@ -389,7 +389,7 @@ void Join::doComputeJoinWithFullScanDummyRight(const IdTable& ndr,
   }
   // Do the scan for the final element.
   LOG(TRACE) << "Inner scan with ID: " << currentJoinId << endl;
-  IdTable jr(2);
+  IdTable jr(2, _executionContext->getAllocator());
   scan(currentJoinId, &jr);
   LOG(TRACE) << "Got #items: " << jr.size() << endl;
   // Build the cross product.

src/engine/Operation.cpp

@@ -57,9 +57,10 @@ shared_ptr<const ResultTable> Operation::getResult(bool isRoot) {
       _executionContext->_pinSubtrees || pinChildIndexScanSizes;
   LOG(TRACE) << "Check cache for Operation result" << endl;
   LOG(TRACE) << "Using key: \n" << cacheKey << endl;
-  auto [newResult, existingResult] = (pinResult)
-                                         ? cache.tryEmplacePinned(cacheKey)
-                                         : cache.tryEmplace(cacheKey);
+  auto [newResult, existingResult] =
+      (pinResult)
+          ? cache.tryEmplacePinned(cacheKey, _executionContext->getAllocator())
+          : cache.tryEmplace(cacheKey, _executionContext->getAllocator());
 
   if (pinChildIndexScanSizes) {
     auto lock = getExecutionContext()->getPinnedSizes().wlock();
@@ -73,6 +74,9 @@ shared_ptr<const ResultTable> Operation::getResult(bool isRoot) {
 
   if (newResult) {
     LOG(TRACE) << "Not in the cache, need to compute result" << endl;
+    LOG(DEBUG) << "Free Megabytes in the limit before computing the result:"
+               << (_executionContext->getAllocator().numFreeBytes() >> 20)
+               << std::endl;
     // Passing the raw pointer here is ok as the result shared_ptr remains
     // in scope
     try {