Located Triples

src/engine/Server.cpp

@@ -13,7 +13,10 @@
 #include "engine/ExportQueryExecutionTrees.h"
 #include "engine/QueryPlanner.h"
 #include "global/RuntimeParameters.h"
+#include "index/IndexImpl.h"
+#include "index/LocatedTriples.h"
 #include "util/AsioHelpers.h"
+#include "util/CancellationHandle.h"
 #include "util/MemorySize/MemorySize.h"
 #include "util/OnDestructionDontThrowDuringStackUnwinding.h"
 #include "util/ParseableDuration.h"
@@ -582,6 +585,141 @@ Awaitable<void> Server::sendStreamableResponse(
   }
 }
 
+// _____________________________________________________________________________
+// TODO: move somewhere else?
+nlohmann::json Server::executeUpdateQuery(
+    const ParsedQuery& query, const QueryExecutionTree& qet,
+    const ad_utility::Timer& requestTimer,
+    SharedCancellationHandle cancellationHandle, Index& index) {
+  AD_CONTRACT_CHECK(query.hasUpdateClause());
+  parsedQuery::UpdateClause update = query.updateClause();
+
+  ad_utility::Timer step{ad_utility::Timer::Started};
+
+  std::shared_ptr<const Result> res = qet.getResult();
+  auto timeQueryBodyExecution = step.msecs();
+  step.start();
+
+  auto& vocab = qet.getQec()->getIndex().getVocab();
+  LocalVocab& localVocab = index.deltaTriples().localVocab();
+  using IdOrVariable = std::variant<Id, Variable>;
+
+  auto transformSparqlTripleComponent =
+      [&vocab, &localVocab](TripleComponent component) -> IdOrVariable {
+    if (component.isVariable()) {
+      return std::move(component.getVariable());
+    } else {
+      return std::move(component).toValueId(vocab, localVocab);
+    }
+  };
+  auto transformSparqlTripleSimple =
+      [&transformSparqlTripleComponent](SparqlTripleSimple triple) {
+        return std::array<IdOrVariable, 3>{
+            transformSparqlTripleComponent(std::move(triple.s_)),
+            transformSparqlTripleComponent(std::move(triple.p_)),
+            transformSparqlTripleComponent(std::move(triple.o_))};
+      };
+  std::vector<std::array<IdOrVariable, 3>> toInsertTemplates =
+      ad_utility::transform(std::move(update.toInsert_),
+                            transformSparqlTripleSimple);
+  std::vector<std::array<IdOrVariable, 3>> toDeleteTemplates =
+      ad_utility::transform(std::move(update.toDelete_),
+                            transformSparqlTripleSimple);
+  auto timeTemplatePrepration = step.msecs();
+  step.start();
+
+  auto resolveVariable = [](const ConstructQueryExportContext& context,
+                            IdOrVariable idOrVar) -> std::optional<Id> {
+    if (std::holds_alternative<Variable>(idOrVar)) {
+      auto var = std::get<Variable>(std::move(idOrVar));
+      if (!context._variableColumns.contains(var)) {
+        return std::nullopt;
+      } else {
+        return context._res.idTable().operator()(
+            context._row, context._variableColumns.at(var).columnIndex_);
+      }
+    } else if (std::holds_alternative<Id>(idOrVar)) {
+      return std::get<Id>(idOrVar);
+    } else {
+      AD_FAIL();
+    }
+  };
+
+  std::vector<IdTriple<0>> toInsert;
+  std::vector<IdTriple<0>> toDelete;
+  toInsert.reserve(res->idTable().size() * toInsertTemplates.size());
+  toDelete.reserve(res->idTable().size() * toDeleteTemplates.size());
+  // Result size is size(query result) x num template rows
+  // TODO: use ExportQueryExecutionTrees::getRowIndices as iterator
+  for (size_t i : std::views::iota((size_t)0, res->idTable().size())) {
+    ConstructQueryExportContext context{i, *res, qet.getVariableColumns(),
+                                        qet.getQec()->getIndex()};
+    for (const auto& [s, p, o] : toInsertTemplates) {
+      auto subject = resolveVariable(context, s);
+      auto predicate = resolveVariable(context, p);
+      auto object = resolveVariable(context, o);
+      if (!subject.has_value() || !predicate.has_value() ||
+          !object.has_value()) {
+        continue;
+      }
+
+      toInsert.emplace_back(std::array<Id, 3>{
+          subject.value(), predicate.value(), object.value()});
+      cancellationHandle->throwIfCancelled();
+    }
+
+    for (const auto& [s, p, o] : toDeleteTemplates) {
+      auto subject = resolveVariable(context, s);
+      auto predicate = resolveVariable(context, p);
+      auto object = resolveVariable(context, o);
+      if (!subject.has_value() || !predicate.has_value() ||
+          !object.has_value()) {
+        continue;
+      }
+
+      toDelete.emplace_back(std::array<Id, 3>{
+          subject.value(), predicate.value(), object.value()});
+      cancellationHandle->throwIfCancelled();
+    }
+  }
+
+  auto timeMaterializeUpdateTriples = step.msecs();
+  step.start();
+
+  index.deltaTriples().insertTriples(cancellationHandle, std::move(toInsert));
+  index.deltaTriples().deleteTriples(cancellationHandle, std::move(toDelete));
+  auto timeDeltaTriples = step.msecs();
+  step.start();
+
+  nlohmann::json j;
+  j["query"] = query._originalString;
+  j["status"] = "ERROR";
+  j["exception"] =
+      "Not supported: SPARQL 1.1 Update currently not supported by QLever.";
+  j["warnings"] = qet.collectWarnings();
+
+  j["runtimeInformation"]["meta"] = nlohmann::ordered_json(
+      qet.getRootOperation()->getRuntimeInfoWholeQuery());
+  RuntimeInformation runtimeInformation = qet.getRootOperation()->runtimeInfo();
+  runtimeInformation.addLimitOffsetRow(
+      query._limitOffset, std::chrono::milliseconds::zero(), false);
+  runtimeInformation.addDetail("executed-implicitly-during-query-export", true);
+  j["runtimeInformation"]["query_execution_tree"] =
+      nlohmann::ordered_json(runtimeInformation);
+
+  j["time"]["total"] = std::to_string(requestTimer.msecs().count()) + "ms";
+  j["time"]["queryBody"] =
+      std::to_string(timeQueryBodyExecution.count()) + "ms";
+  j["time"]["computeResult"] = j["time"]["total"];
+  j["time"]["templatePreparation"] =
+      std::to_string(timeTemplatePrepration.count()) + "ms";
+  j["time"]["updateTripleMaterialization"] =
+      std::to_string(timeMaterializeUpdateTriples.count()) + "ms";
+  j["time"]["deltaTriples"] = std::to_string(timeDeltaTriples.count()) + "ms";
+
+  return j;
+}
+
 // ____________________________________________________________________________
 boost::asio::awaitable<void> Server::processQuery(
     const ParamValueMap& params, ad_utility::Timer& requestTimer,
@@ -718,6 +856,13 @@ boost::asio::awaitable<void> Server::processQuery(
     plannedQuery.value().parsedQuery_._limitOffset._offset -=
         qet.getRootOperation()->getLimit()._offset;
 
+    if (plannedQuery.value().parsedQuery_.hasUpdateClause()) {
+      nlohmann::basic_json resp = Server::executeUpdateQuery(
+          plannedQuery.value().parsedQuery_, qet, requestTimer,
+          std::move(cancellationHandle), index_);
+      co_return co_await sendJson(std::move(resp), responseStatus);
+    }
+
     // This actually processes the query and sends the result in the requested
     // format.
     switch (mediaType.value()) {

src/engine/Server.h

@@ -233,4 +233,10 @@ class Server {
       ad_utility::MediaType mediaType, const PlannedQuery& plannedQuery,
       const QueryExecutionTree& qet,
       SharedCancellationHandle cancellationHandle) const;
+
+  nlohmann::json executeUpdateQuery(const ParsedQuery& query,
+                                    const QueryExecutionTree& qet,
+                                    const ad_utility::Timer& requestTimer,
+                                    SharedCancellationHandle cancellationHandle,
+                                    Index& index);
 };

src/engine/idTable/IdTable.h

@@ -751,6 +751,17 @@ class IdTableStatic
     *(static_cast<Base*>(this)) = std::move(b);
     return *this;
   }
+
+  // This operator is only for debugging and testing. It returns a
+  // human-readable representation.
+  friend std::ostream& operator<<(std::ostream& os,
+                                  const IdTableStatic& idTable) {
+    os << "{ ";
+    std::ranges::copy(
+        idTable, std::ostream_iterator<columnBasedIdTable::Row<Id>>(os, " "));
+    os << "}";
+    return os;
+  }
 };
 
 // This was previously implemented as an alias (`using IdTable =

src/engine/idTable/IdTableRow.h

@@ -10,6 +10,7 @@
 #include <variant>
 #include <vector>
 
+#include "global/Id.h"
 #include "util/Enums.h"
 #include "util/Exception.h"
 #include "util/Forward.h"
@@ -93,6 +94,17 @@ class Row {
     std::ranges::copy(*this, result.begin());
     return result;
   }
+
+  // This operator is only for debugging and testing. It returns a
+  // human-readable representation.
+  friend std::ostream& operator<<(std::ostream& os, const Row& idTableRow)
+      requires(std::is_same_v<T, Id>) {
+    os << "(";
+    for (size_t i = 0; i < idTableRow.numColumns(); ++i) {
+      os << idTableRow[i] << (i < idTableRow.numColumns() - 1 ? " " : ")");
+    }
+    return os;
+  }
 };
 
 // The following two classes store a reference to a row in the underlying

src/engine/sparqlExpressions/SparqlExpressionValueGetters.cpp

@@ -24,6 +24,7 @@ NumericValue NumericValueGetter::operator()(
       // functions.
       return static_cast<int64_t>(id.getBool());
     case Datatype::Undefined:
+    case Datatype::Sentinel:
     case Datatype::VocabIndex:
     case Datatype::LocalVocabIndex:
     case Datatype::TextRecordIndex:
@@ -49,6 +50,7 @@ auto EffectiveBooleanValueGetter::operator()(
     case Datatype::Bool:
       return id.getBool() ? True : False;
     case Datatype::Undefined:
+    case Datatype::Sentinel:
     case Datatype::BlankNodeIndex:
       return Undef;
     case Datatype::VocabIndex: {
@@ -142,6 +144,7 @@ IntDoubleStr ToNumericValueGetter::operator()(
     ValueId id, [[maybe_unused]] const EvaluationContext* context) const {
   switch (id.getDatatype()) {
     case Datatype::Undefined:
+    case Datatype::Sentinel:
       return std::monostate{};
     case Datatype::Int:
       return id.getInt();

src/global/IdTriple.h

@@ -1,12 +1,70 @@
 // Copyright 2024, University of Freiburg
 // Chair of Algorithms and Data Structures
-// Authors: Hannah Bast <bast@cs.uni-freiburg.de>
+// Authors:
+//    2023 Hannah Bast <bast@cs.uni-freiburg.de>
+//    2024 Julian Mundhahs <mundhahj@tf.uni-freiburg.de>
 
 #pragma once
 
 #include <array>
+#include <ostream>
 
 #include "global/Id.h"
+#include "index/CompressedRelation.h"
 
-// Should we have an own class for this? We need this at several places.
-using IdTriple = std::array<Id, 3>;
+// TODO<qup42> some matching comparison operators between
+// idtriple/permutedtriple would be nice
+template <size_t N = 0>
+struct IdTriple {
+  // The three IDs that define the triple.
+  std::array<Id, 3> ids_;
+  // Some additional payload of the triple, e.g. which graph it belongs to.
+  std::array<Id, N> payload_;
+
+  explicit IdTriple() = default;
+  explicit IdTriple(
+      const CompressedBlockMetadata::PermutedTriple& permutedTriple)
+      : ids_({permutedTriple.col0Id_, permutedTriple.col1Id_,
+              permutedTriple.col2Id_}),
+        payload_(){};
+
+  explicit IdTriple(const std::array<Id, 3>& ids) requires(N == 0)
+      : ids_(ids), payload_(){};
+
+  explicit IdTriple(const std::array<Id, 3>& ids,
+                    const std::array<Id, N>& payload) requires(N != 0)
+      : ids_(ids), payload_(payload){};
+
+  friend std::ostream& operator<<(std::ostream& os, const IdTriple& triple) {
+    os << "IdTriple(";
+    std::ranges::copy(triple.ids_, std::ostream_iterator<Id>(os, ", "));
+    std::ranges::copy(triple.payload_, std::ostream_iterator<Id>(os, ", "));
+    os << ")";
+    return os;
+  }
+
+  auto operator<=>(const IdTriple&) const = default;
+
+  template <typename H>
+  friend H AbslHashValue(H h, const IdTriple& c) {
+    return H::combine(std::move(h), c.ids_);
+  }
+
+  // TODO<qup42>: should this be a `PermutedTriple`?
+  // Permutes the ID of this triple according to the given permutation given by
+  // its keyOrder.
+  IdTriple<N> permute(const std::array<size_t, 3>& keyOrder) const {
+    std::array<Id, 3> newIds{ids_[keyOrder[0]], ids_[keyOrder[1]],
+                             ids_[keyOrder[2]]};
+    if constexpr (N == 0) {
+      return IdTriple<N>(newIds);
+    } else {
+      return IdTriple<N>(newIds, payload_);
+    }
+  }
+
+  CompressedBlockMetadata::PermutedTriple toPermutedTriple() const
+      requires(N == 0) {
+    return {ids_[0], ids_[1], ids_[2]};
+  }
+};

src/global/ValueId.h

@@ -31,7 +31,8 @@ enum struct Datatype {
   Date,
   WordVocabIndex,
   BlankNodeIndex,
-  MaxValue = BlankNodeIndex
+  Sentinel,
+  MaxValue = Sentinel,
   // Note: Unfortunately we cannot easily get the size of an enum.
   // If members are added to this enum, then the `MaxValue`
   // alias must always be equal to the last member,
@@ -61,6 +62,8 @@ constexpr std::string_view toString(Datatype type) {
       return "Date";
     case Datatype::BlankNodeIndex:
       return "BlankNodeIndex";
+    case Datatype::Sentinel:
+      return "Sentinel";
   }
   // This line is reachable if we cast an arbitrary invalid int to this enum
   AD_FAIL();
@@ -112,6 +115,10 @@ class ValueId {
   /// generic code like in the `visit` method.
   struct UndefinedType {};
 
+  // A struct that represent the single sentinel value. See above for why this
+  // is required.
+  struct SentinelType {};
+
  private:
   // The actual bits.
   T _bits;
@@ -174,6 +181,13 @@ class ValueId {
   [[nodiscard]] UndefinedType getUndefined() const noexcept { return {}; }
   bool isUndefined() const noexcept { return *this == makeUndefined(); }
 
+  constexpr static ValueId makeSentinel() noexcept {
+    return addDatatypeBits(std::bit_cast<uint64_t>(0UL), Datatype::Sentinel);
+  }
+
+  [[nodiscard]] SentinelType getSentinel() const noexcept { return {}; }
+  bool isSentinel() const noexcept { return *this == makeSentinel(); }
+
   /// Create a `ValueId` for a double value. The conversion will reduce the
   /// precision of the mantissa of an IEEE double precision floating point
   /// number from 53 to 49 significant bits.
@@ -319,6 +333,8 @@ class ValueId {
         return std::invoke(visitor, getDate());
       case Datatype::BlankNodeIndex:
         return std::invoke(visitor, getBlankNodeIndex());
+      case Datatype::Sentinel:
+        return std::invoke(visitor, getSentinel());
     }
     AD_FAIL();
   }
@@ -329,10 +345,13 @@ class ValueId {
     ostr << toString(id.getDatatype())[0] << ':';
     if (id.getDatatype() == Datatype::Undefined) {
       return ostr << id.getBits();
+    } else if (id.getDatatype() == Datatype::Sentinel) {
+      return ostr << "Sentinel";
     }
 
     auto visitor = [&ostr]<typename T>(T&& value) {
-      if constexpr (ad_utility::isSimilar<T, ValueId::UndefinedType>) {
+      if constexpr (ad_utility::isSimilar<T, ValueId::UndefinedType> ||
+                    ad_utility::isSimilar<T, ValueId::SentinelType>) {
         // already handled above
         AD_FAIL();
       } else if constexpr (ad_utility::isSimilar<T, double> ||