Experiment alternative storage

libs/CMakeLists.txt

@@ -336,6 +336,11 @@ mark_as_advanced(MGCXX_INCLUDE_DIR TANTIVY_TEXT_SEARCH_LIBRARY MGCXX_TEXT_SEARCH
 import_library(tantivy_text_search STATIC ${TANTIVY_TEXT_SEARCH_LIBRARY} ${MGCXX_INCLUDE_DIR} mgcxx-proj)
 import_library(mgcxx_text_search STATIC ${MGCXX_TEXT_SEARCH_LIBRARY} ${MGCXX_INCLUDE_DIR} mgcxx-proj)
 
-
 # setup strong_type (cmake sub_directory tolerant)
 add_subdirectory(strong_type EXCLUDE_FROM_ALL)
+
+# Setup GraphAr
+import_external_library(graphar SHARED
+  ${CMAKE_CURRENT_SOURCE_DIR}/graphar/cpp/lib/libgraphar.so
+  ${CMAKE_CURRENT_SOURCE_DIR}/graphar/cpp/include
+  SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/graphar/cpp)

libs/setup.sh

@@ -157,6 +157,7 @@ declare -A primary_urls=(
   ["nuraft"]="http://$local_cache_host/git/NuRaft.git"
   ["asio"]="http://$local_cache_host/git/asio.git"
   ["mgcxx"]="http://$local_cache_host/git/mgcxx.git"
+  ["graphar"]="http://$local_cache_host/git/GraphAr.git"
   ["strong_type"]="http://$local_cache_host/git/strong_type.git"
 )
 
@@ -191,6 +192,7 @@ declare -A secondary_urls=(
   ["asio"]="https://github.com/chriskohlhoff/asio.git"
   ["mgcxx"]="https://github.com/memgraph/mgcxx.git"
   ["strong_type"]="https://github.com/rollbear/strong_type.git"
+  ["graphar"]="https://github.com/apache/incubator-graphar.git"
 )
 
 # antlr
@@ -337,3 +339,7 @@ repo_clone_try_double "${primary_urls[mgcxx]}" "${secondary_urls[mgcxx]}" "mgcxx
 # strong_type v14
 strong_type_ref="v14"
 repo_clone_try_double "${primary_urls[strong_type]}" "${secondary_urls[strong_type]}" "strong_type" "$strong_type_ref"
+
+# GraphAr 2024-07-02
+graphar_tag="v0.12.0"
+repo_clone_try_double "${primary_urls[graphar]}" "${secondary_urls[graphar]}" "graphar" "$graphar_tag" true

src/CMakeLists.txt

@@ -9,7 +9,7 @@ add_subdirectory(kvstore)
 add_subdirectory(telemetry)
 add_subdirectory(communication)
 add_subdirectory(memory)
-add_subdirectory(storage/v2)
+add_subdirectory(storage)
 add_subdirectory(integrations)
 add_subdirectory(query)
 add_subdirectory(glue)

src/flags/experimental.cpp

@@ -48,6 +48,7 @@ auto const canonicalize_string = [](auto &&rng) {
 namespace memgraph::flags {
 
 auto const mapping = std::map{std::pair{"text-search"sv, Experiments::TEXT_SEARCH},
+                              std::pair{"alternative-storage"sv, Experiments::ALTERNATIVE_STORAGE},
                               std::pair{"high-availability"sv, Experiments::HIGH_AVAILABILITY}};
 
 auto ExperimentsInstance() -> Experiments & {

src/flags/experimental.hpp

@@ -26,6 +26,7 @@ namespace memgraph::flags {
 enum class Experiments : uint8_t {
   TEXT_SEARCH = 1 << 0,
   HIGH_AVAILABILITY = 1 << 1,
+  ALTERNATIVE_STORAGE = 1 << 2,
 };
 
 bool AreExperimentsEnabled(Experiments experiments);

src/memgraph.cpp

@@ -557,6 +557,7 @@ int main(int argc, char **argv) {
                                            *auth_, FLAGS_data_recovery_on_startup
 #endif
   );
+  auto custom_storage = std::make_unique<memgraph::storage::custom_storage::Storage>();
 
   // Note: Now that all system's subsystems are initialised (dbms & auth)
   //       We can now initialise the recovery of replication (which will include those subsystems)
@@ -590,6 +591,8 @@ int main(int argc, char **argv) {
 
   auto &interpreter_context_ = memgraph::query::InterpreterContextHolder::GetInstance();
   MG_ASSERT(db_acc, "Failed to access the main database");
+  // TODO(gitbuda): Init moved here because tests are constructing the interpreter context.
+  interpreter_context_.custom_storage = custom_storage.get();
 
   memgraph::query::procedure::gModuleRegistry.SetModulesDirectory(memgraph::flags::ParseQueryModulesDirectory(),
                                                                   FLAGS_data_directory);

src/query/CMakeLists.txt

@@ -78,6 +78,7 @@ target_sources(mg-query
     string_helpers.hpp
 )
 
+add_subdirectory(custom_cursors)
 
 target_link_libraries(mg-query
     PUBLIC
@@ -96,6 +97,7 @@ target_link_libraries(mg-query
     mg::system
     mg-flags
     mg-dbms
+    mg_custom_cursors
     mg-events
     PRIVATE
     mg-module-support

src/query/context.hpp

@@ -22,6 +22,7 @@
 #include "query/trigger.hpp"
 #include "utils/async_timer.hpp"
 
+#include "storage/custom_storage/storage.hpp"
 #include "query/frame_change.hpp"
 #include "query/hops_limit.hpp"
 
@@ -79,6 +80,7 @@ inline std::vector<storage::LabelId> NamesToLabels(const std::vector<std::string
 }
 
 struct ExecutionContext {
+  memgraph::storage::custom_storage::Storage *custom_storage{nullptr};
   DbAccessor *db_accessor{nullptr};
   SymbolTable symbol_table;
   EvaluationContext evaluation_context;

src/query/custom_cursors/CMakeLists.txt

@@ -0,0 +1,9 @@
+set(mg_custom_cursors_sources
+  create_node.cpp
+  once.cpp
+  scanall.cpp
+  produce.cpp
+)
+add_library(mg_custom_cursors STATIC ${mg_custom_cursors_sources})
+# TODO(gitbuda): Having mg-storage-v2 here is broken.
+target_link_libraries(mg_custom_cursors mg-utils mg-storage-v2 mg_custom_storage mg-query)

src/query/custom_cursors/all.hpp

@@ -0,0 +1,17 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include "query/custom_cursors/create_node.hpp"
+#include "query/custom_cursors/once.hpp"
+#include "query/custom_cursors/produce.hpp"
+#include "query/custom_cursors/scanall.hpp"

src/query/custom_cursors/bfs.hpp

@@ -0,0 +1,14 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+// TODO(gitbuda): To make BFS fast -> global cache required.

src/query/custom_cursors/create_edge.hpp

@@ -0,0 +1,15 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+// TODO(gitbuda): To make this Cypher CREATE edge fast -> add global cache.
+// TODO(gitbuda): To make create edge fast -> introduce a new semantic.

src/query/custom_cursors/create_node.cpp

@@ -0,0 +1,83 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "query/custom_cursors/create_node.hpp"
+#include "query/context.hpp"
+#include "query/custom_cursors/utils.hpp"
+#include "query/interpret/eval.hpp"
+#include "query/interpret/frame.hpp"
+#include "query/plan/operator.hpp"
+#include "query/plan/scoped_profile.hpp"
+#include "storage/custom_storage/types.hpp"
+#include "utils/logging.hpp"
+
+namespace memgraph::query::custom_cursors {
+
+struct QueryVertex {};
+
+// Creates a vertex on this GraphDb. Returns a reference to vertex placed on the
+// frame.
+QueryVertex CreateVertex(const plan::NodeCreationInfo &node_info, Frame *frame, ExecutionContext &context) {
+  auto &dba = *context.db_accessor;
+  if (node_info.labels.size() != 1) {
+    throw QueryRuntimeException(
+        "0 or multiple labels not yet supported under CreateNode. You have to provide exactly 1 lable for any given "
+        "vertex/node.");
+  }
+  // NOTE: Evaluator should use the latest accessors, as modified in this query, when
+  // setting properties on new nodes.
+  // NOTE: Evaluator is using query::DBAccessor of default storage mode (IN_MEM_TX), for props mapping & storage mode.
+  ExpressionEvaluator evaluator(frame, context.symbol_table, context.evaluation_context, context.db_accessor,
+                                storage::View::NEW);
+  // TODO: PropsSetChecked allocates a PropertyValue, make it use context.memory
+  // when we update PropertyValue with custom allocator.
+  std::map<storage::PropertyId, storage::PropertyValue> properties;
+  if (const auto *node_info_properties = std::get_if<plan::PropertiesMapList>(&node_info.properties)) {
+    for (const auto &[key, value_expression] : *node_info_properties) {
+      properties.emplace(key, value_expression->Accept(evaluator));
+    }
+  } else {
+    auto property_map = evaluator.Visit(*std::get<ParameterLookup *>(node_info.properties));
+    for (const auto &[key, value] : property_map.ValueMap()) {
+      properties.emplace(dba.NameToProperty(key), value);
+    }
+  }
+  // TODO(gitbuda): Put vertex on the frame. (*frame)[node_info.symbol] = new_node;
+  // (*frame)[node_info.symbol] = new_node;
+  // return (*frame)[node_info.symbol].ValueVertex();
+
+  // TODO(gitbuda): node_info.labels change type -> add the transformation.
+  // auto new_node = memgraph::storage::custom_storage::Vertex{.labels = node_info.labels, .properties = properties};
+  auto new_node = memgraph::storage::custom_storage::Vertex{.labels = {}, .properties = properties};
+  auto *vertex_ptr = context.custom_storage->AddVertex(std::move(new_node));
+  SPDLOG_WARN("{}", context.custom_storage->VerticesNo());
+  return QueryVertex{};
+}
+
+CreateNodeCursor::CreateNodeCursor(const plan::CreateNode &logical_operator, plan::UniqueCursorPtr input_cursor)
+    : logical_operator_(logical_operator), input_cursor_(std::move(input_cursor)) {}
+
+bool CreateNodeCursor::Pull(Frame &frame, ExecutionContext &context) {
+  utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
+  memgraph::query::plan::ScopedProfile profile{ComputeProfilingKey(this), "CreateNode", &context};
+  SPDLOG_WARN("CreateNodeCursor::Pull");
+  if (input_cursor_->Pull(frame, context)) {
+    CreateVertex(logical_operator_.node_info_, &frame, context);
+    return true;
+  }
+  return false;
+}
+
+void CreateNodeCursor::Shutdown() { input_cursor_->Shutdown(); }
+
+void CreateNodeCursor::Reset() { input_cursor_->Reset(); }
+
+}  // namespace memgraph::query::custom_cursors

src/query/custom_cursors/create_node.hpp

@@ -0,0 +1,39 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include "query/plan/cursor.hpp"
+
+// TASKS:
+//   1. check label creation access rule
+//   2. create vertex
+//   3. inform the trigger
+
+namespace memgraph::query::plan {
+class CreateNode;
+}  // namespace memgraph::query::plan
+
+namespace memgraph::query::custom_cursors {
+
+class CreateNodeCursor : public memgraph::query::plan::Cursor {
+ public:
+  explicit CreateNodeCursor(const plan::CreateNode &logical_operator, plan::UniqueCursorPtr input_cursor);
+  bool Pull(Frame &frame, ExecutionContext &context) override;
+  void Shutdown() override;
+  void Reset() override;
+
+ private:
+  const plan::CreateNode &logical_operator_;
+  const plan::UniqueCursorPtr input_cursor_;
+};
+
+}  // namespace memgraph::query::custom_cursors

src/query/custom_cursors/once.cpp

@@ -0,0 +1,36 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "query/custom_cursors/once.hpp"
+#include "query/context.hpp"
+#include "query/custom_cursors/utils.hpp"
+#include "query/interpret/frame.hpp"
+#include "query/plan/scoped_profile.hpp"
+#include "utils/logging.hpp"
+
+namespace memgraph::query::custom_cursors {
+
+bool OnceCursor::Pull(Frame & /*unused*/, ExecutionContext &context) {
+  utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
+  memgraph::query::plan::ScopedProfile profile{ComputeProfilingKey(this), "Once", &context};
+  SPDLOG_WARN("OnceCursor::Pull");
+  if (!did_pull_) {
+    did_pull_ = true;
+    return true;
+  }
+  return false;
+}
+
+void OnceCursor::Shutdown() {}
+
+void OnceCursor::Reset() { did_pull_ = false; }
+
+}  // namespace memgraph::query::custom_cursors

src/query/custom_cursors/once.hpp

@@ -0,0 +1,29 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include "query/plan/cursor.hpp"
+
+namespace memgraph::query::custom_cursors {
+
+class OnceCursor : public memgraph::query::plan::Cursor {
+ public:
+  OnceCursor() = default;
+  bool Pull(Frame & /*unused*/, ExecutionContext &context) override;
+  void Shutdown() override;
+  void Reset() override;
+
+ private:
+  bool did_pull_{false};
+};
+
+}  // namespace memgraph::query::custom_cursors

src/query/custom_cursors/produce.cpp

@@ -0,0 +1,34 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "query/custom_cursors/produce.hpp"
+#include "query/context.hpp"
+#include "query/custom_cursors/utils.hpp"
+#include "query/interpret/frame.hpp"
+#include "spdlog/spdlog.h"
+#include "utils/logging.hpp"
+
+namespace memgraph::query::custom_cursors {
+
+ProduceCursor::ProduceCursor(plan::UniqueCursorPtr input_cursor) : input_cursor_(std::move(input_cursor)) {}
+
+bool ProduceCursor::Pull(Frame &frame, ExecutionContext &context) {
+  utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
+  memgraph::query::plan::ScopedProfile profile{ComputeProfilingKey(this), "Produce", &context};
+  SPDLOG_WARN("Produce");
+  return input_cursor_->Pull(frame, context);
+}
+
+void ProduceCursor::Shutdown() { input_cursor_->Shutdown(); }
+
+void ProduceCursor::Reset() { input_cursor_->Reset(); }
+
+}  // namespace memgraph::query::custom_cursors