[WIP] Introduce grin apis

.devcontainer/devcontainer.json

@@ -0,0 +1,67 @@
+
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/javascript-node
+{
+	"name": "Vineyard",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:latest",
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+    "features": {
+        "ghcr.io/devcontainers/features/common-utils:2":{
+            "installZsh": "true",
+			"configureZshAsDefaultShell": "true",
+			"installOhMyZsh": true,
+            "upgradePackages": "false"
+        }
+    },
+	// Configure tool-specific properties.
+	"customizations": {
+		// Configure properties specific to VS Code.
+		"vscode": {
+			"settings": {},
+			"extensions": [
+				"streetsidesoftware.code-spell-checker",
+				"eamodio.gitlens"
+			]
+		}
+	},
+
+	// Set `remoteUser` to `root` to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "root"
+
+	// Uncomment this to enable C++ and Rust debugging in containers
+	// "capAdd": ["SYS_PTRACE"],
+	// "securityOpt": ["seccomp=unconfined"],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [3000],
+
+	// Use 'portsAttributes' to set default properties for specific forwarded ports. 
+	// More info: https://containers.dev/implementors/json_reference/#port-attributes
+	// "portsAttributes": {
+	// 	"9000": {
+	// 		"label": "Hello Remote World",
+	// 		"onAutoForward": "notify"
+	// 	}
+	// },
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "yarn install"
+
+	// Improve performance
+
+	// Uncomment these to mount a folder to a volume
+	// https://code.visualstudio.com/remote/advancedcontainers/improve-performance#_use-a-targeted-named-volume
+	// "mounts": [
+    // 	"source=${localWorkspaceFolderBasename}-node_modules,target=${containerWorkspaceFolder}/node_modules,type=volume"
+	// ],
+	// "postCreateCommand": "sudo chown graphscope node_modules"
+
+
+	// Uncomment these to use a named volume for your entire source tree
+	// https://code.visualstudio.com/remote/advancedcontainers/improve-performance#_use-a-named-volume-for-your-entire-source-tree
+	// "workspaceMount": "source=gs,target=/workspace,type=volume",
+	// "workspaceFolder": "/workspace"
+	// "postCreateCommand": "sudo chown -R graphscope:graphscope /workspace"
+}

.gitmodules

@@ -54,3 +54,6 @@
 	path = modules/graph/thirdparty/libgrape-lite
 	url = https://github.com/alibaba/libgrape-lite.git
 	shallow = true
+[submodule "modules/graph/grin/include"]
+	path = modules/graph/grin/include
+	url = https://github.com/GraphScope/GRIN.git

modules/basic/ds/arrow_utils.cc

@@ -743,6 +743,62 @@
   }
 }
 
+const void* get_arrow_array_data_element(std::shared_ptr<arrow::Array> const& array, unsigned offset) {
+  if (array->type()->Equals(arrow::int8())) {
+    return reinterpret_cast<const void*>(
+        std::dynamic_pointer_cast<arrow::Int8Array>(array)->raw_values() + offset);
+  } else if (array->type()->Equals(arrow::uint8())) {
+    return reinterpret_cast<const void*>(
+        std::dynamic_pointer_cast<arrow::UInt8Array>(array)->raw_values() + offset);
+  } else if (array->type()->Equals(arrow::int16())) {
+    return reinterpret_cast<const void*>(
+        std::dynamic_pointer_cast<arrow::Int16Array>(array)->raw_values() + offset);
+  } else if (array->type()->Equals(arrow::uint16())) {
+    return reinterpret_cast<const void*>(
+        std::dynamic_pointer_cast<arrow::UInt16Array>(array)->raw_values() + offset);
+  } else if (array->type()->Equals(arrow::int32())) {
+    return reinterpret_cast<const void*>(
+        std::dynamic_pointer_cast<arrow::Int32Array>(array)->raw_values() + offset);
+  } else if (array->type()->Equals(arrow::uint32())) {
+    return reinterpret_cast<const void*>(
+        std::dynamic_pointer_cast<arrow::UInt32Array>(array)->raw_values() + offset);
+  } else if (array->type()->Equals(arrow::int64())) {
+    return reinterpret_cast<const void*>(
+        std::dynamic_pointer_cast<arrow::Int64Array>(array)->raw_values() + offset);
+  } else if (array->type()->Equals(arrow::uint64())) {
+    return reinterpret_cast<const void*>(
+        std::dynamic_pointer_cast<arrow::UInt64Array>(array)->raw_values() + offset);
+  } else if (array->type()->Equals(arrow::float32())) {
+    return reinterpret_cast<const void*>(
+        std::dynamic_pointer_cast<arrow::FloatArray>(array)->raw_values() + offset);
+  } else if (array->type()->Equals(arrow::float64())) {
+    return reinterpret_cast<const void*>(
+        std::dynamic_pointer_cast<arrow::DoubleArray>(array)->raw_values() + offset);
+  } else if (array->type()->Equals(arrow::utf8())) {
+    return reinterpret_cast<const void*>(new std::string(
+        std::dynamic_pointer_cast<arrow::StringArray>(array).get()->GetView(offset)));
+  } else if (array->type()->Equals(arrow::large_utf8())) {
+    return reinterpret_cast<const void*>(new std::string(
+        std::dynamic_pointer_cast<arrow::LargeStringArray>(array).get()->GetView(offset)));
+  } else if (array->type()->id() == arrow::Type::LIST) {
+    return reinterpret_cast<const void*>(
+        std::dynamic_pointer_cast<arrow::ListArray>(array).get() + offset);
+  } else if (array->type()->id() == arrow::Type::LARGE_LIST) {
+    return reinterpret_cast<const void*>(
+        std::dynamic_pointer_cast<arrow::LargeListArray>(array).get() + offset);
+  } else if (array->type()->id() == arrow::Type::FIXED_SIZE_LIST) {
+    return reinterpret_cast<const void*>(
+        std::dynamic_pointer_cast<arrow::FixedSizeListArray>(array).get() + offset);
+  } else if (array->type()->Equals(arrow::null())) {
+    return reinterpret_cast<const void*>(
+        std::dynamic_pointer_cast<arrow::NullArray>(array).get() + offset);
+  } else {
+    LOG(ERROR) << "Unsupported arrow array type '" << array->type()->ToString()
+               << "', type id: " << array->type()->id();
+    return NULL;
+  }
+}
+
 Status TypeLoosen(const std::vector<std::shared_ptr<arrow::Schema>>& schemas,
                   std::shared_ptr<arrow::Schema>& schema) {
   int field_num = -1;

modules/basic/ds/arrow_utils.h

@@ -285,6 +285,8 @@ std::string type_name_from_arrow_type(
 
 const void* get_arrow_array_data(std::shared_ptr<arrow::Array> const& array);
 
+const void* get_arrow_array_data_element(std::shared_ptr<arrow::Array> const& array, unsigned offset);
+
 Status TypeLoosen(const std::vector<std::shared_ptr<arrow::Schema>>& schemas,
                   std::shared_ptr<arrow::Schema>& schema);
 

modules/graph/CMakeLists.txt

@@ -8,6 +8,13 @@ file(GLOB_RECURSE VINEYARD_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}"
                                    "*.h")
 
 find_package(MPI REQUIRED)
+find_package(libgrapelite REQUIRED)
+
+find_package(Protobuf REQUIRED)
+include_directories(${Protobuf_INCLUDE_DIRS})
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
+protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${CMAKE_CURRENT_SOURCE_DIR}/grin/include/proto/graph.proto)
+
 
 if(VINEYARD_MOD_SRCS)
     vineyard_generate(
@@ -48,6 +55,8 @@ file(GLOB_RECURSE GRAPH_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}" "fragment/*.cc"
                                                                 "writer/*.cc"
 )
 
+message(${GRAPH_SRC_FILES})
+
 add_library(vineyard_graph ${GRAPH_SRC_FILES})
 target_add_debuginfo(vineyard_graph)
 target_compile_options(vineyard_graph PUBLIC "-fopenmp")
@@ -148,6 +157,7 @@ add_dependencies(vineyard_tests vineyard_graph_tests)
 if(BUILD_VINEYARD_TESTS)
     enable_testing()
     file(GLOB TEST_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/test" "${CMAKE_CURRENT_SOURCE_DIR}/test/*.cc")
+    list(REMOVE_ITEM TEST_FILES "grin_test.cc")
     foreach(f ${TEST_FILES})
         string(REGEX MATCH "^(.*)\\.[^.]*$" dummy ${f})
         set(T_NAME ${CMAKE_MATCH_1})
@@ -168,3 +178,36 @@ if(BUILD_VINEYARD_TESTS)
         add_dependencies(vineyard_graph_tests ${T_NAME})
     endforeach()
 endif()
+
+# grin lib
+file(GLOB_RECURSE GRIN_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}" "grin/*.cc")
+add_library(vineyard_grin ${GRIN_SRC_FILES} ${PROTO_SRCS} ${PROTO_HDRS})
+target_link_libraries(vineyard_grin PRIVATE vineyard_graph ${Protobuf_LIBRARIES})
+install_vineyard_target(vineyard_grin)
+
+
+# grin test
+if(BUILD_VINEYARD_TESTS)
+    enable_testing()
+    file(GLOB TEST_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/test" "${CMAKE_CURRENT_SOURCE_DIR}/test/grin_*.cc")
+    foreach(f ${TEST_FILES})
+        string(REGEX MATCH "^(.*)\\.[^.]*$" dummy ${f})
+        set(T_NAME ${CMAKE_MATCH_1})
+        message(STATUS "Found unit_test - " ${T_NAME})
+        if(BUILD_VINEYARD_TESTS_ALL)
+            add_executable(${T_NAME} test/${T_NAME}.cc)
+        else()
+            add_executable(${T_NAME} EXCLUDE_FROM_ALL test/${T_NAME}.cc)
+        endif()
+        target_link_libraries(${T_NAME} PRIVATE
+                              vineyard_graph
+                              vineyard_grin
+                              ${Protobuf_LIBRARIES}
+                              ${ARROW_SHARED_LIB}
+                              ${MPI_CXX_LIBRARIES})
+        if(${LIBUNWIND_FOUND})
+            target_link_libraries(${T_NAME} PRIVATE ${LIBUNWIND_LIBRARIES})
+        endif()
+        add_test(${T_NAME}, ${T_NAME})
+    endforeach()
+endif()