diff --git a/.github/workflows/build-graphscope-wheels-macos.yml b/.github/workflows/build-graphscope-wheels-macos.yml index 9c5305656904..0fd85afcb94f 100644 --- a/.github/workflows/build-graphscope-wheels-macos.yml +++ b/.github/workflows/build-graphscope-wheels-macos.yml @@ -132,7 +132,7 @@ jobs: run: | source ~/.graphscope_env python3 -m pip install libclang - git clone --single-branch -b v0.12.2 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d + git clone --single-branch -b v0.13.0 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d cd /tmp/v6d git submodule update --init cmake . -DCMAKE_INSTALL_PREFIX=/usr/local \ diff --git a/.github/workflows/gae.yml b/.github/workflows/gae.yml index 5b3a30b756c8..bbfc395e5191 100644 --- a/.github/workflows/gae.yml +++ b/.github/workflows/gae.yml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-20.04 if: ${{ github.repository == 'alibaba/GraphScope' }} container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.12.2 + image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.13.0 options: --shm-size 4096m steps: @@ -38,20 +38,21 @@ jobs: - name: Install latest vineyard if: false run: | + source /home/graphscope/.graphscope_env + sudo pip3 install libclang git clone --single-branch --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d cd /tmp/v6d git submodule update --init mkdir -p build && cd build - cmake .. -DCMAKE_INSTALL_PREFIX=/opt/vineyard \ + cmake .. -DCMAKE_PREFIX_PATH=/opt/graphscope \ + -DCMAKE_INSTALL_PREFIX=/opt/graphscope \ + -DOPENSSL_ROOT_DIR=/opt/graphscope \ -DUSE_EXTERNAL_ETCD_LIBS=OFF \ -DBUILD_SHARED_LIBS=ON \ -DBUILD_VINEYARD_TESTS=OFF make -j$(nproc) sudo make install - # copy to /usr/local, keep the same logic as gsvineyard.Dockerfile - sudo cp -r /opt/vineyard/* /usr/local/ - - name: Build env: RUN_JAVA_TESTS: ON @@ -83,7 +84,7 @@ jobs: export USER_JAR_PATH=${GITHUB_WORKSPACE}/analytical_engine/java/grape-demo/target/grape-demo-${version}-shaded.jar # for giraph test export GIRAPH_JAR_PATH=${GITHUB_WORKSPACE}/analytical_engine/java/grape-giraph/target/grape-giraph-${version}-shaded.jar - + cd ${GITHUB_WORKSPACE}/analytical_engine/build ../test/app_tests.sh --test_dir ${GS_TEST_DIR} diff --git a/.github/workflows/local-ci.yml b/.github/workflows/local-ci.yml index 996bc97ef8ee..4f55ad568ea1 100644 --- a/.github/workflows/local-ci.yml +++ b/.github/workflows/local-ci.yml @@ -232,6 +232,8 @@ jobs: git submodule update --init mkdir -p build && cd build cmake .. -DCMAKE_INSTALL_PREFIX=/opt/vineyard \ + -DCMAKE_PREFIX_PATH=/opt/graphscope \ + -DOPENSSL_ROOT_DIR=/opt/graphscope \ -DUSE_EXTERNAL_ETCD_LIBS=OFF \ -DBUILD_SHARED_LIBS=ON \ -DBUILD_VINEYARD_TESTS=OFF @@ -239,7 +241,7 @@ jobs: sudo make install # copy to /usr/local, keep the same logic as gsvineyard.Dockerfile - sudo cp -r /opt/vineyard/* ${GRAPHSCOPE_HOME}/ + sudo rsync -av /opt/vineyard/ ${GRAPHSCOPE_HOME}/ - name: Build Wheels run: | diff --git a/.github/workflows/networkx-forward-algo-nightly.yml b/.github/workflows/networkx-forward-algo-nightly.yml index fcab4fd22c54..68615b17b2d5 100644 --- a/.github/workflows/networkx-forward-algo-nightly.yml +++ b/.github/workflows/networkx-forward-algo-nightly.yml @@ -16,7 +16,7 @@ jobs: run: shell: bash --noprofile --norc -eo pipefail {0} container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.12.2 + image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.13.0 options: --shm-size 4096m diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 4e930bfbdd31..6ed1feb009a2 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -39,7 +39,7 @@ jobs: fi sudo mkdir /opt/graphscope sudo chown -R $(id -u):$(id -g) /opt/graphscope - ./gs install-deps dev --v6d-version v0.12.2 + ./gs install-deps dev --v6d-version v0.13.0 - name: Setup tmate session if: false diff --git a/analytical_engine/CMakeLists.txt b/analytical_engine/CMakeLists.txt index 461066d4bcdc..362a3d0dbad2 100644 --- a/analytical_engine/CMakeLists.txt +++ b/analytical_engine/CMakeLists.txt @@ -240,7 +240,7 @@ else() endif() # find vineyard after arrow to avoid duplicate target names -find_package(vineyard 0.12.2 REQUIRED) +find_package(vineyard 0.13.0 REQUIRED) include_directories(${VINEYARD_INCLUDE_DIRS}) add_compile_options(-DENABLE_SELECTOR) diff --git a/analytical_engine/core/context/tensor_dataframe_builder.h b/analytical_engine/core/context/tensor_dataframe_builder.h index f5cb09747fa8..940dcd77ff45 100644 --- a/analytical_engine/core/context/tensor_dataframe_builder.h +++ b/analytical_engine/core/context/tensor_dataframe_builder.h @@ -55,30 +55,32 @@ class MPIGlobalTensorBuilder : public vineyard::GlobalTensorBuilder, void AddChunks(std::vector const& chunk_ids) { for (auto& chunk_id : chunk_ids) { - this->local_chunk_ids_.emplace_back(chunk_id); + this->AddChunk(chunk_id); } } - std::shared_ptr _Seal(vineyard::Client& client) override { - std::shared_ptr tensor; + vineyard::Status _Seal(vineyard::Client& client, + std::shared_ptr& object) override { vineyard::ObjectID id = vineyard::InvalidObjectID(); + this->SetGlobal(); if (comm_spec_.worker_id() == 0) { - tensor = std::dynamic_pointer_cast( - vineyard::GlobalTensorBuilder::_Seal(client)); - id = tensor->id(); + RETURN_ON_ERROR(vineyard::GlobalTensorBuilder::_Seal(client, object)); + id = object->id(); + RETURN_ON_ERROR(client_.Persist(id)); } else { - VINEYARD_CHECK_OK(this->Build(client)); + RETURN_ON_ERROR(this->Build(client)); } SyncGlobalObjectID(comm_spec_, id); // this sync can be seen as a barrier if (comm_spec_.worker_id() != 0) { - // FIXME: the aim of `Construct` is to fillup the ObjectSet, needs better + // FIXME: the aim of `Construct` is to fill up the ObjectSet, needs better // design. - tensor = std::make_shared(); + auto tensor = std::make_shared(); vineyard::ObjectMeta meta; - VINEYARD_CHECK_OK(client.GetMetaData(id, meta, true)); + RETURN_ON_ERROR(client.GetMetaData(id, meta, true)); tensor->Construct(meta); + object = std::static_pointer_cast(tensor); } - return tensor; + return vineyard::Status::OK(); } vineyard::Status Build(vineyard::Client& client) override { @@ -114,28 +116,30 @@ class MPIGlobalDataFrameBuilder : public vineyard::GlobalDataFrameBuilder, void AddChunks(std::vector const& chunk_ids) { for (auto& chunk_id : chunk_ids) { - this->local_chunk_ids_.emplace_back(chunk_id); + this->AddChunk(chunk_id); } } - std::shared_ptr _Seal(vineyard::Client& client) override { - std::shared_ptr df; + vineyard::Status _Seal(vineyard::Client& client, + std::shared_ptr& object) override { vineyard::ObjectID id = vineyard::InvalidObjectID(); + this->SetGlobal(); if (comm_spec_.worker_id() == 0) { - df = std::dynamic_pointer_cast( - vineyard::GlobalDataFrameBuilder::_Seal(client)); - id = df->id(); + RETURN_ON_ERROR(vineyard::GlobalDataFrameBuilder::_Seal(client, object)); + id = object->id(); + RETURN_ON_ERROR(client_.Persist(id)); } else { - VINEYARD_CHECK_OK(this->Build(client)); + RETURN_ON_ERROR(this->Build(client)); } SyncGlobalObjectID(comm_spec_, id); // this sync can be seen as a barrier if (comm_spec_.worker_id() != 0) { - df = std::make_shared(); + auto df = std::make_shared(); vineyard::ObjectMeta meta; - VINEYARD_CHECK_OK(client.GetMetaData(id, meta, true)); + RETURN_ON_ERROR(client.GetMetaData(id, meta, true)); df->Construct(meta); + object = std::static_pointer_cast(df); } - return df; + return vineyard::Status::OK(); } vineyard::Status Build(vineyard::Client& client) override { diff --git a/analytical_engine/core/fragment/arrow_flattened_fragment.h b/analytical_engine/core/fragment/arrow_flattened_fragment.h index c2456a97fc1e..41317ea85bb5 100644 --- a/analytical_engine/core/fragment/arrow_flattened_fragment.h +++ b/analytical_engine/core/fragment/arrow_flattened_fragment.h @@ -516,11 +516,16 @@ class ArrowFlattenedFragment { prop_id_t e_prop_id) : fragment_(frag), v_prop_id_(v_prop_id), e_prop_id_(e_prop_id) { ivnum_ = ovnum_ = tvnum_ = 0; - label_id_t vertex_label_num = fragment_->vertex_label_num(); + auto& schema = fragment_->schema(); + label_id_t vertex_label_num = + static_cast(schema.AllVertexEntries().size()); for (label_id_t v_label = 0; v_label < vertex_label_num; v_label++) { - vid_t ivnum = fragment_->GetInnerVerticesNum(v_label); - vid_t ovnum = fragment_->GetOuterVerticesNum(v_label); - vid_t tvnum = fragment_->GetVerticesNum(v_label); + vid_t ivnum = 0, ovnum = 0, tvnum = 0; + if (schema.IsVertexValid(v_label)) { + ivnum = fragment_->GetInnerVerticesNum(v_label); + ovnum = fragment_->GetOuterVerticesNum(v_label); + tvnum = fragment_->GetVerticesNum(v_label); + } ivnums_.push_back(ivnum); ovnums_.push_back(ovnum); tvnums_.push_back(tvnum); @@ -535,13 +540,19 @@ class ArrowFlattenedFragment { union_vertex_range_offset_.resize(2 * vertex_label_num + 1, 0); for (label_id_t v_label = 0; v_label < vertex_label_num; v_label++) { union_vertex_range_offset_[v_label + 1] = - union_vertex_range_offset_[v_label] + - fragment_->GetInnerVerticesNum(v_label); + union_vertex_range_offset_[v_label]; + if (schema.IsVertexValid(v_label)) { + union_vertex_range_offset_[v_label + 1] += + fragment_->GetInnerVerticesNum(v_label); + } } for (label_id_t v_label = 0; v_label < vertex_label_num; v_label++) { union_vertex_range_offset_[v_label + vertex_label_num + 1] = - union_vertex_range_offset_[v_label + vertex_label_num] + - fragment_->GetOuterVerticesNum(v_label); + union_vertex_range_offset_[v_label + vertex_label_num]; + if (schema.IsVertexValid(v_label)) { + union_vertex_range_offset_[v_label + vertex_label_num + 1] += + fragment_->GetOuterVerticesNum(v_label); + } } // init id parser union_id_parser_.Init(fragment_->fnum(), vertex_label_num, @@ -730,8 +741,13 @@ class ArrowFlattenedFragment { std::vector> adj_lists; adj_lists.reserve(fragment_->edge_label_num()); - for (label_id_t e_label = 0; e_label < fragment_->edge_label_num(); - e_label++) { + auto& schema = fragment_->schema(); + label_id_t edge_label_num = + static_cast(schema.AllEdgeEntries().size()); + for (label_id_t e_label = 0; e_label < edge_label_num; e_label++) { + if (!schema.IsEdgeValid(e_label)) { + continue; + } auto adj_list = fragment_->GetOutgoingAdjList(v_, e_label); if (adj_list.NotEmpty()) { adj_lists.push_back(adj_list); @@ -745,8 +761,13 @@ class ArrowFlattenedFragment { std::vector> adj_lists; adj_lists.reserve(fragment_->edge_label_num()); - for (label_id_t e_label = 0; e_label < fragment_->edge_label_num(); - e_label++) { + auto& schema = fragment_->schema(); + label_id_t edge_label_num = + static_cast(schema.AllEdgeEntries().size()); + for (label_id_t e_label = 0; e_label < edge_label_num; e_label++) { + if (!schema.IsEdgeValid(e_label)) { + continue; + } auto adj_list = fragment_->GetIncomingAdjList(v_, e_label); if (adj_list.NotEmpty()) { adj_lists.push_back(adj_list); @@ -758,8 +779,13 @@ class ArrowFlattenedFragment { inline int GetLocalOutDegree(const vertex_t& v) const { vertex_t v_(union_id_parser_.ParseContinuousLid(v.GetValue())); int local_out_degree = 0; - for (label_id_t e_label = 0; e_label < fragment_->edge_label_num(); - e_label++) { + auto& schema = fragment_->schema(); + label_id_t edge_label_num = + static_cast(schema.AllEdgeEntries().size()); + for (label_id_t e_label = 0; e_label < edge_label_num; e_label++) { + if (!schema.IsEdgeValid(e_label)) { + continue; + } local_out_degree += fragment_->GetLocalOutDegree(v_, e_label); } return local_out_degree; @@ -768,8 +794,13 @@ class ArrowFlattenedFragment { inline int GetLocalInDegree(const vertex_t& v) const { vertex_t v_(union_id_parser_.ParseContinuousLid(v.GetValue())); int local_in_degree = 0; - for (label_id_t e_label = 0; e_label < fragment_->edge_label_num(); - e_label++) { + auto& schema = fragment_->schema(); + label_id_t edge_label_num = + static_cast(schema.AllEdgeEntries().size()); + for (label_id_t e_label = 0; e_label < edge_label_num; e_label++) { + if (!schema.IsEdgeValid(e_label)) { + continue; + } local_in_degree += fragment_->GetLocalInDegree(v_, e_label); } return local_in_degree; @@ -779,8 +810,13 @@ class ArrowFlattenedFragment { vertex_t v_(union_id_parser_.ParseContinuousLid(v.GetValue())); std::vector dest_lists; dest_lists.reserve(fragment_->edge_label_num()); - for (label_id_t e_label = 0; e_label < fragment_->edge_label_num(); - e_label++) { + auto& schema = fragment_->schema(); + label_id_t edge_label_num = + static_cast(schema.AllEdgeEntries().size()); + for (label_id_t e_label = 0; e_label < edge_label_num; e_label++) { + if (!schema.IsEdgeValid(e_label)) { + continue; + } dest_lists.push_back(fragment_->IEDests(v_, e_label)); } return dest_list_t(dest_lists); @@ -790,8 +826,13 @@ class ArrowFlattenedFragment { vertex_t v_(union_id_parser_.ParseContinuousLid(v.GetValue())); std::vector dest_lists; dest_lists.reserve(fragment_->edge_label_num()); - for (label_id_t e_label = 0; e_label < fragment_->edge_label_num(); - e_label++) { + auto& schema = fragment_->schema(); + label_id_t edge_label_num = + static_cast(schema.AllEdgeEntries().size()); + for (label_id_t e_label = 0; e_label < edge_label_num; e_label++) { + if (!schema.IsEdgeValid(e_label)) { + continue; + } dest_lists.push_back(fragment_->OEDests(v_, e_label)); } return dest_list_t(dest_lists); @@ -801,8 +842,13 @@ class ArrowFlattenedFragment { vertex_t v_(union_id_parser_.ParseContinuousLid(v.GetValue())); std::vector dest_lists; dest_lists.reserve(fragment_->edge_label_num()); - for (label_id_t e_label = 0; e_label < fragment_->edge_label_num(); - e_label++) { + auto& schema = fragment_->schema(); + label_id_t edge_label_num = + static_cast(schema.AllEdgeEntries().size()); + for (label_id_t e_label = 0; e_label < edge_label_num; e_label++) { + if (!schema.IsEdgeValid(e_label)) { + continue; + } dest_lists.push_back(fragment_->IOEDests(v_, e_label)); } return dest_list_t(dest_lists); diff --git a/analytical_engine/test/run_ctx.cc b/analytical_engine/test/run_ctx.cc index 4607ec1bf916..095859f8ae9a 100644 --- a/analytical_engine/test/run_ctx.cc +++ b/analytical_engine/test/run_ctx.cc @@ -177,15 +177,8 @@ void output_vineyard_tensor(vineyard::Client& client, const std::string& prefix) { auto stored_tensor = std::dynamic_pointer_cast( client.GetObject(tensor_object)); - auto const& shape = stored_tensor->shape(); - auto const& partition_shape = stored_tensor->partition_shape(); auto const& local_chunks = stored_tensor->LocalPartitions(client); - CHECK_EQ(shape.size(), 1); - CHECK_EQ(partition_shape.size(), 1); CHECK_EQ(local_chunks.size(), static_cast(comm_spec.local_num())); - if (comm_spec.worker_id() == 0) { - LOG(INFO) << "tensor shape: " << shape[0] << ", " << partition_shape[0]; - } if (comm_spec.local_id() == 0) { for (auto obj : local_chunks) { @@ -223,15 +216,9 @@ void output_vineyard_dataframe(vineyard::Client& client, const std::string& prefix) { auto stored_dataframe = std::dynamic_pointer_cast( client.GetObject(dataframe_object)); - auto const& partition_shape = stored_dataframe->partition_shape(); auto const& local_chunks = stored_dataframe->LocalPartitions(client); CHECK_EQ(local_chunks.size(), static_cast(comm_spec.local_num())); - if (comm_spec.worker_id() == 0) { - LOG(INFO) << "dataframe shape: " << partition_shape.first << ", " - << partition_shape.second; - } - if (comm_spec.local_id() == 0) { for (auto const& obj : local_chunks) { auto single_dataframe = diff --git a/analytical_engine/test/run_java_app.cc b/analytical_engine/test/run_java_app.cc index cc381bcad352..118dd5a8feb9 100644 --- a/analytical_engine/test/run_java_app.cc +++ b/analytical_engine/test/run_java_app.cc @@ -173,15 +173,8 @@ void output_vineyard_tensor(vineyard::Client& client, vineyard::AnyType& expected_type) { auto stored_tensor = std::dynamic_pointer_cast( client.GetObject(tensor_object)); - auto const& shape = stored_tensor->shape(); - auto const& partition_shape = stored_tensor->partition_shape(); auto const& local_chunks = stored_tensor->LocalPartitions(client); - CHECK_EQ(shape.size(), 1); - CHECK_EQ(partition_shape.size(), 1); CHECK_EQ(local_chunks.size(), static_cast(comm_spec.local_num())); - if (comm_spec.worker_id() == 0) { - VLOG(1) << "tensor shape: " << shape[0] << ", " << partition_shape[0]; - } if (comm_spec.local_id() == 0) { for (auto obj : local_chunks) { diff --git a/charts/graphscope/values.yaml b/charts/graphscope/values.yaml index 4c213d7eaa7f..1bde464091c0 100644 --- a/charts/graphscope/values.yaml +++ b/charts/graphscope/values.yaml @@ -89,7 +89,7 @@ vineyard: image: name: vineyardcloudnative/vineyardd # Overrides the image tag whose default is the chart appVersion. - tag: v0.12.2 + tag: v0.13.0 resources: requests: cpu: 0.5 diff --git a/coordinator/gscoordinator/coordinator.py b/coordinator/gscoordinator/coordinator.py index bb0247bb55a3..0ef8f1dd48a1 100644 --- a/coordinator/gscoordinator/coordinator.py +++ b/coordinator/gscoordinator/coordinator.py @@ -107,8 +107,11 @@ def config_logging(log_level): """ logging.basicConfig(level=logging.CRITICAL) - if log_level: - log_level = log_level.upper() + # `NOTSET` is special as it doesn't show log in Python + if isinstance(log_level, str): + log_level = getattr(logging, log_level.upper()) + if log_level == logging.NOTSET: + log_level = logging.DEBUG logger = logging.getLogger("graphscope") logger.setLevel(log_level) diff --git a/coordinator/gscoordinator/utils.py b/coordinator/gscoordinator/utils.py index 6cd403b2da55..f55f580abe6e 100644 --- a/coordinator/gscoordinator/utils.py +++ b/coordinator/gscoordinator/utils.py @@ -191,7 +191,7 @@ def get_lib_path(app_dir: str, app_name: str) -> str: def get_app_sha256(attr, java_class_path: str): ( app_type, - _, + app_header, app_class, vd_type, _, @@ -200,6 +200,13 @@ def get_app_sha256(attr, java_class_path: str): java_app_class, ) = _codegen_app_info(attr, DEFAULT_GS_CONFIG_FILE, java_class_path) graph_header, graph_type, _ = _codegen_graph_info(attr) + logger.info( + "app type: %s (%s), graph type: %s (%s)", + app_class, + app_header, + graph_type, + graph_header, + ) if app_type == "cpp_pie": app_sha256 = hashlib.sha256( @@ -1538,11 +1545,12 @@ def _codegen_app_info(attr, meta_file: str, java_class_path: str): "{}<{}>".format(real_algo, java_app_template_str), ) + app_info = None for app in config_yaml["app"]: if app["algo"] == algo: app_type = app["type"] # cpp_pie or cython_pregel or cython_pie, java_pie if app_type in ("cpp_pie", "cpp_pregel"): - return ( + app_info = ( app_type, app["src"], f"{app['class_name']}<_GRAPH_TYPE>", @@ -1552,9 +1560,10 @@ def _codegen_app_info(attr, meta_file: str, java_class_path: str): None, None, ) + break if app_type in ("cython_pregel", "cython_pie"): # cython app doesn't have c-header file - return ( + app_info = ( app_type, "", "", @@ -1564,8 +1573,11 @@ def _codegen_app_info(attr, meta_file: str, java_class_path: str): None, None, ) + break - raise KeyError("Algorithm does not exist in the gar resource.") + if app_info is None: + raise KeyError("Algorithm does not exist in the gar resource.") + return app_info # a mapping for class name to header file. @@ -1663,7 +1675,6 @@ def internal_type(t): # The template of vertex map needs special care. raise ValueError( f"Unknown graph type: {graph_def_pb2.GraphTypePb.Name(graph_type)}" ) - logger.info("Codegened graph type: %s, Graph header: %s", graph_fqn, graph_header) return graph_header, graph_fqn, oid_type() diff --git a/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/CMakeLists.txt b/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/CMakeLists.txt index 03c5810cdb99..4f65d065d5bb 100644 --- a/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/CMakeLists.txt +++ b/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/CMakeLists.txt @@ -57,7 +57,7 @@ find_package(Threads REQUIRED) # we need edge src/dst ids in etable. add_definitions(-DENDPOINT_LISTS) -find_package(vineyard 0.12.2 REQUIRED) +find_package(vineyard 0.13.0 REQUIRED) add_library(v6d_native_store global_store_ffi.cc htap_ds_impl.cc graph_builder_ffi.cc diff --git a/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/graph_builder_ffi.cc b/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/graph_builder_ffi.cc index 2d691035e21c..45001fb7c4df 100644 --- a/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/graph_builder_ffi.cc +++ b/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/graph_builder_ffi.cc @@ -115,7 +115,8 @@ ObjectId v6d_build_global_graph_stream(const char *graph_name, size_t size, builder.AddStream(idx, static_cast(object_ids[idx]), instance_ids[idx]); } - auto gs = builder.Seal(client); + std::shared_ptr gs; + VINEYARD_CHECK_OK(builder.Seal(client, gs)); VINEYARD_CHECK_OK(client.Persist(gs->id())); LOG(INFO) << "start build_global_graph_stream create name: name = " << graph_name; diff --git a/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/graph_schema.cc b/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/graph_schema.cc index 6c452384d733..bb424fbd12ac 100644 --- a/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/graph_schema.cc +++ b/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/graph_schema.cc @@ -1,29 +1,18 @@ /** * Copyright 2020 Alibaba Group Holding Limited. - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -/** Copyright 2020 Alibaba Group Holding Limited. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ #include "graph_schema.h" diff --git a/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/graph_schema.h b/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/graph_schema.h index e5ffd6a175f0..e469527bebd3 100644 --- a/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/graph_schema.h +++ b/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/graph_schema.h @@ -1,12 +1,12 @@ /** * Copyright 2020 Alibaba Group Holding Limited. - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/property_graph_stream.cc b/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/property_graph_stream.cc index efc52045e4a8..7f60fffeb2dc 100644 --- a/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/property_graph_stream.cc +++ b/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/property_graph_stream.cc @@ -600,10 +600,11 @@ void GlobalPGStream::Construct(const ObjectMeta& meta) { LOG(INFO) << "local stream chunk size: " << local_streams_.size(); } -std::shared_ptr GlobalPGStreamBuilder::_Seal(Client& client) { +Status GlobalPGStreamBuilder::_Seal(Client& client, std::shared_ptr&object) { VINEYARD_CHECK_OK(this->Build(client)); auto gstream = std::make_shared(); + object = gstream; gstream->total_stream_chunks_ = total_stream_chunks_; gstream->meta_.SetTypeName(type_name()); gstream->meta_.SetGlobal(true); @@ -615,7 +616,7 @@ std::shared_ptr GlobalPGStreamBuilder::_Seal(Client& client) { } VINEYARD_CHECK_OK(client.CreateMetaData(gstream->meta_, gstream->id_)); - return std::dynamic_pointer_cast(gstream); + return Status::OK(); } } // namespace htap diff --git a/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/property_graph_stream.h b/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/property_graph_stream.h index 06ffb8ba174c..d5ccfd0d0826 100644 --- a/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/property_graph_stream.h +++ b/interactive_engine/executor/store/global_query/src/store_impl/v6d/native/property_graph_stream.h @@ -207,7 +207,7 @@ class PropertyGraphOutStream : public Registered { std::unordered_map params{ {"kind", "vertex"}, {"graph_name", graph_name} }; - auto stream_id = RecordBatchStream::Make(client, params); + auto stream_id = StreamBuilder::Make(client, params); s->vertex_stream_ = client.GetObject(stream_id); client.Persist(s->vertex_stream_->id()); @@ -218,7 +218,7 @@ class PropertyGraphOutStream : public Registered { std::unordered_map params{ {"kind", "edge"}, {"graph_name", graph_name} }; - auto stream_id = RecordBatchStream::Make(client, params); + auto stream_id = StreamBuilder::Make(client, params); s->edge_stream_ = client.GetObject(stream_id); client.Persist(s->edge_stream_->id()); @@ -413,7 +413,7 @@ class GlobalPGStreamBuilder : public ObjectBuilder { return Status::OK(); } - std::shared_ptr _Seal(Client& client); + Status _Seal(Client& client, std::shared_ptr&object); private: std::vector stream_chunks_; diff --git a/k8s/Makefile b/k8s/Makefile index 45f97b62b2a1..aae126c5d7c9 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -7,7 +7,7 @@ ifeq ($(REGISTRY),) endif VERSION ?= latest -VINEYARD_VERSION ?= v0.12.2 +VINEYARD_VERSION ?= v0.13.0 # This is the version of builder base image in most cases, except for graphscope-dev BUILDER_VERSION ?= $(VINEYARD_VERSION) # This is the version of runtime base image diff --git a/k8s/actions-runner-controller/manylinux/Makefile b/k8s/actions-runner-controller/manylinux/Makefile index f1c3c80584b9..3d2b279c786d 100644 --- a/k8s/actions-runner-controller/manylinux/Makefile +++ b/k8s/actions-runner-controller/manylinux/Makefile @@ -12,7 +12,7 @@ TARGETPLATFORM ?= $(shell arch) RUNNER_VERSION ?= 2.287.1 DOCKER_VERSION ?= 20.10.12 -VINEYARD_VERSION ?= v0.12.2 +VINEYARD_VERSION ?= v0.13.0 BUILDER_VERSION ?= $(VINEYARD_VERSION) # default list of platforms for which multiarch image is built diff --git a/k8s/dockerfiles/coordinator.Dockerfile b/k8s/dockerfiles/coordinator.Dockerfile index 6d4409799bc4..99e3773bcfc8 100644 --- a/k8s/dockerfiles/coordinator.Dockerfile +++ b/k8s/dockerfiles/coordinator.Dockerfile @@ -50,8 +50,8 @@ RUN sudo mkdir -p /var/log/graphscope \ && sudo chown -R graphscope:graphscope /var/log/graphscope # kubectl v1.19.2 -RUN curl -L -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v1.19.2/bin/linux/amd64/kubectl -RUN chmod +x /usr/local/bin/kubectl +RUN curl -L -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v1.19.2/bin/linux/amd64/kubectl +RUN chmod +x /usr/bin/kubectl COPY --from=builder /home/graphscope/install /opt/graphscope/ RUN python3 -m pip install --no-cache-dir /opt/graphscope/*.whl && rm -rf /opt/graphscope/ @@ -59,6 +59,7 @@ COPY --from=builder /opt/openmpi /opt/openmpi COPY ./interactive_engine/assembly/src/bin/graphscope/giectl /opt/graphscope/bin/giectl COPY ./k8s/utils/kube_ssh /usr/local/bin/kube_ssh +RUN sudo chmod a+wrx /tmp USER graphscope WORKDIR /home/graphscope diff --git a/k8s/dockerfiles/graphscope-store.Dockerfile b/k8s/dockerfiles/graphscope-store.Dockerfile index 1ea3f6fd2536..91274ba09f59 100644 --- a/k8s/dockerfiles/graphscope-store.Dockerfile +++ b/k8s/dockerfiles/graphscope-store.Dockerfile @@ -30,6 +30,8 @@ RUN tar -zxf /tmp/groot.tar.gz -C /usr/local && rm /tmp/groot.tar.gz RUN useradd -m graphscope -u 1001 \ && echo 'graphscope ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +RUN sudo chmod a+wrx /tmp + USER graphscope WORKDIR /home/graphscope diff --git a/k8s/dockerfiles/interactive-experimental.Dockerfile b/k8s/dockerfiles/interactive-experimental.Dockerfile index 7cc63f37fc08..fe09fd84c244 100644 --- a/k8s/dockerfiles/interactive-experimental.Dockerfile +++ b/k8s/dockerfiles/interactive-experimental.Dockerfile @@ -23,6 +23,7 @@ COPY --from=builder /home/graphscope/GraphScope/interactive_engine/executor/ir/t RUN yum install -y sudo java-1.8.0-openjdk \ && yum clean all \ && rm -rf /var/cache/yum +RUN sudo chmod a+wrx /tmp RUN useradd -m graphscope -u 1001 \ && echo 'graphscope ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers diff --git a/k8s/dockerfiles/interactive.Dockerfile b/k8s/dockerfiles/interactive.Dockerfile index 2a8ef1f48df3..f5cecdb9bea5 100644 --- a/k8s/dockerfiles/interactive.Dockerfile +++ b/k8s/dockerfiles/interactive.Dockerfile @@ -60,6 +60,7 @@ COPY --from=builder /home/graphscope/install/bin /opt/graphscope/bin COPY --from=builder /home/graphscope/install/conf /opt/graphscope/conf RUN sudo chmod +x /opt/graphscope/bin/* +RUN sudo chmod a+wrx /tmp USER graphscope WORKDIR /home/graphscope diff --git a/k8s/dockerfiles/learning.Dockerfile b/k8s/dockerfiles/learning.Dockerfile index 3ba2bbcad8cc..a4a7369b226d 100644 --- a/k8s/dockerfiles/learning.Dockerfile +++ b/k8s/dockerfiles/learning.Dockerfile @@ -38,6 +38,7 @@ RUN yum install -y centos-release-scl-rh sudo && \ rpm -V $INSTALL_PKGS && \ yum -y clean all --enablerepo='*' && \ rm -rf /var/cache/yum +RUN sudo chmod a+wrx /tmp SHELL [ "/usr/bin/scl", "enable", "rh-python38" ] diff --git a/k8s/dockerfiles/vineyard-dev.Dockerfile b/k8s/dockerfiles/vineyard-dev.Dockerfile index c06cdbf080f9..6b180fc4268a 100644 --- a/k8s/dockerfiles/vineyard-dev.Dockerfile +++ b/k8s/dockerfiles/vineyard-dev.Dockerfile @@ -44,7 +44,8 @@ WORKDIR /home/graphscope COPY ./gs ./gs ARG VINEYARD_VERSION=main -RUN ./gs install-deps dev --for-analytical --v6d-version=$VINEYARD_VERSION && \ +RUN sudo chmod a+wrx /tmp && \ + ./gs install-deps dev --for-analytical --v6d-version=$VINEYARD_VERSION && \ sudo yum clean all -y && \ sudo rm -fr /var/cache/yum diff --git a/k8s/dockerfiles/vineyard-runtime.Dockerfile b/k8s/dockerfiles/vineyard-runtime.Dockerfile index babe5713c567..39c6b74b18b6 100644 --- a/k8s/dockerfiles/vineyard-runtime.Dockerfile +++ b/k8s/dockerfiles/vineyard-runtime.Dockerfile @@ -25,6 +25,7 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64 RUN yum install -y sudo libunwind-devel libgomp && \ yum clean all -y --enablerepo='*' && \ rm -rf /var/cache/yum +RUN sudo chmod a+wrx /tmp RUN useradd -m graphscope -u 1001 \ && echo 'graphscope ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers diff --git a/k8s/internal/Makefile b/k8s/internal/Makefile index f0529ede481e..755c4d0c8403 100644 --- a/k8s/internal/Makefile +++ b/k8s/internal/Makefile @@ -34,7 +34,7 @@ GRAPHSCOPE_HOME ?= /usr/local INSTALL_PREFIX ?= /opt/graphscope VERSION ?= latest -VINEYARD_VERSION ?= v0.12.2 +VINEYARD_VERSION ?= v0.13.0 PROFILE ?= release CI ?= false diff --git a/python/graphscope/config.py b/python/graphscope/config.py index eaa3df576142..cc74c51ffa6b 100644 --- a/python/graphscope/config.py +++ b/python/graphscope/config.py @@ -68,7 +68,7 @@ class GSConfig(object): # vineyard resource configuration # image for vineyard container - k8s_vineyard_image = "vineyardcloudnative/vineyardd:v0.12.2" + k8s_vineyard_image = "vineyardcloudnative/vineyardd:v0.13.0" k8s_vineyard_daemonset = None k8s_vineyard_cpu = 0.5 k8s_vineyard_mem = "512Mi" diff --git a/python/graphscope/tests/unittest/test_app.py b/python/graphscope/tests/unittest/test_app.py index 51805f810adc..479a94357326 100644 --- a/python/graphscope/tests/unittest/test_app.py +++ b/python/graphscope/tests/unittest/test_app.py @@ -463,3 +463,39 @@ def test_louvain_on_projected_graph(arrow_property_graph_undirected): ) ctx = louvain(g) ctx.to_dataframe({"node": "v.id", "r": "r"}) + + +def test_pagerank_on_projected_projected(ldbc_graph): + pg1 = ldbc_graph.project( + vertices={"post": [], "tag": [], "tagclass": []}, + edges={"hasTag": [], "isSubclassOf": []}, + ) + pg2 = pg1.project(vertices={"tagclass": []}, edges={"isSubclassOf": []}) + pr_context = graphscope.pagerank_nx(pg2, alpha=0.85, max_iter=100, tol=1e-06) + df = pr_context.to_dataframe(selector={"id": "v.id", "dist": "r"}) + assert df.shape == (71, 2) # V(tagclass) + + # check existence of OIDs + for oid in [349, 211, 239, 0, 98, 233, 41, 243, 242, 67]: + assert oid in df["id"].values + + +def test_pagerank_on_flatten(ldbc_graph): + pg = ldbc_graph.project(vertices={"post": [], "tag": []}, edges={"hasTag": []}) + pr_context = graphscope.pagerank_nx(pg, alpha=0.85, max_iter=100, tol=1e-06) + df = pr_context.to_dataframe(selector={"id": "v.id", "dist": "r"}) + assert df.shape == (95056, 2) # V(post) + V(tag) + + # check existence of OIDs + for oid in [ + 618475290624, + 3, + 412316860420, + 412316860421, + 412316860422, + 16075, + 16076, + 16077, + 16078, + ]: + assert oid in df["id"].values diff --git a/python/graphscope/tests/unittest/test_graph.py b/python/graphscope/tests/unittest/test_graph.py index 24a15a02af47..20b8251ddb8f 100644 --- a/python/graphscope/tests/unittest/test_graph.py +++ b/python/graphscope/tests/unittest/test_graph.py @@ -481,6 +481,19 @@ def test_project_subgraph(arrow_modern_graph): assert graph.schema.get_vertex_properties("person")[0].name == "pr" +def test_project_project(ldbc_graph): + pg1 = ldbc_graph.project( + vertices={"post": [], "tag": [], "tagclass": []}, + edges={"hasTag": [], "isSubclassOf": []}, + ) + assert pg1.schema.vertex_labels == ["post", "tag", "tagclass"] + assert pg1.schema.edge_labels == ["isSubclassOf", "hasTag"] + + pg2 = pg1.project(vertices={"tagclass": []}, edges={"isSubclassOf": []}) + assert pg2.schema.vertex_labels == ["tagclass"] + assert pg2.schema.edge_labels == ["isSubclassOf"] + + def test_error_on_project(arrow_property_graph, ldbc_graph): graph = arrow_property_graph g2 = graph.project(vertices={"v0": []}, edges={"e0": []}) diff --git a/scripts/install_deps.sh b/scripts/install_deps.sh index 93746bcc06c4..36140eb72cc3 100755 --- a/scripts/install_deps.sh +++ b/scripts/install_deps.sh @@ -14,8 +14,8 @@ readonly GREEN="\033[0;32m" readonly NC="\033[0m" # No Color readonly GRAPE_BRANCH="master" # libgrape-lite branch -readonly V6D_VERSION="0.12.2" # vineyard version -readonly V6D_BRANCH="v0.12.2" # vineyard branch +readonly V6D_VERSION="0.13.0" # vineyard version +readonly V6D_BRANCH="v0.13.0" # vineyard branch readonly OUTPUT_ENV_FILE="${HOME}/.graphscope_env" IS_IN_WSL=false && [[ ! -z "${IS_WSL}" || ! -z "${WSL_DISTRO_NAME}" ]] && IS_IN_WSL=true