App now can carry cmake options, and enabled wcc on string oid. (#2130)

coordinator/gscoordinator/template/CMakeLists.template

@@ -12,6 +12,7 @@ option(PROJECT_FRAME "Whether to build project frame" False)
 option(ENABLE_PREGEL_COMBINE "Whether enable combinator in pregel app." False)
 option(NETWORKX "networkx on?" ON)
 option(ENABLE_JAVA_SDK "Build with support for java sdk" OFF)
+option(WCC_USE_GID "Whether use gid as component id in wcc" OFF)
 
 execute_process(COMMAND uname -m OUTPUT_VARIABLE SYSTEM_PROCESSOR)
 string(REGEX REPLACE "\n$" "" SYSTEM_PROCESSOR "${SYSTEM_PROCESSOR}")
@@ -54,6 +55,10 @@ if (ENABLE_PREGEL_COMBINE)
     add_definitions(-D_ENABLE_COMBINE)
 endif ()
 
+if (WCC_USE_GID)
+    add_definitions(-DWCC_USE_GID)
+endif ()
+
 include(CheckCXXCompilerFlag)
 include(GNUInstallDirs)
 

coordinator/gscoordinator/utils.py

@@ -278,6 +278,10 @@ def compile_app(
 
     os.chdir(app_dir)
 
+    extra_options = []
+    if types_pb2.CMAKE_EXTRA_OPTIONS in attr:
+        extra_options = attr[types_pb2.CMAKE_EXTRA_OPTIONS].s.decode("utf-8").split(" ")
+
     module_name = ""
     # Output directory for java codegen
     java_codegen_out_dir = ""
@@ -289,6 +293,7 @@ def compile_app(
         f"-DNETWORKX={engine_config['networkx']}",
         f"-DCMAKE_PREFIX_PATH='{GRAPHSCOPE_HOME};{OPAL_PREFIX}'",
     ]
+    cmake_commands.extend(extra_options)
     if os.environ.get("GRAPHSCOPE_ANALYTICAL_DEBUG", "") == "1":
         cmake_commands.append("-DCMAKE_BUILD_TYPE=Debug")
     if app_type == "java_pie":

k8s/precompile.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-import glob
+import sys
 import hashlib
 import multiprocessing
 import os
@@ -65,9 +65,18 @@ def cmake_and_make(cmake_commands):
         shutil.rmtree("CMakeFiles")
     except subprocess.CalledProcessError as e:
         print(e.stderr)
+        raise
 
+def get_lib_path(app_dir, app_name):
+    if sys.platform == "linux" or sys.platform == "linux2":
+        return app_dir / f"lib{app_name}.so"
+    elif sys.platform == "darwin":
+        return app_dir / f"lib{app_name}.dylib"
+    else:
+        raise RuntimeError(f"Unsupported platform {sys.platform}")
 
 def cmake_graph(graph_class):
+    print("Start to compile", graph_class)
     library_name = compute_sig(graph_class)
     library_dir = WORKSPACE / library_name
     library_dir.mkdir(exist_ok=True)
@@ -89,11 +98,15 @@ def cmake_graph(graph_class):
         cmake_commands.append("-DPROJECT_FRAME=True")
 
     cmake_and_make(cmake_commands)
-    print("Finished compiling", graph_class)
+    print("-- Finished compiling", graph_class)
 
 
 def cmake_app(app):
-    algo, graph_class = app
+    if len(app) == 3:
+        algo, graph_class, extra_option = app
+    else:
+        algo, graph_class = app
+        extra_option = []
     if "ArrowFragment" in graph_class:
         graph_header = "vineyard/graph/fragment/arrow_fragment.h"
     elif "ArrowProjectedFragment" in graph_class:
@@ -106,12 +119,16 @@ def cmake_app(app):
         raise ValueError("Not supported graph class %s" % graph_class)
 
     app_type, app_header, app_class = get_app_info(algo)
+    print("Start to compile", app_class, graph_class)
     assert app_type == "cpp_pie", "Only support cpp_pie currently."
-
     library_name = compute_sig(f"{app_type}.{app_class}.{graph_class}")
     library_dir = WORKSPACE / library_name
     library_dir.mkdir(exist_ok=True)
     os.chdir(library_dir)
+    # library = get_lib_path(library_dir, library_name)
+    # if library.exists():
+    #     print("-- Application already exists, skipped.")
+    #     return
     cmakelists_file = library_dir / "CMakeLists.txt"
     with open(CMAKELISTS_TEMPLATE, mode="r") as template:
         content = template.read()
@@ -126,9 +143,10 @@ def cmake_app(app):
         with open(cmakelists_file, mode="w") as f:
             f.write(content)
     cmake_commands = ["cmake", ".", "-DNETWORKX=" + NETWORKX]
+    cmake_commands.extend(extra_option)
 
     cmake_and_make(cmake_commands)
-    print("Finished compiling", app_class, graph_class)
+    print("-- Finished compiling", app_class, graph_class)
 
 
 def get_app_info(algo: str):
@@ -219,7 +237,7 @@ def compile_cpp_pie_app():
             ("pagerank", psuee),
             ("pagerank", pluee),
             ("pagerank", plull),
-            ("wcc", psuee),
+            ("wcc", psuee, ["-DWCC_USE_GID=ON"]),
             ("wcc", pluee),
             ("wcc", plull),
             ("sssp", psuel),
@@ -285,8 +303,8 @@ def compile_cpp_pie_app():
     flued = flatten_template.format("int64_t", "uint64_t", "grape::EmptyType", "double")
     targets.extend(
         [
-            ("pagerank", fsuee),
-            ("pagerank", fluee),
+            # ("pagerank", fsuee), pagerank cannot run on flatten fragment
+            # ("pagerank", fluee),
             ("sssp", fsuel),
             ("sssp", fsued),
             ("sssp", fluel),

python/graphscope/analytical/app/wcc.py

@@ -16,13 +16,16 @@
 # limitations under the License.
 #
 
+import logging
 
 from graphscope.framework.app import AppAssets
 from graphscope.framework.app import not_compatible_for
 from graphscope.framework.app import project_to_simple
 
 __all__ = ["wcc"]
 
+logger = logging.getLogger("graphscope")
+
 
 @project_to_simple
 @not_compatible_for("arrow_property", "dynamic_property")
@@ -49,6 +52,12 @@ def wcc(graph):
         >>> c = graphscope.wcc(pg)
         >>> sess.close()
     """
+    cmake_extra_options = None
     if graph.oid_type == "std::string":
-        raise RuntimeError("Wcc algorithm cannot run on the graph with 'string' type")
-    return AppAssets(algo="wcc", context="vertex_data")(graph)
+        logger.warning(
+            "WCC algorithm will output int value as component ID on graphs that has 'string' type as ID"
+        )
+        cmake_extra_options = "-DWCC_USE_GID=ON"
+    return AppAssets(
+        algo="wcc", context="vertex_data", cmake_extra_options=cmake_extra_options
+    )(graph)

python/graphscope/client/session.py

@@ -21,7 +21,6 @@
 
 import atexit
 import base64
-import concurrent.futures
 import contextlib
 import json
 import logging

python/graphscope/framework/app.py

@@ -163,7 +163,7 @@ class AppAssets(DAGNode):
         "labeled_vertex_property",
     ]
 
-    def __init__(self, algo, context=None, gar=None):
+    def __init__(self, algo, context=None, gar=None, cmake_extra_options=None):
         """Init assets of the algorithm.
 
         Args:
@@ -182,13 +182,15 @@ def __init__(self, algo, context=None, gar=None):
         self._meta = {}
 
         # used for gar resource
-        if gar and isinstance(gar, (BytesIO, bytes)):
+        if gar is not None and isinstance(gar, (BytesIO, bytes)):
             self._gar = gar if isinstance(gar, bytes) else gar.getvalue()
             self._extract_meta_info()
         else:
             # built_in apps has no gar resource.
             self._gar = None
 
+        self._cmake_extra_options = cmake_extra_options
+
         if self._context_type not in self._support_context_type:
             raise InvalidArgumentError(
                 "Unsupport context type: {0}".format(self._context_type)
@@ -269,6 +271,10 @@ def to_gar(cls, path):
     def bytes(cls):
         return cls._gar
 
+    @property
+    def cmake_extra_options(self):
+        return self._cmake_extra_options
+
     @property
     def signature(self):
         """Generate a signature of the app assets by its algo name (and gar resources).

python/graphscope/framework/dag_utils.py

@@ -54,11 +54,17 @@ def bind_app(graph, app_assets):
         analytical engine how to build the app.
     """
     inputs = [graph.op, app_assets.op]
+    config = {}
+    config[types_pb2.APP_ALGO] = utils.s_to_attr(app_assets.algo)
+    if app_assets.cmake_extra_options is not None:
+        config[types_pb2.CMAKE_EXTRA_OPTIONS] = utils.s_to_attr(
+            app_assets.cmake_extra_options
+        )
     op = Operation(
         graph.session_id,
         types_pb2.BIND_APP,
         inputs=inputs,
-        config={types_pb2.APP_ALGO: utils.s_to_attr(app_assets.algo)},
+        config=config,
         output_types=types_pb2.BOUND_APP,
     )
     return op

python/graphscope/proto/types.proto

@@ -205,6 +205,7 @@ enum ParamKey {
   AXIS = 106;
   GAR = 107;
   TYPE_SIGNATURE = 108;
+  CMAKE_EXTRA_OPTIONS = 109;
 
   REPORT_TYPE = 200;
   MODIFY_TYPE = 201;

python/graphscope/tests/unittest/test_app.py

@@ -398,6 +398,8 @@ def test_run_app_on_string_oid_graph(p2p_project_directed_graph_string):
     ctx = sssp(p2p_project_directed_graph_string, src="6")
     r1 = ctx.to_dataframe({"node": "v.id", "r": "r"})
     assert r1[r1["node"] == "6"].r.values[0] == 0.0
+    ctx = wcc(p2p_project_directed_graph_string)
+    r1 = ctx.to_dataframe({"node": "v.id", "r": "r"})
 
 
 @pytest.mark.skipif("FULL_TEST_SUITE" not in os.environ, reason="Run in nightly CI")