refactor engine selection interface to enabled_engines (#2410)

Session interface changed to
```python
sess = graphscope.session(enabled_engines="analytical,interactive,learning")
```
or
```python
sess = graphscope.session(enabled_engines="gae,gie,gle")
```

charts/graphscope/templates/coordinator.yaml

@@ -110,14 +110,8 @@ spec:
           - "False"
           - "--k8s_delete_namespace"
           - "False"
-          - "--k8s_with_analytical"
-          - {{ .Values.engines.analytical.enabled | quote }}
-          - "--k8s_with_analytical_java"
-          - {{ .Values.engines.analytical_java.enabled | quote }}
-          - "--k8s_with_interactive"
-          - {{ .Values.engines.interactive.enabled | quote }}
-          - "--k8s_with_learning"
-          - {{ .Values.engines.learning.enabled | quote }}
+          - "--k8s_enabled_engines"
+          - {{ .Values.engines.enabled_engines }}
           - "--k8s_with_dataset"
           - {{ .Values.engines.dataset.enabled | quote }}
 

charts/graphscope/values.yaml

@@ -65,22 +65,7 @@ engines:
   # Available options of log_level: INFO, DEBUG
   log_level: INFO
 
-  analytical:
-    enabled: True
-    image:
-      name: analytical
-  analytical_java:
-    enabled: False
-    image:
-      name: analytical-java
-  interactive:
-    enabled: True
-    image:
-      name: interactive
-  learning:
-    enabled: True
-    image:
-      name: learning
+  enabled_engines: "analytical,interactive,learning"
   dataset:
     enabled: False
     image:
@@ -102,7 +87,7 @@ vineyard:
   # The vineyard IPC socket is placed on host at /var/run/vineyard-{namespace}-{release}.
   daemonset: ""
   image:
-    name: ghcr.io/v6d-io/v6d/vineyardd
+    name: vineyardcloudnative/vineyardd
     # Overrides the image tag whose default is the chart appVersion.
     tag: v0.11.2
   resources:
@@ -115,8 +100,6 @@ vineyard:
   ## Init size of vineyard shared memory.
   shared_mem: 8Gi
 
-
-
 withJupyter: true
 jupyter:
   image:

coordinator/gscoordinator/cluster_builder.py

@@ -126,19 +126,19 @@ def __init__(
 
         self._vineyard_daemonset = vineyard_daemonset
 
+        self._with_analytical = with_analytical
+        self._with_analytical_java = with_analytical_java
+        self._with_interactive = with_interactive
+        self._with_learning = with_learning
+        self._with_mars = with_mars
+
         if with_analytical and with_analytical_java:
             logger.warning(
                 "Cannot setup `with_analytical` and `with_analytical_java` at the same time"
             )
             logger.warning("Disabled `analytical`.")
             self._with_analytical = False
 
-        self._with_analytical = with_analytical
-        self._with_analytical_java = with_analytical_java
-        self._with_interactive = with_interactive
-        self._with_learning = with_learning
-        self._with_mars = with_mars
-
         self._glog_level = glog_level
         self._preemptive = preemptive
         self._vineyard_shared_mem = vineyard_shared_mem

coordinator/gscoordinator/coordinator.py

@@ -755,36 +755,10 @@ def parse_sys_args():
         help="The port that etcd server will beind to for accepting peer connections. Defaults to 2380.",
     )
     parser.add_argument(
-        "--k8s_with_analytical",
-        type=str2bool,
-        nargs="?",
-        const=True,
-        default=True,
-        help="Enable analytical engine or not.",
-    )
-    parser.add_argument(
-        "--k8s_with_analytical_java",
-        type=str2bool,
-        nargs="?",
-        const=True,
-        default=True,
-        help="Enable analytical engine with java or not.",
-    )
-    parser.add_argument(
-        "--k8s_with_interactive",
-        type=str2bool,
-        nargs="?",
-        const=True,
-        default=True,
-        help="Enable interactive engine or not.",
-    )
-    parser.add_argument(
-        "--k8s_with_learning",
-        type=str2bool,
-        nargs="?",
-        const=True,
-        default=True,
-        help="Enable learning engine or not.",
+        "--k8s_enabled_engines",
+        type=str,
+        default=None,
+        help="A set of engines to enable",
     )
     parser.add_argument(
         "--k8s_with_mars",
@@ -923,10 +897,7 @@ def get_launcher(args):
             volumes=args.k8s_volumes,
             waiting_for_delete=args.waiting_for_delete,
             with_mars=args.k8s_with_mars,
-            with_analytical=args.k8s_with_analytical,
-            with_analytical_java=args.k8s_with_analytical_java,
-            with_interactive=args.k8s_with_interactive,
-            with_learning=args.k8s_with_learning,
+            enabled_engines=args.k8s_enabled_engines,
         )
     elif args.cluster_type == "hosts":
         launcher = LocalLauncher(

coordinator/gscoordinator/kubernetes_launcher.py

@@ -102,10 +102,7 @@ def __init__(
         volumes=None,
         waiting_for_delete=None,
         with_mars=False,
-        with_analytical=True,
-        with_analytical_java=False,
-        with_interactive=True,
-        with_learning=True,
+        enabled_engines="",
         **kwargs,
     ):
 
@@ -156,10 +153,31 @@ def __init__(
 
         self._waiting_for_delete = waiting_for_delete
 
-        self._with_analytical = with_analytical
-        self._with_analytical_java = with_analytical_java
-        self._with_interactive = with_interactive
-        self._with_learning = with_learning
+        self._with_analytical = False
+        self._with_analytical_java = False
+        self._with_interactive = False
+        self._with_learning = False
+        engines = set([item.strip() for item in enabled_engines.split(",")])
+        valid_engines = set(
+            "analytical,analytical-java,interactive,learning,gae,gae-java,gie,gle".split(
+                ","
+            )
+        )
+
+        for item in engines:
+            if item not in valid_engines:
+                raise ValueError(
+                    f"Not a valid engine name: {item}, valid engines are {valid_engines}"
+                )
+            if item == "analytical" or item == "gae":
+                self._with_analytical = True
+            if item == "interactive" or item == "gie":
+                self._with_interactive = True
+            if item == "learning" or item == "gle":
+                self._with_learning = True
+            if item == "analytical-java" or item == "gae-java":
+                self._with_analytical_java = True
+
         self._with_mars = with_mars
         self._mars_scheduler_cpu = mars_scheduler_cpu
         self._mars_scheduler_mem = mars_scheduler_mem
@@ -216,10 +234,10 @@ def __init__(
             vineyard_shared_mem=vineyard_shared_mem,
             volumes=volumes,
             with_mars=with_mars,
-            with_analytical=with_analytical,
-            with_analytical_java=with_analytical_java,
-            with_interactive=with_interactive,
-            with_learning=with_learning,
+            with_analytical=self._with_analytical,
+            with_analytical_java=self._with_analytical_java,
+            with_interactive=self._with_interactive,
+            with_learning=self._with_learning,
         )
 
         self._vineyard_service_endpoint = None

coordinator/requirements.txt

@@ -1,10 +1,10 @@
 etcd-distro>=3.5.1
-graphscope-client>=0.11.0
+graphscope-client>=0.19.0
 grpcio;python_version>="3.11"
 grpcio<=1.43.0,>=1.40.0;python_version<"3.11"
 grpcio-tools;python_version>="3.11"
 grpcio-tools<=1.43.0,>=1.40.0;python_version<"3.11"
-kubernetes~=12.0.1
+kubernetes>=24.2.0
 protobuf;python_version>="3.11"
 protobuf>=3.15.0,<=3.18.1;python_version<"3.11"
 PyYAML

python/graphscope/client/session.py

@@ -317,11 +317,8 @@ def __init__(
         k8s_waiting_for_delete=gs_config.k8s_waiting_for_delete,
         timeout_seconds=gs_config.timeout_seconds,
         dangling_timeout_seconds=gs_config.dangling_timeout_seconds,
+        enabled_engines=gs_config.enabled_engines,
         with_mars=gs_config.with_mars,
-        with_analytical=gs_config.with_analytical,
-        with_analytical_java=gs_config.with_analytical_java,
-        with_interactive=gs_config.with_interactive,
-        with_learning=gs_config.with_learning,
         with_dataset=gs_config.with_dataset,
         reconnect=False,
         hosts=["localhost"],
@@ -420,17 +417,8 @@ def __init__(
             with_mars (bool, optional):
                 Launch graphscope with mars. Defaults to False.
 
-            with_analytical (bool, optional):
-                Launch graphscope with analytical engine. Defaults to True.
-
-            with_analytical_java (bool, optional):
-                Launch graphscope with analytical engine with java support. Defaults to False.
-
-            with_interactive (bool, optional):
-                Launch graphscope with interactive engine. Defaults to True.
-
-            with_learning (bool, optional):
-                Launch graphscope with learning engine. Defaults to True.
+            enabled_engines (str, optional):
+                Select a subset of engines to enable. Only make sense in k8s mode.
 
             with_dataset (bool, optional):
                 Create a container and mount aliyun demo dataset bucket to the path `/dataset`.
@@ -568,16 +556,13 @@ def __init__(
             "k8s_mars_scheduler_mem",
             "k8s_coordinator_pod_node_selector",
             "k8s_engine_pod_node_selector",
-            "with_mars",
-            "with_analytical",
-            "with_analytical_java",
-            "with_interactive",
-            "with_learning",
+            "enabled_engines",
             "reconnect",
             "k8s_volumes",
             "k8s_waiting_for_delete",
             "timeout_seconds",
             "dangling_timeout_seconds",
+            "with_mars",
             "with_dataset",
             "hosts",
         )
@@ -608,16 +593,23 @@ def __init__(
         self._dag = Dag()
 
         # mars cannot work with run-on-local mode
-        if self._cluster_type == types_pb2.HOSTS and self._config_params["with_mars"]:
-            raise NotImplementedError(
-                "Mars cluster cannot be launched along with local GraphScope deployment"
-            )
-        if with_analytical and with_analytical_java:
-            logger.warning(
-                "Cannot setup `with_analytical` and `with_analytical_java` at the same time"
+        if self._cluster_type == types_pb2.HOSTS:
+            if self._config_params["with_mars"]:
+                logger.warning(
+                    "Mars cluster cannot be launched along with local GraphScope deployment"
+                )
+        if self._cluster_type == types_pb2.K8S:
+            engines = set([item.strip() for item in enabled_engines.split(",")])
+            valid_engines = set(
+                "analytical,analytical-java,interactive,learning,gae,gae-java,gie,gle".split(
+                    ","
+                )
             )
-            logger.warning("Disabled `analytical`.")
-            self._config_params["with_analytical"] = False
+            for item in engines:
+                if item not in valid_engines:
+                    raise ValueError(
+                        f"Not a valid engine name: {item}, valid engines are {valid_engines}"
+                    )
 
         # deprecated params handle
         for param in self._deprecated_params:
@@ -1332,11 +1324,9 @@ def set_option(**kwargs):
         - k8s_mars_worker_mem
         - k8s_mars_scheduler_cpu
         - k8s_mars_scheduler_mem
+        - enabled_engines
         - with_mars
-        - with_analytical
-        - with_analytical_java
-        - with_interactive
-        - with_learning
+        - with_dataset
         - k8s_volumes
         - k8s_waiting_for_delete
         - timeout_seconds
@@ -1389,11 +1379,9 @@ def get_option(key):
         - k8s_mars_worker_mem
         - k8s_mars_scheduler_cpu
         - k8s_mars_scheduler_mem
+        - enabled_engines
         - with_mars
-        - with_analytical
-        - with_analytical_java
-        - with_interactive
-        - with_learning
+        - with_dataset
         - k8s_volumes
         - k8s_waiting_for_delete
         - timeout_seconds
@@ -1410,7 +1398,7 @@ def get_option(key):
     """
     if hasattr(gs_config, key):
         return getattr(gs_config, key)
-    raise ValueError("No such option {} exists.".format(key))
+    raise ValueError(f"No such option {key}.")
 
 
 def default_session(session):

python/graphscope/config.py

@@ -88,12 +88,12 @@ class GSConfig(object):
     k8s_coordinator_pod_node_selector = None
     k8s_engine_pod_node_selector = None
 
+    # Enabled engines, default to all 3 engines
+    # Available options: analytical, analytical-java, interactive, learning
+    enabled_engines = "analytical,interactive,learning"
+
     # launch graphscope with mars
     with_mars = False
-    with_analytical = True
-    with_analytical_java = False
-    with_interactive = True
-    with_learning = True
     # Demo dataset related
     with_dataset = False
 

python/graphscope/deploy/kubernetes/cluster.py

@@ -90,10 +90,7 @@ def __init__(
         k8s_coordinator_pod_node_selector=None,
         k8s_engine_pod_node_selector=None,
         with_mars=None,
-        with_analytical=None,
-        with_analytical_java=None,
-        with_interactive=None,
-        with_learning=None,
+        enabled_engines=None,
         k8s_volumes=None,
         timeout_seconds=600,
         dangling_timeout_seconds=None,
@@ -380,14 +377,8 @@ def _get_coordinator_args(self):
             str(self._saved_locals["k8s_mars_scheduler_mem"]),
             "--k8s_with_mars",
             str(self._saved_locals["with_mars"]),
-            "--k8s_with_analytical",
-            str(self._saved_locals["with_analytical"]),
-            "--k8s_with_analytical_java",
-            str(self._saved_locals["with_analytical_java"]),
-            "--k8s_with_interactive",
-            str(self._saved_locals["with_interactive"]),
-            "--k8s_with_learning",
-            str(self._saved_locals["with_learning"]),
+            "--k8s_enabled_engines",
+            str(self._saved_locals["enabled_engines"]),
             "--k8s_with_dataset",
             str(self._saved_locals["with_dataset"]),
             "--timeout_seconds",

python/graphscope/tests/kubernetes/test_demo_script.py

@@ -332,3 +332,13 @@ def test_helm_installation():
     g = sess.g()
     assert g is not None
     sess.close()
+
+
+def test_modualize():
+    sess = graphscope.session(
+        num_workers=1,
+        k8s_image_registry=get_gs_registry_on_ci_env(),
+        k8s_image_tag=get_gs_tag_on_ci_env(),
+        enabled_engines="interactive",
+    )
+    sess.close()