From 0491d277902f3499d03490e842d3cfc852dfac32 Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Mon, 6 Feb 2023 15:59:22 +0800 Subject: [PATCH] refactor engine selection interface to enabled_engines (#2410) Session interface changed to ```python sess = graphscope.session(enabled_engines="analytical,interactive,learning") ``` or ```python sess = graphscope.session(enabled_engines="gae,gie,gle") ``` --- charts/graphscope/templates/coordinator.yaml | 10 +-- charts/graphscope/values.yaml | 21 +----- coordinator/gscoordinator/cluster_builder.py | 12 ++-- coordinator/gscoordinator/coordinator.py | 39 ++--------- .../gscoordinator/kubernetes_launcher.py | 42 ++++++++---- coordinator/requirements.txt | 4 +- python/graphscope/client/session.py | 64 ++++++++----------- python/graphscope/config.py | 8 +-- .../graphscope/deploy/kubernetes/cluster.py | 15 +---- .../tests/kubernetes/test_demo_script.py | 10 +++ python/requirements-dev.txt | 1 - 11 files changed, 90 insertions(+), 136 deletions(-) diff --git a/charts/graphscope/templates/coordinator.yaml b/charts/graphscope/templates/coordinator.yaml index f11bc20184e7..e2b7597d8656 100644 --- a/charts/graphscope/templates/coordinator.yaml +++ b/charts/graphscope/templates/coordinator.yaml @@ -110,14 +110,8 @@ spec: - "False" - "--k8s_delete_namespace" - "False" - - "--k8s_with_analytical" - - {{ .Values.engines.analytical.enabled | quote }} - - "--k8s_with_analytical_java" - - {{ .Values.engines.analytical_java.enabled | quote }} - - "--k8s_with_interactive" - - {{ .Values.engines.interactive.enabled | quote }} - - "--k8s_with_learning" - - {{ .Values.engines.learning.enabled | quote }} + - "--k8s_enabled_engines" + - {{ .Values.engines.enabled_engines }} - "--k8s_with_dataset" - {{ .Values.engines.dataset.enabled | quote }} diff --git a/charts/graphscope/values.yaml b/charts/graphscope/values.yaml index 03bdb0f19333..4ea4de83b3a5 100644 --- a/charts/graphscope/values.yaml +++ b/charts/graphscope/values.yaml @@ -65,22 +65,7 @@ engines: # Available options of log_level: INFO, DEBUG log_level: INFO - analytical: - enabled: True - image: - name: analytical - analytical_java: - enabled: False - image: - name: analytical-java - interactive: - enabled: True - image: - name: interactive - learning: - enabled: True - image: - name: learning + enabled_engines: "analytical,interactive,learning" dataset: enabled: False image: @@ -102,7 +87,7 @@ vineyard: # The vineyard IPC socket is placed on host at /var/run/vineyard-{namespace}-{release}. daemonset: "" image: - name: ghcr.io/v6d-io/v6d/vineyardd + name: vineyardcloudnative/vineyardd # Overrides the image tag whose default is the chart appVersion. tag: v0.11.2 resources: @@ -115,8 +100,6 @@ vineyard: ## Init size of vineyard shared memory. shared_mem: 8Gi - - withJupyter: true jupyter: image: diff --git a/coordinator/gscoordinator/cluster_builder.py b/coordinator/gscoordinator/cluster_builder.py index 6a22edbe9d52..2b655f3c30eb 100644 --- a/coordinator/gscoordinator/cluster_builder.py +++ b/coordinator/gscoordinator/cluster_builder.py @@ -126,6 +126,12 @@ def __init__( self._vineyard_daemonset = vineyard_daemonset + self._with_analytical = with_analytical + self._with_analytical_java = with_analytical_java + self._with_interactive = with_interactive + self._with_learning = with_learning + self._with_mars = with_mars + if with_analytical and with_analytical_java: logger.warning( "Cannot setup `with_analytical` and `with_analytical_java` at the same time" @@ -133,12 +139,6 @@ def __init__( logger.warning("Disabled `analytical`.") self._with_analytical = False - self._with_analytical = with_analytical - self._with_analytical_java = with_analytical_java - self._with_interactive = with_interactive - self._with_learning = with_learning - self._with_mars = with_mars - self._glog_level = glog_level self._preemptive = preemptive self._vineyard_shared_mem = vineyard_shared_mem diff --git a/coordinator/gscoordinator/coordinator.py b/coordinator/gscoordinator/coordinator.py index 1c3dc79b6b98..bb0247bb55a3 100644 --- a/coordinator/gscoordinator/coordinator.py +++ b/coordinator/gscoordinator/coordinator.py @@ -755,36 +755,10 @@ def parse_sys_args(): help="The port that etcd server will beind to for accepting peer connections. Defaults to 2380.", ) parser.add_argument( - "--k8s_with_analytical", - type=str2bool, - nargs="?", - const=True, - default=True, - help="Enable analytical engine or not.", - ) - parser.add_argument( - "--k8s_with_analytical_java", - type=str2bool, - nargs="?", - const=True, - default=True, - help="Enable analytical engine with java or not.", - ) - parser.add_argument( - "--k8s_with_interactive", - type=str2bool, - nargs="?", - const=True, - default=True, - help="Enable interactive engine or not.", - ) - parser.add_argument( - "--k8s_with_learning", - type=str2bool, - nargs="?", - const=True, - default=True, - help="Enable learning engine or not.", + "--k8s_enabled_engines", + type=str, + default=None, + help="A set of engines to enable", ) parser.add_argument( "--k8s_with_mars", @@ -923,10 +897,7 @@ def get_launcher(args): volumes=args.k8s_volumes, waiting_for_delete=args.waiting_for_delete, with_mars=args.k8s_with_mars, - with_analytical=args.k8s_with_analytical, - with_analytical_java=args.k8s_with_analytical_java, - with_interactive=args.k8s_with_interactive, - with_learning=args.k8s_with_learning, + enabled_engines=args.k8s_enabled_engines, ) elif args.cluster_type == "hosts": launcher = LocalLauncher( diff --git a/coordinator/gscoordinator/kubernetes_launcher.py b/coordinator/gscoordinator/kubernetes_launcher.py index 3845925a81a3..e426c68d91f2 100644 --- a/coordinator/gscoordinator/kubernetes_launcher.py +++ b/coordinator/gscoordinator/kubernetes_launcher.py @@ -102,10 +102,7 @@ def __init__( volumes=None, waiting_for_delete=None, with_mars=False, - with_analytical=True, - with_analytical_java=False, - with_interactive=True, - with_learning=True, + enabled_engines="", **kwargs, ): @@ -156,10 +153,31 @@ def __init__( self._waiting_for_delete = waiting_for_delete - self._with_analytical = with_analytical - self._with_analytical_java = with_analytical_java - self._with_interactive = with_interactive - self._with_learning = with_learning + self._with_analytical = False + self._with_analytical_java = False + self._with_interactive = False + self._with_learning = False + engines = set([item.strip() for item in enabled_engines.split(",")]) + valid_engines = set( + "analytical,analytical-java,interactive,learning,gae,gae-java,gie,gle".split( + "," + ) + ) + + for item in engines: + if item not in valid_engines: + raise ValueError( + f"Not a valid engine name: {item}, valid engines are {valid_engines}" + ) + if item == "analytical" or item == "gae": + self._with_analytical = True + if item == "interactive" or item == "gie": + self._with_interactive = True + if item == "learning" or item == "gle": + self._with_learning = True + if item == "analytical-java" or item == "gae-java": + self._with_analytical_java = True + self._with_mars = with_mars self._mars_scheduler_cpu = mars_scheduler_cpu self._mars_scheduler_mem = mars_scheduler_mem @@ -216,10 +234,10 @@ def __init__( vineyard_shared_mem=vineyard_shared_mem, volumes=volumes, with_mars=with_mars, - with_analytical=with_analytical, - with_analytical_java=with_analytical_java, - with_interactive=with_interactive, - with_learning=with_learning, + with_analytical=self._with_analytical, + with_analytical_java=self._with_analytical_java, + with_interactive=self._with_interactive, + with_learning=self._with_learning, ) self._vineyard_service_endpoint = None diff --git a/coordinator/requirements.txt b/coordinator/requirements.txt index 0e42efcb621d..2b8bb5d49bd4 100644 --- a/coordinator/requirements.txt +++ b/coordinator/requirements.txt @@ -1,10 +1,10 @@ etcd-distro>=3.5.1 -graphscope-client>=0.11.0 +graphscope-client>=0.19.0 grpcio;python_version>="3.11" grpcio<=1.43.0,>=1.40.0;python_version<"3.11" grpcio-tools;python_version>="3.11" grpcio-tools<=1.43.0,>=1.40.0;python_version<"3.11" -kubernetes~=12.0.1 +kubernetes>=24.2.0 protobuf;python_version>="3.11" protobuf>=3.15.0,<=3.18.1;python_version<"3.11" PyYAML diff --git a/python/graphscope/client/session.py b/python/graphscope/client/session.py index 7cec53a47b4f..b435d17f98af 100755 --- a/python/graphscope/client/session.py +++ b/python/graphscope/client/session.py @@ -317,11 +317,8 @@ def __init__( k8s_waiting_for_delete=gs_config.k8s_waiting_for_delete, timeout_seconds=gs_config.timeout_seconds, dangling_timeout_seconds=gs_config.dangling_timeout_seconds, + enabled_engines=gs_config.enabled_engines, with_mars=gs_config.with_mars, - with_analytical=gs_config.with_analytical, - with_analytical_java=gs_config.with_analytical_java, - with_interactive=gs_config.with_interactive, - with_learning=gs_config.with_learning, with_dataset=gs_config.with_dataset, reconnect=False, hosts=["localhost"], @@ -420,17 +417,8 @@ def __init__( with_mars (bool, optional): Launch graphscope with mars. Defaults to False. - with_analytical (bool, optional): - Launch graphscope with analytical engine. Defaults to True. - - with_analytical_java (bool, optional): - Launch graphscope with analytical engine with java support. Defaults to False. - - with_interactive (bool, optional): - Launch graphscope with interactive engine. Defaults to True. - - with_learning (bool, optional): - Launch graphscope with learning engine. Defaults to True. + enabled_engines (str, optional): + Select a subset of engines to enable. Only make sense in k8s mode. with_dataset (bool, optional): Create a container and mount aliyun demo dataset bucket to the path `/dataset`. @@ -568,16 +556,13 @@ def __init__( "k8s_mars_scheduler_mem", "k8s_coordinator_pod_node_selector", "k8s_engine_pod_node_selector", - "with_mars", - "with_analytical", - "with_analytical_java", - "with_interactive", - "with_learning", + "enabled_engines", "reconnect", "k8s_volumes", "k8s_waiting_for_delete", "timeout_seconds", "dangling_timeout_seconds", + "with_mars", "with_dataset", "hosts", ) @@ -608,16 +593,23 @@ def __init__( self._dag = Dag() # mars cannot work with run-on-local mode - if self._cluster_type == types_pb2.HOSTS and self._config_params["with_mars"]: - raise NotImplementedError( - "Mars cluster cannot be launched along with local GraphScope deployment" - ) - if with_analytical and with_analytical_java: - logger.warning( - "Cannot setup `with_analytical` and `with_analytical_java` at the same time" + if self._cluster_type == types_pb2.HOSTS: + if self._config_params["with_mars"]: + logger.warning( + "Mars cluster cannot be launched along with local GraphScope deployment" + ) + if self._cluster_type == types_pb2.K8S: + engines = set([item.strip() for item in enabled_engines.split(",")]) + valid_engines = set( + "analytical,analytical-java,interactive,learning,gae,gae-java,gie,gle".split( + "," + ) ) - logger.warning("Disabled `analytical`.") - self._config_params["with_analytical"] = False + for item in engines: + if item not in valid_engines: + raise ValueError( + f"Not a valid engine name: {item}, valid engines are {valid_engines}" + ) # deprecated params handle for param in self._deprecated_params: @@ -1332,11 +1324,9 @@ def set_option(**kwargs): - k8s_mars_worker_mem - k8s_mars_scheduler_cpu - k8s_mars_scheduler_mem + - enabled_engines - with_mars - - with_analytical - - with_analytical_java - - with_interactive - - with_learning + - with_dataset - k8s_volumes - k8s_waiting_for_delete - timeout_seconds @@ -1389,11 +1379,9 @@ def get_option(key): - k8s_mars_worker_mem - k8s_mars_scheduler_cpu - k8s_mars_scheduler_mem + - enabled_engines - with_mars - - with_analytical - - with_analytical_java - - with_interactive - - with_learning + - with_dataset - k8s_volumes - k8s_waiting_for_delete - timeout_seconds @@ -1410,7 +1398,7 @@ def get_option(key): """ if hasattr(gs_config, key): return getattr(gs_config, key) - raise ValueError("No such option {} exists.".format(key)) + raise ValueError(f"No such option {key}.") def default_session(session): diff --git a/python/graphscope/config.py b/python/graphscope/config.py index 079334f7d2d4..f902f90536b1 100644 --- a/python/graphscope/config.py +++ b/python/graphscope/config.py @@ -88,12 +88,12 @@ class GSConfig(object): k8s_coordinator_pod_node_selector = None k8s_engine_pod_node_selector = None + # Enabled engines, default to all 3 engines + # Available options: analytical, analytical-java, interactive, learning + enabled_engines = "analytical,interactive,learning" + # launch graphscope with mars with_mars = False - with_analytical = True - with_analytical_java = False - with_interactive = True - with_learning = True # Demo dataset related with_dataset = False diff --git a/python/graphscope/deploy/kubernetes/cluster.py b/python/graphscope/deploy/kubernetes/cluster.py index 030ff65ca7fe..edd08d600eec 100644 --- a/python/graphscope/deploy/kubernetes/cluster.py +++ b/python/graphscope/deploy/kubernetes/cluster.py @@ -90,10 +90,7 @@ def __init__( k8s_coordinator_pod_node_selector=None, k8s_engine_pod_node_selector=None, with_mars=None, - with_analytical=None, - with_analytical_java=None, - with_interactive=None, - with_learning=None, + enabled_engines=None, k8s_volumes=None, timeout_seconds=600, dangling_timeout_seconds=None, @@ -380,14 +377,8 @@ def _get_coordinator_args(self): str(self._saved_locals["k8s_mars_scheduler_mem"]), "--k8s_with_mars", str(self._saved_locals["with_mars"]), - "--k8s_with_analytical", - str(self._saved_locals["with_analytical"]), - "--k8s_with_analytical_java", - str(self._saved_locals["with_analytical_java"]), - "--k8s_with_interactive", - str(self._saved_locals["with_interactive"]), - "--k8s_with_learning", - str(self._saved_locals["with_learning"]), + "--k8s_enabled_engines", + str(self._saved_locals["enabled_engines"]), "--k8s_with_dataset", str(self._saved_locals["with_dataset"]), "--timeout_seconds", diff --git a/python/graphscope/tests/kubernetes/test_demo_script.py b/python/graphscope/tests/kubernetes/test_demo_script.py index 3a98d22b09ee..61f9c52575d4 100644 --- a/python/graphscope/tests/kubernetes/test_demo_script.py +++ b/python/graphscope/tests/kubernetes/test_demo_script.py @@ -332,3 +332,13 @@ def test_helm_installation(): g = sess.g() assert g is not None sess.close() + + +def test_modualize(): + sess = graphscope.session( + num_workers=1, + k8s_image_registry=get_gs_registry_on_ci_env(), + k8s_image_tag=get_gs_tag_on_ci_env(), + enabled_engines="interactive", + ) + sess.close() diff --git a/python/requirements-dev.txt b/python/requirements-dev.txt index 26f8c9bbe201..b844ad035fab 100644 --- a/python/requirements-dev.txt +++ b/python/requirements-dev.txt @@ -16,6 +16,5 @@ sphinx-copybutton sphinx-panels sphinxemoji sphinxext-opengraph -tomli docutils==0.16 wheel