From 1e602fc23d5ea3378a6da4cc605644d90b71d26e Mon Sep 17 00:00:00 2001 From: DongZe Li <9546726@qq.com> Date: Wed, 1 Sep 2021 20:27:50 +0800 Subject: [PATCH] Remove the legacy code related to graph manager and make default user 'graphscope' in graphscope-store image (#748) * Fix nightly dockerfile * Fix permission problem of graphscope-store image when using non-root user --- .github/workflows/ci.yml | 6 +---- .github/workflows/docker_nightly.yml | 11 -------- .github/workflows/graphscope-store.Dockerfile | 14 ++++------ README-zh.md | 5 ++-- README.md | 5 ++-- .../templates/coordinator/statefulset.yaml | 20 +++++++------- .../templates/frontend/statefulset.yaml | 22 +++++++-------- .../templates/ingestor/statefulset.yaml | 22 +++++++-------- .../templates/store/statefulset.yaml | 22 +++++++-------- k8s/graphscope-store.Dockerfile | 27 +++++++++---------- k8s/graphscope.Dockerfile | 2 +- proto/types.proto | 2 -- python/graphscope/client/session.py | 8 ------ python/graphscope/config.py | 4 --- python/graphscope/framework/dag_utils.py | 6 +---- python/graphscope/interactive/query.py | 3 --- python/tests/kubernetes/test_demo_script.py | 4 +-- python/tests/kubernetes/test_with_gaia.py | 4 +-- scripts/prepare_env.sh | 4 --- scripts/test.sh | 13 ++------- 20 files changed, 71 insertions(+), 133 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0df2f7594cee..ccebf97a88e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -416,8 +416,7 @@ jobs: - name: Clean if: always() run: | - sudo docker rmi -f registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} \ - registry.cn-hongkong.aliyuncs.com/graphscope/maxgraph_standalone_manager:${{ github.sha }} || true + sudo docker rmi -f registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${{ github.sha }} || true helm uninstall graphscope || true kubectl delete pod graphscope-test-rpc-service --wait=false || true @@ -519,9 +518,6 @@ jobs: echo "source ~/.cargo/env" >> ~/.bashrc source ~/.bashrc rustup component add rustfmt - sudo yum update -y - sudo yum install -y xz - sudo ln -s /bin/true /bin/apt-get - uses: actions/checkout@v2.3.2 diff --git a/.github/workflows/docker_nightly.yml b/.github/workflows/docker_nightly.yml index 7e8a279c33b2..9d5927fcf2a5 100644 --- a/.github/workflows/docker_nightly.yml +++ b/.github/workflows/docker_nightly.yml @@ -17,11 +17,6 @@ jobs: cd ${GITHUB_WORKSPACE} sudo make graphscope - - name: Build interactive_manager image - run: | - cd ${GITHUB_WORKSPACE} - sudo make interactive_manager - - name: Build graphscope-store image run: | cd ${GITHUB_WORKSPACE} @@ -32,7 +27,6 @@ jobs: short_sha=$(git rev-parse --short HEAD) echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV echo "GS_IMAGE=graphscope/graphscope:${short_sha}" >> $GITHUB_ENV - echo "GIE_MANAGER_IMAGE=graphscope/maxgraph_standalone_manager:${short_sha}" >> $GITHUB_ENV - name: Kubernetes test env: @@ -53,21 +47,16 @@ jobs: echo ${{ secrets.ALIYUN_TOKEN }} | sudo docker login --username=grape_dev registry.cn-hongkong.aliyuncs.com --password-stdin sudo docker tag graphscope/graphscope:${SHORT_SHA} \ registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:nightly - sudo docker tag graphscope/maxgraph_standalone_manager:${SHORT_SHA} \ - registry.cn-hongkong.aliyuncs.com/graphscope/maxgraph_standalone_manager:nightly sudo docker tag graphscope/graphscope-store:${SHORT_SHA} \ registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-store:nightly sudo docker push registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:nightly - sudo docker push registry.cn-hongkong.aliyuncs.com/graphscope/maxgraph_standalone_manager:nightly sudo docker push registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-store:nightly - name: Clean if: always() run: | sudo docker rmi -f graphscope/graphscope:${SHORT_SHA} \ - graphscope/maxgraph_standalone_manager:${SHORT_SHA} \ graphscope/graphscope-store:${SHORT_SHA} \ registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:nightly \ - registry.cn-hongkong.aliyuncs.com/graphscope/maxgraph_standalone_manager:nightly \ registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-store:nightly || true diff --git a/.github/workflows/graphscope-store.Dockerfile b/.github/workflows/graphscope-store.Dockerfile index fc513d9e13af..6351f743b290 100644 --- a/.github/workflows/graphscope-store.Dockerfile +++ b/.github/workflows/graphscope-store.Dockerfile @@ -1,13 +1,9 @@ -FROM registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-runtime:latest-root-user +FROM registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-runtime:latest -RUN mkdir -p /home/maxgraph/{bin,conf} COPY ./opt/graphscope/ /usr/local/ -COPY ./maxgraph /home/maxgraph/maxgraph +COPY ./maxgraph /usr/local/maxgraph COPY ./ready_probe.sh /tmp/ready_probe.sh -COPY ./interactive_engine/bin/giectl /home/maxgraph/bin/giectl -COPY ./interactive_engine/conf/* /home/maxgraph/conf/ +COPY ./interactive_engine/bin/giectl /usr/local/bin/giectl +COPY ./interactive_engine/conf/* /usr/local/conf/ -ENV GRAPHSCOPE_HOME=/home/maxgraph -ENV GRAPHSCOPE_RUNTIME=/tmp/graphscope - -WORKDIR /home/maxgraph/ +ENV GRAPHSCOPE_HOME=/usr/local diff --git a/README-zh.md b/README-zh.md index 26bc5c5253af..42c97758fc83 100644 --- a/README-zh.md +++ b/README-zh.md @@ -322,8 +322,7 @@ GraphScope 包含一个 [Dockerfile](k8s/graphscope.Dockerfile)，可以构建 D # ./scripts/prepare_dev.sh make graphscope -make interactive_manager -# by default, the built image is tagged as graphscope/graphscope:SHORTSHA and graphscope/maxgraph_standalone_manager:SHORTSHA +# by default, the built image is tagged as graphscope/graphscope:SHORTSHA ``` ### 构建客户端 Python 包 @@ -351,7 +350,7 @@ python3 setup.py install ./scripts/test.sh --all # run all test cases on your built image -./scripts/test.sh --all --gs_image graphscope/graphscope:SHORTSHA --gie_manager_image graphscope/maxgraph_standalone_manager:SHORTSHA +./scripts/test.sh --all --gs_image graphscope/graphscope:SHORTSHA # or run the selected cases on a certain module. e.g., ./scripts/test.sh --python diff --git a/README.md b/README.md index 29b7a69c7bd2..7b9bb72f233c 100644 --- a/README.md +++ b/README.md @@ -328,8 +328,7 @@ a `runtime-base` image. To build images with latest version of GraphScope, go to # ./scripts/prepare_dev.sh make graphscope -make interactive_manager -# by default, the built image is tagged as graphscope/graphscope:SHORTSHA and graphscope/maxgraph_standalone_manager:SHORTSHA +# by default, the built image is tagged as graphscope/graphscope:SHORTSHA ``` ### Building client library @@ -364,7 +363,7 @@ You may run the whole test suite with commands: ./scripts/test.sh --all # run all test cases on your built image -./scripts/test.sh --all --gs_image graphscope/graphscope:SHORTSHA --gie_manager_image graphscope/maxgraph_standalone_manager:SHORTSHA +./scripts/test.sh --all --gs_image graphscope/graphscope:SHORTSHA # or run the selected cases on a certain module. e.g., ./scripts/test.sh --python diff --git a/charts/graphscope-store/templates/coordinator/statefulset.yaml b/charts/graphscope-store/templates/coordinator/statefulset.yaml index 90bde75da05f..c499589208ac 100644 --- a/charts/graphscope-store/templates/coordinator/statefulset.yaml +++ b/charts/graphscope-store/templates/coordinator/statefulset.yaml @@ -68,16 +68,16 @@ spec: [[ `hostname` =~ -([0-9]+)$ ]] || exit 1 ordinal=${BASH_REMATCH[1]} - sed -e "s/ROLE/$(ROLE)/g" \ - -e "s/INDEX/${ordinal}/g" \ - -e "s/KAFKA_SERVERS/$(KAFKA_SERVERS)/g" \ - -e "s/FRONTEND/$(DNS_NAME_PREFIX_FRONTEND)/g" \ - -e "s/INGESTOR/$(DNS_NAME_PREFIX_INGESTOR)/g" \ - -e "s/COORDINATOR/$(DNS_NAME_PREFIX_COORDINATOR)/g" \ - -e "s/STORE/$(DNS_NAME_PREFIX_STORE)/g" \ - -e "s@LOG4RS_CONFIG@${GRAPHSCOPE_HOME}/conf/log4rs.yml@g" \ - /etc/graphscope-store/my.cnf.tpl > /etc/graphscope-store/my.cnf - LOG_NAME=graphscope-store.log /home/maxgraph/bin/giectl maxgraph /etc/graphscope-store/my.cnf coordinator + sudo sed -e "s/ROLE/$(ROLE)/g" \ + -e "s/INDEX/${ordinal}/g" \ + -e "s/KAFKA_SERVERS/$(KAFKA_SERVERS)/g" \ + -e "s/FRONTEND/$(DNS_NAME_PREFIX_FRONTEND)/g" \ + -e "s/INGESTOR/$(DNS_NAME_PREFIX_INGESTOR)/g" \ + -e "s/COORDINATOR/$(DNS_NAME_PREFIX_COORDINATOR)/g" \ + -e "s/STORE/$(DNS_NAME_PREFIX_STORE)/g" \ + -e "s@LOG4RS_CONFIG@${GRAPHSCOPE_HOME}/conf/log4rs.yml@g" \ + /etc/graphscope-store/my.cnf.tpl | sudo tee /etc/graphscope-store/my.cnf + LOG_NAME=graphscope-store.log /usr/local/bin/giectl maxgraph /etc/graphscope-store/my.cnf coordinator tail -f /dev/null env: - name: DEBUG diff --git a/charts/graphscope-store/templates/frontend/statefulset.yaml b/charts/graphscope-store/templates/frontend/statefulset.yaml index 94ae5b275f23..fdac8bff1af3 100644 --- a/charts/graphscope-store/templates/frontend/statefulset.yaml +++ b/charts/graphscope-store/templates/frontend/statefulset.yaml @@ -68,18 +68,18 @@ spec: [[ `hostname` =~ -([0-9]+)$ ]] || exit 1 ordinal=${BASH_REMATCH[1]} - sed -e "s/GRAPH_NAME/$(GRAPH_NAME)/g" \ - -e "s/ROLE/$(ROLE)/g" \ - -e "s/INDEX/${ordinal}/g" \ - -e "s/KAFKA_SERVERS/$(KAFKA_SERVERS)/g" \ - -e "s/FRONTEND/$(DNS_NAME_PREFIX_FRONTEND)/g" \ - -e "s/INGESTOR/$(DNS_NAME_PREFIX_INGESTOR)/g" \ - -e "s/COORDINATOR/$(DNS_NAME_PREFIX_COORDINATOR)/g" \ - -e "s/STORE/$(DNS_NAME_PREFIX_STORE)/g" \ - -e "s@LOG4RS_CONFIG@${GRAPHSCOPE_HOME}/conf/log4rs.yml@g" \ - /etc/graphscope-store/my.cnf.tpl > /etc/graphscope-store/my.cnf + sudo sed -e "s/GRAPH_NAME/$(GRAPH_NAME)/g" \ + -e "s/ROLE/$(ROLE)/g" \ + -e "s/INDEX/${ordinal}/g" \ + -e "s/KAFKA_SERVERS/$(KAFKA_SERVERS)/g" \ + -e "s/FRONTEND/$(DNS_NAME_PREFIX_FRONTEND)/g" \ + -e "s/INGESTOR/$(DNS_NAME_PREFIX_INGESTOR)/g" \ + -e "s/COORDINATOR/$(DNS_NAME_PREFIX_COORDINATOR)/g" \ + -e "s/STORE/$(DNS_NAME_PREFIX_STORE)/g" \ + -e "s@LOG4RS_CONFIG@${GRAPHSCOPE_HOME}/conf/log4rs.yml@g" \ + /etc/graphscope-store/my.cnf.tpl | sudo tee /etc/graphscope-store/my.cnf - LOG_NAME=graphscope-store.log /home/maxgraph/bin/giectl maxgraph /etc/graphscope-store/my.cnf $(EXECUTOR) + LOG_NAME=graphscope-store.log /usr/local/bin/giectl maxgraph /etc/graphscope-store/my.cnf $(EXECUTOR) tail -f /dev/null env: - name: DEBUG diff --git a/charts/graphscope-store/templates/ingestor/statefulset.yaml b/charts/graphscope-store/templates/ingestor/statefulset.yaml index 2747f0eb8c0d..eeab5519e7f9 100644 --- a/charts/graphscope-store/templates/ingestor/statefulset.yaml +++ b/charts/graphscope-store/templates/ingestor/statefulset.yaml @@ -68,18 +68,18 @@ spec: [[ `hostname` =~ -([0-9]+)$ ]] || exit 1 ordinal=${BASH_REMATCH[1]} - sed -e "s/GRAPH_NAME/$(GRAPH_NAME)/g" \ - -e "s/ROLE/$(ROLE)/g" \ - -e "s/INDEX/${ordinal}/g" \ - -e "s/KAFKA_SERVERS/$(KAFKA_SERVERS)/g" \ - -e "s/FRONTEND/$(DNS_NAME_PREFIX_FRONTEND)/g" \ - -e "s/INGESTOR/$(DNS_NAME_PREFIX_INGESTOR)/g" \ - -e "s/COORDINATOR/$(DNS_NAME_PREFIX_COORDINATOR)/g" \ - -e "s/STORE/$(DNS_NAME_PREFIX_STORE)/g" \ - -e "s@LOG4RS_CONFIG@${GRAPHSCOPE_HOME}/conf/log4rs.yml@g" \ - /etc/graphscope-store/my.cnf.tpl > /etc/graphscope-store/my.cnf + sudo sed -e "s/GRAPH_NAME/$(GRAPH_NAME)/g" \ + -e "s/ROLE/$(ROLE)/g" \ + -e "s/INDEX/${ordinal}/g" \ + -e "s/KAFKA_SERVERS/$(KAFKA_SERVERS)/g" \ + -e "s/FRONTEND/$(DNS_NAME_PREFIX_FRONTEND)/g" \ + -e "s/INGESTOR/$(DNS_NAME_PREFIX_INGESTOR)/g" \ + -e "s/COORDINATOR/$(DNS_NAME_PREFIX_COORDINATOR)/g" \ + -e "s/STORE/$(DNS_NAME_PREFIX_STORE)/g" \ + -e "s@LOG4RS_CONFIG@${GRAPHSCOPE_HOME}/conf/log4rs.yml@g" \ + /etc/graphscope-store/my.cnf.tpl | sudo tee /etc/graphscope-store/my.cnf - LOG_NAME=graphscope-store.log /home/maxgraph/bin/giectl maxgraph /etc/graphscope-store/my.cnf ingestor + LOG_NAME=graphscope-store.log /usr/local/bin/giectl maxgraph /etc/graphscope-store/my.cnf ingestor tail -f /dev/null env: - name: DEBUG diff --git a/charts/graphscope-store/templates/store/statefulset.yaml b/charts/graphscope-store/templates/store/statefulset.yaml index c256d0afe935..abe8d8e2f58e 100644 --- a/charts/graphscope-store/templates/store/statefulset.yaml +++ b/charts/graphscope-store/templates/store/statefulset.yaml @@ -68,18 +68,18 @@ spec: [[ `hostname` =~ -([0-9]+)$ ]] || exit 1 ordinal=${BASH_REMATCH[1]} - sed -e "s/GRAPH_NAME/$(GRAPH_NAME)/g" \ - -e "s/ROLE/$(ROLE)/g" \ - -e "s/INDEX/${ordinal}/g" \ - -e "s/KAFKA_SERVERS/$(KAFKA_SERVERS)/g" \ - -e "s/FRONTEND/$(DNS_NAME_PREFIX_FRONTEND)/g" \ - -e "s/INGESTOR/$(DNS_NAME_PREFIX_INGESTOR)/g" \ - -e "s/COORDINATOR/$(DNS_NAME_PREFIX_COORDINATOR)/g" \ - -e "s/STORE/$(DNS_NAME_PREFIX_STORE)/g" \ - -e "s@LOG4RS_CONFIG@${GRAPHSCOPE_HOME}/conf/log4rs.yml@g" \ - /etc/graphscope-store/my.cnf.tpl > /etc/graphscope-store/my.cnf + sudo sed -e "s/GRAPH_NAME/$(GRAPH_NAME)/g" \ + -e "s/ROLE/$(ROLE)/g" \ + -e "s/INDEX/${ordinal}/g" \ + -e "s/KAFKA_SERVERS/$(KAFKA_SERVERS)/g" \ + -e "s/FRONTEND/$(DNS_NAME_PREFIX_FRONTEND)/g" \ + -e "s/INGESTOR/$(DNS_NAME_PREFIX_INGESTOR)/g" \ + -e "s/COORDINATOR/$(DNS_NAME_PREFIX_COORDINATOR)/g" \ + -e "s/STORE/$(DNS_NAME_PREFIX_STORE)/g" \ + -e "s@LOG4RS_CONFIG@${GRAPHSCOPE_HOME}/conf/log4rs.yml@g" \ + /etc/graphscope-store/my.cnf.tpl | sudo tee /etc/graphscope-store/my.cnf - LOG_NAME=graphscope-store.log /home/maxgraph/bin/giectl maxgraph /etc/graphscope-store/my.cnf $(EXECUTOR) + LOG_NAME=graphscope-store.log /usr/local/bin/giectl maxgraph /etc/graphscope-store/my.cnf $(EXECUTOR) tail -f /dev/null env: - name: DEBUG diff --git a/k8s/graphscope-store.Dockerfile b/k8s/graphscope-store.Dockerfile index ae8c43e491b9..2481bb790167 100644 --- a/k8s/graphscope-store.Dockerfile +++ b/k8s/graphscope-store.Dockerfile @@ -10,18 +10,19 @@ ENV NETWORKX=$NETWORKX ARG profile=debug ENV profile=$profile -COPY . /root/gs -COPY ./interactive_engine/deploy/docker/dockerfile/maven.settings.xml /root/.m2/settings.xml +COPY . /home/graphscope/gs +COPY ./interactive_engine/deploy/docker/dockerfile/maven.settings.xml /home/graphscope/.m2/settings.xml -RUN wget --no-verbose https://golang.org/dl/go1.15.5.linux-amd64.tar.gz && \ - tar -C /usr/local -xzf go1.15.5.linux-amd64.tar.gz && \ +RUN sudo chown -R graphscope:graphscope /home/graphscope/gs /home/graphscope/.m2 && \ + wget --no-verbose https://golang.org/dl/go1.15.5.linux-amd64.tar.gz && \ + sudo tar -C /usr/local -xzf go1.15.5.linux-amd64.tar.gz && \ curl -sf -L https://static.rust-lang.org/rustup.sh | \ sh -s -- -y --profile minimal --default-toolchain 1.54.0 && \ echo "source ~/.cargo/env" >> ~/.bashrc \ && source ~/.bashrc \ && rustup component add rustfmt \ && echo "build with profile: $profile" \ - && cd /root/gs/interactive_engine \ + && cd /home/graphscope/gs/interactive_engine \ && export CMAKE_PREFIX_PATH=/opt/graphscope \ && export LIBRARY_PATH=$LIBRARY_PATH:/opt/graphscope/lib \ && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/graphscope/lib \ @@ -35,13 +36,9 @@ FROM registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-runtime:latest COPY ./k8s/ready_probe.sh /tmp/ready_probe.sh COPY --from=builder /opt/graphscope /usr/local/ -COPY --from=builder /root/gs/interactive_engine/distribution/target/maxgraph.tar.gz /tmp/maxgraph.tar.gz -RUN mkdir -p /home/maxgraph \ - && tar -zxf /tmp/maxgraph.tar.gz -C /home/maxgraph -RUN mkdir -p /home/maxgraph/{bin,conf} -COPY --from=builder /root/gs/interactive_engine/bin/giectl /home/maxgraph/bin/giectl -COPY --from=builder /root/gs/interactive_engine/conf/* /home/maxgraph/conf/ - -ENV GRAPHSCOPE_HOME=/home/maxgraph -ENV GRAPHSCOPE_RUNTIME=/tmp/graphscope -WORKDIR /home/maxgraph/ +COPY --from=builder /home/graphscope/gs/interactive_engine/distribution/target/maxgraph.tar.gz /tmp/maxgraph.tar.gz +RUN sudo tar -zxf /tmp/maxgraph.tar.gz -C /usr/local +COPY --from=builder /home/graphscope/gs/interactive_engine/bin/giectl /usr/local/bin/giectl +COPY --from=builder /home/graphscope/gs/interactive_engine/conf/* /usr/local/conf/ + +ENV GRAPHSCOPE_HOME=/usr/local diff --git a/k8s/graphscope.Dockerfile b/k8s/graphscope.Dockerfile index a66579cd6e67..14d86b483696 100644 --- a/k8s/graphscope.Dockerfile +++ b/k8s/graphscope.Dockerfile @@ -4,7 +4,7 @@ # the result image includes all runtime stuffs of graphscope, with analytical engine, # learning engine and interactive engine installed. -ARG BASE_VERSION=latest +ARG BASE_VERSION=v0.2.6 FROM registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:$BASE_VERSION as builder SHELL ["/usr/bin/scl", "enable", "devtoolset-7"] diff --git a/proto/types.proto b/proto/types.proto index ae2c70767ef5..4fcff63f6249 100644 --- a/proto/types.proto +++ b/proto/types.proto @@ -170,8 +170,6 @@ enum ParamKey { DISTRIBUTED = 27; SCHEMA_PATH = 31; - GIE_GREMLIN_SERVER_CPU = 32; - GIE_GREMLIN_SERVER_MEM = 33; GIE_GREMLIN_ENGINE_PARAMS = 34; GIE_GREMLIN_QUERY_MESSAGE = 35; GIE_GREMLIN_REQUEST_OPTIONS = 36; diff --git a/python/graphscope/client/session.py b/python/graphscope/client/session.py index 3685bf211139..0d78a4ad2212 100755 --- a/python/graphscope/client/session.py +++ b/python/graphscope/client/session.py @@ -1206,10 +1206,6 @@ def set_option(**kwargs): - k8s_mars_worker_mem - k8s_mars_scheduler_cpu - k8s_mars_scheduler_mem - - k8s_gie_gremlin_server_cpu - Minimum number of CPU cores request for gremlin pod. Defaults to 0.5. - - k8s_gie_gremlin_server_mem - Minimum number of memory request for gremlin pod. Defaults to '512Mi'. - with_mars - enable_gaia - k8s_volumes @@ -1265,10 +1261,6 @@ def get_option(key): - k8s_mars_worker_mem - k8s_mars_scheduler_cpu - k8s_mars_scheduler_mem - - k8s_gie_gremlin_server_cpu - Minimum number of CPU cores request for gremlin pod. Defaults to 0.5. - - k8s_gie_gremlin_server_mem - Minimum number of memory request for gremlin pod. Defaults to '512Mi'. - with_mars - enable_gaia - k8s_volumes diff --git a/python/graphscope/config.py b/python/graphscope/config.py index b6bf55a90d77..72587e76d2dd 100644 --- a/python/graphscope/config.py +++ b/python/graphscope/config.py @@ -53,10 +53,6 @@ class GSConfig(object): k8s_etcd_cpu = 1.0 k8s_etcd_mem = "512Mi" - # GIE gremlin server resource configuration - k8s_gie_gremlin_server_cpu = 0.5 - k8s_gie_gremlin_server_mem = "512Mi" - # vineyard resource configuration k8s_vineyard_daemonset = "none" k8s_vineyard_cpu = 0.2 diff --git a/python/graphscope/framework/dag_utils.py b/python/graphscope/framework/dag_utils.py index 5731a1b0da5a..bc422bb382d8 100644 --- a/python/graphscope/framework/dag_utils.py +++ b/python/graphscope/framework/dag_utils.py @@ -925,7 +925,7 @@ def graph_to_dataframe(graph, selector=None, vertex_range=None): return op -def create_interactive_query(graph, engine_params, cpu, mem, enable_gaia): +def create_interactive_query(graph, engine_params, enable_gaia): """Create a interactive engine that query on the :code:`graph` Args: @@ -934,15 +934,11 @@ def create_interactive_query(graph, engine_params, cpu, mem, enable_gaia): engine_params (dict, optional): Configuration to startup the interactive engine. See detail in: `interactive_engine/deploy/docker/dockerfile/executor.vineyard.properties` - cpu (float): The number of CPU cores for gremlin server. - mem (str): The number of mem for gremlin server. Returns: An op to create a interactive engine based on a graph. """ config = {} - config[types_pb2.GIE_GREMLIN_SERVER_CPU] = utils.f_to_attr(cpu) - config[types_pb2.GIE_GREMLIN_SERVER_MEM] = utils.s_to_attr(mem) if engine_params is not None: config[types_pb2.GIE_GREMLIN_ENGINE_PARAMS] = utils.s_to_attr( json.dumps(engine_params) diff --git a/python/graphscope/interactive/query.py b/python/graphscope/interactive/query.py index b0d4c231bdea..a8699c1e9c83 100644 --- a/python/graphscope/interactive/query.py +++ b/python/graphscope/interactive/query.py @@ -26,7 +26,6 @@ from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection from gremlin_python.process.anonymous_traversal import traversal -from graphscope.config import GSConfig as gs_config from graphscope.framework.dag import DAGNode from graphscope.framework.dag_utils import close_interactive_query from graphscope.framework.dag_utils import create_interactive_query @@ -139,8 +138,6 @@ def __init__(self, session, graph, engine_params=None, enable_gaia=False): self._op = create_interactive_query( self._graph, self._engine_params, - gs_config.k8s_gie_gremlin_server_cpu, - gs_config.k8s_gie_gremlin_server_mem, enable_gaia, ) # add op to dag diff --git a/python/tests/kubernetes/test_demo_script.py b/python/tests/kubernetes/test_demo_script.py index 2c81cc49b2e1..ca10861514fb 100644 --- a/python/tests/kubernetes/test_demo_script.py +++ b/python/tests/kubernetes/test_demo_script.py @@ -31,9 +31,7 @@ from graphscope.framework.graph import Graph from graphscope.framework.loader import Loader -graphscope.set_option( - show_log=True, k8s_gie_gremlin_server_cpu=1.0, k8s_gie_gremlin_server_mem="4Gi" -) +graphscope.set_option(show_log=True) logger = logging.getLogger("graphscope") diff --git a/python/tests/kubernetes/test_with_gaia.py b/python/tests/kubernetes/test_with_gaia.py index 2348554ae1c4..af402df681e2 100644 --- a/python/tests/kubernetes/test_with_gaia.py +++ b/python/tests/kubernetes/test_with_gaia.py @@ -25,9 +25,7 @@ from graphscope.config import GSConfig as gs_config from graphscope.dataset.modern_graph import load_modern_graph -graphscope.set_option( - show_log=True, k8s_gie_gremlin_server_cpu=1.0, k8s_gie_gremlin_server_mem="4Gi" -) +graphscope.set_option(show_log=True) logger = logging.getLogger("graphscope") diff --git a/scripts/prepare_env.sh b/scripts/prepare_env.sh index f35440ea1a30..273143d303b2 100755 --- a/scripts/prepare_env.sh +++ b/scripts/prepare_env.sh @@ -252,15 +252,11 @@ launch_k8s_cluster() { pull_images() { log "Pulling GraphScope images." sudo docker pull registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${image_tag} || true - sudo docker pull registry.cn-hongkong.aliyuncs.com/graphscope/maxgraph_standalone_manager:${image_tag} || true - sudo docker pull zookeeper:3.4.14 || true sudo docker pull quay.io/coreos/etcd:v3.4.13 || true log "GraphScope images pulled successfully." log "Loading images into kind cluster." sudo kind load docker-image registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${image_tag} || true - sudo kind load docker-image registry.cn-hongkong.aliyuncs.com/graphscope/maxgraph_standalone_manager:${image_tag} || true - sudo kind load docker-image zookeeper:3.4.14 || true sudo kind load docker-image quay.io/coreos/etcd:v3.4.13 || true log "GraphScope images loaded into kind cluster successfully." } diff --git a/scripts/test.sh b/scripts/test.sh index bc4099cafac0..f3eebf6f6ab7 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -12,7 +12,6 @@ platform=$(awk -F= '/^NAME/{print $2}' /etc/os-release) test_dir="" gs_image="registry.cn-hongkong.aliyuncs.com/graphscope/graphscope:${version}" -gie_manager_image="registry.cn-hongkong.aliyuncs.com/graphscope/maxgraph_standalone_manager:${version}" test_GIE=0 test_on_k8s=0 @@ -44,7 +43,6 @@ function usage() { -h, --help show this help message and exit -t, --test_dir DIRECTORY the existing test data dir. -g, --gs_image DOCKER_IMAGE GraphScope engine's docker image uri - -i, --gie_manager_image DOCKER_IMAGE GIE graph manager docker image uri --all run all tests --gie run graph interactive engine tests --python run python tests @@ -81,7 +79,6 @@ function get_test_data() { # Globals: # graphscope_home # gs_image -# gie_manager_image # Arguments: # None # Outputs: @@ -90,7 +87,7 @@ function get_test_data() { ########################## function run_gie_test() { pushd "${graphscope_home}"/interactive_engine/tests - ./function_test.sh 8111 1 ${gs_image} ${gie_manager_image} + ./function_test.sh 8111 1 ${gs_image} res=$(grep "failed" ./target/surefire-reports/testng-results.xml) if [[ ${res} == *"failed=\"0\""* ]]; then @@ -115,9 +112,8 @@ function run_gie_test() { ########################## function run_k8s_test() { export GS_IMAGE="${gs_image}" # let session use specified image tag - export GIE_MANAGER_IMAGE="${gie_manager_image}" - python3 -m pytest -v "${graphscope_home}"/python/graphscope/deploy/tests + python3 -m pytest -v "${graphscope_home}"/python/tests/kubernetes/ } while [[ $# -gt 0 ]]; do @@ -138,11 +134,6 @@ while [[ $# -gt 0 ]]; do shift shift ;; - -i | --gie_manager_image) - gie_manager_image="$2" - shift - shift - ;; --all) test_GIE=1 test_on_k8s=1