From 39c572bfb4bbefd7a6df47e5bb45ad0d270da684 Mon Sep 17 00:00:00 2001 From: DongZe Li <9546726@qq.com> Date: Tue, 7 Jun 2022 11:44:06 +0800 Subject: [PATCH] Merge gs-jython and gs-lib into graphscope package (#1680) --- .../build-graphscope-wheels-linux.yml | 8 ++-- .github/workflows/ci.yml | 16 +++---- coordinator/foo/__init__.py | 2 +- coordinator/setup.py | 48 ++++--------------- k8s/Makefile | 8 ---- k8s/graphscope.Dockerfile | 4 +- 6 files changed, 24 insertions(+), 62 deletions(-) diff --git a/.github/workflows/build-graphscope-wheels-linux.yml b/.github/workflows/build-graphscope-wheels-linux.yml index 277c52bc0d8e..e5fcdcd68239 100644 --- a/.github/workflows/build-graphscope-wheels-linux.yml +++ b/.github/workflows/build-graphscope-wheels-linux.yml @@ -166,10 +166,10 @@ jobs: # install graphscope tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz - pushd coordinator/dist/wheelhouse + pushd coordinator/dist python3 -m pip install ./*.whl --user popd - pushd coordinator/dist + pushd coordinator/dist/wheelhouse python3 -m pip install ./*.whl --user popd @@ -222,10 +222,10 @@ jobs: # install graphscope tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz - pushd coordinator/dist/wheelhouse + pushd coordinator/dist python3 -m pip install ./*.whl --user popd - pushd coordinator/dist + pushd coordinator/dist/wheelhouse python3 -m pip install ./*.whl --user || true popd diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c55f75db9a7..df8b3d53b377 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -286,10 +286,10 @@ jobs: # install graphscope tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz - pushd coordinator/dist/wheelhouse + pushd coordinator/dist python3 -m pip install ./*.whl popd - pushd coordinator/dist + pushd coordinator/dist/wheelhouse python3 -m pip install ./*.whl popd @@ -345,10 +345,10 @@ jobs: # install graphscope tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz - pushd coordinator/dist/wheelhouse + pushd coordinator/dist python3 -m pip install ./*.whl popd - pushd coordinator/dist + pushd coordinator/dist/wheelhouse python3 -m pip install ./*.whl popd @@ -426,10 +426,10 @@ jobs: # install graphscope tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz - pushd coordinator/dist/wheelhouse + pushd coordinator/dist python3 -m pip install ./*.whl popd - pushd coordinator/dist + pushd coordinator/dist/wheelhouse python3 -m pip install ./*.whl popd @@ -511,10 +511,10 @@ jobs: # install graphscope tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz - pushd coordinator/dist/wheelhouse + pushd coordinator/dist python3 -m pip install ./*.whl popd - pushd coordinator/dist + pushd coordinator/dist/wheelhouse python3 -m pip install ./*.whl popd diff --git a/coordinator/foo/__init__.py b/coordinator/foo/__init__.py index 22ffe6315654..6e2f3be1b2bb 100644 --- a/coordinator/foo/__init__.py +++ b/coordinator/foo/__init__.py @@ -16,5 +16,5 @@ # limitations under the License. # -"""Dummy package dir for gs-lib gs-engine gs-jython package +"""Dummy package dir for gs-engine package """ diff --git a/coordinator/setup.py b/coordinator/setup.py index 07f04fcbf58e..73d18a4d7c15 100644 --- a/coordinator/setup.py +++ b/coordinator/setup.py @@ -62,8 +62,6 @@ def get_lib_suffix(): GRAPHSCOPE_REQUIRED_PACKAGES = [ f"gs-coordinator == {version}", - f"gs-jython == {version}", - f"gs-lib == {version}", f"gs-engine == {version}", f"gs-include == {version}", f"gs-apps == {version}", @@ -81,12 +79,11 @@ def _get_extra_data(): # into site-packages/graphscope.runtime # # For shrink the package size less than "100M", we split graphscope into - # 1) gs-coordinator: include python releated code of gscoordinator - # 2) gs-lib: libs exclude jython-standalone/groovy/grpc/curator/hadoop/gremlin/conscrypt**.jar - # 3) gs-jython: other libs not included in gs-lib - # 4) gs-include: header files and full-openmpi - # 5) gs-engine: other runtime info such as 'bin', 'conf' - # 6) gs-apps: precompiled builtin applications + # 1) graphscope: libs include *.so and *.jar + # 2) gs-coordinator: include python releated code of gscoordinator + # 3) gs-include: header files and full-openmpi + # 4) gs-engine: other runtime info such as 'bin', 'conf' + # 5) gs-apps: precompiled builtin applications def __get_openmpi_prefix(): openmpi_prefix = "" @@ -113,42 +110,15 @@ def __get_openmpi_prefix(): # data format: # {"source_dir": "package_dir"} or # {"source_dir": (package_dir, [exclude_list])} - if name == "gs-lib": - # exclude jython-standalone-**.jar + if name == "graphscope": + # lib data = { - "/opt/graphscope/lib/": ( - os.path.join(RUNTIME_ROOT, "lib"), - # exclude lists - [ - "jython-standalone", - "groovy", - "grpc", - "curator", - "hadoop", - "gremlin", - "conscrypt", - "bcprov", - "maxgraph", - ], - ), + "/opt/graphscope/lib/": os.path.join(RUNTIME_ROOT, "lib"), "/usr/local/lib/libvineyard_internal_registry.{0}".format( get_lib_suffix() ): os.path.join(RUNTIME_ROOT, "lib"), } - elif name == "gs-jython": - data = { - "/opt/graphscope/lib/jython-standalone*.jar": os.path.join( - RUNTIME_ROOT, "lib" - ), - "/opt/graphscope/lib/*groovy*.jar": os.path.join(RUNTIME_ROOT, "lib"), - "/opt/graphscope/lib/*grpc*.jar": os.path.join(RUNTIME_ROOT, "lib"), - "/opt/graphscope/lib/*curator*.jar": os.path.join(RUNTIME_ROOT, "lib"), - "/opt/graphscope/lib/*hadoop*.jar": os.path.join(RUNTIME_ROOT, "lib"), - "/opt/graphscope/lib/*gremlin*.jar": os.path.join(RUNTIME_ROOT, "lib"), - "/opt/graphscope/lib/*conscrypt*.jar": os.path.join(RUNTIME_ROOT, "lib"), - "/opt/graphscope/lib/*bcprov*.jar": os.path.join(RUNTIME_ROOT, "lib"), - "/opt/graphscope/lib/*maxgraph*.jar": os.path.join(RUNTIME_ROOT, "lib"), - } + elif name == "gs-engine": data = { "/opt/graphscope/bin/": os.path.join(RUNTIME_ROOT, "bin"), diff --git a/k8s/Makefile b/k8s/Makefile index feaaff8f5470..738c25e59db9 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -112,16 +112,12 @@ graphscope-darwin-py3: sudo strip /opt/graphscope/bin/executor && \ sudo strip /opt/graphscope/bin/gaia_executor && \ export DYLD_LIBRARY_PATH=/usr/local/lib:$$DYLD_LIBRARY_PATH && \ - package_name=gs-lib python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \ - rm -fr build && \ package_name=gs-include python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \ rm -fr build && \ package_name=gs-apps python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \ rm -fr build && \ package_name=gs-engine python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \ rm -fr build && \ - package_name=gs-jython python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \ - rm -fr build && \ package_name=gs-coordinator python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \ rm -fr build && \ package_name=graphscope python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \ @@ -143,16 +139,12 @@ graphscope-manylinux2014-py3-nodocker: sudo strip /opt/graphscope/bin/gaia_executor && \ sudo strip /opt/graphscope/lib/*.so && \ strip /tmp/gs/builtin/*/*.so && \ - package_name=gs-lib python3 setup.py bdist_wheel && \ - rm -fr build && \ package_name=gs-include python3 setup.py bdist_wheel && \ rm -fr build && \ package_name=gs-apps python3 setup.py bdist_wheel && \ rm -fr build && \ package_name=gs-engine python3 setup.py bdist_wheel && \ rm -fr build && \ - package_name=gs-jython python3 setup.py bdist_wheel && \ - rm -fr build && \ package_name=gs-coordinator python3 setup.py bdist_wheel && \ rm -fr build && \ package_name=graphscope python3 setup.py bdist_wheel && \ diff --git a/k8s/graphscope.Dockerfile b/k8s/graphscope.Dockerfile index 94f9271de0c1..6446d81aa1d5 100644 --- a/k8s/graphscope.Dockerfile +++ b/k8s/graphscope.Dockerfile @@ -59,10 +59,10 @@ RUN cd /home/graphscope/gs && \ pushd artifacts/python/dist/wheelhouse; \ for f in * ; do python3 -m pip install --no-cache-dir $f; done || true; \ popd; \ - pushd artifacts/coordinator/dist/wheelhouse; \ + pushd artifacts/coordinator/dist; \ python3 -m pip install --no-cache-dir ./*.whl; \ popd; \ - pushd artifacts/coordinator/dist; \ + pushd artifacts/coordinator/dist/wheelhouse; \ python3 -m pip install --no-cache-dir ./*.whl; \ popd; \ else \