Skip to content

Commit

Permalink
Merge branch 'main' into acezen/clustering-optimize
Browse files Browse the repository at this point in the history
  • Loading branch information
@acezen
acezen committed Jun 7, 2022
2 parents b526062 + 39c572b commit e98b70b
Show file tree
Hide file tree
Showing 6 changed files with 24 additions and 62 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/build-graphscope-wheels-linux.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,10 +166,10 @@ jobs:
# install graphscope
tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
pushd coordinator/dist/wheelhouse
pushd coordinator/dist
python3 -m pip install ./*.whl --user
popd
pushd coordinator/dist
pushd coordinator/dist/wheelhouse
python3 -m pip install ./*.whl --user
popd
Expand Down Expand Up @@ -222,10 +222,10 @@ jobs:
# install graphscope
tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
pushd coordinator/dist/wheelhouse
pushd coordinator/dist
python3 -m pip install ./*.whl --user
popd
pushd coordinator/dist
pushd coordinator/dist/wheelhouse
python3 -m pip install ./*.whl --user || true
popd
Expand Down
16 changes: 8 additions & 8 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,10 +286,10 @@ jobs:
# install graphscope
tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
pushd coordinator/dist/wheelhouse
pushd coordinator/dist
python3 -m pip install ./*.whl
popd
pushd coordinator/dist
pushd coordinator/dist/wheelhouse
python3 -m pip install ./*.whl
popd
Expand Down Expand Up @@ -345,10 +345,10 @@ jobs:
# install graphscope
tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
pushd coordinator/dist/wheelhouse
pushd coordinator/dist
python3 -m pip install ./*.whl
popd
pushd coordinator/dist
pushd coordinator/dist/wheelhouse
python3 -m pip install ./*.whl
popd
Expand Down Expand Up @@ -426,10 +426,10 @@ jobs:
# install graphscope
tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
pushd coordinator/dist/wheelhouse
pushd coordinator/dist
python3 -m pip install ./*.whl
popd
pushd coordinator/dist
pushd coordinator/dist/wheelhouse
python3 -m pip install ./*.whl
popd
Expand Down Expand Up @@ -511,10 +511,10 @@ jobs:
# install graphscope
tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
pushd coordinator/dist/wheelhouse
pushd coordinator/dist
python3 -m pip install ./*.whl
popd
pushd coordinator/dist
pushd coordinator/dist/wheelhouse
python3 -m pip install ./*.whl
popd
Expand Down
2 changes: 1 addition & 1 deletion coordinator/foo/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@
# limitations under the License.
#

"""Dummy package dir for gs-lib gs-engine gs-jython package
"""Dummy package dir for gs-engine package
"""
48 changes: 9 additions & 39 deletions coordinator/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,6 @@ def get_lib_suffix():

GRAPHSCOPE_REQUIRED_PACKAGES = [
f"gs-coordinator == {version}",
f"gs-jython == {version}",
f"gs-lib == {version}",
f"gs-engine == {version}",
f"gs-include == {version}",
f"gs-apps == {version}",
Expand All @@ -81,12 +79,11 @@ def _get_extra_data():
# into site-packages/graphscope.runtime
#
# For shrink the package size less than "100M", we split graphscope into
# 1) gs-coordinator: include python releated code of gscoordinator
# 2) gs-lib: libs exclude jython-standalone/groovy/grpc/curator/hadoop/gremlin/conscrypt**.jar
# 3) gs-jython: other libs not included in gs-lib
# 4) gs-include: header files and full-openmpi
# 5) gs-engine: other runtime info such as 'bin', 'conf'
# 6) gs-apps: precompiled builtin applications
# 1) graphscope: libs include *.so and *.jar
# 2) gs-coordinator: include python releated code of gscoordinator
# 3) gs-include: header files and full-openmpi
# 4) gs-engine: other runtime info such as 'bin', 'conf'
# 5) gs-apps: precompiled builtin applications

def __get_openmpi_prefix():
openmpi_prefix = ""
Expand All @@ -113,42 +110,15 @@ def __get_openmpi_prefix():
# data format:
# {"source_dir": "package_dir"} or
# {"source_dir": (package_dir, [exclude_list])}
if name == "gs-lib":
# exclude jython-standalone-**.jar
if name == "graphscope":
# lib
data = {
"/opt/graphscope/lib/": (
os.path.join(RUNTIME_ROOT, "lib"),
# exclude lists
[
"jython-standalone",
"groovy",
"grpc",
"curator",
"hadoop",
"gremlin",
"conscrypt",
"bcprov",
"maxgraph",
],
),
"/opt/graphscope/lib/": os.path.join(RUNTIME_ROOT, "lib"),
"/usr/local/lib/libvineyard_internal_registry.{0}".format(
get_lib_suffix()
): os.path.join(RUNTIME_ROOT, "lib"),
}
elif name == "gs-jython":
data = {
"/opt/graphscope/lib/jython-standalone*.jar": os.path.join(
RUNTIME_ROOT, "lib"
),
"/opt/graphscope/lib/*groovy*.jar": os.path.join(RUNTIME_ROOT, "lib"),
"/opt/graphscope/lib/*grpc*.jar": os.path.join(RUNTIME_ROOT, "lib"),
"/opt/graphscope/lib/*curator*.jar": os.path.join(RUNTIME_ROOT, "lib"),
"/opt/graphscope/lib/*hadoop*.jar": os.path.join(RUNTIME_ROOT, "lib"),
"/opt/graphscope/lib/*gremlin*.jar": os.path.join(RUNTIME_ROOT, "lib"),
"/opt/graphscope/lib/*conscrypt*.jar": os.path.join(RUNTIME_ROOT, "lib"),
"/opt/graphscope/lib/*bcprov*.jar": os.path.join(RUNTIME_ROOT, "lib"),
"/opt/graphscope/lib/*maxgraph*.jar": os.path.join(RUNTIME_ROOT, "lib"),
}

elif name == "gs-engine":
data = {
"/opt/graphscope/bin/": os.path.join(RUNTIME_ROOT, "bin"),
Expand Down
8 changes: 0 additions & 8 deletions k8s/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,16 +112,12 @@ graphscope-darwin-py3:
sudo strip /opt/graphscope/bin/executor && \
sudo strip /opt/graphscope/bin/gaia_executor && \
export DYLD_LIBRARY_PATH=/usr/local/lib:$$DYLD_LIBRARY_PATH && \
package_name=gs-lib python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \
rm -fr build && \
package_name=gs-include python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \
rm -fr build && \
package_name=gs-apps python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \
rm -fr build && \
package_name=gs-engine python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \
rm -fr build && \
package_name=gs-jython python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \
rm -fr build && \
package_name=gs-coordinator python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \
rm -fr build && \
package_name=graphscope python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \
Expand All @@ -143,16 +139,12 @@ graphscope-manylinux2014-py3-nodocker:
sudo strip /opt/graphscope/bin/gaia_executor && \
sudo strip /opt/graphscope/lib/*.so && \
strip /tmp/gs/builtin/*/*.so && \
package_name=gs-lib python3 setup.py bdist_wheel && \
rm -fr build && \
package_name=gs-include python3 setup.py bdist_wheel && \
rm -fr build && \
package_name=gs-apps python3 setup.py bdist_wheel && \
rm -fr build && \
package_name=gs-engine python3 setup.py bdist_wheel && \
rm -fr build && \
package_name=gs-jython python3 setup.py bdist_wheel && \
rm -fr build && \
package_name=gs-coordinator python3 setup.py bdist_wheel && \
rm -fr build && \
package_name=graphscope python3 setup.py bdist_wheel && \
Expand Down
4 changes: 2 additions & 2 deletions k8s/graphscope.Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,10 @@ RUN cd /home/graphscope/gs && \
pushd artifacts/python/dist/wheelhouse; \
for f in * ; do python3 -m pip install --no-cache-dir $f; done || true; \
popd; \
pushd artifacts/coordinator/dist/wheelhouse; \
pushd artifacts/coordinator/dist; \
python3 -m pip install --no-cache-dir ./*.whl; \
popd; \
pushd artifacts/coordinator/dist; \
pushd artifacts/coordinator/dist/wheelhouse; \
python3 -m pip install --no-cache-dir ./*.whl; \
popd; \
else \
Expand Down

0 comments on commit e98b70b

Please sign in to comment.