diff --git a/.github/workflows/build-graphscope-wheels-linux.yml b/.github/workflows/build-graphscope-wheels-linux.yml index c6fa72f77ef0..0566860d2ce0 100644 --- a/.github/workflows/build-graphscope-wheels-linux.yml +++ b/.github/workflows/build-graphscope-wheels-linux.yml @@ -12,7 +12,7 @@ on: - "v*" env: - REGISTRY: registry-vpc.cn-hongkong.aliyuncs.com + REGISTRY: registry.cn-hongkong.aliyuncs.com jobs: build-wheels: @@ -116,10 +116,7 @@ jobs: # make jupyter-image cd .. - docker build \ - --build-arg CI=${CI} \ - -t graphscope/jupyter:${SHORT_SHA} \ - -f ./k8s/internal/jupyter.Dockerfile . + docker build --build-arg CI=${CI} -t graphscope/jupyter:${SHORT_SHA} -f ./k8s/internal/jupyter.Dockerfile . # dataset image doesn't changed, we can just use the latest one # make dataset-image diff --git a/Makefile b/Makefile index 9fd048f08bd4..025a86e57dcf 100644 --- a/Makefile +++ b/Makefile @@ -58,8 +58,8 @@ clean: rm -rf $(ANALYTICAL_BUILD_DIR) $(ANALYTICAL_DIR)/proto cd $(ANALYTICAL_DIR)/java && mvn clean - cd $(INTERACTIVE_DIR) && mvn clean - # TODO: use maven clean to clean ir target + cd $(INTERACTIVE_DIR) && mvn clean || true + # TODO: use maven clean to clean ir target rm -rf $(INTERACTIVE_DIR)/executor/ir/target rm -rf $(LEARNING_BUILD_DIR) $(LEARNING_DIR)/proto/*.h $(LEARNING_DIR)/proto/*.cc diff --git a/k8s/build_scripts/build_vineyard.sh b/k8s/build_scripts/build_vineyard.sh index bdbf81e120a2..f18ba8ff0941 100755 --- a/k8s/build_scripts/build_vineyard.sh +++ b/k8s/build_scripts/build_vineyard.sh @@ -8,7 +8,7 @@ echo "Installing libgrape-lite" cd ${WORKDIR} && \ git clone -b master https://github.com/alibaba/libgrape-lite.git --depth=1 && \ cd libgrape-lite && \ - cmake . && \ + cmake . -DCMAKE_INSTALL_PREFIX=/opt/vineyard && \ make -j$(nproc) && \ make install && \ rm -rf ${WORKDIR}/libgrape-lite @@ -19,14 +19,17 @@ cd ${WORKDIR} && \ git clone -b v0.10.2 https://github.com/v6d-io/v6d.git --depth=1 && \ pushd v6d && \ git submodule update --init && \ - cmake . -DCMAKE_INSTALL_PREFIX=/usr/local \ + cmake . -DCMAKE_PREFIX_PATH=/opt/vineyard \ + -DCMAKE_INSTALL_PREFIX=/opt/vineyard \ -DBUILD_VINEYARD_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ -DBUILD_VINEYARD_PYTHON_BINDINGS=ON && \ make -j$(nproc) && \ make install && \ - strip /usr/local/bin/vineyard* /usr/local/lib/libvineyard* && \ - popd && \ + strip /opt/vineyard/bin/vineyard* /opt/vineyard/lib/libvineyard* && \ python3 setup.py bdist_wheel && \ python3 setup_io.py bdist_wheel && \ - pip3 install dist/* + pip3 install dist/* && \ + sudo cp -r /opt/vineyard/* /usr/local/ && \ rm -rf ${WORKDIR}/v6d + diff --git a/k8s/build_scripts/build_vineyard_dependencies.sh b/k8s/build_scripts/build_vineyard_dependencies.sh index 09c591e889ae..5fc790d34c42 100755 --- a/k8s/build_scripts/build_vineyard_dependencies.sh +++ b/k8s/build_scripts/build_vineyard_dependencies.sh @@ -10,16 +10,28 @@ cd ${WORKDIR} && \ bash cmake-3.24.3-linux-x86_64.sh --prefix=/usr/local --skip-license && \ rm -rf ${WORKDIR}/cmake-3.24.3-linux-x86_64.sh -# install openmpi v4.1.4 +# install openmpi v4.0.5 echo "Installing openmpi" cd ${WORKDIR} && \ - wget -q https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.4.tar.gz && \ - tar zxvf openmpi-4.1.4.tar.gz && \ - cd openmpi-4.1.4 && \ - ./configure --enable-mpi-cxx --disable-dlopen --prefix=/usr/local && \ + wget -q https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.5.tar.gz && \ + tar zxvf openmpi-4.0.5.tar.gz && \ + cd openmpi-4.0.5 && \ + ./configure --enable-mpi-cxx --disable-dlopen --prefix=/opt/openmpi && \ make -j$(nproc) && \ make install && \ - rm -rf ${WORKDIR}/openmpi-4.1.4 ${WORKDIR}/openmpi-4.1.4.tar.gz + cp -rs /opt/openmpi/* /usr/local/ && \ + rm -rf ${WORKDIR}/openmpi-4.0.5 ${WORKDIR}/openmpi-4.0.5.tar.gz + +# gflags v2.2.2 +echo "Installing gflags" +cd ${WORKDIR} && \ + wget -q https://github.com/gflags/gflags/archive/v2.2.2.tar.gz && \ + tar zxvf v2.2.2.tar.gz && \ + cd gflags-2.2.2 && \ + cmake . -DBUILD_SHARED_LIBS=ON && \ + make -j$(nproc) && \ + make install && \ + rm -rf ${WORKDIR}/v2.2.2.tar.gz ${WORKDIR}/gflags-2.2.2 # GLOG 0.6.0 echo "Installing glog" @@ -81,17 +93,6 @@ cd ${WORKDIR} && \ make install && \ rm -rf ${WORKDIR}/arrow-apache-arrow-9.0.0 ${WORKDIR}/apache-arrow-9.0.0.tar.gz -# gflags v2.2.2 -echo "Installing gflags" -cd ${WORKDIR} && \ - wget -q https://github.com/gflags/gflags/archive/v2.2.2.tar.gz && \ - tar zxvf v2.2.2.tar.gz && \ - cd gflags-2.2.2 && \ - cmake . -DBUILD_SHARED_LIBS=ON && \ - make -j$(nproc) && \ - make install && \ - rm -rf ${WORKDIR}/v2.2.2.tar.gz ${WORKDIR}/gflags-2.2.2 - # Boost 1.74.0, required by vineyard echo "Installing boost" cd ${WORKDIR} && \ diff --git a/k8s/dockerfiles/graphscope-dev-base.Dockerfile b/k8s/dockerfiles/graphscope-dev-base.Dockerfile index 7cc539a7b758..68c0f74cae03 100644 --- a/k8s/dockerfiles/graphscope-dev-base.Dockerfile +++ b/k8s/dockerfiles/graphscope-dev-base.Dockerfile @@ -6,7 +6,7 @@ FROM vineyardcloudnative/manylinux-llvm:2014-11.0.0 AS llvm FROM $REGISTRY/graphscope/manylinux2014:2022-08-16-53df7cb # yum install dependencies -RUN yum install -y perl which sudo wget libunwind-devel vim zip java-1.8.0-openjdk-devel msgpack-devel rapidjson-devel libuuid-devel && \ +RUN yum install -y perl which sudo wget libunwind-devel vim zip java-1.8.0-openjdk-devel msgpack-devel rapidjson-devel libuuid-devel openssh-clients && \ yum clean all -y && \ rm -fr /var/cache/yum diff --git a/k8s/internal/Makefile b/k8s/internal/Makefile index 04e9deeeed0f..78221cc4c65f 100644 --- a/k8s/internal/Makefile +++ b/k8s/internal/Makefile @@ -11,12 +11,17 @@ ifeq ($(REGISTRY),) endif +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S), Linux) + XARGS_EMPTY_FLAG := --no-run-if-empty +endif + # Default null TARGET_SERVER_PACKAGE := TARGET_CLIENT_PACKAGE := ifeq ($(UNAME_S), Linux) TARGET_SERVER_PACKAGE := graphscope-manylinux2014-py3 - TARGET_CLIENT_PACKAGE := graphscope-client-manylinux2014-py3 + TARGET_CLIENT_PACKAGE := graphscope-client-manylinux2014-py3 else TARGET_SERVER_PACKAGE := graphscope-darwin-py3 TARGET_CLIENT_PACKAGE := graphscope-client-darwin-py3 @@ -108,7 +113,7 @@ graphscope-manylinux2014-py3-nodocker: package_name=graphscope python3 setup.py bdist_wheel && \ cd dist && \ for wheel in `ls ./*.whl`; do \ - auditwheel repair $$wheel --plat=manylinux2014_x86_64 && rm $$wheel; \ + (auditwheel repair $$wheel --plat=manylinux2014_x86_64 && rm $$wheel) || true; \ done graphscope-manylinux2014-py3: @@ -116,7 +121,7 @@ graphscope-manylinux2014-py3: docker run --rm -v $(WORKING_DIR)/../..:/work \ ${REGISTRY}/graphscope/graphscope-dev:${VINEYARD_VERSION} \ bash -c 'source ~/.bashrc && \ - cd /work/k8s && \ + cd /work/k8s/internal && \ make graphscope-manylinux2014-py3-nodocker' graphscope-client-manylinux2014-py3-nodocker: @@ -144,6 +149,7 @@ graphscope-client-manylinux2014-py3-nodocker: graphscope-client-darwin-py3: cd $(WORKING_DIR)/../../learning_engine/graph-learn && \ git submodule update --init third_party/pybind11 && \ + cd graphlearn && \ rm -rf cmake-build && \ mkdir -p cmake-build && \ cd cmake-build && \ @@ -163,5 +169,5 @@ graphscope-client-manylinux2014-py3: docker pull ${REGISTRY}/graphscope/graphscope-dev:${VINEYARD_VERSION} docker run --rm -v $(WORKING_DIR)/../..:/work \ ${REGISTRY}/graphscope/graphscope-dev:${VINEYARD_VERSION} \ - bash -c 'cd /work/k8s && make graphscope-client-manylinux2014-py3-nodocker' + bash -c 'cd /work/k8s/internal && make graphscope-client-manylinux2014-py3-nodocker' diff --git a/scripts/install_deps.sh b/scripts/install_deps.sh index e232a5873511..b89bfd6a03f8 100755 --- a/scripts/install_deps.sh +++ b/scripts/install_deps.sh @@ -390,6 +390,7 @@ write_envs_config() { { echo "export CC=${homebrew_prefix}/opt/llvm/bin/clang" echo "export CXX=${homebrew_prefix}/opt/llvm/bin/clang++" + echo "export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=${CC}" if [ -z "${JAVA_HOME}" ]; then echo "export JAVA_HOME=\$(/usr/libexec/java_home -v11)" fi @@ -759,6 +760,7 @@ install_dependencies() { export OPENSSL_SSL_LIBRARY=${homebrew_prefix}/opt/openssl/lib/libssl.dylib export CC=${homebrew_prefix}/opt/llvm/bin/clang export CXX=${homebrew_prefix}/opt/llvm/bin/clang++ + export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=${CC} export CPPFLAGS=-I${homebrew_prefix}/opt/llvm/include fi