diff --git a/.github/workflows/build-graphscope-wheels-linux.yml b/.github/workflows/build-graphscope-wheels-linux.yml index b9d04bcd8dc2..b77cb081bccd 100644 --- a/.github/workflows/build-graphscope-wheels-linux.yml +++ b/.github/workflows/build-graphscope-wheels-linux.yml @@ -160,7 +160,8 @@ jobs: needs: [build-wheels] strategy: matrix: - python-version: [3.7, 3.8, 3.9] + # The protobuf version of tenorflow and grpcio is conflicts in python 3.11 + python-version: ['3.7', '3.8', '3.9', '3.10'] steps: - uses: actions/download-artifact@v3 @@ -175,6 +176,7 @@ jobs: - name: Prepare Environment shell: bash run: | + echo "Python version: ${{ matrix.python-version }}" # install graphscope-client cd artifacts tar -zxf ./wheel-${{ github.sha }}/client.tar.gz @@ -192,7 +194,11 @@ jobs: popd # install tensorflow - python3 -m pip install pytest "tensorflow<=2.5.2" "pandas<1.5.0" --user + if [ "${{ matrix.python-version }}" == "3.10" ]; then + python3 -m pip install pytest "tensorflow<=2.8.4" "pandas" --user + else + python3 -m pip install pytest "tensorflow<=2.5.2" "pandas" --user + fi # install java sudo apt update -y && sudo apt install openjdk-11-jdk -y @@ -248,7 +254,8 @@ jobs: popd # install tensorflow - python3 -m pip install pytest "tensorflow<=2.5.2" "pandas<1.5.0" --user + python3 -m pip install pytest "tensorflow<=2.5.2" "pandas<1.5.0" --user + python3 -m pip install -U wrapt --user # install jdk dnf install -y java-11-openjdk-devel diff --git a/.github/workflows/build-graphscope-wheels-macos.yml b/.github/workflows/build-graphscope-wheels-macos.yml index 72ab92c27277..6a335f3e0eb2 100644 --- a/.github/workflows/build-graphscope-wheels-macos.yml +++ b/.github/workflows/build-graphscope-wheels-macos.yml @@ -64,7 +64,7 @@ jobs: runs-on: macos-10.15 strategy: matrix: - python-version: ['3.7', '3.8', '3.9'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v3 @@ -117,7 +117,7 @@ jobs: needs: [build-wheels, build-client-wheels] strategy: matrix: - python-version: ['3.7', '3.8', '3.9'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/download-artifact@v3 @@ -164,7 +164,8 @@ jobs: needs: [build-wheels, build-client-wheels] strategy: matrix: - python-version: ['3.7', '3.8', '3.9'] + # The protobuf version of tenorflow and grpcio is conflicts in python 3.11 + python-version: ['3.7', '3.8', '3.9', '3.10'] steps: - uses: actions/download-artifact@v3 @@ -191,7 +192,11 @@ jobs: popd # install tensorflow - python3 -m pip install pytest "tensorflow<=2.5.2" "pandas<1.5.0" --user + if [ "${{ matrix.python-version }}" == "3.10" ]; then + python3 -m pip install pytest "tensorflow<=2.8.4" "pandas" --user + else + python3 -m pip install pytest "tensorflow<=2.5.2" "pandas<1.5.0" --user + fi - name: Run Minimum Test env: diff --git a/coordinator/requirements.txt b/coordinator/requirements.txt index 9ab7ce13f944..0e42efcb621d 100644 --- a/coordinator/requirements.txt +++ b/coordinator/requirements.txt @@ -1,12 +1,15 @@ etcd-distro>=3.5.1 graphscope-client>=0.11.0 -grpcio<=1.43.0,>=1.40.0 -grpcio-tools<=1.43.0,>=1.40.0 +grpcio;python_version>="3.11" +grpcio<=1.43.0,>=1.40.0;python_version<"3.11" +grpcio-tools;python_version>="3.11" +grpcio-tools<=1.43.0,>=1.40.0;python_version<"3.11" kubernetes~=12.0.1 -protobuf>=3.15.0,<=3.18.1 +protobuf;python_version>="3.11" +protobuf>=3.15.0,<=3.18.1;python_version<"3.11" PyYAML -vineyard>=0.11.6; sys_platform != "win32" -vineyard-io>=0.11.6; sys_platform != "win32" +vineyard>=0.11.1; sys_platform != "win32" +vineyard-io>=0.11.1; sys_platform != "win32" prometheus-client>=0.14.1 setuptools>=65.5.1 packaging diff --git a/coordinator/setup.py b/coordinator/setup.py index b2395e300455..822857a711e4 100644 --- a/coordinator/setup.py +++ b/coordinator/setup.py @@ -373,8 +373,6 @@ def parse_version(root, **kwargs): }, setup_requires=[ "setuptools_scm>=5.0.0", - "grpcio<=1.43.0,>=1.40.0", - "grpcio-tools<=1.43.0,>=1.40.0", ], package_dir=parsed_package_dir(), packages=parsed_packages(), diff --git a/k8s/internal/Makefile b/k8s/internal/Makefile index 591b1d923e4d..b538ff8e3be9 100644 --- a/k8s/internal/Makefile +++ b/k8s/internal/Makefile @@ -127,6 +127,7 @@ graphscope-manylinux2014-py3: cd /work/k8s/internal && \ make graphscope-manylinux2014-py3-nodocker' + graphscope-client-manylinux2014-py3-nodocker: cd $(WORKING_DIR)/../../learning_engine/graph-learn && \ git submodule update --init third_party/pybind11 && \ @@ -137,11 +138,17 @@ graphscope-client-manylinux2014-py3-nodocker: cmake -DKNN=OFF -DWITH_VINEYARD=ON -DTESTING=OFF .. && \ make graphlearn_shared -j`nproc` && \ export LD_LIBRARY_PATH=$(WORKING_DIR)/../../learning_engine/graph-learn/graphlearn/built/lib:$$LD_LIBRARY_PATH && \ - for py in cp37-cp37m cp38-cp38 cp39-cp39 ; do \ + for py in cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311; do \ cd $(WORKING_DIR)/../../python; \ export PATH=/opt/python/$$py/bin:$$PATH; \ pip3 install -U pip; \ - pip3 install "numpy==1.18.5" "pandas<1.5.0" "grpcio<=1.43.0,>=1.40.0" "grpcio-tools<=1.43.0,>=1.40.0" wheel "auditwheel==5.0.0"; \ + if [ "$$py" == "cp311-cp311" ]; then \ + pip3 install "numpy" "pandas" "grpcio" "grpcio-tools" wheel "auditwheel==5.0.0"; \ + elif [ "$$py" == "cp310-cp310" ]; then \ + pip3 install "numpy==1.21.6" "pandas" "grpcio<=1.43.0,>=1.40.0" "grpcio-tools<=1.43.0,>=1.40.0" wheel "auditwheel==5.0.0"; \ + else \ + pip3 install "numpy==1.18.5" "pandas" "grpcio<=1.43.0,>=1.40.0" "grpcio-tools<=1.43.0,>=1.40.0" wheel "auditwheel==5.0.0"; \ + fi; \ rm -rf build; \ rm -rf dist/*.whl; \ python3 setup.py bdist_wheel; \ @@ -160,8 +167,15 @@ graphscope-client-darwin-py3: make graphlearn_shared -j`nproc` && \ export DYLD_LIBRARY_PATH=$(WORKING_DIR)/../../learning_engine/graph-learn/graphlearn/built/lib:/usr/local/lib:$$DYLD_LIBRARY_PATH && \ cd $(WORKING_DIR)/../../python && \ + py=$$(python3 -V 2>&1 | awk '{print $$2}' | awk -F '.' '{print $$1$$2}') && \ pip3 install -U pip && \ - pip3 install "numpy==1.18.5" "pandas<1.5.0" "grpcio<=1.43.0,>=1.40.0" "grpcio-tools<=1.43.0,>=1.40.0" delocate wheel && \ + if [ "$$py" == "311" ]; then \ + pip3 install "numpy" "pandas" "grpcio" "grpcio-tools" delocate wheel; \ + elif [ "$$py" == "310" ]; then \ + pip3 install "numpy==1.21.6" "pandas" "grpcio<=1.43.0,>=1.40.0" "grpcio-tools<=1.43.0,>=1.40.0" delocate wheel; \ + else \ + pip3 install "numpy==1.18.5" "pandas" "grpcio<=1.43.0,>=1.40.0" "grpcio-tools<=1.43.0,>=1.40.0" delocate wheel; \ + fi; \ rm -rf build dist/*.whl || true && \ python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \ for wheel in `ls dist/*.whl`; do \ diff --git a/k8s/utils/precompile.py b/k8s/utils/precompile.py index 563ea7434a88..8a0f71841ee2 100755 --- a/k8s/utils/precompile.py +++ b/k8s/utils/precompile.py @@ -458,7 +458,7 @@ def parse_sys_args(): ) return parser.parse_args() -WORKSPACE = os.path.join("/", tempfile.gettempprefix(), "gs", "builtin") +WORKSPACE = Path(os.path.join("/", tempfile.gettempprefix(), "gs", "builtin")).resolve() if __name__ == "__main__": args = parse_sys_args() diff --git a/python/graphscope/tests/minitest/test_min.py b/python/graphscope/tests/minitest/test_min.py index da952cd21912..8340c8ac6ac1 100644 --- a/python/graphscope/tests/minitest/test_min.py +++ b/python/graphscope/tests/minitest/test_min.py @@ -126,6 +126,10 @@ def simple_flow(sess, ogbn_small_script): interactive = sess.gremlin(graph) papers = interactive.execute(ogbn_small_script).one() + # waiting for graphlearn support python3.10 + if sys.version_info[1] >= 10: + return + # GLE on ogbn_mag_small graph paper_features = [] for i in range(128): @@ -217,6 +221,10 @@ def test_demo_with_default_session(ogbn_small_script): sub_graph = sub_graph.add_column(ret1, {"kcore": "r"}) sub_graph = sub_graph.add_column(ret2, {"tc": "r"}) + # waiting for graphlearn support python3.10 + if sys.version_info[1] >= 10: + return + # GLE on ogbn_mag_small graph paper_features = [] for i in range(128): diff --git a/python/requirements.txt b/python/requirements.txt index 73b612aba12e..46b6283eaa37 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,7 +1,9 @@ Cython==3.0a6 gremlinpython>=3.4.9 -grpcio<=1.43.0,>=1.40.0 -grpcio-tools<=1.43.0,>=1.40.0 +grpcio;python_version>="3.11" +grpcio<=1.43.0,>=1.40.0;python_version<"3.11" +grpcio-tools;python_version>="3.11" +grpcio-tools<=1.43.0,>=1.40.0;python_version<"3.11" kubernetes msgpack nest_asyncio @@ -11,12 +13,14 @@ numpy orjson packaging pandas -protobuf>=3.15.0,<=3.18.1 +protobuf;python_version>="3.11" +protobuf>=3.15.0,<=3.18.1;python_version<"3.11" psutil -pyarrow<=6.0.0 +pyarrow;python_version>="3.11" +pyarrow<=6.0.0;python_version<"3.11" pysimdjson PyYAML scipy tqdm -vineyard>=0.5.2; sys_platform != "win32" -vineyard-io>=0.5.2; sys_platform != "win32" +vineyard>=0.11.1; sys_platform != "win32" +vineyard-io>=0.11.1; sys_platform != "win32" diff --git a/python/setup.py b/python/setup.py index 32f955bbbe55..0005858f925e 100644 --- a/python/setup.py +++ b/python/setup.py @@ -288,9 +288,6 @@ def parse_version(root, **kwargs): }, setup_requires=[ "setuptools_scm>=5.0.0", - "grpcio<=1.43.0,>=1.40.0", - "grpcio-tools<=1.43.0,>=1.40.0", - "numpy", ], package_dir=resolve_graphscope_package_dir(), packages=find_graphscope_packages(), diff --git a/scripts/install_deps.sh b/scripts/install_deps.sh index 901a75071cab..f483aea66425 100755 --- a/scripts/install_deps.sh +++ b/scripts/install_deps.sh @@ -398,6 +398,7 @@ write_envs_config() { echo "export OPENSSL_ROOT_DIR=${homebrew_prefix}/opt/openssl" echo "export OPENSSL_LIBRARIES=${homebrew_prefix}/opt/openssl/lib" echo "export OPENSSL_SSL_LIBRARY=${homebrew_prefix}/opt/openssl/lib/libssl.dylib" + echo "export LDFLAGS=\"$LDFLAGS -Wl,-rpath,${homebrew_prefix}/opt/libomp/lib -L${homebrew_prefix}/opt/libomp/lib\"" } >> ${OUTPUT_ENV_FILE} elif [[ "${PLATFORM}" == *"Ubuntu"* ]]; then @@ -556,7 +557,7 @@ install_cppkafka() { if [[ "${PLATFORM}" == *"Darwin"* ]]; then declare -r homebrew_prefix=$(brew --prefix) - export LDFLAGS="-L${homebrew_prefix}/opt/openssl@3/lib" + export LDFLAGS="${LDFLAGS} -L${homebrew_prefix}/opt/openssl@3/lib" export CPPFLAGS="-I${homebrew_prefix}/opt/openssl@3/include" fi @@ -762,6 +763,7 @@ install_dependencies() { export CXX=${homebrew_prefix}/opt/llvm/bin/clang++ export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=${CC} export CPPFLAGS=-I${homebrew_prefix}/opt/llvm/include + export LDFLAGS="${LDFLAGS} -Wl,-rpath,${homebrew_prefix}/opt/libomp/lib -L${homebrew_prefix}/opt/libomp/lib" fi log "Installing python packages for vineyard codegen."