Enhance the install_deps.sh script (#1873)

.github/workflows/nightly.yml

@@ -31,7 +31,7 @@ jobs:
         if [ -z "${r}" ];then export hn=$(hostname); sudo -E bash -c 'echo "127.0.0.1 ${hn}" >> /etc/hosts'; fi
         cat /etc/hosts
 
-        /bin/bash ${GITHUB_WORKSPACE}/scripts/install_deps.sh --dev --verbose
+        /bin/bash ${GITHUB_WORKSPACE}/scripts/install_deps.sh --dev --grape-jdk --verbose
 
     - name: Build GraphScope
       run: |

README.md

@@ -47,7 +47,7 @@ Visit our website at [graphscope.io](https://graphscope.io) to learn more.
 
 We provide a [Playground](https://try.graphscope.app) with a managed JupyterLab. [Try GraphScope](https://try.graphscope.app) straight away in your browser!
 
-GraphScope supports run in standalone mode or on clusters managed by [Kubernetes](https://kubernetes.io/) within containers. For quickly getting started, 
+GraphScope supports run in standalone mode or on clusters managed by [Kubernetes](https://kubernetes.io/) within containers. For quickly getting started,
 let's begin with the standalone mode.
 
 
@@ -230,7 +230,7 @@ config = {
 train(config, lg)
 ```
 
-A python script with the entire process is availabe [here](https://colab.research.google.com/github/alibaba/GraphScope/blob/main/tutorials/1_node_classification_on_citation.ipynb), you may try it out by yourself. 
+A python script with the entire process is availabe [here](https://colab.research.google.com/github/alibaba/GraphScope/blob/main/tutorials/1_node_classification_on_citation.ipynb), you may try it out by yourself.
 
 
 ## Processing Large Graph on Kubernetes Cluster
@@ -250,11 +250,11 @@ Alternatively, you can set up a local k8s cluster for testing with [Kind](https:
 
 If you did not install the `graphscope` package in the above step, you can install a subset of the whole package with client functions only.
 
-```bash 
+```bash
 pip3 install graphscope-client
 ```
 
-Next, let's revisit the example by running on a cluster instead. 
+Next, let's revisit the example by running on a cluster instead.
 
 <div align="center">
     <img src="https://graphscope.io/docs/_images/how-it-works.png" width="600" alt="how-it-works" />
@@ -326,13 +326,13 @@ Please note that we have not hardened this release for production use and it lac
 
 ### Building on local
 
-To build graphscope Python package and the engine bineries, you need to install some dependencies and build tools.
+To build graphscope Python package and the engine binaries, you need to install some dependencies and build tools.
 
 ```bash
 ./scripts/install_deps.sh --dev
 
 # With argument --cn to speed up the download if you are in China.
-./scripts/install_deps.sh --dev --cn 
+./scripts/install_deps.sh --dev --cn
 ```
 
 Then you can build GraphScope with pre-configured `make` commands.

docs/developer_guide.rst

@@ -27,7 +27,7 @@ make their changes to the code and build GraphScope with command:
     sudo docker run --shm-size 10240m -it registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:latest /bin/bash
 
     git clone https://github.com/alibaba/GraphScope.git
-    
+
     # download dataset for test
     git clone https://github.com/GraphScope/gstest.git
 
@@ -59,7 +59,7 @@ make their changes to the code and build with command to build graphscope image:
     make graphscope-dev-image
 
 This command triggers the building process. It will build the current source code in a container with
-image `graphscope-vineyard`, and copy built binaries into a new image based from `graphscope-runtime`. 
+image `graphscope-vineyard`, and copy built binaries into a new image based from `graphscope-runtime`.
 The generated releasing image is tagged as ``graphscope/graphscope:SHORTSHA``
 
 GraphScope python client is separate with the engines image. If you are developing python client and
@@ -83,6 +83,80 @@ To test the newly built binaries, manually open a session and assigned your imag
     # ...
 
 
+Building and Testing GraphScope on local
+---------------------------------------------------
+
+To build graphscope Python package and the engine binaries, some dependencies and build tools need to be installed.
+
+To make life easier, we provide a script to install the dependencies and build tools.
+the script is supported on the following 64-bit systems:
+
+- Ubuntu 18.04 or later
+- CentOS 8 or later
+- macOS 11.2.1 (Big Sur) or later, with both Intel chip and Apple M1 chip
+
+The script would install following dependencies or tools which needed by GraphScope building:
+- C++ compiler (gcc or llvm)
+- cmake (>=3.1)
+- java sdk (>=8)
+- maven
+- boost (>=1.66)
+- apache-arrow
+- rust (> 1.52.0)
+- etcd
+- openmpi
+- protobuf
+- grpc
+- libgrape-lite (the core of analytical engine)
+- vineyard
+- fastFFI (for grape-jdk)
+
+First, you need to ``git clone`` the latest version of code from our `repo <https://github.com/alibaba/GraphScope>`_
+and run the command:
+
+.. code:: bash
+    cd GraphScope
+    ./scripts/install_deps.sh --dev
+
+    # With argument --cn to speed up the download if you are in China.
+    ./scripts/install_deps.sh --dev --cn
+
+
+The `install_deps.sh` does not install the grape-jdk dependency `fastFFI` by default.
+If you don't care about `grape jdk <https://github.com/alibaba/GraphScope/blob/main/analytical_engine/java/README.md>`,
+you can ignore this phase. But if you want to use the grape jdk of GraphScope,
+you need to run the command after install the default dependencies.
+
+.. code::bash
+    # source the environment created by command above
+    source ~/.graphscope_env
+    # install the grape-jdk dependency
+    ./script/install_deps.sh --grape-jdk
+
+
+Then you can build GraphScope with pre-configured `make` commands.
+
+```bash
+# to make graphscope whole package, including python package + engine binaries.
+make graphscope
+
+# or make the engine components
+# make gie
+# make gae
+# make gle
+```
+
+To test the newly built binaries, manually open a session:
+
+.. code:: python
+
+    import graphscope
+
+    sess = graphscope.session(cluster_type="hosts")
+
+    # ...
+
+
 Build Python Wheels
 -------------------
 
@@ -137,6 +211,8 @@ Note that if you want to build wheel packages for different Python versions, you
 version of Python using `conda <https://docs.conda.io/en/latest/>`_ or `pyenv <https://github.com/pyenv/pyenv>`_.
 
 
+
+
 Code Format
 -----------
 

scripts/install_deps.sh

@@ -26,6 +26,7 @@ PLATFORM=
 OS_VERSION=
 VERBOSE=false
 CN_MIRROR=false
+GRAPE_JDK=false
 packages_to_install=()
 
 err() {
@@ -63,6 +64,7 @@ cat <<END
     --verbose            Print the debug logging information
     --k8s                Install the dependencies for running GraphScope on k8s locally
     --dev                Install the dependencies for build GraphScope on local
+    --grape-jdk          Install the dependecies for GraphScope grape jdk on local
     --cn                 Use tsinghua mirror for brew when install dependencies on macOS
 END
 }
@@ -169,6 +171,7 @@ init_basic_packages() {
       libprotobuf-dev
       librdkafka-dev
       libre2-dev
+      libc-ares-dev
       libsnappy-dev
       libssl-dev
       libunwind-dev
@@ -402,7 +405,6 @@ write_envs_config() {
         echo "export JAVA_HOME=/usr/lib/jvm/default-java"
       fi
       echo "export PATH=\${JAVA_HOME}/bin:\$HOME/.cargo/bin:\$PATH"
-      echo "export LLVM11_HOME=${homebrew_prefix}/opt/llvm/"
     } >> ${OUTPUT_ENV_FILE}
   else
     {
@@ -411,7 +413,6 @@ write_envs_config() {
         echo "export JAVA_HOME=/usr/lib/jvm/java"
       fi
       echo "export PATH=\${JAVA_HOME}/bin:\$HOME/.cargo/bin:\$PATH"
-      echo "export LLVM11_HOME=${homebrew_prefix}/opt/llvm/"
     } >> ${OUTPUT_ENV_FILE}
   fi
 }
@@ -491,26 +492,45 @@ install_vineyard() {
 ##########################
 install_fastFFI() {
   log "Building and installing fastFFI."
+  if [[ -d /opt/fastFFI && -d ${HOME}/.m2/repository/com/alibaba/fastffi ]]; then
+    log "fastFFI already installed, skip."
+    return 0
+  fi
 
-  pushd /opt/
+  if [ -z "${PLATFORM}" ]; then
+    get_os_version
+  fi
   if [[ -d /opt/fastFFI ]]; then
-    log "Found /opt/fastFFI exists, remove it."
     sudo rm -fr /opt/fastFFI
   fi
-  sudo git clone https://github.com/alibaba/fastFFI.git 
+  sudo git clone https://github.com/alibaba/fastFFI.git /opt/fastFFI
   sudo chown -R $(id -u):$(id -g) /opt/fastFFI
-  pushd fastFFI
+  pushd /opt/fastFFI
   git checkout a166c6287f2efb938c27fb01b3d499932d484f9c
 
   if [[ "${PLATFORM}" == *"Darwin"* ]]; then
+    declare -r homebrew_prefix=$(brew --prefix)
+    export LLVM11_HOME=${homebrew_prefix}/opt/llvm/
+
     # only install "ffi&annotation-processsor" on macOS, because
     # llvm4jni not compatible for Apple M1 chip.
     mvn clean install -DskipTests -pl :ffi,annotation-processor,llvm4jni-runtime -am --quiet
+  elif [[ "${PLATFORM}" == *"Ubuntu"* ]]; then
+    sudo apt update -y
+    sudo apt-get install -y llvm-11-dev lld-11 clang-11
+    export LLVM11_HOME=/usr/lib/llvm-11
+    export PATH=${LLVM11_HOME}/bin:${PATH}
+    mvn clean install -DskipTests --quiet
   else
-    export PATH=${PATH}:${LLVM11_HOME}/bin
-    mvn clean install -DskipTests
+    sudo dnf install -y llvm lld clang
+    export LLVM11_HOME=/usr/include/llvm
+    mvn clean install -DskipTests --quiet
   fi
-  popd
+
+  {
+    echo "export LLVM11_HOME=${LLVM11_HOME}"
+  } >> ${OUTPUT_ENV_FILE}
+
   popd
 }
 
@@ -605,10 +625,7 @@ install_dependencies() {
 
     if [[ "${packages_to_install[*]}" =~ "apache-arrow" ]]; then
       log "Installing apache-arrow."
-      sudo dnf install -y epel-release || sudo dnf install -y \
-        https://dl.fedoraproject.org/pub/epel/epel-release-latest-$(cut -d: -f5 /etc/system-release-cpe | cut -d. -f1).noarch.rpm
-      sudo dnf install -y https://apache.jfrog.io/artifactory/arrow/centos/$(cut -d: -f5 /etc/system-release-cpe | cut -d. -f1)/apache-arrow-release-latest.rpm
-      sudo dnf install -y arrow-devel
+      sudo dnf install -y libarrow-devel
       # remove apache-arrow from packages_to_install
       packages_to_install=("${packages_to_install[@]/apache-arrow}")
     fi
@@ -693,6 +710,8 @@ install_dependencies() {
     popd
     rm -fr /tmp/grpc
 
+    export LD_LIBRARY_PATH=/usr/local/lib
+
   elif [[ "${PLATFORM}" == *"Darwin"* ]]; then
     if [[ ${CN_MIRROR} == true ]]; then
       # set brew to tsinghua mirror
@@ -740,7 +759,6 @@ install_dependencies() {
     export CC=${homebrew_prefix}/opt/llvm/bin/clang
     export CXX=${homebrew_prefix}/opt/llvm/bin/clang++
     export CPPFLAGS=-I${homebrew_prefix}/opt/llvm/include
-    export LLVM11_HOME=${homebrew_prefix}/opt/llvm/
   fi
 
   log "Installing python packages for vineyard codegen."
@@ -753,8 +771,6 @@ install_dependencies() {
 
   install_cppkafka
 
-  install_fastFFI
-
   log "Output environments config file ${OUTPUT_ENV_FILE}"
   write_envs_config
 }
@@ -917,6 +933,7 @@ install_deps_dev() {
       --verbose)         VERBOSE=true; readonly VERBOSE; ;;
       --cn)              CN_MIRROR=true; readonly CN_MIRROR; ;;
       --vineyard_prefix) DEPS_PREFIX=$1; readonly DEPS_PREFIX; shift ;;
+      --grape-jdk)       GRAPE_JDK=true; readonly GRAPE_JDK; ;;
       *)
         echo "unrecognized option '${arg}'"
         usage; exit;;
@@ -941,6 +958,17 @@ install_deps_dev() {
   $ source ${OUTPUT_ENV_FILE}
   $ make graphscope\n
   to build and develop GraphScope."
+  if [[ ${GRAPE_JDK} == true ]]; then
+    install_fastFFI
+  else
+    {
+      echo "export ENABLE_JAVA_SDK=OFF"
+    } >> ${OUTPUT_ENV_FILE}
+    succ_msg=${succ_msg}"\n
+    Note: For simplify, The script doesn't install grape-jdk dependency by default. If you want to use grape jdk, use command:\n
+    $ ./install_deps.sh --grape-jdk
+    to install grape-jdk dependency."
+  fi
   succ "${succ_msg}"
 }
 
@@ -1007,6 +1035,7 @@ while test $# -ne 0; do
     --verbose)        VERBOSE=true; readonly VERBOSE; ;;
     --cn)             CN_MIRROR=true; readonly CN_MIRROR; ;;
     --dev) install_deps_dev "$@"; exit;;
+    --grape-jdk) install_fastFFI "$@"; exit;;
     --k8s) install_deps_k8s "$@"; exit;;
     *)
       echo "unrecognized option '${arg}'"