Add a local deploy and test ci on MacOS (#512)

.github/workflows/mac.yml

@@ -0,0 +1,143 @@
+name: GraphScope CI on MacOS
+
+on:
+  # Trigger the workflow on push or pull request, but only for the main branch
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - 'gnn_engine/**'
+      - '**.md'
+      - '**.rst'
+      - 'docs/**'
+      - 'research/**'
+      - 'scripts/**'
+      - 'tutorials/**'
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - 'gnn_engine/**'
+      - '**.md'
+      - '**.rst'
+      - 'docs/**'
+      - 'research/**'
+      - 'scripts/**'
+      - 'tutorials/**'
+
+jobs:
+  build:
+    runs-on: macOS-latest
+    env:
+      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
+      OPENSSL_ROOT_DIR: /usr/local/opt/openssl
+      OPENSSL_LIBRARIES: /usr/local/opt/openssl/lib
+      OPENSSL_SSL_LIBRARY: /usr/local/opt/openssl/lib/libssl.dylib
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+
+    - name: Setup tmate session
+      if: false
+      uses: mxschmitt/action-tmate@v2
+
+    - name: Install Dependencies for MacOS
+      run: |
+        pushd ${GITHUB_WORKSPACE}
+        brew install llvm@9
+
+        # write envs to .env
+        {
+          echo "export CC=/usr/local/opt/llvm@9/bin/clang"
+          echo "export CXX=/usr/local/opt/llvm@9/bin/clang++"
+          echo "export LDFLAGS=-L/usr/local/opt/llvm@9/lib"
+          echo "export CPPFLAGS=-I/usr/local/opt/llvm@9/include"
+          echo "export PATH=/usr/local/opt/llvm@9/bin:${JAVA_HOME}/bin:$PATH"
+        } >> .env
+
+        # install dependency
+        source .env
+        /bin/bash ./scripts/install_dependencies.sh
+        popd
+
+    - name: Install libgrape-lite
+      run: |
+        source ${GITHUB_WORKSPACE}/.env
+        git clone -b master --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git
+        cd libgrape-lite
+        mkdir build && cd build
+        cmake ..
+        make -j$(sysctl -n hw.ncpu)
+        sudo make install
+
+    - name: Setup tmate session
+      if: false
+      uses: mxschmitt/action-tmate@v2
+
+    - name: Install libvineyard
+      run: |
+        source ${GITHUB_WORKSPACE}/.env
+        git clone -b main-v0.2.5 --single-branch --depth=1 https://github.com/alibaba/libvineyard.git
+        cd libvineyard
+        git submodule update --init
+        mkdir build && pushd build
+        cmake .. -DBUILD_VINEYARD_PYTHON_BINDINGS=ON -DBUILD_SHARED_LIBS=ON -DBUILD_VINEYARD_IO_OSS=ON -DBUILD_VINEYARD_TESTS=OFF
+        make -j$(sysctl -n hw.ncpu)
+        make vineyard_client_python -j$(sysctl -n hw.ncpu)
+        sudo make install
+        popd
+        python3 setup.py bdist_wheel
+        pip3 install ./dist/*.whl
+
+    - name: Build GraphScope
+      run: |
+        source ${GITHUB_WORKSPACE}/.env
+        export PATH=/usr/local/opt/gnu-sed/libexec/gnubin:${PATH}:/usr/local/zookeeper/bin
+
+        # builde and install gae and gie
+        make gae gie
+
+        pushd python
+        pip3 install -U setuptools --user
+        pip3 install -r requirements.txt -r requirements-dev.txt --user
+        python3 setup.py bdist_wheel
+        pip3 install -U ./dist/*.whl --user
+        popd
+        pushd coordinator
+        pip3 install -r requirements.txt -r requirements-dev.txt --user
+        python3 setup.py bdist_wheel
+        pip3 install -U ./dist/*.whl --user
+        popd
+
+    - name: DEBUG
+      if: false
+      uses: mxschmitt/action-tmate@v2
+
+    - name: Run Local Test
+      run: |
+        source ${GITHUB_WORKSPACE}/.env
+        # prelaunch the etcd
+        /usr/local/bin/etcd --data-dir=/tmp/default.etcd&
+
+        export PATH=/usr/local/opt/gnu-sed/libexec/gnubin:${JAVA_HOME}/bin:${PATH}:/usr/local/zookeeper/bin
+        export GRAPHSCOPE_PREFIX=/usr/local
+        export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
+
+        export GS_TEST_DIR=${GITHUB_WORKSPACE}/gstest
+        git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR}
+        python3 -m pytest -s -v ${GITHUB_WORKSPACE}/python/tests/test_run_locally.py
+
+        # delete etcd dir
+        rm -fr /tmp/default.etcd || true
+
+    - name: Upload GIE Log
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: gie-log
+        path: /tmp/graphscope/
+
+    - name: Setup tmate session debug
+      if: false
+      uses: mxschmitt/action-tmate@v2

Makefile

@@ -10,6 +10,9 @@ NETWORKX                    ?= OFF
 # client build options
 WITH_LEARNING_ENGINE        ?= OFF
 
+# testing build option
+BUILD_TEST                  ?= OFF
+
 .PHONY: all
 all: graphscope
 
@@ -52,7 +55,7 @@ coordinator: client
 gae:
 	mkdir -p $(WORKING_DIR)/analytical_engine/build
 	cd $(WORKING_DIR)/analytical_engine/build && \
-	cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) -DNETWORKX=$(NETWORKX) .. && \
+	cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) -DNETWORKX=$(NETWORKX) -DBUILD_TESTS=${BUILD_TEST} .. && \
 	make -j`nproc` && \
 	sudo make install
 
@@ -81,7 +84,7 @@ gle:
 	cd $(WORKING_DIR)/learning_engine/graph-learn && \
 	git submodule update --init third_party/pybind11 && \
 	mkdir -p cmake-build && cd cmake-build && \
-	cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) -DWITH_VINEYARD=ON -DTESTING=OFF .. && \
+	cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) -DWITH_VINEYARD=ON -DTESTING=${BUILD_TEST} .. && \
 	make -j`nproc` && \
 	sudo make install
 

interactive_engine/src/common/rust/common/src/util/ip.rs

@@ -1,12 +1,12 @@
 //
 //! Copyright 2020 Alibaba Group Holding Limited.
-//! 
+//!
 //! Licensed under the Apache License, Version 2.0 (the "License");
 //! you may not use this file except in compliance with the License.
 //! You may obtain a copy of the License at
-//! 
+//!
 //!     http://www.apache.org/licenses/LICENSE-2.0
-//! 
+//!
 //! Unless required by applicable law or agreed to in writing, software
 //! distributed under the License is distributed on an "AS IS" BASIS,
 //! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,23 +16,28 @@
 use std::process::Command;
 
 pub fn get_local_ip() -> String {
-    let output = match Command::new("hostname").args(&["-I"]).output() {
-        Ok(ok) => ok,
-        Err(_) => {
-            return String::from("127.0.0.1");
-        }
-    };
+    if cfg!(target_os = "linux") {
+        let output = match Command::new("hostname").args(&["-I"]).output() {
+            Ok(ok) => ok,
+            Err(_) => {
+                return String::from("127.0.0.1");
+            }
+        };
 
-    let stdout = match String::from_utf8(output.stdout) {
-        Ok(ok) => ok,
-        Err(_) => {
-            return String::from("127.0.0.1");
-        }
-    };
+        let stdout = match String::from_utf8(output.stdout) {
+            Ok(ok) => ok,
+            Err(_) => {
+                return String::from("127.0.0.1");
+            }
+        };
 
-    let ips: Vec<&str> = stdout.trim().split(" ").collect::<Vec<&str>>();
-    let first = ips.first();
-    return first.map(|s| s.to_string()).unwrap_or("127.0.0.1".to_string()); 
+        let ips: Vec<&str> = stdout.trim().split(" ").collect::<Vec<&str>>();
+        let first = ips.first();
+        return first.map(|s| s.to_string()).unwrap_or("127.0.0.1".to_string());
+    } else {
+        // Mac can only deploy on local.
+        return String::from("127.0.0.1");
+    }
 }
 
 

interactive_engine/src/executor/runtime/src/bin/executor.rs

@@ -1,12 +1,12 @@
 //
 //! Copyright 2020 Alibaba Group Holding Limited.
-//! 
+//!
 //! Licensed under the Apache License, Version 2.0 (the "License");
 //! you may not use this file except in compliance with the License.
 //! You may obtain a copy of the License at
-//! 
+//!
 //!     http://www.apache.org/licenses/LICENSE-2.0
-//! 
+//!
 //! Unless required by applicable law or agreed to in writing, software
 //! distributed under the License is distributed on an "AS IS" BASIS,
 //! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -81,15 +81,11 @@ fn main() {
     let worker_num = store_config.timely_worker_per_process;
     let store_config = Arc::new(store_config);
     if store_config.graph_type.to_lowercase().eq(VINEYARD_GRAPH) {
-        if cfg!(target_os = "linux") {
-            info!("Start executor with vineyard graph object id {:?}", store_config.vineyard_graph_id);
-            use maxgraph_runtime::store::ffi::FFIGraphStore;
-            let ffi_store = FFIGraphStore::new(store_config.vineyard_graph_id, worker_num as i32);
-            let partition_manager = ffi_store.get_partition_manager();
-            run_main(store_config, Arc::new(ffi_store), Arc::new(partition_manager));
-        } else {
-            unimplemented!("Mac not support vineyard graph")
-        }
+        info!("Start executor with vineyard graph object id {:?}", store_config.vineyard_graph_id);
+        use maxgraph_runtime::store::ffi::FFIGraphStore;
+        let ffi_store = FFIGraphStore::new(store_config.vineyard_graph_id, worker_num as i32);
+        let partition_manager = ffi_store.get_partition_manager();
+        run_main(store_config, Arc::new(ffi_store), Arc::new(partition_manager));
     } else {
         unimplemented!("only start vineyard graph from executor")
     }
@@ -126,7 +122,7 @@ fn run_main<V, VI, E, EI>(store_config: Arc<StoreConfig>,
     server_manager = Box::new(pegasus_server_manager);
     let _manager_guards = ServerManager::start_server(server_manager, store_config.clone(), Box::new(recover_prepare)).unwrap();
 
-    // waiting for initial task_partition_manager 
+    // waiting for initial task_partition_manager
     let mut timeout_count = 0;
     loop {
         match task_partition_manager.read() {

python/tests/test_run_locally.py

@@ -33,9 +33,11 @@
 from graphscope.dataset.ogbn_mag import load_ogbn_mag
 from graphscope.framework.graph import Graph
 from graphscope.framework.loader import Loader
-from graphscope.learning.examples import GCN
-from graphscope.learning.graphlearn.python.model.tf.optimizer import get_tf_optimizer
-from graphscope.learning.graphlearn.python.model.tf.trainer import LocalTFTrainer
+
+if sys.platform == "linux":
+    from graphscope.learning.examples import GCN
+    from graphscope.learning.graphlearn.python.model.tf import optimizer
+    from graphscope.learning.graphlearn.python.model.tf.trainer import LocalTFTrainer
 
 graphscope.set_option(show_log=True)
 graphscope.set_option(initializing_interactive_engine=False)
@@ -65,7 +67,7 @@ def model_fn():
     trainer = LocalTFTrainer(
         model_fn,
         epoch=config["epoch"],
-        optimizer=get_tf_optimizer(
+        optimizer=optimizer.get_tf_optimizer(
             config["learning_algo"], config["learning_rate"], config["weight_decay"]
         ),
     )
@@ -105,6 +107,10 @@ def demo(sess, ogbn_mag_small):
     sub_graph = sub_graph.add_column(ret1, {"kcore": "r"})
     sub_graph = sub_graph.add_column(ret2, {"tc": "r"})
 
+    # MacOS skip the GLE test
+    if sys.platform == "darwin":
+        return
+
     # GLE on ogbn_mag_small graph
     paper_features = []
     for i in range(128):
@@ -202,6 +208,7 @@ def test_demo(ogbn_mag_small):
     sess.close()
 
 
+@pytest.mark.skipif(sys.platform == "darwin", reason="Mac no need to run this test.")
 def test_multiple_session(ogbn_mag_small):
     sess1 = graphscope.session(cluster_type="hosts", num_workers=1)
     assert sess1.info["status"] == "active"

scripts/install_dependencies.sh

@@ -110,7 +110,7 @@ function install_dependencies() {
     pushd /tmp/fmt-7.0.3
     mkdir build && cd build
     cmake .. -DBUILD_SHARED_LIBS=ON
-    make install -j
+    make install -j`nporc`
     popd
 
     # install folly
@@ -119,7 +119,7 @@ function install_dependencies() {
     pushd /tmp/folly-2020.10.19.00
     mkdir _build && cd _build
     cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON ..
-    make install -j
+    make install -j`nproc`
     popd
 
     # install zookeeper
@@ -145,7 +145,7 @@ function install_dependencies() {
 
   if [[ "${platform}" == *"Darwin"* ]]; then
     brew install cmake double-conversion etcd protobuf apache-arrow openmpi boost glog gflags \
-      zstd snappy lz4 openssl@1.1 libevent fmt autoconf go maven
+      zstd snappy lz4 openssl@1.1 libevent fmt autoconf maven gnu-sed go@1.15
 
     # export openssl library
     export OPENSSL_ROOT_DIR="/usr/local/opt/openssl"
@@ -158,12 +158,18 @@ function install_dependencies() {
     pushd /tmp/folly-2020.10.19.00
     mkdir _build && cd _build
     cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON ..
-    make install -j
+    make install -j$(sysctl -n hw.physicalcpu)
     popd
 
+    # install zookeeper
+    wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz -P /tmp
+    tar xf /tmp/zookeeper-3.4.14.tar.gz -C /tmp/
+    cp /tmp/zookeeper-3.4.14/conf/zoo_sample.cfg /tmp/zookeeper-3.4.14/conf/zoo.cfg
+    sudo cp -r /tmp/zookeeper-3.4.14 /usr/local/zookeeper || true
+
     # install python packages for vineyard codegen
     pip3 install -U pip --user
-    pip3 install grpc-tools libclang parsec setuptools wheel twine --user
+    pip3 install grpcio-tools libclang parsec setuptools wheel twine --user
   fi
 
   check_dependencies_version