Install jdk 11 on MacOS CI and fixes the GAE link failure in nightly …

…CI on MacOS (#2965)

- Use Java 11 on CI as it is required since
30e45e0
- Explicitly link against protobuf on MacOS

Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>

.github/workflows/build-graphscope-wheels-linux.yml

@@ -21,7 +21,7 @@ env:
 
 jobs:
   build-wheels-for-arm64:
-    if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope')
+    if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch')
     runs-on: [self-hosted, Linux, ARM64]
 
     steps:
@@ -86,13 +86,14 @@ jobs:
 
     - name: Publish distribution to PyPI
       uses: pypa/gh-action-pypi-publish@v1.4.2
+      if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope')
       with:
         user: __token__
         password: ${{ secrets.PYPI_PASSWORD }}
         packages_dir: upload_pypi/
 
   build-wheels:
-    if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope')
+    if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch')
     runs-on: ubuntu-20.04
 
     steps:
@@ -163,6 +164,7 @@ jobs:
 
     - name: Publish distribution to PyPI
       uses: pypa/gh-action-pypi-publish@v1.4.2
+      if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope')
       with:
         user: __token__
         password: ${{ secrets.PYPI_PASSWORD }}
@@ -236,7 +238,7 @@ jobs:
         sudo docker push ${{ env.REGISTRY }}/graphscope/jupyter:${tag}
 
   ubuntu-python-test:
-    if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
+    if: ${{ (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') }}
     runs-on: ubuntu-20.04
     needs: [build-wheels]
     strategy:
@@ -292,7 +294,7 @@ jobs:
         python3 -m pytest -s -v $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/tests/minitest
 
   centos-test:
-    if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
+    if: ${{ (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') }}
     runs-on: ubuntu-20.04
     needs: [build-wheels]
     container:

.github/workflows/build-graphscope-wheels-macos.yml

@@ -18,7 +18,7 @@ concurrency:
 
 jobs:
   build-wheels:
-    if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope')
+    if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch')
     runs-on: macos-11
     steps:
     - uses: actions/checkout@v3
@@ -39,6 +39,12 @@ jobs:
         ./gs install-deps dev
         echo "export PATH=/usr/local/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env
 
+    - name: Install JDK 11
+      uses: actions/setup-java@v3
+      with:
+        distribution: 'zulu' # See 'Supported distributions' for available options
+        java-version: '11'
+
     - name: Reinstall glog
       run: |
         # Essential cause glog installed by brew is broken and produce corrupted messages
@@ -115,7 +121,7 @@ jobs:
         retention-days: 5
 
   build-client-wheels:
-    if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope')
+    if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch')
     runs-on: macos-11
     strategy:
       matrix:
@@ -245,8 +251,8 @@ jobs:
         packages_dir: upload_pypi/
 
   python-test:
-    if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
-    runs-on: macos-latest
+    if: ${{ (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') }}
+    runs-on: macos-11
     needs: [build-wheels, build-client-wheels]
     strategy:
       fail-fast: false

Makefile

@@ -141,6 +141,7 @@ interactive: $(INTERACTIVE_DIR)/assembly/target/graphscope.tar.gz
 
 $(INTERACTIVE_DIR)/assembly/target/graphscope.tar.gz:
 	cd $(INTERACTIVE_DIR) && \
+	mvn --version && \
 	mvn package -DskipTests -Drust.compile.mode=$(BUILD_TYPE) -P graphscope -Drevision=$(VERSION) --quiet
 
 learning-install: learning

coordinator/gscoordinator/op_executor.py

@@ -735,7 +735,6 @@ def _process_serialize_graph(self, op: op_def_pb2.OpDef):
         deployment, hosts = self._launcher.get_vineyard_stream_info()
         path = op.attr[types_pb2.GRAPH_SERIALIZATION_PATH].s.decode()
         obj_id = op.attr[types_pb2.VINEYARD_ID].i
-        logger.info("serialize graph %d  to %s", obj_id, path)
         vineyard.io.serialize(
             path,
             vineyard.ObjectID(obj_id),
@@ -746,7 +745,6 @@ def _process_serialize_graph(self, op: op_def_pb2.OpDef):
             deployment=deployment,
             hosts=hosts,
         )
-        logger.info("Finish serialization")
         return op_def_pb2.OpResult(code=OK, key=op.key)
 
     def _process_deserialize_graph(self, op: op_def_pb2.OpDef):
@@ -772,7 +770,6 @@ def _process_deserialize_graph(self, op: op_def_pb2.OpDef):
         vineyard_ipc_socket = engine_config["vineyard_socket"]
         deployment, hosts = self._launcher.get_vineyard_stream_info()
         path = op.attr[types_pb2.GRAPH_SERIALIZATION_PATH].s.decode()
-        logger.info("Deserialize graph from %s", path)
         graph_id = vineyard.io.deserialize(
             path,
             type="global",
@@ -782,7 +779,6 @@ def _process_deserialize_graph(self, op: op_def_pb2.OpDef):
             deployment=deployment,
             hosts=hosts,
         )
-        logger.info("Finish deserialization, graph id: %d", graph_id)
         # create graph_def
         # run create graph on analytical engine
         create_graph_op = create_single_op_dag(
@@ -811,7 +807,6 @@ def _process_deserialize_graph(self, op: op_def_pb2.OpDef):
                 raise AnalyticalEngineInternalError(e.details())
             else:
                 raise
-        logger.info("response head, %s , body %s", response_head, response_body)
         response_head.head.results[0].key = op.key
         return response_head.head.results[0]
 

coordinator/gscoordinator/template/CMakeLists.template

@@ -288,6 +288,33 @@ if (APPLE AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
     set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup")
 endif()
 
+# address the "illegal thread local variable reference to regular symbol" error
+# in recent version of brew installed protobuf on MacOS
+if (APPLE)
+    # find bundled protobuf library
+    if (EXISTS "${ANALYTICAL_ENGINE_HOME}/lib/libprotobuf.dylib")
+        set(Protobuf_LIBRARIES "${ANALYTICAL_ENGINE_HOME}/lib/libprotobuf.dylib")
+    elseif(EXISTS "${GRAPHSCOPE_ANALYTICAL_HOME}/lib/libprotobuf.dylib")
+        set(Protobuf_LIBRARIES "${GRAPHSCOPE_ANALYTICAL_HOME}/lib/libprotobuf.dylib")
+    else()
+        # search libprotobuf.xx.xxx.dylib in graphscope.libs
+        file(GLOB PROTOBUF_LIBS "${ANALYTICAL_ENGINE_HOME}/../graphscope.dylibs/lib/libprotobuf.*.dylib")
+        if (PROTOBUF_LIBS)
+            list(GET PROTOBUF_LIBS 0 Protobuf_LIBRARIES)
+        endif()
+    endif()
+    if (NOT Protobuf_LIBRARIES)
+        # search system-wide protobuf
+        find_package (Protobuf QUIET)
+        if (NOT Protobuf_FOUND)
+            find_package (protobuf CONFIG REQUIRED)
+        endif()
+    endif()
+    message("Using Protobuf_LIBRARIES: ${Protobuf_LIBRARIES}")
+else()
+    set(Protobuf_LIBRARIES)
+endif()
+
 set(GRAPH_TYPE "$_graph_type")
 # message("GRAPH_TYPE = ${GRAPH_TYPE}")
 
@@ -349,10 +376,10 @@ elseif (JAVA_PIE_APP)
             message(FATAL_ERROR "Preprocess failed")
         endif()
     endif()
-# find jni---------------------------------------------------------------------
-     find_package(JNI REQUIRED)
-     include_directories(SYSTEM ${JAVA_INCLUDE_PATH})
-     include_directories(SYSTEM ${JAVA_INCLUDE_PATH2})
+    # find jni---------------------------------------------------------------------
+    find_package(JNI REQUIRED)
+    include_directories(SYSTEM ${JAVA_INCLUDE_PATH})
+    include_directories(SYSTEM ${JAVA_INCLUDE_PATH2})
 
     # 1. cmake
     file(GLOB_RECURSE FILES_NEED_COMPILE "${OUTPUT_DIR}/SOURCE_OUTPUT/*.cc" ${ANALYTICAL_JAVASDK_CC})
@@ -420,6 +447,10 @@ if (OpenMP_FOUND)
     target_link_libraries(${FRAME_NAME} OpenMP::OpenMP_CXX)
 endif ()
 
+if (APPLE AND Protobuf_LIBRARIES)
+    target_link_libraries(${FRAME_NAME} ${Protobuf_LIBRARIES})
+endif ()
+
 # include the inplace cmake config at the end, to avoid affecting the CMake search path
 include("${ANALYTICAL_ENGINE_HOME}/build/graphscope-analytical-config.cmake" OPTIONAL)