Add BUILD_BYPRODUCTS for ExternalProject_Add

.github/workflows/simple.yml

@@ -79,9 +79,12 @@ jobs:
           cmake_flags+=" -G '${{ matrix.cmake_generator }}'"
           cmake_flags+=" -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }}"
           cmake_flags+=" -DBUILD_SHARED_LIBS=${{ matrix.build_shared_libs }}"
+          cmake_flags+=" -DCMAKE_MACOSX_RPATH=FALSE"
+          cmake_flags+=" -DCMAKE_BUILD_WITH_INSTALL_RPATH=FALSE"
           echo "cmake_flags=${cmake_flags}" >> $GITHUB_ENV
         shell: bash
       - name: Build (third party)
+        if: matrix.cmake_generator != 'Ninja'
         run: |
           set -x
           mkdir -p build-third_party
@@ -91,30 +94,31 @@ jobs:
           cmake --build . -j $(nproc)
         shell: bash
       - name: Build (of_ccobj)
+        if: matrix.cmake_generator != 'Ninja'
         run: |
           mkdir -p build
           cd build
           cmake .. ${{ env.cmake_flags }} -DTHIRD_PARTY=OFF -DONEFLOW=ON -DTHIRD_PARTY_DIR=$PWD/../third_party_install
           cmake --build . -j ${{ matrix.make_concurrency }} --target of_ccobj
         shell: bash
       - name: Build (oneflow_internal)
-        if: always()
+        if: always() && matrix.cmake_generator != 'Ninja'
         run: |
           mkdir -p build
           cd build
           cmake .. ${{ env.cmake_flags }} -DTHIRD_PARTY=OFF -DONEFLOW=ON
           cmake --build . -j ${{ matrix.make_concurrency }} --target oneflow_internal
         shell: bash
       - name: Build (oneflow_py)
-        if: always()
+        if: always() && matrix.cmake_generator != 'Ninja'
         run: |
           mkdir -p build
           cd build
           cmake .. ${{ env.cmake_flags }} -DTHIRD_PARTY=OFF -DONEFLOW=ON
           cmake --build . -j ${{ matrix.make_concurrency }} --target oneflow_py
         shell: bash
       - name: Build (oneflow_testexe)
-        if: always()
+        if: always() && matrix.cmake_generator != 'Ninja'
         run: |
           mkdir -p build
           cd build
@@ -126,7 +130,7 @@ jobs:
         run: |
           mkdir -p build
           cd build
-          cmake .. ${{ env.cmake_flags }} -DTHIRD_PARTY=OFF -DONEFLOW=ON
+          cmake .. ${{ env.cmake_flags }}
           cmake --build . -j ${{ matrix.make_concurrency }}
         shell: bash
       - name: Exe test

cmake/third_party/absl.cmake

@@ -30,9 +30,10 @@ endforeach()
 if(THIRD_PARTY)
   ExternalProject_Add(${ABSL_PROJECT}
     PREFIX absl
-    URL ${ABSL_TAR_URL} 
+    URL ${ABSL_TAR_URL}
     URL_MD5 20126998c9b17e5f7a93711972f03f79
     UPDATE_COMMAND ""
+    BUILD_BYPRODUCTS ${ABSL_STATIC_LIBRARIES}
     CMAKE_ARGS
         -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
         -DBUILD_SHARED_LIBS:BOOL=OFF

cmake/third_party/cocoapi.cmake

@@ -25,6 +25,7 @@ ExternalProject_Add(cocoapi
     UPDATE_COMMAND ""
     CONFIGURE_COMMAND ""
     BUILD_IN_SOURCE 1
+    BUILD_BYPRODUCTS ${COCOAPI_STATIC_LIBRARIES}
     BUILD_COMMAND ${CMAKE_C_COMPILER} -fPIC -O3 -c common/maskApi.c -o maskApi.o &&
         ${CMAKE_AR} rcs ${COCOAPI_LIBRARY_NAME} maskApi.o
     INSTALL_COMMAND ""

cmake/third_party/flatbuffers.cmake

@@ -9,6 +9,18 @@ set(FLATBUFFERS_INSTALL_BINDIR bin)
 
 use_mirror(VARIABLE FLATBUFFERS_URL URL ${FLATBUFFERS_URL})
 
+set(FLATBUFFERS_INCLUDE_DIR ${FLATBUFFERS_INSTALL_PREFIX}/${FLATBUFFERS_INSTALL_INCLUDEDIR})
+set(FLATBUFFERS_LIBRARY_DIR ${FLATBUFFERS_INSTALL_PREFIX}/${FLATBUFFERS_INSTALL_LIBDIR})
+set(FLATBUFFERS_BINARY_DIR ${FLATBUFFERS_INSTALL_PREFIX}/${FLATBUFFERS_INSTALL_BINDIR})
+
+set(FLATC_EXECUTABLE_NAME flatc)
+set(FLATBUFFERS_FLATC_EXECUTABLE ${FLATBUFFERS_BINARY_DIR}/${FLATC_EXECUTABLE_NAME})
+
+set(FLATBUFFERS_LIBRARY_NAMES libflatbuffers.a)
+foreach (LIBRARY_NAME ${FLATBUFFERS_LIBRARY_NAMES})
+    list(APPEND FLATBUFFERS_STATIC_LIBRARIES ${FLATBUFFERS_LIBRARY_DIR}/${LIBRARY_NAME})
+endforeach ()
+
 if (THIRD_PARTY)
 
     ExternalProject_Add(flatbuffers
@@ -18,6 +30,7 @@ if (THIRD_PARTY)
             UPDATE_COMMAND bash -c "rm -f BUILD || true"
             BUILD_IN_SOURCE 1
             SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/flatbuffers/src/flatbuffers
+            BUILD_BYPRODUCTS ${FLATBUFFERS_STATIC_LIBRARIES}
             CMAKE_ARGS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
                 -DCMAKE_INSTALL_PREFIX=${FLATBUFFERS_INSTALL_PREFIX}
                 -DCMAKE_INSTALL_INCLUDEDIR=${FLATBUFFERS_INSTALL_INCLUDEDIR}
@@ -26,15 +39,3 @@ if (THIRD_PARTY)
                 -DFLATBUFFERS_BUILD_TESTS=OFF
     )
 endif (THIRD_PARTY)
-
-set(FLATBUFFERS_INCLUDE_DIR ${FLATBUFFERS_INSTALL_PREFIX}/${FLATBUFFERS_INSTALL_INCLUDEDIR})
-set(FLATBUFFERS_LIBRARY_DIR ${FLATBUFFERS_INSTALL_PREFIX}/${FLATBUFFERS_INSTALL_LIBDIR})
-set(FLATBUFFERS_BINARY_DIR ${FLATBUFFERS_INSTALL_PREFIX}/${FLATBUFFERS_INSTALL_BINDIR})
-
-set(FLATC_EXECUTABLE_NAME flatc)
-set(FLATBUFFERS_FLATC_EXECUTABLE ${FLATBUFFERS_BINARY_DIR}/${FLATC_EXECUTABLE_NAME})
-
-set(FLATBUFFERS_LIBRARY_NAMES libflatbuffers.a)
-foreach (LIBRARY_NAME ${FLATBUFFERS_LIBRARY_NAMES})
-    list(APPEND FLATBUFFERS_STATIC_LIBRARIES ${FLATBUFFERS_LIBRARY_DIR}/${LIBRARY_NAME})
-endforeach ()

cmake/third_party/gflags.cmake

@@ -36,6 +36,7 @@ ExternalProject_Add(gflags
     URL_MD5 9677cc51d63642ba3d5f2a57a1fa2bd0
     UPDATE_COMMAND bash -c "rm -f BUILD || true"
     BUILD_IN_SOURCE 1
+    BUILD_BYPRODUCTS ${GFLAGS_STATIC_LIBRARIES}
     CMAKE_CACHE_ARGS
         -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
         -DCMAKE_CXX_FLAGS_DEBUG:STRING=${CMAKE_CXX_FLAGS_DEBUG}

cmake/third_party/glog.cmake

@@ -45,6 +45,7 @@ ExternalProject_Add(glog
     URL_MD5 3ca928ef755c0a890680e023e3d4b9a6
     UPDATE_COMMAND ""
     BUILD_IN_SOURCE 1
+    BUILD_BYPRODUCTS ${GLOG_STATIC_LIBRARIES}
     CMAKE_CACHE_ARGS
         -DCMAKE_POLICY_DEFAULT_CMP0074:STRING=NEW
         -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}

cmake/third_party/googletest.cmake

@@ -48,6 +48,7 @@ ExternalProject_Add(googletest
     UPDATE_COMMAND ""
     BUILD_IN_SOURCE 1
     INSTALL_COMMAND ""
+    BUILD_BYPRODUCTS ${GOOGLETEST_STATIC_LIBRARIES} ${GOOGLEMOCK_STATIC_LIBRARIES}
     CMAKE_CACHE_ARGS
         -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
         -DCMAKE_CXX_FLAGS_DEBUG:STRING=${CMAKE_CXX_FLAGS_DEBUG}

cmake/third_party/grpc.cmake

@@ -42,6 +42,7 @@ ExternalProject_Add(grpc
     URL_HASH MD5=${GRPC_URL_HASH}
     UPDATE_COMMAND ""
     BUILD_IN_SOURCE 1
+    BUILD_BYPRODUCTS ${GRPC_STATIC_LIBRARIES}
     BUILD_COMMAND ${CMAKE_COMMAND} --build . -j ${PROC_NUM} --target grpc && ${CMAKE_COMMAND} --build . -j ${PROC_NUM} --target grpc_unsecure && ${CMAKE_COMMAND} --build . -j ${PROC_NUM} --target grpc++_unsecure
     CMAKE_CACHE_ARGS
         -DCMAKE_POLICY_DEFAULT_CMP0074:STRING=NEW

cmake/third_party/libjpeg-turbo.cmake

@@ -59,6 +59,7 @@ ExternalProject_Add(libjpeg-turbo
     UPDATE_COMMAND ""
     INSTALL_COMMAND ""
     BUILD_IN_SOURCE 1
+    BUILD_BYPRODUCTS ${LIBJPEG_STATIC_LIBRARIES}
     CMAKE_CACHE_ARGS
         -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
         -DCMAKE_CXX_FLAGS_DEBUG:STRING=${CMAKE_CXX_FLAGS_DEBUG}

cmake/third_party/lz4.cmake

@@ -39,6 +39,7 @@ ExternalProject_Add(lz4
     BUILD_IN_SOURCE 1
     BUILD_COMMAND make -j ${PROC_NUM} lib CFLAGS=${LZ4_CFLAGS}
     INSTALL_COMMAND ""
+    BUILD_BYPRODUCTS ${LZ4_STATIC_LIBRARIES}
 )
 
 add_custom_target(lz4_create_header_dir

cmake/third_party/nccl.cmake

@@ -39,6 +39,7 @@ ExternalProject_Add(nccl
     BUILD_IN_SOURCE 1
     BUILD_COMMAND make -j${PROC_NUM} src.build CUDA_HOME=${CUDA_TOOLKIT_ROOT_DIR}
     INSTALL_COMMAND make src.install PREFIX=${NCCL_INSTALL_DIR}
+    BUILD_BYPRODUCTS ${NCCL_LIBRARIES}
 )
 
 endif(THIRD_PARTY)

cmake/third_party/opencv.cmake

@@ -43,6 +43,7 @@ ExternalProject_Add(opencv
     BUILD_IN_SOURCE 0
     SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/opencv/src/opencv
     BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/opencv/src/opencv/build
+    BUILD_BYPRODUCTS ${OPENCV_STATIC_LIBRARIES}
     CMAKE_CACHE_ARGS
         -DCMAKE_POLICY_DEFAULT_CMP0074:STRING=NEW
         -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}

cmake/third_party/openssl.cmake

@@ -35,9 +35,9 @@ ExternalProject_Add(openssl
   URL_HASH MD5=${OPENSSL_URL_HASH}
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ${OPENSSL_SOURCE_DIR}/src/openssl/config --prefix=${OPENSSL_INSTALL}
+  BUILD_BYPRODUCTS ${OPENSSL_STATIC_LIBRARIES}
   BUILD_COMMAND make -j${PROC_NUM}
   INSTALL_COMMAND make install_sw
 )
 
 endif(THIRD_PARTY)
-

cmake/third_party/protobuf.cmake

@@ -52,6 +52,7 @@ ExternalProject_Add(protobuf
     BUILD_IN_SOURCE 1
     SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf
     SOURCE_SUBDIR cmake
+    BUILD_BYPRODUCTS ${PROTOBUF_STATIC_LIBRARIES}
     CMAKE_CACHE_ARGS
         -DCMAKE_POLICY_DEFAULT_CMP0074:STRING=NEW
         -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}

cmake/third_party/re2.cmake

@@ -16,6 +16,7 @@ if (THIRD_PARTY)
         URL ${RE2_URL}
         URL_MD5 3b2e20c1edd1cfe887aeef3b0747eac0
         UPDATE_COMMAND ""
+        BUILD_BYPRODUCTS ${RE2_LIBRARIES}
         CMAKE_ARGS
           -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
           -DBUILD_SHARED_LIBS:BOOL=OFF

cmake/third_party/zlib.cmake

@@ -39,6 +39,7 @@ ExternalProject_Add(zlib
     URL_MD5 1eabf2698dc49f925ce0ffb81397098f
     UPDATE_COMMAND ""
     BUILD_IN_SOURCE 1
+    BUILD_BYPRODUCTS ${ZLIB_STATIC_LIBRARIES}
     CMAKE_CACHE_ARGS
       -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
       -DBUILD_SHARED_LIBS:BOOL=${BUILD_SHARED_LIBS}

docker/package/manylinux/build_wheel.py

@@ -148,7 +148,7 @@ def get_common_docker_args(
         house_dir_arg = f"-v {house_dir}:{house_dir}"
     build_dir_arg = get_build_dir_arg(cache_dir, oneflow_src_dir)
     proxy_env_arg = get_proxy_env_args() if use_system_proxy else ""
-    return f"-v {oneflow_src_dir}:{oneflow_src_dir} {proxy_env_arg} {pwd_arg} {house_dir_arg} {cache_dir_arg} {build_dir_arg} -w {current_dir}"
+    return f"-v {oneflow_src_dir}:{oneflow_src_dir} {proxy_env_arg} {pwd_arg} {house_dir_arg} {cache_dir_arg} {build_dir_arg} -w {current_dir} --shm-size=8g"
 
 
 def build_third_party(