From ded8d1ee1332591381fdad13cfb1e5145c54fcf5 Mon Sep 17 00:00:00 2001 From: jackalcooper Date: Fri, 25 Jun 2021 16:08:17 +0000 Subject: [PATCH 1/8] Add BUILD_BYPRODUCTS for ExternalProject_Add --- cmake/third_party/absl.cmake | 3 ++- cmake/third_party/cocoapi.cmake | 1 + cmake/third_party/flatbuffers.cmake | 25 +++++++++++++------------ cmake/third_party/gflags.cmake | 1 + cmake/third_party/glog.cmake | 1 + cmake/third_party/googletest.cmake | 1 + cmake/third_party/grpc.cmake | 1 + cmake/third_party/libjpeg-turbo.cmake | 1 + cmake/third_party/lz4.cmake | 1 + cmake/third_party/nccl.cmake | 1 + cmake/third_party/opencv.cmake | 1 + cmake/third_party/openssl.cmake | 2 +- cmake/third_party/protobuf.cmake | 1 + cmake/third_party/re2.cmake | 1 + cmake/third_party/zlib.cmake | 1 + 15 files changed, 28 insertions(+), 14 deletions(-) diff --git a/cmake/third_party/absl.cmake b/cmake/third_party/absl.cmake index 417a2f265de..e11b79751d2 100644 --- a/cmake/third_party/absl.cmake +++ b/cmake/third_party/absl.cmake @@ -30,9 +30,10 @@ endforeach() if(THIRD_PARTY) ExternalProject_Add(${ABSL_PROJECT} PREFIX absl - URL ${ABSL_TAR_URL} + URL ${ABSL_TAR_URL} URL_MD5 20126998c9b17e5f7a93711972f03f79 UPDATE_COMMAND "" + BUILD_BYPRODUCTS ${ABSL_STATIC_LIBRARIES} CMAKE_ARGS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS:BOOL=OFF diff --git a/cmake/third_party/cocoapi.cmake b/cmake/third_party/cocoapi.cmake index d3b12a1fb4c..734c5b7bf5e 100644 --- a/cmake/third_party/cocoapi.cmake +++ b/cmake/third_party/cocoapi.cmake @@ -25,6 +25,7 @@ ExternalProject_Add(cocoapi UPDATE_COMMAND "" CONFIGURE_COMMAND "" BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS ${COCOAPI_STATIC_LIBRARIES} BUILD_COMMAND ${CMAKE_C_COMPILER} -fPIC -O3 -c common/maskApi.c -o maskApi.o && ${CMAKE_AR} rcs ${COCOAPI_LIBRARY_NAME} maskApi.o INSTALL_COMMAND "" diff --git a/cmake/third_party/flatbuffers.cmake b/cmake/third_party/flatbuffers.cmake index 11909da9fd9..398de64f442 100644 --- a/cmake/third_party/flatbuffers.cmake +++ b/cmake/third_party/flatbuffers.cmake @@ -9,6 +9,18 @@ set(FLATBUFFERS_INSTALL_BINDIR bin) use_mirror(VARIABLE FLATBUFFERS_URL URL ${FLATBUFFERS_URL}) +set(FLATBUFFERS_INCLUDE_DIR ${FLATBUFFERS_INSTALL_PREFIX}/${FLATBUFFERS_INSTALL_INCLUDEDIR}) +set(FLATBUFFERS_LIBRARY_DIR ${FLATBUFFERS_INSTALL_PREFIX}/${FLATBUFFERS_INSTALL_LIBDIR}) +set(FLATBUFFERS_BINARY_DIR ${FLATBUFFERS_INSTALL_PREFIX}/${FLATBUFFERS_INSTALL_BINDIR}) + +set(FLATC_EXECUTABLE_NAME flatc) +set(FLATBUFFERS_FLATC_EXECUTABLE ${FLATBUFFERS_BINARY_DIR}/${FLATC_EXECUTABLE_NAME}) + +set(FLATBUFFERS_LIBRARY_NAMES libflatbuffers.a) +foreach (LIBRARY_NAME ${FLATBUFFERS_LIBRARY_NAMES}) + list(APPEND FLATBUFFERS_STATIC_LIBRARIES ${FLATBUFFERS_LIBRARY_DIR}/${LIBRARY_NAME}) +endforeach () + if (THIRD_PARTY) ExternalProject_Add(flatbuffers @@ -18,6 +30,7 @@ if (THIRD_PARTY) UPDATE_COMMAND bash -c "rm -f BUILD || true" BUILD_IN_SOURCE 1 SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/flatbuffers/src/flatbuffers + BUILD_BYPRODUCTS ${FLATBUFFERS_STATIC_LIBRARIES} CMAKE_ARGS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${FLATBUFFERS_INSTALL_PREFIX} -DCMAKE_INSTALL_INCLUDEDIR=${FLATBUFFERS_INSTALL_INCLUDEDIR} @@ -26,15 +39,3 @@ if (THIRD_PARTY) -DFLATBUFFERS_BUILD_TESTS=OFF ) endif (THIRD_PARTY) - -set(FLATBUFFERS_INCLUDE_DIR ${FLATBUFFERS_INSTALL_PREFIX}/${FLATBUFFERS_INSTALL_INCLUDEDIR}) -set(FLATBUFFERS_LIBRARY_DIR ${FLATBUFFERS_INSTALL_PREFIX}/${FLATBUFFERS_INSTALL_LIBDIR}) -set(FLATBUFFERS_BINARY_DIR ${FLATBUFFERS_INSTALL_PREFIX}/${FLATBUFFERS_INSTALL_BINDIR}) - -set(FLATC_EXECUTABLE_NAME flatc) -set(FLATBUFFERS_FLATC_EXECUTABLE ${FLATBUFFERS_BINARY_DIR}/${FLATC_EXECUTABLE_NAME}) - -set(FLATBUFFERS_LIBRARY_NAMES libflatbuffers.a) -foreach (LIBRARY_NAME ${FLATBUFFERS_LIBRARY_NAMES}) - list(APPEND FLATBUFFERS_STATIC_LIBRARIES ${FLATBUFFERS_LIBRARY_DIR}/${LIBRARY_NAME}) -endforeach () diff --git a/cmake/third_party/gflags.cmake b/cmake/third_party/gflags.cmake index ceaba978061..c2f94a67917 100644 --- a/cmake/third_party/gflags.cmake +++ b/cmake/third_party/gflags.cmake @@ -36,6 +36,7 @@ ExternalProject_Add(gflags URL_MD5 9677cc51d63642ba3d5f2a57a1fa2bd0 UPDATE_COMMAND bash -c "rm -f BUILD || true" BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS ${GFLAGS_STATIC_LIBRARIES} CMAKE_CACHE_ARGS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_CXX_FLAGS_DEBUG:STRING=${CMAKE_CXX_FLAGS_DEBUG} diff --git a/cmake/third_party/glog.cmake b/cmake/third_party/glog.cmake index ed594b14a77..c487bdadc73 100644 --- a/cmake/third_party/glog.cmake +++ b/cmake/third_party/glog.cmake @@ -45,6 +45,7 @@ ExternalProject_Add(glog URL_MD5 3ca928ef755c0a890680e023e3d4b9a6 UPDATE_COMMAND "" BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS ${GLOG_STATIC_LIBRARIES} CMAKE_CACHE_ARGS -DCMAKE_POLICY_DEFAULT_CMP0074:STRING=NEW -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} diff --git a/cmake/third_party/googletest.cmake b/cmake/third_party/googletest.cmake index 8d3c25984c9..71021de066b 100644 --- a/cmake/third_party/googletest.cmake +++ b/cmake/third_party/googletest.cmake @@ -48,6 +48,7 @@ ExternalProject_Add(googletest UPDATE_COMMAND "" BUILD_IN_SOURCE 1 INSTALL_COMMAND "" + BUILD_BYPRODUCTS ${GOOGLETEST_STATIC_LIBRARIES} ${GOOGLEMOCK_STATIC_LIBRARIES} CMAKE_CACHE_ARGS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_CXX_FLAGS_DEBUG:STRING=${CMAKE_CXX_FLAGS_DEBUG} diff --git a/cmake/third_party/grpc.cmake b/cmake/third_party/grpc.cmake index d09cdad1292..878da486a0b 100644 --- a/cmake/third_party/grpc.cmake +++ b/cmake/third_party/grpc.cmake @@ -42,6 +42,7 @@ ExternalProject_Add(grpc URL_HASH MD5=${GRPC_URL_HASH} UPDATE_COMMAND "" BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS ${GRPC_STATIC_LIBRARIES} BUILD_COMMAND ${CMAKE_COMMAND} --build . -j ${PROC_NUM} --target grpc && ${CMAKE_COMMAND} --build . -j ${PROC_NUM} --target grpc_unsecure && ${CMAKE_COMMAND} --build . -j ${PROC_NUM} --target grpc++_unsecure CMAKE_CACHE_ARGS -DCMAKE_POLICY_DEFAULT_CMP0074:STRING=NEW diff --git a/cmake/third_party/libjpeg-turbo.cmake b/cmake/third_party/libjpeg-turbo.cmake index 4b9464b8c81..f5abcae6f51 100644 --- a/cmake/third_party/libjpeg-turbo.cmake +++ b/cmake/third_party/libjpeg-turbo.cmake @@ -59,6 +59,7 @@ ExternalProject_Add(libjpeg-turbo UPDATE_COMMAND "" INSTALL_COMMAND "" BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS ${LIBJPEG_STATIC_LIBRARIES} CMAKE_CACHE_ARGS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_CXX_FLAGS_DEBUG:STRING=${CMAKE_CXX_FLAGS_DEBUG} diff --git a/cmake/third_party/lz4.cmake b/cmake/third_party/lz4.cmake index e01bc57c557..48a4f8fbb3e 100644 --- a/cmake/third_party/lz4.cmake +++ b/cmake/third_party/lz4.cmake @@ -39,6 +39,7 @@ ExternalProject_Add(lz4 BUILD_IN_SOURCE 1 BUILD_COMMAND make -j ${PROC_NUM} lib CFLAGS=${LZ4_CFLAGS} INSTALL_COMMAND "" + BUILD_BYPRODUCTS ${LZ4_STATIC_LIBRARIES} ) add_custom_target(lz4_create_header_dir diff --git a/cmake/third_party/nccl.cmake b/cmake/third_party/nccl.cmake index 92d56df508b..403bb956886 100644 --- a/cmake/third_party/nccl.cmake +++ b/cmake/third_party/nccl.cmake @@ -39,6 +39,7 @@ ExternalProject_Add(nccl BUILD_IN_SOURCE 1 BUILD_COMMAND make -j${PROC_NUM} src.build CUDA_HOME=${CUDA_TOOLKIT_ROOT_DIR} INSTALL_COMMAND make src.install PREFIX=${NCCL_INSTALL_DIR} + BUILD_BYPRODUCTS ${NCCL_LIBRARIES} ) endif(THIRD_PARTY) diff --git a/cmake/third_party/opencv.cmake b/cmake/third_party/opencv.cmake index fe924df85c9..cd850e62d36 100644 --- a/cmake/third_party/opencv.cmake +++ b/cmake/third_party/opencv.cmake @@ -43,6 +43,7 @@ ExternalProject_Add(opencv BUILD_IN_SOURCE 0 SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/opencv/src/opencv BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/opencv/src/opencv/build + BUILD_BYPRODUCTS ${OPENCV_STATIC_LIBRARIES} CMAKE_CACHE_ARGS -DCMAKE_POLICY_DEFAULT_CMP0074:STRING=NEW -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} diff --git a/cmake/third_party/openssl.cmake b/cmake/third_party/openssl.cmake index 666c1cd97fb..a3f85f2b479 100644 --- a/cmake/third_party/openssl.cmake +++ b/cmake/third_party/openssl.cmake @@ -35,9 +35,9 @@ ExternalProject_Add(openssl URL_HASH MD5=${OPENSSL_URL_HASH} UPDATE_COMMAND "" CONFIGURE_COMMAND ${OPENSSL_SOURCE_DIR}/src/openssl/config --prefix=${OPENSSL_INSTALL} + BUILD_BYPRODUCTS ${OPENSSL_STATIC_LIBRARIES} BUILD_COMMAND make -j${PROC_NUM} INSTALL_COMMAND make install_sw ) endif(THIRD_PARTY) - diff --git a/cmake/third_party/protobuf.cmake b/cmake/third_party/protobuf.cmake index e5c2d60e2fd..df76cfdcd43 100644 --- a/cmake/third_party/protobuf.cmake +++ b/cmake/third_party/protobuf.cmake @@ -52,6 +52,7 @@ ExternalProject_Add(protobuf BUILD_IN_SOURCE 1 SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf SOURCE_SUBDIR cmake + BUILD_BYPRODUCTS ${PROTOBUF_STATIC_LIBRARIES} CMAKE_CACHE_ARGS -DCMAKE_POLICY_DEFAULT_CMP0074:STRING=NEW -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} diff --git a/cmake/third_party/re2.cmake b/cmake/third_party/re2.cmake index 5ba0043ef7e..ee0e41e02fd 100644 --- a/cmake/third_party/re2.cmake +++ b/cmake/third_party/re2.cmake @@ -16,6 +16,7 @@ if (THIRD_PARTY) URL ${RE2_URL} URL_MD5 3b2e20c1edd1cfe887aeef3b0747eac0 UPDATE_COMMAND "" + BUILD_BYPRODUCTS ${RE2_LIBRARIES} CMAKE_ARGS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS:BOOL=OFF diff --git a/cmake/third_party/zlib.cmake b/cmake/third_party/zlib.cmake index cb4fc54e2f8..0542dc31a48 100644 --- a/cmake/third_party/zlib.cmake +++ b/cmake/third_party/zlib.cmake @@ -39,6 +39,7 @@ ExternalProject_Add(zlib URL_MD5 1eabf2698dc49f925ce0ffb81397098f UPDATE_COMMAND "" BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS ${ZLIB_STATIC_LIBRARIES} CMAKE_CACHE_ARGS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS:BOOL=${BUILD_SHARED_LIBS} From 72414a9af66541c53cdbec231625efc3b9441655 Mon Sep 17 00:00:00 2001 From: tsai Date: Mon, 28 Jun 2021 16:19:09 +0800 Subject: [PATCH 2/8] build in one step if it is ninja --- .github/workflows/simple.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/simple.yml b/.github/workflows/simple.yml index 5caa1796ec3..61869a9dec7 100644 --- a/.github/workflows/simple.yml +++ b/.github/workflows/simple.yml @@ -81,6 +81,7 @@ jobs: echo "cmake_flags=${cmake_flags}" >> $GITHUB_ENV shell: bash - name: Build (third party) + if: matrix.cmake_generator != "Ninja" run: | set -x mkdir -p build-third_party @@ -90,6 +91,7 @@ jobs: cmake --build . -j $(nproc) shell: bash - name: Build (of_ccobj) + if: matrix.cmake_generator != "Ninja" run: | mkdir -p build cd build @@ -97,7 +99,7 @@ jobs: cmake --build . -j ${{ matrix.make_concurrency }} --target of_ccobj shell: bash - name: Build (oneflow_internal) - if: always() + if: always() && matrix.cmake_generator != "Ninja" run: | mkdir -p build cd build @@ -105,7 +107,7 @@ jobs: cmake --build . -j ${{ matrix.make_concurrency }} --target oneflow_internal shell: bash - name: Build (oneflow_py) - if: always() + if: always() && matrix.cmake_generator != "Ninja" run: | mkdir -p build cd build @@ -113,7 +115,7 @@ jobs: cmake --build . -j ${{ matrix.make_concurrency }} --target oneflow_py shell: bash - name: Build (oneflow_testexe) - if: always() + if: always() && matrix.cmake_generator != "Ninja" run: | mkdir -p build cd build From eb09c981ca5c0ee10d8545f2485304d200ae5cbd Mon Sep 17 00:00:00 2001 From: tsai Date: Mon, 28 Jun 2021 16:22:18 +0800 Subject: [PATCH 3/8] fix yml syntax --- .github/workflows/simple.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/simple.yml b/.github/workflows/simple.yml index 61869a9dec7..049fb113e70 100644 --- a/.github/workflows/simple.yml +++ b/.github/workflows/simple.yml @@ -81,7 +81,7 @@ jobs: echo "cmake_flags=${cmake_flags}" >> $GITHUB_ENV shell: bash - name: Build (third party) - if: matrix.cmake_generator != "Ninja" + if: matrix.cmake_generator != 'Ninja' run: | set -x mkdir -p build-third_party @@ -91,7 +91,7 @@ jobs: cmake --build . -j $(nproc) shell: bash - name: Build (of_ccobj) - if: matrix.cmake_generator != "Ninja" + if: matrix.cmake_generator != 'Ninja' run: | mkdir -p build cd build @@ -99,7 +99,7 @@ jobs: cmake --build . -j ${{ matrix.make_concurrency }} --target of_ccobj shell: bash - name: Build (oneflow_internal) - if: always() && matrix.cmake_generator != "Ninja" + if: always() && matrix.cmake_generator != 'Ninja' run: | mkdir -p build cd build @@ -107,7 +107,7 @@ jobs: cmake --build . -j ${{ matrix.make_concurrency }} --target oneflow_internal shell: bash - name: Build (oneflow_py) - if: always() && matrix.cmake_generator != "Ninja" + if: always() && matrix.cmake_generator != 'Ninja' run: | mkdir -p build cd build @@ -115,7 +115,7 @@ jobs: cmake --build . -j ${{ matrix.make_concurrency }} --target oneflow_py shell: bash - name: Build (oneflow_testexe) - if: always() && matrix.cmake_generator != "Ninja" + if: always() && matrix.cmake_generator != 'Ninja' run: | mkdir -p build cd build From fb735c083d45ab93f7b252d0852539e4967ce7b7 Mon Sep 17 00:00:00 2001 From: tsai Date: Mon, 28 Jun 2021 16:36:31 +0800 Subject: [PATCH 4/8] limit max-parallel --- .github/workflows/simple.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/simple.yml b/.github/workflows/simple.yml index 049fb113e70..1da71420870 100644 --- a/.github/workflows/simple.yml +++ b/.github/workflows/simple.yml @@ -26,6 +26,7 @@ jobs: needs: [cancel_previous] strategy: fail-fast: false + max-parallel: 8 matrix: test_suite: ["mac", "ubuntu"] cmake_generator: ["Ninja", "Unix Makefiles"] From cc6fafdbaeb89b69158c2a7e4e002b6c87fef0e9 Mon Sep 17 00:00:00 2001 From: tsai Date: Mon, 28 Jun 2021 16:37:39 +0800 Subject: [PATCH 5/8] limit max-parallel --- .github/workflows/simple.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/simple.yml b/.github/workflows/simple.yml index 1da71420870..0661d53a92f 100644 --- a/.github/workflows/simple.yml +++ b/.github/workflows/simple.yml @@ -26,7 +26,7 @@ jobs: needs: [cancel_previous] strategy: fail-fast: false - max-parallel: 8 + max-parallel: 6 matrix: test_suite: ["mac", "ubuntu"] cmake_generator: ["Ninja", "Unix Makefiles"] From 65479f0f57fd8e5c37949b88710e32f3079f13a1 Mon Sep 17 00:00:00 2001 From: tsai Date: Mon, 28 Jun 2021 21:28:34 +0800 Subject: [PATCH 6/8] rm useless --- .github/workflows/simple.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/simple.yml b/.github/workflows/simple.yml index 0661d53a92f..2518af6a9a7 100644 --- a/.github/workflows/simple.yml +++ b/.github/workflows/simple.yml @@ -128,7 +128,7 @@ jobs: run: | mkdir -p build cd build - cmake .. ${{ env.cmake_flags }} -DTHIRD_PARTY=OFF -DONEFLOW=ON + cmake .. ${{ env.cmake_flags }} cmake --build . -j ${{ matrix.make_concurrency }} shell: bash - name: Exe test From 2f5f11848487cf2bc1309749ddca670cb49713b6 Mon Sep 17 00:00:00 2001 From: tsai Date: Tue, 29 Jun 2021 00:26:46 +0800 Subject: [PATCH 7/8] refine --- .github/workflows/simple.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/simple.yml b/.github/workflows/simple.yml index 2518af6a9a7..aadad18a2e1 100644 --- a/.github/workflows/simple.yml +++ b/.github/workflows/simple.yml @@ -79,6 +79,8 @@ jobs: cmake_flags+=" -G '${{ matrix.cmake_generator }}'" cmake_flags+=" -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }}" cmake_flags+=" -DBUILD_SHARED_LIBS=${{ matrix.build_shared_libs }}" + cmake_flags+=" -DCMAKE_MACOSX_RPATH=FALSE" + cmake_flags+=" -DCMAKE_BUILD_WITH_INSTALL_RPATH=FALSE" echo "cmake_flags=${cmake_flags}" >> $GITHUB_ENV shell: bash - name: Build (third party) From 64017e9583144af9b1a02c8d1b0a29471c13557e Mon Sep 17 00:00:00 2001 From: tsai Date: Tue, 29 Jun 2021 20:34:52 +0800 Subject: [PATCH 8/8] add --shm-size=8g --- docker/package/manylinux/build_wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/package/manylinux/build_wheel.py b/docker/package/manylinux/build_wheel.py index 503fdd332ba..eaf1b2a8b7d 100644 --- a/docker/package/manylinux/build_wheel.py +++ b/docker/package/manylinux/build_wheel.py @@ -148,7 +148,7 @@ def get_common_docker_args( house_dir_arg = f"-v {house_dir}:{house_dir}" build_dir_arg = get_build_dir_arg(cache_dir, oneflow_src_dir) proxy_env_arg = get_proxy_env_args() if use_system_proxy else "" - return f"-v {oneflow_src_dir}:{oneflow_src_dir} {proxy_env_arg} {pwd_arg} {house_dir_arg} {cache_dir_arg} {build_dir_arg} -w {current_dir}" + return f"-v {oneflow_src_dir}:{oneflow_src_dir} {proxy_env_arg} {pwd_arg} {house_dir_arg} {cache_dir_arg} {build_dir_arg} -w {current_dir} --shm-size=8g" def build_third_party(