Fix macOS support and run macOS build in Simple CI (#4947)

* update cmake * check in fix * add default for apple * skip and generate_api for in mac ci * fix yml * fix has_grad template * update flatbuffer * fix align * revert * revert * refine * add FLATBUFFERS_BUILD_TESTS=OFF * revert * refine * add note * Update flatbuffers.cmake * refine * rm log * FLATBUFFERS_BUILD_TESTS=OFF * Update flatbuffers.cmake * restore * refine * fix ndarry std::min * restore * refine * update test config * run doctest in cpu * fix vm::TestResourceDescScope * dont run doctest because it seems to involve more * add simple ci * refine * refine * add ubuntu * refine * refine * add todo * refine * refine cache * build oneflow if linux * refine * refine * fix cache order * refine cache key * refine * rm skip * add instruction on readme * refine * refine * revert * refine * refine * refine * Update CMakeLists.txt * add * Fix protobuf debug postfix * refine * fix * refine * refine * refine * refine * add config in matrix * refine * refine * refine * fix * add badge * refine * refine * refine * opencv diable intel ITT * rm RelWithDebInfo * refine * refine * rm intel ITT * dont use realpath to prevent failure * refine * fix * don't build Ninja with Release to reduce product * revert * refine order * refine order * refine order * refine order * less deps * refine * rm mkl note * refine * refine * link py libs if building shared lib on mac * refine * add handy name * refine * refine * refine * refine * refine * refine * quick fix * refine * refine * refine * refine Co-authored-by: strint <xiaoyulink@gmail.com> Co-authored-by: oneflow-ci-bot <69100618+oneflow-ci-bot@users.noreply.github.com> Co-authored-by: liujuncheng <liujuncheng1022@gmail.com>
Oneflow-Inc · Jun 25, 2021 · 695dd7e · 695dd7e
1 parent e47a831
commit 695dd7e
Show file tree

Hide file tree

Showing 22 changed files with 93 additions and 36 deletions.
diff --git a/.github/workflows/simple.yml b/.github/workflows/simple.yml
@@ -81,7 +81,6 @@ jobs:
           echo "cmake_flags=${cmake_flags}" >> $GITHUB_ENV
         shell: bash
       - name: Build (third party)
-        if: startsWith(runner.os, 'Linux')
         run: |
           set -x
           mkdir -p build-third_party
@@ -91,46 +90,55 @@ jobs:
           cmake --build . -j $(nproc)
         shell: bash
       - name: Build (of_ccobj)
-        if: startsWith(runner.os, 'Linux')
         run: |
           mkdir -p build
           cd build
           cmake .. ${{ env.cmake_flags }} -DTHIRD_PARTY=OFF -DONEFLOW=ON -DTHIRD_PARTY_DIR=$PWD/../third_party_install
           cmake --build . -j ${{ matrix.make_concurrency }} --target of_ccobj
         shell: bash
       - name: Build (oneflow_internal)
-        if: startsWith(runner.os, 'Linux')
+        if: always()
         run: |
           mkdir -p build
           cd build
           cmake .. ${{ env.cmake_flags }} -DTHIRD_PARTY=OFF -DONEFLOW=ON
           cmake --build . -j ${{ matrix.make_concurrency }} --target oneflow_internal
         shell: bash
       - name: Build (oneflow_py)
-        if: startsWith(runner.os, 'Linux')
+        if: always()
         run: |
           mkdir -p build
           cd build
           cmake .. ${{ env.cmake_flags }} -DTHIRD_PARTY=OFF -DONEFLOW=ON
           cmake --build . -j ${{ matrix.make_concurrency }} --target oneflow_py
         shell: bash
+      - name: Build (oneflow_testexe)
+        if: always()
+        run: |
+          mkdir -p build
+          cd build
+          cmake .. ${{ env.cmake_flags }} -DTHIRD_PARTY=OFF -DONEFLOW=ON
+          cmake --build . -j ${{ matrix.make_concurrency }} --target oneflow_testexe
+        shell: bash
       - name: Build (ALL)
-        if: startsWith(runner.os, 'Linux')
+        if: always()
         run: |
           mkdir -p build
           cd build
           cmake .. ${{ env.cmake_flags }} -DTHIRD_PARTY=OFF -DONEFLOW=ON
           cmake --build . -j ${{ matrix.make_concurrency }}
         shell: bash
       - name: Exe test
-        if: startsWith(runner.os, 'Linux')
+        if: always()
+        continue-on-error: true
         run: |
           mkdir -p build
           cd build
           ./bin/oneflow_testexe
         shell: bash
       - name: Op test
-        if: startsWith(runner.os, 'Linux')
+        if: always()
+        continue-on-error: true
         run: |
           source build/source.sh
           ONEFLOW_TEST_GITHUB_HOSTED=1 ONEFLOW_TEST_CPU_ONLY=1 bash ci/test/1node_op_test.sh

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -22,13 +22,20 @@ option(BUILD_CUDA "" ON)
 option(BUILD_TESTING "" OFF)
 option(WITH_XLA "Option to build with XLA" OFF)
 option(WITH_TENSORRT "Option to build with TensorRT" OFF)
+option(WITH_COCOAPI "Option to build with COCO API" ON)
 option(BUILD_GIT_VERSION "" ON)
 option(BUILD_PROFILER "" OFF)
 option(OF_SOFTMAX_USE_FAST_MATH "" ON)
 set(RPC_BACKEND "GRPC,LOCAL" CACHE STRING "")
 set(THIRD_PARTY_MIRROR "" CACHE STRING "")
 set(PIP_INDEX_MIRROR "" CACHE STRING "")
 
+if (APPLE)
+  set(RPC_BACKEND "LOCAL")
+  set(BUILD_CUDA OFF)
+  set(WITH_COCOAPI OFF)
+endif()
+
 if (CMAKE_BUILD_TYPE MATCHES Debug)
   set(CUDNN_STATIC OFF CACHE BOOL "")
 else ()
@@ -68,6 +75,9 @@ endif()
 if (WITH_TENSORRT)
   add_definitions(-DWITH_TENSORRT)
 endif()
+if (WITH_COCOAPI)
+  add_definitions(-DWITH_COCOAPI)
+endif()
 if (USE_CXX11_ABI)
   add_definitions(-D_GLIBCXX_USE_CXX11_ABI=1)
 else()
@@ -178,7 +188,7 @@ if (THIRD_PARTY)
 else()
   add_custom_target(prepare_oneflow_third_party ALL)
 endif()
-
+add_custom_target(oneflow_deps ALL DEPENDS prepare_oneflow_third_party)
 # skip oneflow cmake to avoid errors caused by the absences of python-dev, proto src
 if (ONEFLOW)
   include(oneflow)

diff --git a/README.md b/README.md
@@ -104,11 +104,15 @@
   - For more useful flags, plese run the script with flag `--help` or refer to the source code of the script.
 
 - #### Option 3: Build on bare metal
-  - Install dependencies. For instance, on Ubuntu 20.04, run:
-    ```
-    sudo apt install -y libmkl-full-dev nasm libc++-11-dev libncurses5 g++ gcc cmake gdb python3-pip
-    ```
-    If there is a prompt, it is recommended to select the option to make mkl the default BLAS library.
+  - Install dependencies
+    - on Ubuntu 20.04, run:
+      ```
+      sudo apt install -y libopenblas-dev nasm g++ gcc python3-pip cmake
+      ```
+    - on macOS, run:
+      ```
+      brew install nasm
+      ```
   - In the root directory of OneFlow source code, run:
 
     ```

diff --git a/ci/test/1node_op_test.sh b/ci/test/1node_op_test.sh
@@ -20,7 +20,7 @@ for CHUNK in 1
 do
 	export ONEFLOW_TEST_DEVICE_NUM=${CHUNK}
     python3 $src_dir/ci/test/parallel_run.py \
-        --gpu_num=${gpu_num} \
+        --gpu_num="${gpu_num}" \
         --dir=test/ops \
         --timeout=1 \
         --verbose \

diff --git a/ci/test/doctest.sh b/ci/test/doctest.sh
@@ -2,14 +2,16 @@
 set -xe
 export PYTHONUNBUFFERED=1
 src_dir=${ONEFLOW_SRC_DIR:-"$PWD"}
+test_tmp_dir=${ONEFLOW_TEST_TMP_DIR:-"./test_tmp_dir"}
 
-cd `python3 -c "import oneflow; print(oneflow.__path__[0])"`
-cd ..
+mkdir -p ${test_tmp_dir}
+cd ${test_tmp_dir}
+python3 -c 'import oneflow; f=open("oneflow_path.txt", "w"); f.write(oneflow.__path__[0])'
 
 gpu_num=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)
 python3 $src_dir/ci/test/parallel_run.py \
     --gpu_num=${gpu_num} \
-    --dir=${PWD} \
+    --dir=$(cat oneflow_path.txt) \
     --timeout=1 \
     --verbose \
     --chunk=1 \

diff --git a/cmake/oneflow.cmake b/cmake/oneflow.cmake
@@ -251,16 +251,16 @@ if (USE_CLANG_FORMAT)
 endif()
 
 if (BUILD_SHARED_LIBS)
-  get_filename_component(GLOG_RPATH "${GLOG_STATIC_LIBRARIES}" DIRECTORY)
-  get_filename_component(PB_RPATH "${PROTOBUF_LIBRARY_DIR}" DIRECTORY)
   target_link_libraries(of_ccobj of_protoobj of_cfgobj ${ONEFLOW_CUDA_LIBS} glog_imported)
-  set_target_properties(of_ccobj PROPERTIES INSTALL_RPATH "${GLOG_RPATH} ${PB_RPATH}")
 endif()
 
 # py ext lib
 add_library(of_pyext_obj ${of_pyext_obj_cc})
 target_include_directories(of_pyext_obj PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS})
 target_link_libraries(of_pyext_obj of_ccobj)
+if(BUILD_SHARED_LIBS AND APPLE)
+  target_link_libraries(of_pyext_obj ${Python3_LIBRARIES})
+endif()
 add_dependencies(of_pyext_obj of_ccobj)
 
 if(APPLE)

diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake
@@ -11,7 +11,9 @@ include(glog)
 include(libjpeg-turbo)
 include(opencv)
 include(eigen)
-include(cocoapi)
+if (WITH_COCOAPI)
+  include(cocoapi)
+endif()
 include(half)
 include(re2)
 include(json)
@@ -158,8 +160,6 @@ set(oneflow_third_party_dependencies
   libpng_copy_headers_to_destination
   opencv_copy_libs_to_destination
   eigen
-  cocoapi_copy_headers_to_destination
-  cocoapi_copy_libs_to_destination
   half_copy_headers_to_destination
   re2
   json_copy_headers_to_destination
@@ -168,6 +168,11 @@ set(oneflow_third_party_dependencies
   lz4_copy_headers_to_destination
 )
 
+if (WITH_COCOAPI)
+  list(APPEND oneflow_third_party_dependencies cocoapi_copy_headers_to_destination)
+  list(APPEND oneflow_third_party_dependencies cocoapi_copy_libs_to_destination)
+endif()
+
 if (RPC_BACKEND MATCHES "GRPC")
   list(APPEND oneflow_third_party_dependencies grpc)
 endif()

diff --git a/cmake/third_party/opencv.cmake b/cmake/third_party/opencv.cmake
@@ -18,9 +18,6 @@ else()
     set(OPENCV_BUILD_3RDPARTY_LIBRARY_DIR ${OPENCV_INSTALL_DIR}/share/OpenCV/3rdparty/${CMAKE_INSTALL_LIBDIR})
     set(OPENCV_LIBRARY_NAMES libopencv_imgproc.a libopencv_imgcodecs.a libopencv_core.a)
     set(OPENCV_3RDPARTY_LIBRARY_NAMES libIlmImf.a liblibjasper.a liblibpng.a liblibtiff.a liblibwebp.a)
-    if (NOT APPLE)
-      set(OPENCV_3RDPARTY_LIBRARY_NAMES ${OPENCV_3RDPARTY_LIBRARY_NAMES} libittnotify.a)
-    endif()
 endif()
 
 foreach(LIBRARY_NAME ${OPENCV_LIBRARY_NAMES})
@@ -110,7 +107,7 @@ ExternalProject_Add(opencv
         -DBUILD_OPENEXR:BOOL=ON
         -DBUILD_TBB:BOOL=ON
         -DBUILD_IPP_IW:BOOL=OFF
-        -DWITH_ITT:BOOL=ON
+        -DWITH_ITT:BOOL=OFF
         -DBUILD_opencv_flann:BOOL=OFF
         -DBUILD_opencv_ml:BOOL=OFF
         -DBUILD_opencv_objdetect:BOOL=OFF

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -5,5 +5,6 @@ wheel
 tqdm
 requests
 jinja2
-opencv-python==4.2.0.34; sys_platform != 'darwin'
+opencv-python; python_version >= "3.9" and sys_platform != 'darwin'
+opencv-python==4.2.0.34; python_version < '3.9' and sys_platform != 'darwin'
 PyYAML
diff --git a/oneflow/core/eager/transport_blob_instruction_type.cpp b/oneflow/core/eager/transport_blob_instruction_type.cpp
@@ -13,6 +13,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+#ifdef __linux__
+
 #include "oneflow/core/eager/transport_blob_instruction_type.h"
 #include "oneflow/core/vm/instruction_operand.msg.h"
 #include "oneflow/core/vm/object_wrapper.h"
@@ -179,3 +181,5 @@ COMMAND(vm::RegisterInstructionType<ReceiveBlobInstructionType>("ReceiveBlob"));
 
 }  // namespace vm
 }  // namespace oneflow
+
+#endif  // __linux__
diff --git a/oneflow/core/eager/transport_blob_instruction_type.h b/oneflow/core/eager/transport_blob_instruction_type.h
@@ -13,6 +13,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+#ifdef __linux__
+
 #ifndef ONEFLOW_CORE_EAGER_SEND_BLOB_INSTRUCTION_TYPE_H_
 #define ONEFLOW_CORE_EAGER_SEND_BLOB_INSTRUCTION_TYPE_H_
 
@@ -69,3 +71,5 @@ class ReceiveBlobInstructionType : public vm::InstructionType {
 }  // namespace oneflow
 
 #endif  // ONEFLOW_CORE_EAGER_SEND_BLOB_INSTRUCTION_TYPE_H_
+
+#endif  // __linux__
diff --git a/oneflow/core/eager/transport_blob_instruction_type_test.cpp b/oneflow/core/eager/transport_blob_instruction_type_test.cpp
@@ -68,6 +68,7 @@ struct ReceiveRequest {
 HashMap<uint64_t, SendRequest> token2send_request;
 HashMap<uint64_t, ReceiveRequest> token2recv_request;
 
+#ifdef __linux__
 class TestSendBlobInstructionType : public SendBlobInstructionType {
  public:
   TestSendBlobInstructionType() = default;
@@ -125,6 +126,7 @@ class TestReceiveBlobInstructionType : public ReceiveBlobInstructionType {
   }
 };
 COMMAND(vm::RegisterInstructionType<TestReceiveBlobInstructionType>("TestReceiveBlob"));
+#endif  // __linux__
 
 using InstructionMsgList = OBJECT_MSG_LIST(vm::InstructionMsg, instr_msg_link);
 
@@ -258,6 +260,7 @@ void DestroyNumProcessPerNode() { Global<NumProcessPerNode>::Delete(); }
 
 }  // namespace
 
+#ifdef __linux__
 TEST(SendReceiveInstructionType, naive) {
   InitNumProcessPerNode();
 #ifdef WITH_CUDA
@@ -306,6 +309,7 @@ TEST(SendReceiveInstructionType, naive) {
   ASSERT_TRUE(token2recv_request.find(body_token) != token2recv_request.end());
   DestroyNumProcessPerNode();
 }
+#endif  // __linux__
 
 }  // namespace test
 }  // namespace vm

diff --git a/oneflow/core/job/env_global_objects_scope.cpp b/oneflow/core/job/env_global_objects_scope.cpp
@@ -143,16 +143,20 @@ Maybe<void> EnvGlobalObjectsScope::Init(const EnvProto& env_proto) {
   Global<vm::VirtualMachineScope>::New(Global<ResourceDesc, ForSession>::Get()->resource());
   Global<EagerJobBuildAndInferCtxMgr>::New();
   if (!Global<ResourceDesc, ForSession>::Get()->enable_dry_run()) {
+#ifdef __linux__
     Global<EpollCommNet>::New();
     Global<Transport>::New();
+#endif  // __linux__
   }
   return Maybe<void>::Ok();
 }
 
 EnvGlobalObjectsScope::~EnvGlobalObjectsScope() {
   if (!Global<ResourceDesc, ForSession>::Get()->enable_dry_run()) {
+#ifdef __linux__
     Global<Transport>::Delete();
     Global<EpollCommNet>::Delete();
+#endif  // __linux__
   }
   Global<EagerJobBuildAndInferCtxMgr>::Delete();
   Global<vm::VirtualMachineScope>::Delete();

diff --git a/oneflow/core/ndarray/cpu_concat_var_ndarray.h b/oneflow/core/ndarray/cpu_concat_var_ndarray.h
@@ -62,7 +62,8 @@ class CpuConcatVarNdarray : public CpuNdarray<T, NDIMS> {
     size_t var_contiguous_size = 0;
     var_ndarray(var_index).GetMutPtrAndContiguousSize(var_offset, ptr, &var_contiguous_size);
     *size = std::min(var_contiguous_size,
-                     contiguous_lens_[var_index] - var_offset % contiguous_lens_[var_index]);
+                     static_cast<size_t>(contiguous_lens_[var_index]
+                                         - var_offset % contiguous_lens_[var_index]));
   }
 
  private:

diff --git a/oneflow/core/platform/include/wrapper.h b/oneflow/core/platform/include/wrapper.h
@@ -29,7 +29,9 @@ class DynamicLibrary {
 
   static std::unique_ptr<DynamicLibrary> Load(const std::vector<std::string>& names);
   void* LoadSym(const char* name);
+#ifdef __linux__
   std::string AbsolutePath();
+#endif  // __linux__
 
  private:
   DynamicLibrary(void* handle) : handle_(handle){};

diff --git a/oneflow/core/platform/lib/wrapper.cpp b/oneflow/core/platform/lib/wrapper.cpp
@@ -15,7 +15,10 @@ limitations under the License.
 */
 #include "oneflow/core/platform/include/wrapper.h"
 #include <dlfcn.h>
+
+#ifdef __linux__
 #include <link.h>
+#endif  // __linux__
 
 namespace oneflow {
 namespace platform {
@@ -41,7 +44,9 @@ std::unique_ptr<DynamicLibrary> DynamicLibrary::Load(const std::vector<std::stri
     void* handle = dlopen(name.c_str(), RTLD_LOCAL | RTLD_NOW);
     if (handle != nullptr) {
       DynamicLibrary* lib = new DynamicLibrary(handle);
-      std::cerr << "loaded library: " << lib->AbsolutePath() << "\n";
+#ifdef __linux__
+      std::cout << "loaded library: " << lib->AbsolutePath() << "\n";
+#endif  // __linux__
       return std::unique_ptr<DynamicLibrary>(lib);
     }
   }
@@ -50,11 +55,13 @@ std::unique_ptr<DynamicLibrary> DynamicLibrary::Load(const std::vector<std::stri
 
 void* DynamicLibrary::LoadSym(const char* name) { return OpenSymbol(handle_, name); }
 
+#ifdef __linux__
 std::string DynamicLibrary::AbsolutePath() {
   struct link_map* map;
   dlinfo(handle_, RTLD_DI_LINKMAP, &map);
   return map->l_name;
 }
+#endif  // __linux__
 
 DynamicLibrary::~DynamicLibrary() { dlclose(handle_); }
 

diff --git a/oneflow/core/transport/transport.h b/oneflow/core/transport/transport.h
@@ -13,6 +13,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+#ifdef __linux__
+
 #ifndef ONEFLOW_CORE_TRANSPORT_TRANSPORT_H_
 #define ONEFLOW_CORE_TRANSPORT_TRANSPORT_H_
 
@@ -137,3 +139,5 @@ class Transport {
 }  // namespace oneflow
 
 #endif  // ONEFLOW_CORE_TRANSPORT_TRANSPORT_H_
+
+#endif  // __linux__