pybind11 integration (#3517)

* naive integration

* fix order

* make cxx flags global

* check module load

* fmt

* rm example code

* always add pybind11 in cmake

* mv cxx flags up

* rename internal2 =>  oneflow api

* rm old deprecated code

* update setup.py

* add of_pybind_obj_cc

* mv pybind within cpp rule

* check pathname

* rm unused include header

Co-authored-by: tsai <caishenghang@oneflow.org>

CMakeLists.txt

@@ -63,6 +63,34 @@ list(APPEND CMAKE_MODULE_PATH ${oneflow_cmake_dir})
 include(util)
 include(proto2cpp)
 include(swig)
+
+if(WIN32)
+  set(CMAKE_BUILD_TYPE Debug)
+  add_definitions(-DNOMINMAX -D_WIN32_WINNT=0x0A00 -DLANG_CXX11 -DCOMPILER_MSVC -D__VERSION__=\"MSVC\")
+  add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DWIN32_LEAN_AND_MEAN -DNOGDI -DPLATFORM_WINDOWS -D_ITERATOR_DEBUG_LEVEL=0)
+  add_definitions(/bigobj /nologo /EHsc /GF /FC /MP /Gm-)
+  add_definitions(-DGOOGLE_GLOG_DLL_DECL=)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP")
+
+  foreach(flag_var
+      CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
+      CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
+      CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
+    if(${flag_var} MATCHES "/MD")
+      string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
+    endif()
+  endforeach()
+
+  #set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS} /DEBUG:FASTLINK")
+  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /D_ITERATOR_DEBUG_LEVEL=0")
+else()
+  set(EXTRA_CXX_FLAGS "-std=c++11 -Wall -Wno-sign-compare -Wno-unused-function -fPIC")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXX_FLAGS}")
+  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${EXTRA_CXX_FLAGS}")
+  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${EXTRA_CXX_FLAGS}")
+  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${EXTRA_CXX_FLAGS}")
+endif()
+
 include(third_party)
 
 if (BUILD_CUDA)
@@ -100,29 +128,6 @@ endif()
 
 message("-- CMAKE_CXX_COMPILER_VERSION: " ${CMAKE_CXX_COMPILER_VERSION})
 
-if(WIN32)
-  set(CMAKE_BUILD_TYPE Debug)
-  add_definitions(-DNOMINMAX -D_WIN32_WINNT=0x0A00 -DLANG_CXX11 -DCOMPILER_MSVC -D__VERSION__=\"MSVC\")
-  add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DWIN32_LEAN_AND_MEAN -DNOGDI -DPLATFORM_WINDOWS -D_ITERATOR_DEBUG_LEVEL=0)
-  add_definitions(/bigobj /nologo /EHsc /GF /FC /MP /Gm-)
-  add_definitions(-DGOOGLE_GLOG_DLL_DECL=)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP")
-
-  foreach(flag_var
-      CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE
-      CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
-      CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
-    if(${flag_var} MATCHES "/MD")
-      string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
-    endif()
-  endforeach()
-
-  #set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS} /DEBUG:FASTLINK")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /D_ITERATOR_DEBUG_LEVEL=0")
-else()
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wno-sign-compare -Wno-unused-function -fPIC")
-endif()
-
 if (THIRD_PARTY)
   add_custom_target(prepare_oneflow_third_party ALL DEPENDS ${oneflow_third_party_dependencies})
 else()

cmake/oneflow.cmake

@@ -1,3 +1,5 @@
+include(pybind11)
+
 # main cpp
 # TODO(tsai): skip for now, fail to link when building CPU only
 if (BUILD_CUDA)
@@ -125,6 +127,9 @@ foreach(oneflow_single_file ${oneflow_all_src})
     if("${oneflow_single_file}" MATCHES "^${PROJECT_SOURCE_DIR}/oneflow/(core|user|xrt)/.*_test\\.cpp$")
       # test file
       list(APPEND of_all_test_cc ${oneflow_single_file})
+    elseif("${oneflow_single_file}" MATCHES "^${PROJECT_SOURCE_DIR}/oneflow/.*\\.pybind\\.cpp$")
+      list(APPEND of_pybind_obj_cc ${oneflow_single_file})
+      set(group_this ON)
     else()
       # not test file
       list(FIND of_main_cc ${oneflow_single_file} main_found)
@@ -275,10 +280,10 @@ endforeach()
 RELATIVE_SWIG_GENERATE_CPP(SWIG_SRCS SWIG_HDRS
                               ${PROJECT_SOURCE_DIR}
                               ${of_all_rel_swigs})
-oneflow_add_library(oneflow_internal SHARED ${SWIG_SRCS} ${SWIG_HDRS} ${of_main_cc})
+pybind11_add_module(oneflow_internal ${PROJECT_SOURCE_DIR}/oneflow/api/python/init.cpp ${of_pybind_obj_cc} ${SWIG_SRCS} ${SWIG_HDRS} ${of_main_cc})
 set_target_properties(oneflow_internal PROPERTIES PREFIX "_")
 set_target_properties(oneflow_internal PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/python_scripts/oneflow")
-target_link_libraries(oneflow_internal ${of_libs} ${oneflow_third_party_libs})
+target_link_libraries(oneflow_internal PRIVATE ${of_libs} ${oneflow_third_party_libs})
 target_include_directories(oneflow_internal PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS})
 
 set(of_pyscript_dir "${PROJECT_BINARY_DIR}/python_scripts")

cmake/pybind11.cmake

@@ -0,0 +1,11 @@
+include(FetchContent)
+
+FetchContent_Declare(
+    pybind11
+    URL https://github.com/pybind/pybind11/archive/v2.5.0.zip
+)
+FetchContent_GetProperties(pybind11)
+if(NOT pybind11_POPULATED)
+    FetchContent_Populate(pybind11)
+    add_subdirectory(${pybind11_SOURCE_DIR} ${pybind11_BINARY_DIR})
+endif()

cmake/third_party/gflags.cmake

@@ -32,10 +32,6 @@ set (GFLAGS_PUBLIC_H
 
 if (THIRD_PARTY)
 
-set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fPIC")
-set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fPIC")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
-
 ExternalProject_Add(gflags
     PREFIX gflags
     URL ${gflags_URL}
@@ -46,7 +42,8 @@ ExternalProject_Add(gflags
         -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
         -DCMAKE_CXX_FLAGS_DEBUG:STRING=${CMAKE_CXX_FLAGS_DEBUG}
         -DCMAKE_CXX_FLAGS_RELEASE:STRING=${CMAKE_CXX_FLAGS_RELEASE}
-        -DCMAKE_CXX_FLAGS:STRING=-fPIC
+        -DCMAKE_CXX_FLAGS_RELWITHDEBINFO:STRING=${CMAKE_CXX_FLAGS_RELWITHDEBINFO}
+        -DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}
         -DGFLAGS_NAMESPACE:STRING=gflags
 )
 

cmake/third_party/googletest.cmake

@@ -48,8 +48,12 @@ ExternalProject_Add(googletest
     INSTALL_COMMAND ""
     CMAKE_CACHE_ARGS
         -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
+        -DCMAKE_CXX_FLAGS_DEBUG:STRING=${CMAKE_CXX_FLAGS_DEBUG}
+        -DCMAKE_CXX_FLAGS_RELEASE:STRING=${CMAKE_CXX_FLAGS_RELEASE}
+        -DCMAKE_CXX_FLAGS_RELWITHDEBINFO:STRING=${CMAKE_CXX_FLAGS_RELWITHDEBINFO}
+        -DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}
         -DBUILD_GMOCK:BOOL=ON
-	-DBUILD_GTEST:BOOL=OFF  # gmock includes gtest
+	      -DBUILD_GTEST:BOOL=OFF  # gmock includes gtest
         -DCMAKE_CXX_FLAGS_DEBUG:STRING=${CMAKE_CXX_FLAGS_DEBUG}
         #-Dgtest_force_shared_crt:BOOL=ON  #default value is OFF
 )

oneflow/api/python/init.cpp

@@ -0,0 +1,27 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+#include <pybind11/pybind11.h>
+#include "oneflow/core/job/job_build_and_infer_ctx_mgr.h"
+
+namespace py = pybind11;
+
+namespace oneflow {
+
+PYBIND11_MODULE(oneflow_api, m) {
+  m.def("EagerExecutionEnabled", []() { return EagerExecutionEnabled(); });
+}
+
+}  // namespace oneflow

oneflow/init.py

@@ -15,6 +15,36 @@
 """
 # __init__.py, rename to avoid being added to PYTHONPATH
 from __future__ import absolute_import
+
+
+def import_secondary_module(name, path):
+    import importlib.machinery
+    import importlib.util
+
+    loader = importlib.machinery.ExtensionFileLoader(name, path)
+    spec = importlib.util.spec_from_loader(name, loader)
+    module = importlib.util.module_from_spec(spec)
+    loader.exec_module(module)
+    return module
+
+
+def import_oneflow_internal2():
+    import oneflow
+    import os
+    from os.path import dirname
+    import imp
+
+    fp, pathname, description = imp.find_module(
+        "_oneflow_internal", [dirname(__file__)]
+    )
+    assert os.path.isfile(pathname)
+    return import_secondary_module("oneflow_api", pathname)
+
+
+oneflow_api = import_oneflow_internal2()
+del import_secondary_module
+del import_oneflow_internal2
+
 from oneflow.python.version import __version__
 
 from oneflow.core.job.job_set_pb2 import ConfigProto

oneflow/python/framework/c_api_util.py

@@ -31,6 +31,9 @@
 from oneflow.core.framework.config_def_pb2 import ConfigDef
 from oneflow.core.job.inter_user_job_info_pb2 import InterUserJobInfo
 from oneflow.python.framework.job_build_and_infer_error import JobBuildAndInferError
+import oneflow
+
+oneflow_api = oneflow.oneflow_api
 
 
 def RegisterWatcherOnlyOnce(watcher):
@@ -84,7 +87,7 @@ def EnableEagerEnvironment(enable_eager_execution):
 
 
 def EagerExecutionEnabled():
-    return oneflow_internal.EagerExecutionEnabled()
+    return oneflow_api.EagerExecutionEnabled()
 
 
 def IsEnvInited():

oneflow/python/oneflow_internal.h

@@ -49,8 +49,6 @@ void EnableEagerEnvironment(bool enable_eager_execution) {
   *Global<bool, EagerExecution>::Get() = enable_eager_execution;
 }
 
-bool EagerExecutionEnabled() { return oneflow::EagerExecutionEnabled(); }
-
 bool IsEnvInited() {
   using namespace oneflow;
   return Global<EnvGlobalObjectsScope>::Get() != nullptr;

setup.py

@@ -65,23 +65,19 @@ def has_ext_modules(self):
     os.path.relpath(p, "{}/python_scripts/oneflow".format(args.build_dir))
     for p in include_files
 ]
-package_data = {"oneflow": ["_oneflow_internal.so"] + include_files}
-
-if args.with_xla:
-    packages += ["oneflow.libs"]
-    package_dir["oneflow.libs"] = "third_party/tensorflow/lib"
-    package_data["oneflow.libs"] = ["libtensorflow_framework.so.1", "libxla_core.so"]
-    # Patchelf >= 0.9 is required.
-    oneflow_internal_so = "{}/python_scripts/oneflow/_oneflow_internal.so".format(
-        args.build_dir
-    )
-    rpath = os.popen("patchelf --print-rpath " + oneflow_internal_so).read()
-    command = "patchelf --set-rpath '$ORIGIN/:$ORIGIN/libs/:%s' %s" % (
-        rpath.strip(),
-        oneflow_internal_so,
+
+
+def get_oneflow_internal_so_path():
+    import imp
+
+    fp, pathname, description = imp.find_module(
+        "_oneflow_internal", ["{}/python_scripts/oneflow".format(args.build_dir)]
     )
-    if os.system(command) != 0:
-        raise Exception("Patchelf set rpath failed. command is: %s" % command)
+    assert os.path.isfile(pathname)
+    return os.path.relpath(pathname, "{}/python_scripts/oneflow".format(args.build_dir))
+
+
+package_data = {"oneflow": [get_oneflow_internal_so_path()] + include_files}
 
 
 def get_version():