From f5564457a21c6b162f425e311cfb623ce4882769 Mon Sep 17 00:00:00 2001 From: Mengwei Liu Date: Mon, 7 Oct 2024 17:37:12 -0700 Subject: [PATCH] Find portable_lib.so in pip package during cmake build (#5961) Summary: * Rename `_portable_lib.cpython-3..so` to `_portable_lib.so` so it can be found by CMake `find_library()`. This can be achieved by setting `SETUPTOOLS_EXT_SUFFIX`. * Since `executorch-config.cmake` is also being used to find installed libraries such as `executorch.a`, `xnnpack_backend.a`, add a condition to tell if `executorch-config.cmake` is being used in cmake-out or site-packages. Pull Request resolved: https://github.com/pytorch/executorch/pull/5961 Reviewed By: metascroy Differential Revision: D64014291 Pulled By: larryliu0820 fbshipit-source-id: 2757f2883d3f836e9efd45676f792c12f742e63d (cherry picked from commit 7337f8e6daa1252e13b6424d2589bc06c1c9298e) --- build/executorch-config.cmake | 171 ++++++++++++++---------- build/packaging/env_var_script_linux.sh | 4 + build/packaging/env_var_script_m1.sh | 4 + setup.py | 5 + 4 files changed, 117 insertions(+), 67 deletions(-) diff --git a/build/executorch-config.cmake b/build/executorch-config.cmake index 4376c9e5e77..2491ccc21a9 100644 --- a/build/executorch-config.cmake +++ b/build/executorch-config.cmake @@ -9,80 +9,117 @@ # is: # # find_package(executorch REQUIRED) - +# ------- +# +# Finds the ExecuTorch library +# +# This will define the following variables: +# +# EXECUTORCH_FOUND -- True if the system has the Torch library +# EXECUTORCH_INCLUDE_DIRS -- The include directories for torch +# EXECUTORCH_LIBRARIES -- Libraries to link against +# cmake_minimum_required(VERSION 3.19) -set(_root "${CMAKE_CURRENT_LIST_DIR}/../..") -set(required_lib_list executorch executorch_no_prim_ops portable_kernels) -foreach(lib ${required_lib_list}) - set(lib_var "LIB_${lib}") - add_library(${lib} STATIC IMPORTED) - find_library( - ${lib_var} ${lib} - HINTS "${_root}" - CMAKE_FIND_ROOT_PATH_BOTH +# Find prebuilt libportable_lib.so. If found, assuming current file is inside +# a pip package: +# /executorch/executorch-config.cmake. +# If not found, assuming current file is inside cmake-out: +# /cmake/ExecuTorch/executorch-config.cmake +find_library(_portable_lib_LIBRARY _portable_lib.so PATHS "${CMAKE_CURRENT_LIST_DIR}/extension/pybindings/") +message(WARNING "${CMAKE_CURRENT_LIST_DIR}/extension/pybindings/") +set(EXECUTORCH_LIBRARIES) +if(_portable_lib_LIBRARY) + # Assuming current file is /executorch/executorch-config.cmake + message(WARNING "portable library is found") + list(APPEND EXECUTORCH_LIBRARIES _portable_lib) + add_library(_portable_lib STATIC IMPORTED) + set(EXECUTORCH_INCLUDE_DIRS ${CMAKE_CURRENT_LIST_DIR}/include) + set_target_properties(_portable_lib PROPERTIES + IMPORTED_LOCATION "${_portable_lib_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${EXECUTORCH_INCLUDE_DIRS}" + CXX_STANDARD 17 ) - set_target_properties(${lib} PROPERTIES IMPORTED_LOCATION "${${lib_var}}") - target_include_directories(${lib} INTERFACE ${_root}) -endforeach() +else() + # Assuming current file is /cmake/ExecuTorch/executorch-config.cmake + message(WARNING "portable library is not found") + set(_root "${CMAKE_CURRENT_LIST_DIR}/../..") + set(required_lib_list executorch executorch_core portable_kernels) + foreach(lib ${required_lib_list}) + set(lib_var "LIB_${lib}") + add_library(${lib} STATIC IMPORTED) + find_library( + ${lib_var} ${lib} + HINTS "${_root}" + CMAKE_FIND_ROOT_PATH_BOTH + ) + set_target_properties(${lib} PROPERTIES IMPORTED_LOCATION "${${lib_var}}") + target_include_directories(${lib} INTERFACE ${_root}) + endforeach() -target_link_libraries(executorch INTERFACE executorch_no_prim_ops) + target_link_libraries(executorch INTERFACE executorch_core) -if(CMAKE_BUILD_TYPE MATCHES "Debug") - set(FLATCCRT_LIB flatccrt_d) -else() - set(FLATCCRT_LIB flatccrt) -endif() + if(CMAKE_BUILD_TYPE MATCHES "Debug") + set(FLATCCRT_LIB flatccrt_d) + else() + set(FLATCCRT_LIB flatccrt) + endif() -set(lib_list - etdump - bundled_program - extension_data_loader - ${FLATCCRT_LIB} - coremldelegate - mpsdelegate - qnn_executorch_backend - portable_ops_lib - extension_module - extension_module_static - extension_runner_util - extension_tensor - extension_threadpool - xnnpack_backend - XNNPACK - cpuinfo - pthreadpool - vulkan_backend - optimized_kernels - cpublas - eigen_blas - optimized_ops_lib - optimized_native_cpu_ops_lib - quantized_kernels - quantized_ops_lib - quantized_ops_aot_lib -) -foreach(lib ${lib_list}) - # Name of the variable which stores result of the find_library search - set(lib_var "LIB_${lib}") - find_library( - ${lib_var} ${lib} - HINTS "${_root}" - CMAKE_FIND_ROOT_PATH_BOTH + set(lib_list + etdump + bundled_program + extension_data_loader + ${FLATCCRT_LIB} + coremldelegate + mpsdelegate + qnn_executorch_backend + portable_ops_lib + extension_module + extension_module_static + extension_runner_util + extension_tensor + extension_threadpool + extension_training + xnnpack_backend + # Start XNNPACK Lib Deps + XNNPACK + microkernels-prod + kleidiai + # End XNNPACK Lib Deps + cpuinfo + pthreadpool + vulkan_backend + optimized_kernels + cpublas + eigen_blas + optimized_ops_lib + optimized_native_cpu_ops_lib + quantized_kernels + quantized_ops_lib + quantized_ops_aot_lib ) - if(NOT ${lib_var}) - message("${lib} library is not found. - If needed rebuild with the proper options in CMakeLists.txt" + foreach(lib ${lib_list}) + # Name of the variable which stores result of the find_library search + set(lib_var "LIB_${lib}") + find_library( + ${lib_var} ${lib} + HINTS "${_root}" + CMAKE_FIND_ROOT_PATH_BOTH ) - else() - if("${lib}" STREQUAL "extension_module" AND (NOT CMAKE_TOOLCHAIN_IOS)) - add_library(${lib} SHARED IMPORTED) + if(NOT ${lib_var}) + message("${lib} library is not found. + If needed rebuild with the proper options in CMakeLists.txt" + ) else() - # Building a share library on iOS requires code signing, so it's easier to - # keep all libs as static when CMAKE_TOOLCHAIN_IOS is used - add_library(${lib} STATIC IMPORTED) + if("${lib}" STREQUAL "extension_module" AND (NOT CMAKE_TOOLCHAIN_IOS)) + add_library(${lib} SHARED IMPORTED) + else() + # Building a share library on iOS requires code signing, so it's easier to + # keep all libs as static when CMAKE_TOOLCHAIN_IOS is used + add_library(${lib} STATIC IMPORTED) + endif() + set_target_properties(${lib} PROPERTIES IMPORTED_LOCATION "${${lib_var}}") + target_include_directories(${lib} INTERFACE ${_root}) endif() - set_target_properties(${lib} PROPERTIES IMPORTED_LOCATION "${${lib_var}}") - target_include_directories(${lib} INTERFACE ${_root}) - endif() -endforeach() + endforeach() +endif() diff --git a/build/packaging/env_var_script_linux.sh b/build/packaging/env_var_script_linux.sh index 6379dee6b5a..24f2fcb3c72 100644 --- a/build/packaging/env_var_script_linux.sh +++ b/build/packaging/env_var_script_linux.sh @@ -11,6 +11,10 @@ # Enable pybindings so that users can execute ExecuTorch programs from python. export EXECUTORCH_BUILD_PYBIND=1 +# Override extension suffix to be ".so", skipping package info such as +# "cpython-311-x86_64-linux-gnu" +export SETUPTOOLS_EXT_SUFFIX=".so" + # Ensure that CMAKE_ARGS is defined before referencing it. Defaults to empty # if not defined. export CMAKE_ARGS="${CMAKE_ARGS:-}" diff --git a/build/packaging/env_var_script_m1.sh b/build/packaging/env_var_script_m1.sh index da1192455f6..b9e3e6cf543 100644 --- a/build/packaging/env_var_script_m1.sh +++ b/build/packaging/env_var_script_m1.sh @@ -11,6 +11,10 @@ # Enable pybindings so that users can execute ExecuTorch programs from python. export EXECUTORCH_BUILD_PYBIND=1 +# Override extension suffix to be ".so", skipping package info such as +# "cpython-311-darwin" +export SETUPTOOLS_EXT_SUFFIX=".so" + # Ensure that CMAKE_ARGS is defined before referencing it. Defaults to empty # if not defined. export CMAKE_ARGS="${CMAKE_ARGS:-}" diff --git a/setup.py b/setup.py index f6adb4f86c3..4b4251dce8c 100644 --- a/setup.py +++ b/setup.py @@ -423,6 +423,11 @@ def run(self): "devtools/bundled_program/schema/scalar_type.fbs", "devtools/bundled_program/serialize/scalar_type.fbs", ), + # Install executorch-config.cmake to the root of the package. + ( + "build/executorch-config.cmake", + "executorch-config.cmake", + ), ] for src, dst in src_to_dst: dst = os.path.join(dst_root, dst)