Use linker tools on OSX & Linux to limit exports (#4651)

Makefile

@@ -921,9 +921,15 @@ else
 LIBHALIDE_SONAME_FLAGS=
 endif
 
+ifeq ($(UNAME), Linux)
+LIBHALIDE_EXPORTS=-Wl,--version-script=$(ROOT_DIR)/src/exported_symbols.linux
+else
+LIBHALIDE_EXPORTS=-Wl,-exported_symbols_list $(ROOT_DIR)/src/exported_symbols.osx
+endif
+
 $(BIN_DIR)/libHalide.$(SHARED_EXT): $(OBJECTS) $(INITIAL_MODULES)
 	@mkdir -p $(@D)
-	$(CXX) -shared $(OBJECTS) $(INITIAL_MODULES) $(LLVM_LIBS_FOR_SHARED_LIBHALIDE) $(LLVM_SYSTEM_LIBS) $(COMMON_LD_FLAGS) $(INSTALL_NAME_TOOL_LD_FLAGS) $(LIBHALIDE_SONAME_FLAGS) -o $(BIN_DIR)/libHalide.$(SHARED_EXT)
+	$(CXX) -shared $(LIBHALIDE_EXPORTS) $(OBJECTS) $(INITIAL_MODULES) $(LLVM_LIBS_FOR_SHARED_LIBHALIDE) $(LLVM_SYSTEM_LIBS) $(COMMON_LD_FLAGS) $(INSTALL_NAME_TOOL_LD_FLAGS) $(LIBHALIDE_SONAME_FLAGS) -o $(BIN_DIR)/libHalide.$(SHARED_EXT)
 ifeq ($(UNAME), Darwin)
 	install_name_tool -id $(CURDIR)/$(BIN_DIR)/libHalide.$(SHARED_EXT) $(BIN_DIR)/libHalide.$(SHARED_EXT)
 endif

cmake/TargetExportScript.cmake

@@ -0,0 +1,49 @@
+# Note: in CMake 3.18+ there is a CheckLinkerFlags module that should be used to replace this.
+# Sadly, CMake does not attempt to detect the underlying linker and people can try to use, eg.
+# gold or lld via CMAKE_CXX_FLAGS.
+include(CheckCXXSourceCompiles)
+
+function(target_export_script TARGET)
+    set(options)
+    set(oneValueArgs LINK_EXE APPLE_LD GNU_LD)
+    set(multiValueArgs)
+    cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+
+    set(dummy_source [[ int main() { return 0; } ]])
+    set(is_shared "$<STREQUAL:$<TARGET_PROPERTY:TYPE>,SHARED_LIBRARY>")
+
+    # CMake doesn't recognize MSVC/link.exe's unknown-option warning.
+    set(extra_errors FAIL_REGEX "LNK4044: unrecognized option")
+
+    ## More linkers support the GNU syntax (ld, lld, gold), so try it first.
+    set(version_script "LINKER:--version-script=${ARG_GNU_LD}")
+
+    set(CMAKE_REQUIRED_LINK_OPTIONS "${version_script}")
+    check_cxx_source_compiles("${dummy_source}" LINKER_HAS_FLAG_VERSION_SCRIPT ${extra_errors})
+
+    if (LINKER_HAS_FLAG_VERSION_SCRIPT)
+        target_link_options(${TARGET} PRIVATE "$<${is_shared}:${version_script}>")
+        set_property(TARGET ${TARGET} APPEND PROPERTY LINK_DEPENDS "$<${is_shared}:${ARG_GNU_LD}>")
+        return()
+    endif ()
+
+    ## The Apple linker expects a different flag.
+    set(exported_symbols_list "LINKER:-exported_symbols_list,${ARG_APPLE_LD}")
+
+    set(CMAKE_REQUIRED_LINK_OPTIONS "${exported_symbols_list}")
+    check_cxx_source_compiles("${dummy_source}" LINKER_HAS_FLAG_EXPORTED_SYMBOLS_LIST ${extra_errors})
+
+    if (LINKER_HAS_FLAG_EXPORTED_SYMBOLS_LIST)
+        target_link_options(${TARGET} PRIVATE "$<${is_shared}:${exported_symbols_list}>")
+        set_property(TARGET ${TARGET} APPEND PROPERTY LINK_DEPENDS "$<${is_shared}:${ARG_APPLE_LD}>")
+        return()
+    endif ()
+
+    ## TODO: implement something similar for Windows/link.exe
+    # https://github.com/halide/Halide/issues/4651
+
+    ## Warn the user if we were supposed to have been able to attach a linker script.
+    if (BUILD_SHARED_LIBS AND NOT MSVC)
+        message(WARNING "Unknown linker! Could not attach Halide linker script.")
+    endif ()
+endfunction()

src/CMakeLists.txt

@@ -377,6 +377,13 @@ target_compile_definitions(Halide
                            $<$<STREQUAL:$<TARGET_PROPERTY:TYPE>,STATIC_LIBRARY>:Halide_STATIC_DEFINE>
                            $<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:WITH_INTROSPECTION>)
 
+include(TargetExportScript)
+## TODO: implement something similar for Windows/link.exe
+# https://github.com/halide/Halide/issues/4651
+target_export_script(Halide
+                     APPLE_LD "${CMAKE_CURRENT_LIST_DIR}/exported_symbols.osx"
+                     GNU_LD "${CMAKE_CURRENT_LIST_DIR}/exported_symbols.linux")
+
 set_target_properties(Halide PROPERTIES
                       POSITION_INDEPENDENT_CODE ON
                       VERSION ${Halide_VERSION}

src/exported_symbols.linux

@@ -0,0 +1,15 @@
+{
+    global:
+        # Everything at global namespace beginning with halide_
+        halide_* ;
+
+        # Everything in Halide:: namespace (including nested namespace)
+        # (Don't use _Z*6Halide as that can match things like std::sort()
+        # with Halide types in the specialization)
+        _Z?6Halide* ;
+        _Z??6Halide* ;
+        _Z???6Halide* ;
+
+    local: *;
+};
+

src/exported_symbols.osx

@@ -0,0 +1,7 @@
+# Everything at global namespace beginning with halide_
+halide_*
+
+# Everything in Halide:: namespace (including nested namespace)
+__Z?6Halide*
+__Z??6Halide*
+__Z???6Halide*