[libclc] Refactor build system to allow in-tree builds (#87622)

The previous build system was adding custom "OpenCL" and "LLVM IR"
languages in CMake to build the builtin libraries. This was making it
harder to build in-tree because the tool binaries needed to be present
at configure time.

This commit refactors the build system to use custom commands to build
the bytecode files one by one, and link them all together into the final
bytecode library. It also enables in-tree builds by aliasing the
clang/llvm-link/etc. tool targets to internal targets, which are
imported from the LLVM installation directory when building out of tree.

Diffing (with llvm-diff) all of the final bytecode libraries in an
out-of-tree configuration against those built using the current tip
system shows no changes. Note that there are textual changes to metadata
IDs which confuse regular diff, and that llvm-diff 14 and below may show
false-positives.

This commit also removes a file listed in one of the SOURCEs which
didn't exist and which was preventing the use of
ENABLE_RUNTIME_SUBNORMAL when configuring CMake.

libclc/CMakeLists.txt

@@ -4,6 +4,15 @@ project( libclc VERSION 0.2.0 LANGUAGES CXX C)
 
 set(CMAKE_CXX_STANDARD 17)
 
+# Add path for custom modules
+list( INSERT CMAKE_MODULE_PATH 0 "${PROJECT_SOURCE_DIR}/cmake/modules" )
+
+set( LIBCLC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} )
+set( LIBCLC_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR} )
+set( LIBCLC_OBJFILE_DIR ${LIBCLC_BINARY_DIR}/obj.libclc.dir )
+
+include( AddLibclc )
+
 include( GNUInstallDirs )
 set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS
   amdgcn-amdhsa/lib/SOURCES;
@@ -27,31 +36,51 @@ set( LIBCLC_TARGETS_TO_BUILD "all"
 
 option( ENABLE_RUNTIME_SUBNORMAL "Enable runtime linking of subnormal support." OFF )
 
-find_package(LLVM REQUIRED HINTS "${LLVM_CMAKE_DIR}")
-include(AddLLVM)
+if( LIBCLC_STANDALONE_BUILD OR CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR )
+  # Out-of-tree configuration
+  set( LIBCLC_STANDALONE_BUILD TRUE )
 
-message( STATUS "libclc LLVM version: ${LLVM_PACKAGE_VERSION}" )
+  find_package(LLVM REQUIRED HINTS "${LLVM_CMAKE_DIR}")
+  include(AddLLVM)
 
-if( LLVM_PACKAGE_VERSION VERSION_LESS LIBCLC_MIN_LLVM )
-  message( FATAL_ERROR "libclc needs at least LLVM ${LIBCLC_MIN_LLVM}" )
-endif()
+  message( STATUS "libclc LLVM version: ${LLVM_PACKAGE_VERSION}" )
 
-find_program( LLVM_CLANG clang PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
-find_program( LLVM_AS llvm-as PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
-find_program( LLVM_LINK llvm-link PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
-find_program( LLVM_OPT opt PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
-find_program( LLVM_SPIRV llvm-spirv PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
+  if( LLVM_PACKAGE_VERSION VERSION_LESS LIBCLC_MIN_LLVM )
+    message( FATAL_ERROR "libclc needs at least LLVM ${LIBCLC_MIN_LLVM}" )
+  endif()
+
+  # Import required tools as targets
+  foreach( tool clang llvm-as llvm-link opt )
+    find_program( LLVM_TOOL_${tool} ${tool} PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
+    add_executable( libclc::${tool} IMPORTED GLOBAL )
+    set_target_properties( libclc::${tool} PROPERTIES IMPORTED_LOCATION ${LLVM_TOOL_${tool}} )
+  endforeach()
+else()
+  # In-tree configuration
+  set( LIBCLC_STANDALONE_BUILD FALSE )
+
+  set( LLVM_PACKAGE_VERSION ${LLVM_VERSION} )
 
-# Print toolchain
-message( STATUS "libclc toolchain - clang: ${LLVM_CLANG}" )
-message( STATUS "libclc toolchain - llvm-as: ${LLVM_AS}" )
-message( STATUS "libclc toolchain - llvm-link: ${LLVM_LINK}" )
-message( STATUS "libclc toolchain - opt: ${LLVM_OPT}" )
-message( STATUS "libclc toolchain - llvm-spirv: ${LLVM_SPIRV}" )
-if( NOT LLVM_CLANG OR NOT LLVM_OPT OR NOT LLVM_AS OR NOT LLVM_LINK )
+  # Note that we check this later (for both build types) but we can provide a
+  # more useful error message when built in-tree. We assume that LLVM tools are
+  # always available so don't warn here.
+  if( NOT clang IN_LIST LLVM_ENABLE_PROJECTS )
+    message(FATAL_ERROR "Clang is not enabled, but is required to build libclc in-tree")
+  endif()
+
+  foreach( tool clang llvm-as llvm-link opt )
+    add_executable(libclc::${tool} ALIAS ${tool})
+  endforeach()
+endif()
+
+if( NOT TARGET libclc::clang OR NOT TARGET libclc::opt
+    OR NOT TARGET libclc::llvm-as OR NOT TARGET libclc::llvm-link )
   message( FATAL_ERROR "libclc toolchain incomplete!" )
 endif()
 
+# llvm-spirv is an optional dependency, used to build spirv-* targets.
+find_program( LLVM_SPIRV llvm-spirv PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
+
 # List of all targets. Note that some are added dynamically below.
 set( LIBCLC_TARGETS_ALL
   amdgcn--
@@ -90,24 +119,9 @@ if( "spirv-mesa3d-" IN_LIST LIBCLC_TARGETS_TO_BUILD OR "spirv64-mesa3d-" IN_LIST
   endif()
 endif()
 
-set( CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake )
-set( CMAKE_CLC_COMPILER ${LLVM_CLANG} )
-set( CMAKE_CLC_ARCHIVE ${LLVM_LINK} )
-set( CMAKE_LLAsm_PREPROCESSOR ${LLVM_CLANG} )
-set( CMAKE_LLAsm_COMPILER ${LLVM_AS} )
-set( CMAKE_LLAsm_ARCHIVE ${LLVM_LINK} )
-
 # Construct LLVM version define
 set( LLVM_VERSION_DEFINE "-DHAVE_LLVM=0x${LLVM_VERSION_MAJOR}0${LLVM_VERSION_MINOR}" )
 
-
-# LLVM 13 enables standard includes by default
-if( LLVM_PACKAGE_VERSION VERSION_GREATER_EQUAL 13.0.0 )
-  set( CMAKE_LLAsm_FLAGS "${CMAKE_LLAsm_FLAGS} -cl-no-stdinc" )
-  set( CMAKE_CLC_FLAGS "${CMAKE_CLC_FLAGS} -cl-no-stdinc" )
-endif()
-
-enable_language( CLC LLAsm )
 # This needs to be set before any target that needs it
 # We need to use LLVM_INCLUDE_DIRS here, because if we are linking to an
 # llvm build directory, this includes $src/llvm/include which is where all the
@@ -122,7 +136,7 @@ set(LLVM_LINK_COMPONENTS
   IRReader
   Support
 )
-if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
+if( LIBCLC_STANDALONE_BUILD )
   add_llvm_executable( prepare_builtins utils/prepare-builtins.cpp )
 else()
   add_llvm_utility( prepare_builtins utils/prepare-builtins.cpp )
@@ -167,12 +181,14 @@ install( FILES ${CMAKE_CURRENT_BINARY_DIR}/libclc.pc DESTINATION "${CMAKE_INSTAL
 install( DIRECTORY generic/include/clc DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" )
 
 if( ENABLE_RUNTIME_SUBNORMAL )
-  add_library( subnormal_use_default STATIC
-    generic/lib/subnormal_use_default.ll )
-  add_library( subnormal_disable STATIC
-    generic/lib/subnormal_disable.ll )
-  install( TARGETS subnormal_use_default subnormal_disable ARCHIVE
-    DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
+  foreach( file subnormal_use_default subnormal_disable )
+    link_bc(
+       TARGET ${file}
+       INPUTS ${PROJECT_SOURCE_DIR}/generic/lib/${file}.ll
+    )
+    install( FILES $<TARGET_PROPERTY:${file},TARGET_FILE> ARCHIVE
+      DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
+  endforeach()
 endif()
 
 find_package( Python3 REQUIRED COMPONENTS Interpreter )
@@ -232,131 +248,164 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
 
   # Add the generated convert.cl here to prevent adding the one listed in
   # SOURCES
+  set( objects )   # A "set" of already-added input files
+  set( rel_files ) # Source directory input files, relative to the root dir
+  set( gen_files ) # Generated binary input files, relative to the binary dir
   if( NOT ${ARCH} STREQUAL "spirv" AND NOT ${ARCH} STREQUAL "spirv64" )
     if( NOT ENABLE_RUNTIME_SUBNORMAL AND NOT ${ARCH} STREQUAL "clspv" AND
         NOT ${ARCH} STREQUAL "clspv64" )
-      set( rel_files convert.cl )
-      set( objects convert.cl )
+      list( APPEND gen_files convert.cl )
+      list( APPEND objects convert.cl )
       list( APPEND rel_files generic/lib/subnormal_use_default.ll )
     elseif(${ARCH} STREQUAL "clspv" OR ${ARCH} STREQUAL "clspv64")
-      set( rel_files clspv-convert.cl )
-      set( objects clspv-convert.cl )
+      list( APPEND gen_files clspv-convert.cl )
+      list( APPEND objects clspv-convert.cl )
     endif()
-  else()
-    set( rel_files )
-    set( objects )
   endif()
 
   foreach( l ${source_list} )
     file( READ ${l} file_list )
     string( REPLACE "\n" ";" file_list ${file_list} )
     get_filename_component( dir ${l} DIRECTORY )
     foreach( f ${file_list} )
-      list( FIND objects ${f} found )
-      if( found EQUAL  -1 )
+      # Only add each file once, so that targets can 'specialize' builtins
+      if( NOT ${f} IN_LIST objects )
         list( APPEND objects ${f} )
         list( APPEND rel_files ${dir}/${f} )
-        # FIXME: This should really go away
-        file( TO_CMAKE_PATH ${PROJECT_SOURCE_DIR}/${dir}/${f} src_loc )
-        get_filename_component( fdir ${src_loc} DIRECTORY )
-
-        set_source_files_properties( ${dir}/${f}
-          PROPERTIES COMPILE_FLAGS "-I ${fdir}" )
       endif()
     endforeach()
   endforeach()
 
   foreach( d ${${t}_devices} )
-    # Some targets don't have a specific GPU to target
-    if( ${d} STREQUAL "none" OR ${ARCH} STREQUAL "spirv" OR ${ARCH} STREQUAL "spirv64" )
-      set( mcpu )
-      set( arch_suffix "${t}" )
-    else()
-      set( mcpu "-mcpu=${d}" )
-      set( arch_suffix "${d}-${t}" )
+    get_libclc_device_info(
+      TRIPLE ${t}
+      DEVICE ${d}
+      CPU cpu
+      ARCH_SUFFIX arch_suffix
+      CLANG_TRIPLE clang_triple
+    )
+
+    set( mcpu )
+    if( NOT "${cpu}" STREQUAL "" )
+      set( mcpu "-mcpu=${cpu}" )
     endif()
+
     message( STATUS "  device: ${d} ( ${${d}_aliases} )" )
 
-    if ( ${ARCH} STREQUAL "spirv" OR ${ARCH} STREQUAL "spirv64" )
-      if( ${ARCH} STREQUAL "spirv" )
-        set( t "spir--" )
-      else()
-        set( t "spir64--" )
-      endif()
+    if ( ARCH STREQUAL spirv OR ARCH STREQUAL spirv64 )
       set( build_flags -O0 -finline-hint-functions )
       set( opt_flags )
       set( spvflags --spirv-max-version=1.1 )
-    elseif( ${ARCH} STREQUAL "clspv" )
-      set( t "spir--" )
-      set( build_flags "-Wno-unknown-assumption")
-      set( opt_flags -O3 )
-    elseif( ${ARCH} STREQUAL "clspv64" )
-      set( t "spir64--" )
+    elseif( ARCH STREQUAL clspv OR ARCH STREQUAL clspv64 )
       set( build_flags "-Wno-unknown-assumption")
       set( opt_flags -O3 )
     else()
       set( build_flags )
       set( opt_flags -O3 )
     endif()
 
+    set( LIBCLC_ARCH_OBJFILE_DIR "${LIBCLC_OBJFILE_DIR}/${arch_suffix}" )
+    file( MAKE_DIRECTORY ${LIBCLC_ARCH_OBJFILE_DIR} )
+
+    string( TOUPPER "CLC_${ARCH}" CLC_TARGET_DEFINE )
+
+    list( APPEND build_flags
+      -D__CLC_INTERNAL
+      -D${CLC_TARGET_DEFINE}
+      -I${PROJECT_SOURCE_DIR}/generic/include
+      # FIXME: Fix libclc to not require disabling this noisy warning
+      -Wno-bitwise-conditional-parentheses
+    )
+
+    set( bytecode_files "" )
+    foreach( file IN LISTS gen_files rel_files )
+      # We need to take each file and produce an absolute input file, as well
+      # as a unique architecture-specific output file. We deal with a mix of
+      # different input files, which makes this trickier.
+      if( ${file} IN_LIST gen_files )
+        # Generated files are given just as file names, which we must make
+        # absolute to the binary directory.
+        set( input_file ${CMAKE_CURRENT_BINARY_DIR}/${file} )
+        set( output_file "${LIBCLC_ARCH_OBJFILE_DIR}/${file}.o" )
+      else()
+        # Other files are originally relative to each SOURCE file, which are
+        # then make relative to the libclc root directory. We must normalize
+        # the path (e.g., ironing out any ".."), then make it relative to the
+        # root directory again, and use that relative path component for the
+        # binary path.
+        get_filename_component( abs_path ${file} ABSOLUTE BASE_DIR ${PROJECT_SOURCE_DIR} )
+        file( RELATIVE_PATH root_rel_path ${PROJECT_SOURCE_DIR} ${abs_path} )
+        set( input_file ${PROJECT_SOURCE_DIR}/${file} )
+        set( output_file "${LIBCLC_ARCH_OBJFILE_DIR}/${root_rel_path}.o" )
+      endif()
+
+      get_filename_component( file_dir ${file} DIRECTORY )
+
+      compile_to_bc(
+        TRIPLE ${clang_triple}
+        INPUT ${input_file}
+        OUTPUT ${output_file}
+        EXTRA_OPTS "${mcpu}" -fno-builtin -nostdlib
+                   "${build_flags}" -I${PROJECT_SOURCE_DIR}/${file_dir}
+      )
+      list( APPEND bytecode_files ${output_file} )
+    endforeach()
+
     set( builtins_link_lib_tgt builtins.link.${arch_suffix} )
-    add_library( ${builtins_link_lib_tgt} STATIC ${rel_files} )
-    # Make sure we depend on the pseudo target to prevent
-    # multiple invocations
-    add_dependencies( ${builtins_link_lib_tgt} generate_convert.cl )
-    add_dependencies( ${builtins_link_lib_tgt} clspv-generate_convert.cl )
-    # CMake will turn this include into absolute path
-    target_include_directories( ${builtins_link_lib_tgt} PRIVATE
-      "generic/include" )
-    target_compile_definitions( ${builtins_link_lib_tgt} PRIVATE
-      "__CLC_INTERNAL" )
-    string( TOUPPER "-DCLC_${ARCH}" CLC_TARGET_DEFINE )
-    target_compile_definitions( ${builtins_link_lib_tgt} PRIVATE
-      ${CLC_TARGET_DEFINE} )
-    target_compile_options( ${builtins_link_lib_tgt} PRIVATE  -target
-      ${t} ${mcpu} -fno-builtin -nostdlib ${build_flags} )
-    set_target_properties( ${builtins_link_lib_tgt} PROPERTIES
-      LINKER_LANGUAGE CLC )
-
-    set( obj_suffix ${arch_suffix}.bc )
-    set( builtins_opt_lib_tgt builtins.opt.${obj_suffix} )
-
-    # Add opt target
-    add_custom_command( OUTPUT ${builtins_opt_lib_tgt}
-      COMMAND ${LLVM_OPT} ${opt_flags} -o ${builtins_opt_lib_tgt}
-        $<TARGET_FILE:${builtins_link_lib_tgt}>
-      DEPENDS ${builtins_link_lib_tgt} )
-    add_custom_target( "opt.${obj_suffix}" ALL
-      DEPENDS ${builtins_opt_lib_tgt} )
-
-    if( ${ARCH} STREQUAL "spirv" OR ${ARCH} STREQUAL "spirv64" )
+
+    link_bc(
+      TARGET ${builtins_link_lib_tgt}
+      INPUTS ${bytecode_files}
+    )
+
+    set( builtins_link_lib $<TARGET_PROPERTY:${builtins_link_lib_tgt},TARGET_FILE> )
+
+    if( ARCH STREQUAL spirv OR ARCH STREQUAL spirv64 )
       set( spv_suffix ${arch_suffix}.spv )
-      add_custom_command( OUTPUT "${spv_suffix}"
-        COMMAND ${LLVM_SPIRV} ${spvflags} -o "${spv_suffix}" $<TARGET_FILE:${builtins_link_lib_tgt}>
-        DEPENDS ${builtins_link_lib_tgt} )
+      add_custom_command( OUTPUT ${spv_suffix}
+        COMMAND ${LLVM_SPIRV} ${spvflags} -o ${spv_suffix} ${builtins_link_lib}
+        DEPENDS ${builtins_link_lib_tgt}
+      )
       add_custom_target( "prepare-${spv_suffix}" ALL DEPENDS "${spv_suffix}" )
       install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${spv_suffix}
          DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
     else()
+      set( builtins_opt_lib_tgt builtins.opt.${arch_suffix} )
+
+      # Add opt target
+      add_custom_command( OUTPUT ${builtins_opt_lib_tgt}.bc
+        COMMAND libclc::opt ${opt_flags} -o ${builtins_opt_lib_tgt}.bc
+          ${builtins_link_lib}
+        DEPENDS libclc::opt ${builtins_link_lib_tgt}
+      )
+      add_custom_target( ${builtins_opt_lib_tgt}
+        ALL DEPENDS ${builtins_opt_lib_tgt}.bc
+      )
+      set_target_properties( ${builtins_opt_lib_tgt}
+        PROPERTIES TARGET_FILE ${builtins_opt_lib_tgt}.bc
+      )
+
       # Add prepare target
-      add_custom_command( OUTPUT "${obj_suffix}"
-        COMMAND prepare_builtins -o "${obj_suffix}" ${builtins_opt_lib_tgt}
-        DEPENDS "opt.${obj_suffix}" ${builtins_opt_lib_tgt} prepare_builtins )
-      add_custom_target( "prepare-${obj_suffix}" ALL DEPENDS "${obj_suffix}" )
+      set( obj_suffix ${arch_suffix}.bc )
+      add_custom_command( OUTPUT ${obj_suffix}
+        COMMAND prepare_builtins -o ${obj_suffix}
+          $<TARGET_PROPERTY:${builtins_opt_lib_tgt},TARGET_FILE>
+        DEPENDS ${builtins_opt_lib_tgt} prepare_builtins )
+      add_custom_target( prepare-${obj_suffix} ALL DEPENDS ${obj_suffix} )
 
       # nvptx-- targets don't include workitem builtins
-      if( NOT ${t} MATCHES ".*ptx.*--$" )
+      if( NOT clang_triple MATCHES ".*ptx.*--$" )
         add_test( NAME external-calls-${obj_suffix}
           COMMAND ./check_external_calls.sh ${CMAKE_CURRENT_BINARY_DIR}/${obj_suffix} ${LLVM_TOOLS_BINARY_DIR}
           WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} )
       endif()
 
       install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${obj_suffix} DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
       foreach( a ${${d}_aliases} )
-        set( alias_suffix "${a}-${t}.bc" )
+        set( alias_suffix "${a}-${clang_triple}.bc" )
         add_custom_target( ${alias_suffix} ALL
           COMMAND ${CMAKE_COMMAND} -E create_symlink ${obj_suffix} ${alias_suffix}
-          DEPENDS "prepare-${obj_suffix}" )
+          DEPENDS prepare-${obj_suffix} )
         install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${alias_suffix} DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
       endforeach( a )
     endif()