From c7597b71743a9df6c003fbe85f4b053ae8f6e83f Mon Sep 17 00:00:00 2001 From: Andrei Elovikov Date: Fri, 24 Oct 2025 12:02:08 -0700 Subject: [PATCH 1/7] [SYCL RTC] Use `cmake --install` for `resource.cpp` generation Fixes https://github.com/intel/llvm/issues/20275 That way we achieve all of: * Better "filtering" of what we need, as we don't operate at files/directories level anymore. Applies to both what we want to include and what we don't need (i.e., if provided by components we're not interested in) * Avoid race conditions with other targets creating temp files in the directories we take files from --- sycl-jit/jit-compiler/CMakeLists.txt | 26 +++++++++++++++++++------ sycl-jit/jit-compiler/utils/generate.py | 10 +--------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/sycl-jit/jit-compiler/CMakeLists.txt b/sycl-jit/jit-compiler/CMakeLists.txt index d0f63e27cda63..db4686acd01c9 100644 --- a/sycl-jit/jit-compiler/CMakeLists.txt +++ b/sycl-jit/jit-compiler/CMakeLists.txt @@ -9,25 +9,39 @@ else() set(SYCL_JIT_VIRTUAL_TOOLCHAIN_ROOT "/sycl-jit-toolchain/") endif() -set(SYCL_JIT_RESOURCE_DEPS - sycl-headers # include/sycl - clang # lib/clang/N/include - ${CMAKE_CURRENT_SOURCE_DIR}/utils/generate.py) +set(SYCL_JIT_RESOURCE_DEPS sycl-headers clang) +set(SYCL_JIT_RESOURCE_INSTALL_COMPONENTS sycl-headers OpenCL-Headers clang-resource-headers) if ("libclc" IN_LIST LLVM_ENABLE_PROJECTS) # Somehow just "libclc" doesn't build "remangled-*" (and maybe whatever else). list(APPEND SYCL_JIT_RESOURCE_DEPS libclc libspirv-builtins) # lib/clc/*.bc + list(APPEND SYCL_JIT_RESOURCE_INSTALL_COMPONENTS libspirv-builtins) endif() if ("libdevice" IN_LIST LLVM_ENABLE_PROJECTS) list(APPEND SYCL_JIT_RESOURCE_DEPS libsycldevice) # lib/*.bc + list(APPEND SYCL_JIT_RESOURCE_INSTALL_COMPONENTS libsycldevice) endif() +set(ALL_RTC_PREPARE_RESOURCES_COMMANDS) +foreach(component IN LISTS SYCL_JIT_RESOURCE_INSTALL_COMPONENTS) + list(APPEND ALL_RTC_PREPARE_RESOURCES_COMMANDS + COMMAND ${CMAKE_COMMAND} --install ${CMAKE_BINARY_DIR} --prefix ${CMAKE_CURRENT_BINARY_DIR}/rtc-resources-install --component "${component}" + ) +endforeach() + +add_custom_target(rtc-prepare-resources + DEPENDS ${SYCL_JIT_RESOURCE_DEPS} + ${ALL_RTC_PREPARE_RESOURCES_COMMANDS} + COMMAND_EXPAND_LISTS +) + add_custom_command( OUTPUT ${SYCL_JIT_RESOURCE_CPP} - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/utils/generate.py --toolchain-dir ${CMAKE_BINARY_DIR} --output ${SYCL_JIT_RESOURCE_CPP} --prefix ${SYCL_JIT_VIRTUAL_TOOLCHAIN_ROOT} + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/utils/generate.py --toolchain-dir ${CMAKE_CURRENT_BINARY_DIR}/rtc-resources-install --output ${SYCL_JIT_RESOURCE_CPP} --prefix ${SYCL_JIT_VIRTUAL_TOOLCHAIN_ROOT} DEPENDS - ${SYCL_JIT_RESOURCE_DEPS} + rtc-prepare-resources + ${CMAKE_CURRENT_SOURCE_DIR}/utils/generate.py ) # We use C23/C++26's `#embed` to implement this resource creation, and "current" diff --git a/sycl-jit/jit-compiler/utils/generate.py b/sycl-jit/jit-compiler/utils/generate.py index 7fca23e869716..449723b8fa268 100644 --- a/sycl-jit/jit-compiler/utils/generate.py +++ b/sycl-jit/jit-compiler/utils/generate.py @@ -52,15 +52,7 @@ def process_dir(dir): file_path = os.path.join(root, file) process_file(file_path) - process_dir(os.path.join(args.toolchain_dir, "include/")) - process_dir(os.path.join(args.toolchain_dir, "lib/clang/")) - process_dir(os.path.join(args.toolchain_dir, "lib/clc/")) - - for file in glob.iglob( - "*.bc", root_dir=os.path.join(args.toolchain_dir, "lib") - ): - file_path = os.path.join(args.toolchain_dir, "lib", file) - process_file(file_path) + process_dir(args.toolchain_dir) out.write( f""" From 73e9f6ee11c0bfcc8e9ec6b9a4156706daa08000 Mon Sep 17 00:00:00 2001 From: Andrei Elovikov Date: Fri, 24 Oct 2025 13:26:05 -0700 Subject: [PATCH 2/7] SYCL_JIT_RESOURCE_INSTALL_DIR --- sycl-jit/jit-compiler/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sycl-jit/jit-compiler/CMakeLists.txt b/sycl-jit/jit-compiler/CMakeLists.txt index db4686acd01c9..c74467490c1bb 100644 --- a/sycl-jit/jit-compiler/CMakeLists.txt +++ b/sycl-jit/jit-compiler/CMakeLists.txt @@ -23,10 +23,12 @@ if ("libdevice" IN_LIST LLVM_ENABLE_PROJECTS) list(APPEND SYCL_JIT_RESOURCE_INSTALL_COMPONENTS libsycldevice) endif() +set(SYCL_JIT_RESOURCE_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/rtc-resources-install) + set(ALL_RTC_PREPARE_RESOURCES_COMMANDS) foreach(component IN LISTS SYCL_JIT_RESOURCE_INSTALL_COMPONENTS) list(APPEND ALL_RTC_PREPARE_RESOURCES_COMMANDS - COMMAND ${CMAKE_COMMAND} --install ${CMAKE_BINARY_DIR} --prefix ${CMAKE_CURRENT_BINARY_DIR}/rtc-resources-install --component "${component}" + COMMAND ${CMAKE_COMMAND} --install ${CMAKE_BINARY_DIR} --prefix ${SYCL_JIT_RESOURCE_INSTALL_DIR} --component "${component}" ) endforeach() @@ -38,7 +40,7 @@ add_custom_target(rtc-prepare-resources add_custom_command( OUTPUT ${SYCL_JIT_RESOURCE_CPP} - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/utils/generate.py --toolchain-dir ${CMAKE_CURRENT_BINARY_DIR}/rtc-resources-install --output ${SYCL_JIT_RESOURCE_CPP} --prefix ${SYCL_JIT_VIRTUAL_TOOLCHAIN_ROOT} + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/utils/generate.py --toolchain-dir ${SYCL_JIT_RESOURCE_INSTALL_DIR} --output ${SYCL_JIT_RESOURCE_CPP} --prefix ${SYCL_JIT_VIRTUAL_TOOLCHAIN_ROOT} DEPENDS rtc-prepare-resources ${CMAKE_CURRENT_SOURCE_DIR}/utils/generate.py From 2b47019fc86158f78bdc7d1077882f6bcd994883 Mon Sep 17 00:00:00 2001 From: Andrei Elovikov Date: Fri, 24 Oct 2025 13:26:59 -0700 Subject: [PATCH 3/7] s/ALL_RTC_PREPARE_RESOURCES_COMMANDS/SYCL_JIT_PREPARE_RESOURCE_COMMANDS/ --- sycl-jit/jit-compiler/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sycl-jit/jit-compiler/CMakeLists.txt b/sycl-jit/jit-compiler/CMakeLists.txt index c74467490c1bb..2bf10471e0c01 100644 --- a/sycl-jit/jit-compiler/CMakeLists.txt +++ b/sycl-jit/jit-compiler/CMakeLists.txt @@ -25,16 +25,16 @@ endif() set(SYCL_JIT_RESOURCE_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/rtc-resources-install) -set(ALL_RTC_PREPARE_RESOURCES_COMMANDS) +set(SYCL_JIT_PREPARE_RESOURCE_COMMANDS) foreach(component IN LISTS SYCL_JIT_RESOURCE_INSTALL_COMPONENTS) - list(APPEND ALL_RTC_PREPARE_RESOURCES_COMMANDS + list(APPEND SYCL_JIT_PREPARE_RESOURCE_COMMANDS COMMAND ${CMAKE_COMMAND} --install ${CMAKE_BINARY_DIR} --prefix ${SYCL_JIT_RESOURCE_INSTALL_DIR} --component "${component}" ) endforeach() add_custom_target(rtc-prepare-resources DEPENDS ${SYCL_JIT_RESOURCE_DEPS} - ${ALL_RTC_PREPARE_RESOURCES_COMMANDS} + ${SYCL_JIT_PREPARE_RESOURCE_COMMANDS} COMMAND_EXPAND_LISTS ) From ce42833a838bf88f089fd18d285e8b34e6e31558 Mon Sep 17 00:00:00 2001 From: Andrei Elovikov Date: Fri, 24 Oct 2025 13:38:24 -0700 Subject: [PATCH 4/7] Drop COMMAND_EXPAND_LISTS --- sycl-jit/jit-compiler/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/sycl-jit/jit-compiler/CMakeLists.txt b/sycl-jit/jit-compiler/CMakeLists.txt index 2bf10471e0c01..0b35612e6768c 100644 --- a/sycl-jit/jit-compiler/CMakeLists.txt +++ b/sycl-jit/jit-compiler/CMakeLists.txt @@ -35,7 +35,6 @@ endforeach() add_custom_target(rtc-prepare-resources DEPENDS ${SYCL_JIT_RESOURCE_DEPS} ${SYCL_JIT_PREPARE_RESOURCE_COMMANDS} - COMMAND_EXPAND_LISTS ) add_custom_command( From 648c87a1db301b807fdfd435522e3faf08d66230 Mon Sep 17 00:00:00 2001 From: Andrei Elovikov Date: Fri, 24 Oct 2025 13:47:23 -0700 Subject: [PATCH 5/7] Drop now useless comments --- sycl-jit/jit-compiler/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sycl-jit/jit-compiler/CMakeLists.txt b/sycl-jit/jit-compiler/CMakeLists.txt index 0b35612e6768c..a43d786a08791 100644 --- a/sycl-jit/jit-compiler/CMakeLists.txt +++ b/sycl-jit/jit-compiler/CMakeLists.txt @@ -14,12 +14,12 @@ set(SYCL_JIT_RESOURCE_INSTALL_COMPONENTS sycl-headers OpenCL-Headers clang-resou if ("libclc" IN_LIST LLVM_ENABLE_PROJECTS) # Somehow just "libclc" doesn't build "remangled-*" (and maybe whatever else). - list(APPEND SYCL_JIT_RESOURCE_DEPS libclc libspirv-builtins) # lib/clc/*.bc + list(APPEND SYCL_JIT_RESOURCE_DEPS libclc libspirv-builtins) list(APPEND SYCL_JIT_RESOURCE_INSTALL_COMPONENTS libspirv-builtins) endif() if ("libdevice" IN_LIST LLVM_ENABLE_PROJECTS) - list(APPEND SYCL_JIT_RESOURCE_DEPS libsycldevice) # lib/*.bc + list(APPEND SYCL_JIT_RESOURCE_DEPS libsycldevice) list(APPEND SYCL_JIT_RESOURCE_INSTALL_COMPONENTS libsycldevice) endif() From 7dc63ba755d106986c4d8cd0b9f10a8032564b8e Mon Sep 17 00:00:00 2001 From: Andrei Elovikov Date: Mon, 27 Oct 2025 09:37:26 -0700 Subject: [PATCH 6/7] Re-use components for deps --- sycl-jit/jit-compiler/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sycl-jit/jit-compiler/CMakeLists.txt b/sycl-jit/jit-compiler/CMakeLists.txt index a43d786a08791..a24597921ecce 100644 --- a/sycl-jit/jit-compiler/CMakeLists.txt +++ b/sycl-jit/jit-compiler/CMakeLists.txt @@ -9,17 +9,13 @@ else() set(SYCL_JIT_VIRTUAL_TOOLCHAIN_ROOT "/sycl-jit-toolchain/") endif() -set(SYCL_JIT_RESOURCE_DEPS sycl-headers clang) set(SYCL_JIT_RESOURCE_INSTALL_COMPONENTS sycl-headers OpenCL-Headers clang-resource-headers) if ("libclc" IN_LIST LLVM_ENABLE_PROJECTS) - # Somehow just "libclc" doesn't build "remangled-*" (and maybe whatever else). - list(APPEND SYCL_JIT_RESOURCE_DEPS libclc libspirv-builtins) list(APPEND SYCL_JIT_RESOURCE_INSTALL_COMPONENTS libspirv-builtins) endif() if ("libdevice" IN_LIST LLVM_ENABLE_PROJECTS) - list(APPEND SYCL_JIT_RESOURCE_DEPS libsycldevice) list(APPEND SYCL_JIT_RESOURCE_INSTALL_COMPONENTS libsycldevice) endif() @@ -32,6 +28,10 @@ foreach(component IN LISTS SYCL_JIT_RESOURCE_INSTALL_COMPONENTS) ) endforeach() +set(SYCL_JIT_RESOURCE_DEPS ${SYCL_JIT_RESOURCE_INSTALL_COMPONENTS}) + # OpenCL-Headers doesn't have a corresponding build target: +list(FILTER SYCL_JIT_RESOURCE_DEPS EXCLUDE REGEX "^OpenCL-Headers$") + add_custom_target(rtc-prepare-resources DEPENDS ${SYCL_JIT_RESOURCE_DEPS} ${SYCL_JIT_PREPARE_RESOURCE_COMMANDS} From 7598d3ec710476e986d91c25357079f3fd12e274 Mon Sep 17 00:00:00 2001 From: Andrei Elovikov Date: Mon, 27 Oct 2025 12:01:57 -0700 Subject: [PATCH 7/7] Better track incremental build deps --- sycl-jit/jit-compiler/CMakeLists.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sycl-jit/jit-compiler/CMakeLists.txt b/sycl-jit/jit-compiler/CMakeLists.txt index a24597921ecce..dab40ee9eed58 100644 --- a/sycl-jit/jit-compiler/CMakeLists.txt +++ b/sycl-jit/jit-compiler/CMakeLists.txt @@ -32,9 +32,17 @@ set(SYCL_JIT_RESOURCE_DEPS ${SYCL_JIT_RESOURCE_INSTALL_COMPONENTS}) # OpenCL-Headers doesn't have a corresponding build target: list(FILTER SYCL_JIT_RESOURCE_DEPS EXCLUDE REGEX "^OpenCL-Headers$") +# This is very hacky and I don't quite know what I'm doing, but it's necessary +# to have `resource.cpp` re-generated/re-built when some SYCL header changes. +# +# Inspired by the way `sycl-headers` target is created. +file(GLOB_RECURSE SYCL_JIT_RESOURCE_FILES CONFIGURE_DEPENDS "${SYCL_JIT_RESOURCE_INSTALL_DIR}/*" ) + add_custom_target(rtc-prepare-resources DEPENDS ${SYCL_JIT_RESOURCE_DEPS} ${SYCL_JIT_PREPARE_RESOURCE_COMMANDS} + BYPRODUCTS + ${SYCL_JIT_RESOURCE_FILES} ) add_custom_command( @@ -42,6 +50,8 @@ add_custom_command( COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/utils/generate.py --toolchain-dir ${SYCL_JIT_RESOURCE_INSTALL_DIR} --output ${SYCL_JIT_RESOURCE_CPP} --prefix ${SYCL_JIT_VIRTUAL_TOOLCHAIN_ROOT} DEPENDS rtc-prepare-resources + ${SYCL_JIT_RESOURCE_DEPS} + ${SYCL_JIT_RESOURCE_FILES} ${CMAKE_CURRENT_SOURCE_DIR}/utils/generate.py ) @@ -82,6 +92,8 @@ add_custom_command( ${clang_exe} --target=${LLVM_HOST_TRIPLE} ${SYCL_JIT_RESOURCE_CPP} -I ${CMAKE_CURRENT_SOURCE_DIR}/include -c -o ${SYCL_JIT_RESOURCE_OBJ} ${SYCL_JIT_RESOURCE_CXX_FLAGS} DEPENDS ${SYCL_JIT_RESOURCE_CPP} + ${SYCL_JIT_RESOURCE_DEPS} + ${SYCL_JIT_RESOURCE_FILES} ${CMAKE_CURRENT_SOURCE_DIR}/include/Resource.h )