Merge pull request #80 from CHIP-SPV/cuspv

direct CUDA compilation support

CMakeLists.txt

@@ -90,6 +90,10 @@ if((CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang") OR
     message(FATAL_ERROR "this project requires clang >= 8.0")
   endif()
 
+  if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16.0.0)
+    set(CLANG_VERSION_LESS_16 ON)
+  endif()
+
   if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 15.0.0)
     set(CLANG_VERSION_LESS_15 ON)
   endif()
@@ -99,6 +103,11 @@ if((CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang") OR
             Support for Clang < 14.0 will be discontinued in the future.")
     set(CLANG_VERSION_LESS_14 ON)
   endif()
+
+  if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 13.0.0)
+    set(CLANG_VERSION_LESS_13 ON)
+  endif()
+
 else()
   message(FATAL_ERROR "this project must be compiled with clang. CMAKE_CXX_COMPILER_ID = ${CMAKE_CXX_COMPILER_ID}")
 endif()
@@ -199,14 +208,18 @@ add_subdirectory(bitcode)
 
 enable_testing()
 add_subdirectory(HIP/tests/catch catch)
-
+add_subdirectory(tests/cuda)
 add_subdirectory(./samples samples)
 
 # Make CHIP depend on devicelib_bc and LLVMHipPasses for
 # convenience. The CHIP module itself does not depend on these but
 # HIP program compilation does.
 add_dependencies(CHIP devicelib_bc LLVMHipPasses)
 add_dependencies(samples CHIP)
+
+# CHIP binaries
+add_subdirectory(bin)
+
 # CHIP-SPV CMAKE DEPENDENCIES
 ###############################################################################
 
@@ -223,7 +236,10 @@ target_link_libraries(CHIP INTERFACE ${OpenCL_LIBRARY} ${LevelZero_LIBRARY} ${PT
 target_include_directories(CHIP
   PUBLIC
   "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
-  "$<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/HIP/include;${CMAKE_SOURCE_DIR}/include>"
+  "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/cuspv>"
+  "$<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/HIP/include>"
+  "$<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/include>"
+  "$<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/include/cuspv>"
   PRIVATE
   "${CMAKE_SOURCE_DIR}/src"
   "${CMAKE_SOURCE_DIR}/include"
@@ -262,7 +278,7 @@ set(HIP_OFFLOAD_COMPILE_OPTIONS_
   ${HIP_LINK_DEVICELIB_INSTALL})
 
 # HIP applicaitons need to link against libCHIP.so; add it to rpath
-list(APPEND HIP_OFFLOAD_LINK_OPTIONS_  "-L${LIB_INSTALL_DIR} -lCHIP")
+list(APPEND HIP_OFFLOAD_LINK_OPTIONS_  "-L${LIB_INSTALL_DIR}" "-lCHIP")
 list(APPEND HIP_OFFLOAD_LINK_OPTIONS_  "-lOpenCL")
 list(APPEND HIP_OFFLOAD_LINK_OPTIONS_  "-lze_loader")
 if(SET_RPATH)
@@ -283,18 +299,15 @@ add_library(deviceInternal INTERFACE)
 target_compile_options(deviceInternal INTERFACE 
   ${HIP_ENABLE_SPIRV} 
   ${HIP_OFFLOAD_COMPILE_ONLY_OPTIONS_}
-  ${HIP_OFFLOAD_COMPILE_AND_LINK_OPTIONS_} 
   ${HIP_LINK_DEVICELIB_BUILD})
 target_link_options(deviceInternal INTERFACE 
-  ${HIP_LINK_DEVICELIB_BUILD}
-  ${HIP_OFFLOAD_LINK_OPTIONS})
+  ${HIP_LINK_DEVICELIB_BUILD})
 target_link_libraries(deviceInternal INTERFACE CHIP)
 
 add_library(device INTERFACE)
 target_compile_options(device INTERFACE 
   ${HIP_ENABLE_SPIRV} 
   ${HIP_OFFLOAD_COMPILE_ONLY_OPTIONS_}
-  ${HIP_OFFLOAD_COMPILE_AND_LINK_OPTIONS_} 
   ${HIP_LINK_DEVICELIB_INSTALL})
 target_link_options(device INTERFACE 
   ${HIP_LINK_DEVICELIB_INSTALL})
@@ -399,4 +412,4 @@ install(EXPORT CHIPTargets
   DESTINATION cmake/CHIP)
 
 # CHIP-SPV INSTALLATION AND PACKAGING
-###############################################################################
+###############################################################################

README.md

@@ -105,6 +105,10 @@ If you do not provide this value, `hipcc` will check for existance of the follow
 /opt/rocm
 ```
 
+### Compiling CUDA application directly with CHIP-SPV
+
+Compilation of CUDA sources without changing the sources, can be done in two ways. The first is to replace calls of the nvcc compiler with calls of the wrapper script <CHIP-install-path>/bin/cuspv in Makefiles. The wrapper script will call clang with the correct flags. The other way is possible when using CMake: use `find_package(HIP REQUIRED CONFIG)` and then use `target_link_libraries(<YOUR_TARGET> hip::CHIP hip::host hip::device)`. However the project must be compiled with Clang (a version supported by HIP). Note that it's not necessary to have Nvidia's CUDA installed.
+
 ## Compiling a HIP application using CHIP-SPV
 
 Compiling a HIP application with CHIP-SPV will allow you to execute HIP code on any device that supports SPIR-V, such as Intel GPUs. To compile a HIP application, all you need to do is to use the `hipcc` compiler wrapper provided by this project. In case you have AMD implementation installed as well, you can switch between them by using `HIP_PLATFORM` environment variable.

bin/CMakeLists.txt

@@ -0,0 +1,15 @@
+# Read includes, prepend each one with -I.
+set(prop "$<TARGET_PROPERTY:CHIP,INCLUDE_DIRECTORIES>")
+set(CHIP_INCLUDES "$<$<BOOL:${prop}>:-I$<JOIN:${prop}, -I>>")
+
+# Generate cuspvc tool that is usable within build directory.
+configure_file(cuspvc-build.in cuspvc-build.in @ONLY)
+file(
+  GENERATE
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/cuspvc
+  INPUT ${CMAKE_CURRENT_BINARY_DIR}/cuspvc-build.in)
+
+# Generate cuspv tool that will be installed.
+configure_file(cuspvc-install.in cuspvc-install @ONLY)
+install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/cuspvc-install
+  DESTINATION bin RENAME cuspvc)

bin/cuspvc-build.in

@@ -0,0 +1,10 @@
+#!/bin/bash
+# CUDA-SPIRV compiler - A compiler wrapper for compiling CUDA sources directly.
+#
+# CUDA sources are compiled in HIP mode with an include search path to
+# cuda_runtime.h wrapper which translates CUDA API to HIP API.
+#
+# NOTE: this file is not meant only to be usable in CHIP-SPV build directory.
+export HIP_PLATFORM=spirv
+export PATH=@CMAKE_BINARY_DIR@/bin:$PATH
+hipcc @CHIP_INCLUDES@ "$@"

bin/cuspvc-install.in

@@ -0,0 +1,9 @@
+#!/bin/bash
+# CUDA-SPIRV compiler - A compiler wrapper for compiling CUDA sources directly.
+#
+# CUDA sources are compiled in HIP mode with an include search path to
+# cuda_runtime.h wrapper which translates CUDA API to HIP API.
+export HIP_PLATFORM=spirv
+# Add path for locating hipcc.
+export PATH=@CMAKE_INSTALL_PREFIX@/bin:$PATH
+hipcc -I@CMAKE_INSTALL_PREFIX@/include/cuspv "$@"

bitcode/CMakeLists.txt

@@ -1,15 +1,15 @@
 
 file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/BC")
 
+set(EXTRA_FLAGS)
 # Ugly fix for interactions between clang13+ and igc
-if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 13)
-  set(CLANG_CL_NO_STDINC_FLAG "")
-else()
-  set(CLANG_CL_NO_STDINC_FLAG "-cl-no-stdinc")
+if(NOT CLANG_VERSION_LESS_13)
+  list(APPEND EXTRA_FLAGS "-cl-no-stdinc")
 endif()
 
-if(NOT DEFINED LLVM_VERSION)
-  message(FATAL "Could not determine LLVM version.")
+# disable opaque pointers for LLVM 15 only
+if(NOT CLANG_VERSION_LESS_15 AND CLANG_VERSION_LESS_16)
+  list(APPEND EXTRA_FLAGS ${DISABLE_OPAQUE_PTRS_OPT})
 endif()
 
 if("${LLVM_VERSION}" VERSION_LESS 14.0)
@@ -25,42 +25,29 @@ else()
   set(BC_DESTINATION lib/hip-device-lib)
 endif()
 
-set(BITCODE_COMPILE_FLAGS "${CLANG_CL_NO_STDINC_FLAG}"
+set(BITCODE_COMPILE_FLAGS
   -Xclang -finclude-default-header -O2 -x cl -cl-std=CL2.0
-  --target=${BC_TRIPLE} -emit-llvm ${DISABLE_OPAQUE_PTRS_OPT})
-
-add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/BC/devicelib.bc"
-  DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/devicelib.cl"
-  COMMAND "${CMAKE_CXX_COMPILER}" ${BITCODE_COMPILE_FLAGS}
-  -o "${CMAKE_CURRENT_BINARY_DIR}/BC/devicelib.bc"
-  -c "${CMAKE_CURRENT_SOURCE_DIR}/devicelib.cl"
-  COMMENT "Building devicelib.bc"
-  VERBATIM)
-list(APPEND DEPEND_LIST "${CMAKE_CURRENT_BINARY_DIR}/BC/devicelib.bc")
+  --target=${BC_TRIPLE} -emit-llvm ${EXTRA_FLAGS})
 
-# Support function(s) for printf().
-add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/BC/printf_support.bc"
-  DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/_cl_print_str.cl"
-  COMMAND "${CMAKE_CXX_COMPILER}" ${BITCODE_COMPILE_FLAGS}
-  -o "${CMAKE_CURRENT_BINARY_DIR}/BC/printf_support.bc"
-  -c "${CMAKE_CURRENT_SOURCE_DIR}/_cl_print_str.cl"
-  COMMENT "Building printf_support.bc"
-  VERBATIM)
-list(APPEND DEPEND_LIST "${CMAKE_CURRENT_BINARY_DIR}/BC/printf_support.bc")
+# non-OCML sources
+set(NON_OCML_SOURCES "devicelib" "_cl_print_str" "texture") # "printf_support"
 
-add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/BC/texture.bc"
-  DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/texture.cl"
-  COMMAND "${CMAKE_CXX_COMPILER}" ${BITCODE_COMPILE_FLAGS}
-  -o "${CMAKE_CURRENT_BINARY_DIR}/BC/texture.bc"
-  -c "${CMAKE_CURRENT_SOURCE_DIR}/texture.cl"
-  COMMENT "Building texture.bc"
-  VERBATIM)
-list(APPEND DEPEND_LIST "${CMAKE_CURRENT_BINARY_DIR}/BC/texture.bc")
+foreach(SOURCE IN LISTS NON_OCML_SOURCES)
+  add_custom_command(
+    OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/BC/${SOURCE}.bc"
+    DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE}.cl"
+    COMMAND "${CMAKE_CXX_COMPILER}" ${BITCODE_COMPILE_FLAGS}
+    -o "${CMAKE_CURRENT_BINARY_DIR}/BC/${SOURCE}.bc"
+    -c "${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE}.cl"
+    COMMENT "Building ${SOURCE}.bc"
+    VERBATIM)
+  list(APPEND DEPEND_LIST "${CMAKE_CURRENT_BINARY_DIR}/BC/${SOURCE}.bc")
+endforeach()
 
-# devicelib sources
-set(SOURCES erfcinvD erfcinvF erfcxD erfcxF erfinvD erfinvF i0D i0F i1D i1F j0D j0F j1D j1F ncdfD ncdfF ncdfinvD ncdfinvF nearbyintD nearbyintF rcbrtD rcbrtF rhypotF rhypotD rlen3D rlen3F rlen4D rlen4F scalbD scalbF scalbnD scalbnF tables y0D y0F y1D y1F)
+# OCML sources
+set(OCML_SOURCES erfcinvD erfcinvF erfcxD erfcxF erfinvD erfinvF i0D i0F i1D i1F j0D j0F j1D j1F ncdfD ncdfF ncdfinvD ncdfinvF nearbyintD nearbyintF rcbrtD rcbrtF rhypotF rhypotD rlen3D rlen3F rlen4D rlen4F scalbD scalbF scalbnD scalbnF tables y0D y0F y1D y1F)
 
-foreach(SOURCE IN LISTS SOURCES)
+foreach(SOURCE IN LISTS OCML_SOURCES)
   add_custom_command(
     OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/BC/${SOURCE}.bc"
     DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/OCML/${SOURCE}.cl"

bitcode/devicelib.cl

@@ -769,9 +769,6 @@ EXPORT float CL_NAME2(shfl_down, f)(float var, uint delta) {
   return intel_sub_group_shuffle_down(var, tmp2, delta);
 };
 
-
-
-
 typedef struct {
   intptr_t  image;
   intptr_t  sampler;
@@ -783,4 +780,4 @@ EXPORT float CL_NAME2(tex2D, f)(hipTextureObject_t textureObject,
     __builtin_astype(textureObject->image, read_only image2d_t),
     __builtin_astype(textureObject->sampler, sampler_t),
     (float2)(x, y)).x;
-}
+}

include/cuspv/cuda.h

@@ -0,0 +1,4 @@
+// in CUDA, this header has the driver API definitions.
+// we include both driver & runtime in the same header
+
+#include <cuda_runtime.h>