From ec391a7c47216cf7e75b2e73fe940a34f151505c Mon Sep 17 00:00:00 2001 From: Tim Creech Date: Mon, 11 Jul 2022 14:17:20 -0400 Subject: [PATCH 1/4] [Driver][SYCL] Add support for large device code via a linker script Add -fsycl-huge-device-code as a driver option. When enabled, the driver generates a linker script which directs the linker to place device code later in the binary, which makes it less likely to create a distance larger than that which a PC32 relocation can span. For example, if a 3GB __clang_offload_bundle__ section is placed betwen .text and .rodata, a PC32 relocation to .rodata in the .text is not possible. With this option enabled the device code will be placed after both rather than between them. --- clang/include/clang/Driver/Options.td | 2 + clang/lib/Driver/ToolChains/Gnu.cpp | 49 +++++++++++++++++++ .../sycl-huge-device-code-linker-script.cpp | 12 +++++ 3 files changed, 63 insertions(+) create mode 100644 clang/test/Driver/sycl-huge-device-code-linker-script.cpp diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 6732ba80d02a5..8072d6f64431e 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -2881,6 +2881,8 @@ def fsycl_use_bitcode : Flag<["-"], "fsycl-use-bitcode">, def fno_sycl_use_bitcode : Flag<["-"], "fno-sycl-use-bitcode">, Alias, AliasArgs<["spirv"]>, Flags<[CoreOption]>, HelpText<"Use SPIR-V instead of LLVM bitcode in fat objects">; +def fsycl_huge_device_code : Flag<["-"], "fsycl-huge-device-code">, Flags<[CoreOption]>, + HelpText<"Generate and use a custom linker script for huge device code sections">; def fsycl_link_EQ : Joined<["-"], "fsycl-link=">, Flags<[CC1Option, CoreOption]>, HelpText<"Generate partially linked device and host object to be used at various stages of compilation">, Values<"image,early">; def fsycl_link : Flag<["-"], "fsycl-link">, Alias, diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp index 88f44512d42dc..a303ee41a20aa 100644 --- a/clang/lib/Driver/ToolChains/Gnu.cpp +++ b/clang/lib/Driver/ToolChains/Gnu.cpp @@ -660,6 +660,55 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA, CmdArgs.push_back("-lm"); } + // If requested, use a custom linker script to handle very large device code + // sections. + if (Args.hasArg(options::OPT_fsycl) && + Args.hasArg(options::OPT_fsycl_huge_device_code)) { + // Create temporary linker script. Keep it if save-temps is enabled. + const char *LKS; + SmallString<256> Name = llvm::sys::path::filename(Output.getFilename()); + if (C.getDriver().isSaveTempsEnabled()) { + llvm::sys::path::replace_extension(Name, "ld"); + LKS = C.getArgs().MakeArgString(Name.c_str()); + } else { + llvm::sys::path::replace_extension(Name, ""); + Name = C.getDriver().GetTemporaryPath(Name, "ld"); + LKS = C.addTempFile(C.getArgs().MakeArgString(Name.c_str())); + } + + // Add linker script option to the command. + CmdArgs.push_back("-T"); + CmdArgs.push_back(LKS); + + // If this is not a dry run, create the linker script file. + if (!C.getArgs().hasArg(options::OPT__HASH_HASH_HASH)) { + std::error_code EC; + llvm::raw_fd_ostream ScriptOS(LKS, EC, llvm::sys::fs::OF_None); + + if (EC) { + C.getDriver().Diag(clang::diag::err_unable_to_make_temp) + << EC.message(); + } else { + ScriptOS + << "/*\n" + " * This linker script allows huge (>3GB) device code\n" + " * sections. It has been auto-generated by the SYCL driver.\n" + " */\n" + "SECTIONS\n" + "{\n" + " . = SEGMENT_START(\"sycl-device-code\", .);\n" + " SYCL_DEVICE_CODE ALIGN(CONSTANT (MAXPAGESIZE)) + (. & " + "(CONSTANT (MAXPAGESIZE) - 1)) :\n" + " {\n" + " *(__CLANG_OFFLOAD_BUNDLE__*)\n" + " }\n" + "}\n" + "INSERT AFTER .bss\n"; + ScriptOS.close(); + } + } + } + if (!Args.hasArg(options::OPT_nostdlib, options::OPT_r)) { if (!Args.hasArg(options::OPT_nodefaultlibs)) { if (IsStatic || IsStaticPIE) diff --git a/clang/test/Driver/sycl-huge-device-code-linker-script.cpp b/clang/test/Driver/sycl-huge-device-code-linker-script.cpp new file mode 100644 index 0000000000000..ab68157ff46f4 --- /dev/null +++ b/clang/test/Driver/sycl-huge-device-code-linker-script.cpp @@ -0,0 +1,12 @@ +// UNSUPPORTED: system-windows +// Test the addition of a custom linker script for huge device code. + +// RUN: %clangxx -### -fsycl -fsycl-huge-device-code %s 2>&1 | \ +// RUN: FileCheck --check-prefix=CHECK-LINKER-SCRIPT %s +// CHECK-LINKER-SCRIPT: "-T" "{{.*}}.ld" + +// Also check that a user-provided linker script may be used: +// RUN: %clangxx -### -fsycl -fsycl-huge-device-code %s \ +// RUN: -T custom-user-script.ld 2>&1 | \ +// RUN: FileCheck --check-prefixes=CHECK-USER-SCRIPT %s +// CHECK-USER-SCRIPT: "-T" "custom-user-script.ld" From 6a68ba7fa77e6eb48c8002fec1b6a72e6b97b3ee Mon Sep 17 00:00:00 2001 From: Tim Creech Date: Mon, 15 Aug 2022 15:59:46 -0400 Subject: [PATCH 2/4] Update option name, add docs, add negative option --- clang/include/clang/Driver/Options.td | 8 ++++++-- clang/lib/Driver/ToolChains/Gnu.cpp | 3 ++- clang/test/Driver/sycl-huge-device-code-linker-script.cpp | 4 ++-- sycl/doc/UsersManual.md | 6 ++++++ 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 8072d6f64431e..efec7ad266ede 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -2867,6 +2867,12 @@ defm sycl_instrument_device_code BothFlags<[CC1Option, CoreOption], " Instrumentation and Tracing " "Technology (ITT) instrumentation intrinsics calls " "(experimental)">>; +def fsycl_link_huge_device_code : Flag<["-"], "fsycl-link-huge-device-code">, + Group, Flags<[CoreOption]>, HelpText<"Generate and use a custom" + " linker script for huge device code sections">; +def fno_sycl_link_huge_device_code : Flag<["-"], "fno-sycl-link-huge-device-code">, + Group, Flags<[CoreOption]>, HelpText<"Do not generate or use a" + " custom linker script for huge device code sections (default)">; defm sycl_id_queries_fit_in_int: BoolFOption<"sycl-id-queries-fit-in-int", LangOpts<"SYCLValueFitInMaxInt">, DefaultTrue, PosFlag, NegFlag, @@ -2881,8 +2887,6 @@ def fsycl_use_bitcode : Flag<["-"], "fsycl-use-bitcode">, def fno_sycl_use_bitcode : Flag<["-"], "fno-sycl-use-bitcode">, Alias, AliasArgs<["spirv"]>, Flags<[CoreOption]>, HelpText<"Use SPIR-V instead of LLVM bitcode in fat objects">; -def fsycl_huge_device_code : Flag<["-"], "fsycl-huge-device-code">, Flags<[CoreOption]>, - HelpText<"Generate and use a custom linker script for huge device code sections">; def fsycl_link_EQ : Joined<["-"], "fsycl-link=">, Flags<[CC1Option, CoreOption]>, HelpText<"Generate partially linked device and host object to be used at various stages of compilation">, Values<"image,early">; def fsycl_link : Flag<["-"], "fsycl-link">, Alias, diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp index a303ee41a20aa..3067017cb2485 100644 --- a/clang/lib/Driver/ToolChains/Gnu.cpp +++ b/clang/lib/Driver/ToolChains/Gnu.cpp @@ -663,7 +663,8 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA, // If requested, use a custom linker script to handle very large device code // sections. if (Args.hasArg(options::OPT_fsycl) && - Args.hasArg(options::OPT_fsycl_huge_device_code)) { + Args.hasFlag(options::OPT_fsycl_link_huge_device_code, + options::OPT_fno_sycl_link_huge_device_code, false)) { // Create temporary linker script. Keep it if save-temps is enabled. const char *LKS; SmallString<256> Name = llvm::sys::path::filename(Output.getFilename()); diff --git a/clang/test/Driver/sycl-huge-device-code-linker-script.cpp b/clang/test/Driver/sycl-huge-device-code-linker-script.cpp index ab68157ff46f4..0761fa0cc412a 100644 --- a/clang/test/Driver/sycl-huge-device-code-linker-script.cpp +++ b/clang/test/Driver/sycl-huge-device-code-linker-script.cpp @@ -1,12 +1,12 @@ // UNSUPPORTED: system-windows // Test the addition of a custom linker script for huge device code. -// RUN: %clangxx -### -fsycl -fsycl-huge-device-code %s 2>&1 | \ +// RUN: %clangxx -### -fsycl -fsycl-link-huge-device-code %s 2>&1 | \ // RUN: FileCheck --check-prefix=CHECK-LINKER-SCRIPT %s // CHECK-LINKER-SCRIPT: "-T" "{{.*}}.ld" // Also check that a user-provided linker script may be used: -// RUN: %clangxx -### -fsycl -fsycl-huge-device-code %s \ +// RUN: %clangxx -### -fsycl -fsycl-link-huge-device-code %s \ // RUN: -T custom-user-script.ld 2>&1 | \ // RUN: FileCheck --check-prefixes=CHECK-USER-SCRIPT %s // CHECK-USER-SCRIPT: "-T" "custom-user-script.ld" diff --git a/sycl/doc/UsersManual.md b/sycl/doc/UsersManual.md index 0a089f61ca472..468c47596210a 100644 --- a/sycl/doc/UsersManual.md +++ b/sycl/doc/UsersManual.md @@ -217,6 +217,12 @@ and not recommended to use in production environment. various events inside JIT generated kernels. These device libraries are linked in by default. +**`-f[no-]sycl-link-huge-device-code`** + + Place device code later in the linked binary in order to avoid precluding + 32-bit PC relative relocations between surrounding ELF sections when device + code is larger than 2GiB. This is disabled by default. + ## Intel FPGA specific options **`-fintelfpga`** From 77c2ac154639611c8b8b8f91068eb9e0beae7142 Mon Sep 17 00:00:00 2001 From: Tim Creech Date: Tue, 16 Aug 2022 16:03:52 -0400 Subject: [PATCH 3/4] fixup: don't declare as a CoreOption --- clang/include/clang/Driver/Options.td | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index efec7ad266ede..aa96b0e8b31d5 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -2868,11 +2868,11 @@ defm sycl_instrument_device_code "Technology (ITT) instrumentation intrinsics calls " "(experimental)">>; def fsycl_link_huge_device_code : Flag<["-"], "fsycl-link-huge-device-code">, - Group, Flags<[CoreOption]>, HelpText<"Generate and use a custom" - " linker script for huge device code sections">; + Group, HelpText<"Generate and use a custom linker script for huge" + " device code sections">; def fno_sycl_link_huge_device_code : Flag<["-"], "fno-sycl-link-huge-device-code">, - Group, Flags<[CoreOption]>, HelpText<"Do not generate or use a" - " custom linker script for huge device code sections (default)">; + Group, HelpText<"Do not generate or use a custom linker script" + " for huge device code sections (default)">; defm sycl_id_queries_fit_in_int: BoolFOption<"sycl-id-queries-fit-in-int", LangOpts<"SYCLValueFitInMaxInt">, DefaultTrue, PosFlag, NegFlag, From 2ecdfc51d4d578527d14ffcad3f4c95e2193c1e7 Mon Sep 17 00:00:00 2001 From: Tim Creech Date: Mon, 22 Aug 2022 13:03:19 -0400 Subject: [PATCH 4/4] fixup: document Linux-only support --- sycl/doc/UsersManual.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sycl/doc/UsersManual.md b/sycl/doc/UsersManual.md index 468c47596210a..60c295d503dc2 100644 --- a/sycl/doc/UsersManual.md +++ b/sycl/doc/UsersManual.md @@ -223,6 +223,8 @@ and not recommended to use in production environment. 32-bit PC relative relocations between surrounding ELF sections when device code is larger than 2GiB. This is disabled by default. + NOTE: This option is currently only supported on Linux. + ## Intel FPGA specific options **`-fintelfpga`**