diff --git a/bolt/docs/CommandLineArgumentReference.md b/bolt/docs/CommandLineArgumentReference.md index 7c6e01d669b74..e6b908416308a 100644 --- a/bolt/docs/CommandLineArgumentReference.md +++ b/bolt/docs/CommandLineArgumentReference.md @@ -811,6 +811,15 @@ Specify file name of the runtime instrumentation library +- `--runtime-lib-init-hook=` + + Primary target for hooking runtime library initialization, used in + fallback order of availabiliy in input binary (entry-point -> init + -> init_array) (default: entry_point) + - `entry_point`: use ELF Header Entry Point + - `init`: use ELF DT_INIT entry + - `init_array`: use ELF 1st entry of .init_array + - `--sctc-mode=` Mode for simplify conditional tail calls diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h index 2af1d330b7545..8a90febcea3cc 100644 --- a/bolt/include/bolt/Core/BinaryContext.h +++ b/bolt/include/bolt/Core/BinaryContext.h @@ -807,6 +807,15 @@ class BinaryContext { /// the execution of the binary is completed. std::optional FiniFunctionAddress; + /// DT_INIT. + std::optional InitAddress; + + /// DT_INIT_ARRAY. Only used when DT_INIT is not set. + std::optional InitArrayAddress; + + /// DT_INIT_ARRAYSZ. Only used when DT_INIT is not set. + std::optional InitArraySize; + /// DT_FINI. std::optional FiniAddress; diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h index 0fe2e32b61933..3e5cfde0c4191 100644 --- a/bolt/include/bolt/Rewrite/RewriteInstance.h +++ b/bolt/include/bolt/Rewrite/RewriteInstance.h @@ -93,14 +93,23 @@ class RewriteInstance { /// section allocations if found. void discoverBOLTReserved(); + /// Check whether we should use DT_INIT or DT_INIT_ARRAY for instrumentation. + /// DT_INIT is preferred; DT_INIT_ARRAY is only used when no DT_INIT entry was + /// found. + Error discoverRtInitAddress(); + /// Check whether we should use DT_FINI or DT_FINI_ARRAY for instrumentation. /// DT_FINI is preferred; DT_FINI_ARRAY is only used when no DT_FINI entry was /// found. Error discoverRtFiniAddress(); + /// If DT_INIT_ARRAY is used for instrumentation, update the relocation of its + /// first entry to point to the instrumentation library's init address. + Error updateRtInitReloc(); + /// If DT_FINI_ARRAY is used for instrumentation, update the relocation of its /// first entry to point to the instrumentation library's fini address. - void updateRtFiniReloc(); + Error updateRtFiniReloc(); /// Create and initialize metadata rewriters for this instance. void initializeMetadataManager(); diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp index 5769577aa3f74..2c9a37c9dd51b 100644 --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -80,6 +80,7 @@ namespace opts { extern cl::list HotTextMoveSections; extern cl::opt Hugify; extern cl::opt Instrument; +extern cl::opt InstrumentationSleepTime; extern cl::opt KeepNops; extern cl::opt Lite; extern cl::list ReorderData; @@ -292,6 +293,28 @@ cl::bits GadgetScannersToRun( clEnumValN(GS_ALL, "all", "All implemented scanners")), cl::ZeroOrMore, cl::CommaSeparated, cl::cat(BinaryAnalysisCategory)); +// Primary targets for hooking runtime library initialization hooking +// with fallback to next item in case if current item is not available +// in the input binary. +enum RuntimeLibInitHookTarget : char { + RLIH_ENTRY_POINT = 0, /// Use ELF Header Entry Point + RLIH_INIT = 1, /// Use ELF DT_INIT entry + RLIH_INIT_ARRAY = 2, /// Use ELF 1st entry of .init_array +}; + +cl::opt RuntimeLibInitHook( + "runtime-lib-init-hook", + cl::desc("Primary target for hooking runtime library initialization, used " + "in fallback order of availabiliy in input binary (entry-point -> " + "init -> init_array) (default: entry_point)"), + cl::init(RLIH_ENTRY_POINT), + cl::values(clEnumValN(RLIH_ENTRY_POINT, "entry_point", + "use ELF Header Entry Point"), + clEnumValN(RLIH_INIT, "init", "use ELF DT_INIT entry"), + clEnumValN(RLIH_INIT_ARRAY, "init_array", + "use ELF 1st entry of .init_array")), + cl::ZeroOrMore, cl::cat(BoltOptCategory)); + } // namespace opts // FIXME: implement a better way to mark sections for replacement. @@ -737,9 +760,12 @@ Error RewriteInstance::run() { adjustCommandLineOptions(); discoverFileObjects(); - if (opts::Instrument && !BC->IsStaticExecutable) + if (opts::Instrument && !BC->IsStaticExecutable) { + if (Error E = discoverRtInitAddress()) + return E; if (Error E = discoverRtFiniAddress()) return E; + } preprocessProfileData(); @@ -781,8 +807,12 @@ Error RewriteInstance::run() { updateMetadata(); - if (opts::Instrument && !BC->IsStaticExecutable) - updateRtFiniReloc(); + if (opts::Instrument && !BC->IsStaticExecutable) { + if (Error E = updateRtInitReloc()) + return E; + if (Error E = updateRtFiniReloc()) + return E; + } if (opts::OutputFilename == "/dev/null") { BC->outs() << "BOLT-INFO: skipping writing final binary to disk\n"; @@ -1407,6 +1437,60 @@ void RewriteInstance::discoverBOLTReserved() { NextAvailableAddress = BC->BOLTReserved.start(); } +Error RewriteInstance::discoverRtInitAddress() { + if (BC->HasInterpHeader && opts::RuntimeLibInitHook == opts::RLIH_ENTRY_POINT) + return Error::success(); + + // Use DT_INIT if it's available. + if (BC->InitAddress && opts::RuntimeLibInitHook <= opts::RLIH_INIT) { + BC->StartFunctionAddress = BC->InitAddress; + return Error::success(); + } + + if (!BC->InitArrayAddress || !BC->InitArraySize) { + return createStringError(std::errc::not_supported, + "Instrumentation of shared library needs either " + "DT_INIT or DT_INIT_ARRAY"); + } + + if (*BC->InitArraySize < BC->AsmInfo->getCodePointerSize()) { + return createStringError(std::errc::not_supported, + "Need at least 1 DT_INIT_ARRAY slot"); + } + + ErrorOr InitArraySection = + BC->getSectionForAddress(*BC->InitArrayAddress); + if (auto EC = InitArraySection.getError()) + return errorCodeToError(EC); + + if (const Relocation *Reloc = InitArraySection->getDynamicRelocationAt(0)) { + if (Reloc->isRelative()) { + BC->StartFunctionAddress = Reloc->Addend; + } else { + MCSymbol *Sym = Reloc->Symbol; + if (!Sym) + return createStringError( + std::errc::not_supported, + "Failed to locate symbol for 0 entry of .init_array"); + const BinaryFunction *BF = BC->getFunctionForSymbol(Sym); + if (!BF) + return createStringError( + std::errc::not_supported, + "Failed to locate binary function for 0 entry of .init_array"); + BC->StartFunctionAddress = BF->getAddress() + Reloc->Addend; + } + return Error::success(); + } + + if (const Relocation *Reloc = InitArraySection->getRelocationAt(0)) { + BC->StartFunctionAddress = Reloc->Value; + return Error::success(); + } + + return createStringError(std::errc::not_supported, + "No relocation for first DT_INIT_ARRAY slot"); +} + Error RewriteInstance::discoverRtFiniAddress() { // Use DT_FINI if it's available. if (BC->FiniAddress) { @@ -1415,6 +1499,10 @@ Error RewriteInstance::discoverRtFiniAddress() { } if (!BC->FiniArrayAddress || !BC->FiniArraySize) { + // It is still possible to generate profile without fini hook if + // InstrumentationSleepTime is set + if (opts::InstrumentationSleepTime > 0) + return Error::success(); return createStringError( std::errc::not_supported, "Instrumentation needs either DT_FINI or DT_FINI_ARRAY"); @@ -1444,26 +1532,98 @@ Error RewriteInstance::discoverRtFiniAddress() { "No relocation for first DT_FINI_ARRAY slot"); } -void RewriteInstance::updateRtFiniReloc() { +Error RewriteInstance::updateRtInitReloc() { + if (BC->HasInterpHeader && opts::RuntimeLibInitHook == opts::RLIH_ENTRY_POINT) + return Error::success(); + + // Updating DT_INIT is handled by patchELFDynamic. + if (BC->InitAddress && opts::RuntimeLibInitHook <= opts::RLIH_INIT) + return Error::success(); + + const RuntimeLibrary *RT = BC->getRuntimeLibrary(); + if (!RT || !RT->getRuntimeStartAddress()) + return Error::success(); + + if (!BC->InitArrayAddress) + return Error::success(); + + if (!BC->InitArrayAddress || !BC->InitArraySize) + return createStringError(std::errc::not_supported, + "inconsistent .init_array state"); + + ErrorOr InitArraySection = + BC->getSectionForAddress(*BC->InitArrayAddress); + if (!InitArraySection) + return createStringError(std::errc::not_supported, ".init_array removed"); + + if (std::optional Reloc = + InitArraySection->takeDynamicRelocationAt(0)) { + if (Reloc->isRelative()) { + if (Reloc->Addend != BC->StartFunctionAddress) + return createStringError(std::errc::not_supported, + "inconsistent .init_array dynamic relocation"); + Reloc->Addend = RT->getRuntimeStartAddress(); + InitArraySection->addDynamicRelocation(*Reloc); + } else { + MCSymbol *Sym = Reloc->Symbol; + if (!Sym) + return createStringError( + std::errc::not_supported, + "Failed to locate symbol for 0 entry of .init_array"); + const BinaryFunction *BF = BC->getFunctionForSymbol(Sym); + if (!BF) + return createStringError( + std::errc::not_supported, + "Failed to locate binary function for 0 entry of .init_array"); + if (BF->getAddress() + Reloc->Addend != BC->StartFunctionAddress) + return createStringError(std::errc::not_supported, + "inconsistent .init_array dynamic relocation"); + InitArraySection->addDynamicRelocation(Relocation{ + /*Offset*/ 0, /*Symbol*/ nullptr, /*Type*/ Relocation::getAbs64(), + /*Addend*/ RT->getRuntimeStartAddress(), /*Value*/ 0}); + } + } + // Update the static relocation by adding a pending relocation which will get + // patched when flushPendingRelocations is called in rewriteFile. Note that + // flushPendingRelocations will calculate the value to patch as + // "Symbol + Addend". Since we don't have a symbol, just set the addend to the + // desired value. + InitArraySection->addPendingRelocation(Relocation{ + /*Offset*/ 0, /*Symbol*/ nullptr, /*Type*/ Relocation::getAbs64(), + /*Addend*/ RT->getRuntimeStartAddress(), /*Value*/ 0}); + return Error::success(); +} + +Error RewriteInstance::updateRtFiniReloc() { // Updating DT_FINI is handled by patchELFDynamic. if (BC->FiniAddress) - return; + return Error::success(); const RuntimeLibrary *RT = BC->getRuntimeLibrary(); if (!RT || !RT->getRuntimeFiniAddress()) - return; + return Error::success(); + + // It is still possible to generate profile without fini hook if + // InstrumentationSleepTime is set + if ((!BC->FiniArrayAddress || !BC->FiniArraySize) && + opts::InstrumentationSleepTime > 0) { + return Error::success(); + } - assert(BC->FiniArrayAddress && BC->FiniArraySize && - "inconsistent .fini_array state"); + if (!BC->FiniArrayAddress || !BC->FiniArraySize) + return createStringError(std::errc::not_supported, + "inconsistent .fini_array state"); ErrorOr FiniArraySection = BC->getSectionForAddress(*BC->FiniArrayAddress); - assert(FiniArraySection && ".fini_array removed"); + if (!FiniArraySection) + return createStringError(std::errc::not_supported, ".fini_array removed"); if (std::optional Reloc = FiniArraySection->takeDynamicRelocationAt(0)) { - assert(Reloc->Addend == BC->FiniFunctionAddress && - "inconsistent .fini_array dynamic relocation"); + if (Reloc->Addend != BC->FiniFunctionAddress) + return createStringError(std::errc::not_supported, + "inconsistent .fini_array dynamic relocation"); Reloc->Addend = RT->getRuntimeFiniAddress(); FiniArraySection->addDynamicRelocation(*Reloc); } @@ -1476,6 +1636,7 @@ void RewriteInstance::updateRtFiniReloc() { FiniArraySection->addPendingRelocation(Relocation{ /*Offset*/ 0, /*Symbol*/ nullptr, /*Type*/ Relocation::getAbs64(), /*Addend*/ RT->getRuntimeFiniAddress(), /*Value*/ 0}); + return Error::success(); } void RewriteInstance::registerFragments() { @@ -4838,7 +4999,8 @@ void RewriteInstance::patchELFSectionHeaderTable(ELFObjectFile *File) { ELFEhdrTy NewEhdr = Obj.getHeader(); if (BC->HasRelocations) { - if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary()) + RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary(); + if (RtLibrary && opts::RuntimeLibInitHook == opts::RLIH_ENTRY_POINT) NewEhdr.e_entry = RtLibrary->getRuntimeStartAddress(); else NewEhdr.e_entry = getNewFunctionAddress(NewEhdr.e_entry); @@ -5684,7 +5846,9 @@ void RewriteInstance::patchELFDynamic(ELFObjectFile *File) { if (uint64_t Addr = RtLibrary->getRuntimeFiniAddress()) NewDE.d_un.d_ptr = Addr; } - if (RtLibrary && Dyn.getTag() == ELF::DT_INIT && !BC->HasInterpHeader) { + if (RtLibrary && Dyn.getTag() == ELF::DT_INIT && + (!BC->HasInterpHeader || + opts::RuntimeLibInitHook == opts::RLIH_INIT)) { if (auto Addr = RtLibrary->getRuntimeStartAddress()) { LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Set DT_INIT to 0x" << Twine::utohexstr(Addr) << '\n'); @@ -5760,6 +5924,13 @@ Error RewriteInstance::readELFDynamic(ELFObjectFile *File) { LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Set start function address\n"); BC->StartFunctionAddress = Dyn.getPtr(); } + BC->InitAddress = Dyn.getPtr(); + break; + case ELF::DT_INIT_ARRAY: + BC->InitArrayAddress = Dyn.getPtr(); + break; + case ELF::DT_INIT_ARRAYSZ: + BC->InitArraySize = Dyn.getPtr(); break; case ELF::DT_FINI: BC->FiniAddress = Dyn.getPtr(); diff --git a/bolt/test/AArch64/hook-init.s b/bolt/test/AArch64/hook-init.s new file mode 100644 index 0000000000000..e5b11ecae57cc --- /dev/null +++ b/bolt/test/AArch64/hook-init.s @@ -0,0 +1,206 @@ +## Test the different ways of hooking the init function for instrumentation (via +## entry point, DT_INIT and via DT_INIT_ARRAY). We test the latter for both PIE +## and non-PIE binaries because of the different ways of handling relocations +## (static or dynamic), executable and shared library. +## All tests perform the following steps: +## - Compile and link for the case to be tested +## - Some sanity-checks on the dynamic section and relocations in the binary to +## verify it has the shape we want for testing: +## - INTERP in Program Headers +## - DT_INIT or DT_INIT_ARRAY in dynamic section +## - No relative relocations for non-PIE +## - Instrument (with extra --runtime-lib-init-hook=init/init_array options +## in some cases) +## - Verify generated binary +# REQUIRES: system-linux,bolt-runtime,target=aarch64{{.*}} + +# RUN: %clang %cflags -pie %s -Wl,-q -o %t.exe +# RUN: llvm-readelf -d %t.exe | FileCheck --check-prefix=DYN-INIT %s +# RUN: llvm-readelf -l %t.exe | FileCheck --check-prefix=PH-INTERP %s +# RUN: llvm-readelf -r %t.exe | FileCheck --check-prefix=RELOC-PIE %s +# RUN: llvm-bolt %t.exe -o %t --instrument +# RUN: llvm-readelf -hdrs %t | FileCheck --check-prefix=CHECK-INIT-EP %s +# RUN: llvm-bolt %t.exe -o %t-no-ep --instrument --runtime-lib-init-hook=init +# RUN: llvm-readelf -hdrs %t-no-ep | FileCheck --check-prefix=CHECK-INIT-NO-EP %s +# RUN: llvm-bolt %t.exe -o %t-no-ep --instrument --runtime-lib-init-hook=init_array +# RUN: llvm-readelf -hdrs %t-no-ep | FileCheck --check-prefix=CHECK-INIT-ARRAY-NO-EP %s + +# RUN: %clang -shared %cflags -pie %s -Wl,-q -o %t-shared.exe +# RUN: llvm-readelf -d %t-shared.exe | FileCheck --check-prefix=DYN-INIT %s +# RUN: llvm-readelf -l %t-shared.exe | FileCheck --check-prefix=PH-INTERP-SHARED %s +# RUN: llvm-readelf -r %t-shared.exe | FileCheck --check-prefix=RELOC-SHARED-PIE %s +# RUN: llvm-bolt %t-shared.exe -o %t-shared --instrument +# RUN: llvm-readelf -hdrs %t-shared | FileCheck --check-prefix=CHECK-SHARED-INIT %s + +# RUN: %clang %cflags -pie %s -Wl,-q,-init=0 -o %t-no-init.exe +# RUN: llvm-readelf -d %t-no-init.exe | FileCheck --check-prefix=DYN-NO-INIT %s +# RUN: llvm-readelf -l %t-no-init.exe | FileCheck --check-prefix=PH-INTERP %s +# RUN: llvm-readelf -r %t-no-init.exe | FileCheck --check-prefix=RELOC-PIE %s +# RUN: llvm-bolt %t-no-init.exe -o %t-no-init --instrument +# RUN: llvm-readelf -hdrs %t-no-init | FileCheck --check-prefix=CHECK-NO-INIT-EP %s +# RUN: llvm-bolt %t-no-init.exe -o %t-no-init-no-ep --instrument --runtime-lib-init-hook=init +# RUN: llvm-readelf -hdrs %t-no-init-no-ep | FileCheck --check-prefix=CHECK-NO-INIT-NO-EP %s + +# RUN: %clang -shared %cflags -pie %s -Wl,-q,-init=0 -o %t-shared-no-init.exe +# RUN: llvm-readelf -d %t-shared-no-init.exe | FileCheck --check-prefix=DYN-NO-INIT %s +# RUN: llvm-readelf -l %t-shared-no-init.exe | FileCheck --check-prefix=PH-INTERP-SHARED %s +# RUN: llvm-readelf -r %t-shared-no-init.exe | FileCheck --check-prefix=RELOC-SHARED-PIE %s +# RUN: llvm-bolt %t-shared-no-init.exe -o %t-shared-no-init --instrument +# RUN: llvm-readelf -drs %t-shared-no-init | FileCheck --check-prefix=CHECK-SHARED-NO-INIT %s + +## Create a dummy shared library to link against to force creation of the dynamic section. +# RUN: %clang %cflags %p/../Inputs/stub.c -fPIC -shared -o %t-stub.so +# RUN: %clang %cflags %s -no-pie -Wl,-q,-init=0 %t-stub.so -o %t-no-pie-no-init.exe +# RUN: llvm-readelf -r %t-no-pie-no-init.exe | FileCheck --check-prefix=RELOC-NO-PIE %s +# RUN: llvm-bolt %t-no-pie-no-init.exe -o %t-no-pie-no-init --instrument +# RUN: llvm-readelf -hds %t-no-pie-no-init | FileCheck --check-prefix=CHECK-NO-PIE-NO-INIT-EP %s + +## With init: dynamic section should contain DT_INIT +# DYN-INIT: (INIT) + +## Without init: dynamic section should only contain DT_INIT_ARRAY +# DYN-NO-INIT-NOT: (INIT) +# DYN-NO-INIT: (INIT_ARRAY) +# DYN-NO-INIT: (INIT_ARRAYSZ) + +## With interp program header (executable) +# PH-INTERP: Program Headers: +# PH-INTERP: INTERP + +## Without interp program header (shared library) +# PH-INTERP-SHARED: Program Headers: +# PH-INTERP-SHARED-NOT: INTERP + +## With PIE: binary should have relative relocations +# RELOC-PIE: R_AARCH64_RELATIVE + +## With PIE: binary should have relative relocations +# RELOC-SHARED-PIE: R_AARCH64_ABS64 + +## Without PIE: binary should not have relative relocations +# RELOC-NO-PIE-NOT: R_AARCH64_RELATIVE + +## Check that entry point address is set to __bolt_runtime_start for PIE executable with DT_INIT +# CHECK-INIT-EP: ELF Header: +# CHECK-INIT-EP: Entry point address: 0x[[#%x,EP_ADDR:]] +## Check that the dynamic relocation at .init and .init_array were not patched +# CHECK-INIT-EP: Dynamic section at offset {{.*}} contains {{.*}} entries: +# CHECK-INIT-EP-NOT: (INIT) 0x[[#%x, EP_ADDR]] +# CHECK-INIT-EP-NOT: (INIT_ARRAY) 0x[[#%x, EP_ADDR]] +## Check that the new entry point address points to __bolt_runtime_start +# CHECK-INIT-EP: Symbol table '.symtab' contains {{.*}} entries: +# CHECK-INIT-EP: {{0+}}[[#%x, EP_ADDR]] {{.*}} __bolt_runtime_start + +## Check that DT_INIT address is set to __bolt_runtime_start for PIE executable with DT_INIT +# CHECK-INIT-NO-EP: ELF Header: +# CHECK-INIT-NO-EP: Entry point address: 0x[[#%x,EP_ADDR:]] +## Read Dynamic section DT_INIT and DT_INIT_ARRAY entries +# CHECK-INIT-NO-EP: Dynamic section at offset {{.*}} contains {{.*}} entries: +# CHECK-INIT-NO-EP-DAG: (INIT) 0x[[#%x,INIT:]] +# CHECK-INIT-NO-EP-DAG: (INIT_ARRAY) 0x[[#%x,INIT_ARRAY:]] +## Check if ELF entry point address points to _start symbol and new DT_INIT entry points to __bolt_runtime_start +# CHECK-INIT-NO-EP: Symbol table '.symtab' contains {{.*}} entries: +# CHECK-INIT-NO-EP-DAG: {{0+}}[[#%x, EP_ADDR]] {{.*}} _start +# CHECK-INIT-NO-EP-DAG: {{0+}}[[#%x, INIT]] {{.*}} __bolt_runtime_start + +## Check that 1st entry of DT_INIT_ARRAY is set to __bolt_runtime_start and DT_INIT was not changed +# CHECK-INIT-ARRAY-NO-EP: ELF Header: +# CHECK-INIT-ARRAY-NO-EP: Entry point address: 0x[[#%x,EP_ADDR:]] +## Read Dynamic section DT_INIT and DT_INIT_ARRAY entries +# CHECK-INIT-ARRAY-NO-EP: Dynamic section at offset {{.*}} contains {{.*}} entries: +# CHECK-INIT-ARRAY-NO-EP-DAG: (INIT) 0x[[#%x,INIT:]] +# CHECK-INIT-ARRAY-NO-EP-DAG: (INIT_ARRAY) 0x[[#%x,INIT_ARRAY:]] +## Read the dynamic relocation from 1st entry of .init_array +# CHECK-INIT-ARRAY-NO-EP: Relocation section '.rela.dyn' at offset {{.*}} contains {{.*}} entries +# CHECK-INIT-ARRAY-NO-EP: {{0+}}[[#%x,INIT_ARRAY]] {{.*}} R_AARCH64_RELATIVE [[#%x,INIT_ADDR:]] +# CHECK-INIT-ARRAY-NO-EP-NOT: {{0+}}[[#%x,INIT_ARRAY]] {{.*}} R_AARCH64_RELATIVE [[#%x,INIT]] +## Check that 1st entry of .init_array points to __bolt_runtime_start +# CHECK-INIT-ARRAY-NO-EP: Symbol table '.symtab' contains {{.*}} entries: +# CHECK-INIT-ARRAY-NO-EP-DAG: {{0+}}[[#%x, EP_ADDR]] {{.*}} _start +# CHECK-INIT-ARRAY-NO-EP-DAG: {{[0-9]]*}}: {{0+}}[[#%x, INIT_ADDR]] {{.*}} __bolt_runtime_start + +## Check that entry point address is set to __bolt_runtime_start for PIE executable without DT_INIT +# CHECK-NO-INIT-EP: ELF Header: +# CHECK-NO-INIT-EP: Entry point address: 0x[[#%x,EP_ADDR:]] +## Check that the dynamic relocation at .init and .init_array were not patched +# CHECK-NO-INIT-EP: Dynamic section at offset {{.*}} contains {{.*}} entries: +# CHECK-NO-INIT-EP-NOT: (INIT) 0x[[#%x, EP_ADDR]] +# CHECK-NO-INIT-EP-NOT: (INIT_ARRAY) 0x[[#%x, EP_ADDR]] +## Check that the new entry point address points to __bolt_runtime_start +# CHECK-NO-INIT-EP: Symbol table '.symtab' contains {{.*}} entries: +# CHECK-NO-INIT-EP: {{0+}}[[#%x, EP_ADDR]] {{.*}} __bolt_runtime_start + +## Check that DT_INIT is set to __bolt_runtime_start for shared library with DT_INIT +# CHECK-SHARED-INIT: Dynamic section at offset {{.*}} contains {{.*}} entries: +# CHECK-SHARED-INIT-DAG: (INIT) 0x[[#%x, INIT:]] +# CHECK-SHARED-INIT-DAG: (INIT_ARRAY) 0x[[#%x, INIT_ARRAY:]] +## Check that the dynamic relocation at .init_array was not patched +# CHECK-SHARED-INIT: Relocation section '.rela.dyn' at offset {{.*}} contains {{.*}} entries +# CHECK-SHARED-INIT-NOT: {{0+}}[[#%x, INIT_ARRAY]] {{.*}} R_AARCH64_ABS64 {{0+}}[[#%x, INIT]] +## Check that dynamic section DT_INIT points to __bolt_runtime_start +# CHECK-SHARED-INIT: Symbol table '.symtab' contains {{.*}} entries: +# CHECK-SHARED-INIT: {{0+}}[[#%x, INIT]] {{.*}} __bolt_runtime_start + +## Check that entry point address is set to __bolt_runtime_start for PIE executable without DT_INIT +# CHECK-NO-INIT-NO-EP: ELF Header: +# CHECK-NO-INIT-NO-EP: Entry point address: 0x[[#%x,EP_ADDR:]] +# CHECK-NO-INIT-NO-EP: Dynamic section at offset {{.*}} contains {{.*}} entries: +# CHECK-NO-INIT-NO-EP-NOT: (INIT) +# CHECK-NO-INIT-NO-EP: (INIT_ARRAY) 0x[[#%x,INIT_ARRAY:]] +## Read the dynamic relocation from 1st entry of .init_array +# CHECK-NO-INIT-NO-EP: Relocation section '.rela.dyn' at offset {{.*}} contains {{.*}} entries +# CHECK-NO-INIT-NO-EP: {{0+}}[[#%x,INIT_ARRAY]] {{.*}} R_AARCH64_RELATIVE [[#%x,INIT_ADDR:]] +## Check that 1st entry of .init_array points to __bolt_runtime_start +# CHECK-NO-INIT-NO-EP: Symbol table '.symtab' contains {{.*}} entries: +# CHECK-NO-INIT-NO-EP-DAG: {{0+}}[[#%x, EP_ADDR]] {{.*}} _start +# CHECK-NO-INIT-NO-EP-DAG: {{[0-9]]*}}: {{0+}}[[#%x, INIT_ADDR]] {{.*}} __bolt_runtime_start + +## Check that entry point address is set to __bolt_runtime_start for shared library without DT_INIT +# CHECK-SHARED-NO-INIT: Dynamic section at offset {{.*}} contains {{.*}} entries: +# CHECK-SHARED-NO-INIT-NOT: (INIT) +# CHECK-SHARED-NO-INIT: (INIT_ARRAY) 0x[[#%x,INIT_ARRAY:]] +## Read the dynamic relocation from 1st entry of .init_array +# CHECK-SHARED-NO-INIT: Relocation section '.rela.dyn' at offset {{.*}} contains {{.*}} entries +# CHECK-SHARED-NO-INIT: {{0+}}[[#%x, INIT_ARRAY]] {{.*}} R_AARCH64_ABS64 [[#%x,INIT_ADDR:]] +## Check that 1st entry of .init_array points to __bolt_runtime_start +# CHECK-SHARED-NO-INIT: Symbol table '.symtab' contains {{.*}} entries: +# CHECK-SHARED-NO-INIT: {{[0-9]]*}}: {{0+}}[[#%x, INIT_ADDR]] {{.*}} __bolt_runtime_start + +## Check that entry point address is set to __bolt_runtime_start for non-PIE executable with DT_INIT +# CHECK-NO-PIE-NO-INIT-EP: ELF Header: +# CHECK-NO-PIE-NO-INIT-EP: Entry point address: 0x[[#%x,EP_ADDR:]] +## Check that the dynamic relocation at .init and .init_array were not patched +# CHECK-NO-PIE-NO-INIT-EP: Dynamic section at offset {{.*}} contains {{.*}} entries: +# CHECK-NO-PIE-NO-INIT-EP-NOT: (INIT) 0x[[#%x, EP_ADDR]] +# CHECK-NO-PIE-NO-INIT-EP-NOT: (INIT_ARRAY) 0x[[#%x, EP_ADDR]] +## Check that the new entry point address points to __bolt_runtime_start +# CHECK-NO-PIE-NO-INIT-EP: Symbol table '.symtab' contains {{.*}} entries: +# CHECK-NO-PIE-NO-INIT-EP: {{0+}}[[#%x, EP_ADDR]] {{.*}} __bolt_runtime_start + + .globl _start + .type _start, %function +_start: + # Dummy relocation to force relocation mode. + .reloc 0, R_AARCH64_NONE + ret +.size _start, .-_start + + .globl _init + .type _init, %function +_init: + ret + .size _init, .-_init + + .globl _fini + .type _fini, %function +_fini: + ret + .size _fini, .-_fini + + .section .init_array,"aw" + .align 3 + .dword _init + + .section .fini_array,"aw" + .align 3 + .dword _fini diff --git a/bolt/test/X86/internal-call-instrument-so.s b/bolt/test/X86/internal-call-instrument-so.s index 99e5b29221409..fe23bc61afa32 100644 --- a/bolt/test/X86/internal-call-instrument-so.s +++ b/bolt/test/X86/internal-call-instrument-so.s @@ -5,7 +5,7 @@ # RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o # Delete our BB symbols so BOLT doesn't mark them as entry points # RUN: llvm-strip --strip-unneeded %t.o -# RUN: ld.lld %t.o -o %t.exe -q -shared -fini=_fini +# RUN: ld.lld %t.o -o %t.exe -q -shared -fini=_fini -init=_init # RUN: llvm-bolt --instrument %t.exe --relocs -o %t.out .text @@ -48,6 +48,13 @@ _fini: hlt .size _fini, .-_fini + .globl _init + .type _init, %function + .p2align 4 +_init: + retq + .size _init, .-_init + .data .globl var var: diff --git a/bolt/test/runtime/X86/instrument-wrong-target.s b/bolt/test/runtime/X86/instrument-wrong-target.s index 343d93a89ed13..fa40d43f10a0f 100644 --- a/bolt/test/runtime/X86/instrument-wrong-target.s +++ b/bolt/test/runtime/X86/instrument-wrong-target.s @@ -19,6 +19,13 @@ _start: ret .size _start, .-_start + .globl _init + .type _init, %function + # Force DT_INIT to be created (needed for instrumentation). +_init: + ret + .size _init, .-_init + .globl _fini .type _fini, %function # Force DT_FINI to be created (needed for instrumentation).