diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp index 218e1eff69c468..3ed6f8e1a0034c 100644 --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -207,8 +207,10 @@ class HWAddressSanitizer { void initializeCallbacks(Module &M); + Value *getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val); + Value *getDynamicShadowIfunc(IRBuilder<> &IRB); - Value *getDynamicShadowNonTls(IRBuilder<> &IRB); + Value *getShadowNonTls(IRBuilder<> &IRB); void untagPointerOperand(Instruction *I, Value *Addr); Value *shadowBase(); @@ -294,7 +296,7 @@ class HWAddressSanitizer { Constant *ShadowGlobal; - Value *LocalDynamicShadow = nullptr; + Value *ShadowBase = nullptr; Value *StackBaseTag = nullptr; GlobalValue *ThreadPtrGlobal = nullptr; }; @@ -572,20 +574,27 @@ void HWAddressSanitizer::initializeCallbacks(Module &M) { M.getOrInsertFunction("__hwasan_handle_vfork", IRB.getVoidTy(), IntptrTy); } -Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) { +Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) { // An empty inline asm with input reg == output reg. // An opaque no-op cast, basically. - InlineAsm *Asm = InlineAsm::get( - FunctionType::get(Int8PtrTy, {ShadowGlobal->getType()}, false), - StringRef(""), StringRef("=r,0"), - /*hasSideEffects=*/false); - return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow"); + // This prevents code bloat as a result of rematerializing trivial definitions + // such as constants or global addresses at every load and store. + InlineAsm *Asm = + InlineAsm::get(FunctionType::get(Int8PtrTy, {Val->getType()}, false), + StringRef(""), StringRef("=r,0"), + /*hasSideEffects=*/false); + return IRB.CreateCall(Asm, {Val}, ".hwasan.shadow"); } -Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) { - // Generate code only when dynamic addressing is needed. +Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) { + return getOpaqueNoopCast(IRB, ShadowGlobal); +} + +Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) { if (Mapping.Offset != kDynamicShadowSentinel) - return nullptr; + return getOpaqueNoopCast( + IRB, ConstantExpr::getIntToPtr( + ConstantInt::get(IntptrTy, Mapping.Offset), Int8PtrTy)); if (Mapping.InGlobal) { return getDynamicShadowIfunc(IRB); @@ -621,7 +630,7 @@ void HWAddressSanitizer::getInterestingMemoryOperands( return; // Do not instrument the load fetching the dynamic shadow address. - if (LocalDynamicShadow == I) + if (ShadowBase == I) return; if (LoadInst *LI = dyn_cast(I)) { @@ -685,20 +694,13 @@ void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) { I->setOperand(getPointerOperandIndex(I), UntaggedPtr); } -Value *HWAddressSanitizer::shadowBase() { - if (LocalDynamicShadow) - return LocalDynamicShadow; - return ConstantExpr::getIntToPtr(ConstantInt::get(IntptrTy, Mapping.Offset), - Int8PtrTy); -} - Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) { // Mem >> Scale Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale); if (Mapping.Offset == 0) return IRB.CreateIntToPtr(Shadow, Int8PtrTy); // (Mem >> Scale) + Offset - return IRB.CreateGEP(Int8Ty, shadowBase(), Shadow); + return IRB.CreateGEP(Int8Ty, ShadowBase, Shadow); } void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite, @@ -715,7 +717,7 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite, M, UseShortGranules ? Intrinsic::hwasan_check_memaccess_shortgranules : Intrinsic::hwasan_check_memaccess), - {shadowBase(), Ptr, ConstantInt::get(Int32Ty, AccessInfo)}); + {ShadowBase, Ptr, ConstantInt::get(Int32Ty, AccessInfo)}); return; } @@ -1003,12 +1005,12 @@ Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) { void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) { if (!Mapping.InTls) { - LocalDynamicShadow = getDynamicShadowNonTls(IRB); + ShadowBase = getShadowNonTls(IRB); return; } if (!WithFrameRecord && TargetTriple.isAndroid()) { - LocalDynamicShadow = getDynamicShadowIfunc(IRB); + ShadowBase = getDynamicShadowIfunc(IRB); return; } @@ -1069,12 +1071,12 @@ void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) { // Get shadow base address by aligning RecordPtr up. // Note: this is not correct if the pointer is already aligned. // Runtime library will make sure this never happens. - LocalDynamicShadow = IRB.CreateAdd( + ShadowBase = IRB.CreateAdd( IRB.CreateOr( ThreadLongMaybeUntagged, ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)), ConstantInt::get(IntptrTy, 1), "hwasan.shadow"); - LocalDynamicShadow = IRB.CreateIntToPtr(LocalDynamicShadow, Int8PtrTy); + ShadowBase = IRB.CreateIntToPtr(ShadowBase, Int8PtrTy); } Value *HWAddressSanitizer::readRegister(IRBuilder<> &IRB, StringRef Name) { @@ -1226,7 +1228,7 @@ bool HWAddressSanitizer::sanitizeFunction(Function &F) { IntrinToInstrument.empty()) return Changed; - assert(!LocalDynamicShadow); + assert(!ShadowBase); Instruction *InsertPt = &*F.getEntryBlock().begin(); IRBuilder<> EntryIRB(InsertPt); @@ -1306,7 +1308,7 @@ bool HWAddressSanitizer::sanitizeFunction(Function &F) { instrumentMemIntrinsic(cast(Inst)); } - LocalDynamicShadow = nullptr; + ShadowBase = nullptr; StackBaseTag = nullptr; return true; diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll index d5406117c87416..130e20dfea87f0 100644 --- a/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll +++ b/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll @@ -56,8 +56,7 @@ define i8 @test_load8(i8* %a) sanitize_hwaddress { ; RECOVER: [[CONT]]: -; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %a, i32 0) -; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %a, i32 0) +; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %a, i32 0) ; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4 ; CHECK: ret i8 %[[G]] @@ -108,8 +107,7 @@ define i16 @test_load16(i16* %a) sanitize_hwaddress { ; RECOVER: [[CONT]]: ; ABORT: %[[A:[^ ]*]] = bitcast i16* %a to i8* -; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 1) -; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 1) +; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 1) ; CHECK: %[[G:[^ ]*]] = load i16, i16* %a, align 4 ; CHECK: ret i16 %[[G]] @@ -136,8 +134,7 @@ define i32 @test_load32(i32* %a) sanitize_hwaddress { ; RECOVER: br label ; ABORT: %[[A:[^ ]*]] = bitcast i32* %a to i8* -; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 2) -; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 2) +; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 2) ; CHECK: %[[G:[^ ]*]] = load i32, i32* %a, align 4 ; CHECK: ret i32 %[[G]] @@ -164,8 +161,7 @@ define i64 @test_load64(i64* %a) sanitize_hwaddress { ; RECOVER: br label ; ABORT: %[[A:[^ ]*]] = bitcast i64* %a to i8* -; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 3) -; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 3) +; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 3) ; CHECK: %[[G:[^ ]*]] = load i64, i64* %a, align 8 ; CHECK: ret i64 %[[G]] @@ -192,8 +188,7 @@ define i128 @test_load128(i128* %a) sanitize_hwaddress { ; RECOVER: br label ; ABORT: %[[A:[^ ]*]] = bitcast i128* %a to i8* -; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 4) -; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 4) +; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 4) ; CHECK: %[[G:[^ ]*]] = load i128, i128* %a, align 16 ; CHECK: ret i128 %[[G]] @@ -232,8 +227,7 @@ define void @test_store8(i8* %a, i8 %b) sanitize_hwaddress { ; RECOVER: call void asm sideeffect "brk #2352", "{x0}"(i64 %[[A]]) ; RECOVER: br label -; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %a, i32 16) -; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %a, i32 16) +; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %a, i32 16) ; CHECK: store i8 %b, i8* %a, align 4 ; CHECK: ret void @@ -260,8 +254,7 @@ define void @test_store16(i16* %a, i16 %b) sanitize_hwaddress { ; RECOVER: br label ; ABORT: %[[A:[^ ]*]] = bitcast i16* %a to i8* -; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 17) -; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 17) +; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 17) ; CHECK: store i16 %b, i16* %a, align 4 ; CHECK: ret void @@ -288,8 +281,7 @@ define void @test_store32(i32* %a, i32 %b) sanitize_hwaddress { ; RECOVER: br label ; ABORT: %[[A:[^ ]*]] = bitcast i32* %a to i8* -; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 18) -; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 18) +; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 18) ; CHECK: store i32 %b, i32* %a, align 4 ; CHECK: ret void @@ -316,8 +308,7 @@ define void @test_store64(i64* %a, i64 %b) sanitize_hwaddress { ; RECOVER: br label ; ABORT: %[[A:[^ ]*]] = bitcast i64* %a to i8* -; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 19) -; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 19) +; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 19) ; CHECK: store i64 %b, i64* %a, align 8 ; CHECK: ret void @@ -344,8 +335,7 @@ define void @test_store128(i128* %a, i128 %b) sanitize_hwaddress { ; RECOVER: br label ; ABORT: %[[A:[^ ]*]] = bitcast i128* %a to i8* -; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 20) -; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 20) +; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 20) ; CHECK: store i128 %b, i128* %a, align 16 ; CHECK: ret void diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/kernel.ll b/llvm/test/Instrumentation/HWAddressSanitizer/kernel.ll index 3f5891c1dddbaa..1ff898dcf860e7 100644 --- a/llvm/test/Instrumentation/HWAddressSanitizer/kernel.ll +++ b/llvm/test/Instrumentation/HWAddressSanitizer/kernel.ll @@ -10,6 +10,7 @@ target triple = "aarch64--linux-android" define i8 @test_load(i8* %a) sanitize_hwaddress { ; CHECK-LABEL: @test_load( +; OFFSET: %[[SHADOW:[^ ]*]] = call i8* asm "", "=r,0"(i8* inttoptr (i64 12345678 to i8*)) ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8 @@ -18,7 +19,7 @@ define i8 @test_load(i8* %a) sanitize_hwaddress { ; NOOFFSET: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8* -; OFFSET: %[[E:[^ ]*]] = getelementptr i8, i8* inttoptr (i64 12345678 to i8*), i64 %[[D]] +; OFFSET: %[[E:[^ ]*]] = getelementptr i8, i8* %[[SHADOW]], i64 %[[D]] ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]] ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]