166 changes: 155 additions & 11 deletions llvm/lib/Transforms/IPO/LowerTypeTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/Triple.h"
#include "llvm/Analysis/TypeMetadataUtils.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Function.h"
Expand Down Expand Up @@ -206,24 +207,39 @@ struct ByteArrayInfo {
class GlobalTypeMember final : TrailingObjects<GlobalTypeMember, MDNode *> {
GlobalObject *GO;
size_t NTypes;
// For functions: true if this is a definition (either in the merged module or
// in one of the thinlto modules).
bool IsDefinition;
// For functions: true if this function is either defined or used in a thinlto
// module and its jumptable entry needs to be exported to thinlto backends.
bool IsExported;

friend TrailingObjects;
size_t numTrailingObjects(OverloadToken<MDNode *>) const { return NTypes; }

public:
static GlobalTypeMember *create(BumpPtrAllocator &Alloc, GlobalObject *GO,
bool IsDefinition, bool IsExported,
ArrayRef<MDNode *> Types) {
auto *GTM = static_cast<GlobalTypeMember *>(Alloc.Allocate(
totalSizeToAlloc<MDNode *>(Types.size()), alignof(GlobalTypeMember)));
GTM->GO = GO;
GTM->NTypes = Types.size();
GTM->IsDefinition = IsDefinition;
GTM->IsExported = IsExported;
std::uninitialized_copy(Types.begin(), Types.end(),
GTM->getTrailingObjects<MDNode *>());
return GTM;
}
GlobalObject *getGlobal() const {
return GO;
}
bool isDefinition() const {
return IsDefinition;
}
bool isExported() const {
return IsExported;
}
ArrayRef<MDNode *> types() const {
return makeArrayRef(getTrailingObjects<MDNode *>(), NTypes);
}
Expand Down Expand Up @@ -294,6 +310,7 @@ class LowerTypeTestsModule {
void exportTypeId(StringRef TypeId, const TypeIdLowering &TIL);
TypeIdLowering importTypeId(StringRef TypeId);
void importTypeTest(CallInst *CI);
void importFunction(Function *F, bool isDefinition);

BitSetInfo
buildBitSet(Metadata *TypeId,
Expand Down Expand Up @@ -820,6 +837,41 @@ void LowerTypeTestsModule::importTypeTest(CallInst *CI) {
CI->eraseFromParent();
}

// ThinLTO backend: the function F has a jump table entry; update this module
// accordingly. isDefinition describes the type of the jump table entry.
void LowerTypeTestsModule::importFunction(Function *F, bool isDefinition) {
assert(F->getType()->getAddressSpace() == 0);

// Declaration of a local function - nothing to do.
if (F->isDeclarationForLinker() && isDefinition)
return;

GlobalValue::VisibilityTypes Visibility = F->getVisibility();
std::string Name = F->getName();
Function *FDecl;

if (F->isDeclarationForLinker() && !isDefinition) {
// Declaration of an external function.
FDecl = Function::Create(F->getFunctionType(), GlobalValue::ExternalLinkage,
Name + ".cfi_jt", &M);
FDecl->setVisibility(GlobalValue::HiddenVisibility);
} else {
// Definition.
assert(isDefinition);
F->setName(Name + ".cfi");
F->setLinkage(GlobalValue::ExternalLinkage);
F->setVisibility(GlobalValue::HiddenVisibility);
FDecl = Function::Create(F->getFunctionType(), GlobalValue::ExternalLinkage,
Name, &M);
FDecl->setVisibility(Visibility);
}

if (F->isWeakForLinker())
replaceWeakDeclarationWithJumpTablePtr(F, FDecl);
else
F->replaceAllUsesWith(FDecl);
}

void LowerTypeTestsModule::lowerTypeTestCalls(
ArrayRef<Metadata *> TypeIds, Constant *CombinedGlobalAddr,
const DenseMap<GlobalTypeMember *, uint64_t> &GlobalLayout) {
Expand Down Expand Up @@ -1143,7 +1195,6 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsNative(
// arithmetic that we normally use for globals.

// FIXME: find a better way to represent the jumptable in the IR.

assert(!Functions.empty());

// Build a simple layout based on the regular layout of jump tables.
Expand All @@ -1167,24 +1218,35 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsNative(
// references to the original functions with references to the aliases.
for (unsigned I = 0; I != Functions.size(); ++I) {
Function *F = cast<Function>(Functions[I]->getGlobal());
bool IsDefinition = Functions[I]->isDefinition();

Constant *CombinedGlobalElemPtr = ConstantExpr::getBitCast(
ConstantExpr::getInBoundsGetElementPtr(
JumpTableType, JumpTable,
ArrayRef<Constant *>{ConstantInt::get(IntPtrTy, 0),
ConstantInt::get(IntPtrTy, I)}),
F->getType());
if (F->isDeclarationForLinker()) {
if (Functions[I]->isExported()) {
if (IsDefinition) {
ExportSummary->cfiFunctionDefs().insert(F->getName());
} else {
GlobalAlias *JtAlias = GlobalAlias::create(
F->getValueType(), 0, GlobalValue::ExternalLinkage,
F->getName() + ".cfi_jt", CombinedGlobalElemPtr, &M);
JtAlias->setVisibility(GlobalValue::HiddenVisibility);
ExportSummary->cfiFunctionDecls().insert(F->getName());
}
}
if (!IsDefinition) {
if (F->isWeakForLinker())
replaceWeakDeclarationWithJumpTablePtr(F, CombinedGlobalElemPtr);
else
F->replaceAllUsesWith(CombinedGlobalElemPtr);
} else {
assert(F->getType()->getAddressSpace() == 0);

GlobalAlias *FAlias = GlobalAlias::create(F->getValueType(), 0,
F->getLinkage(), "",
CombinedGlobalElemPtr, &M);
GlobalAlias *FAlias = GlobalAlias::create(
F->getValueType(), 0, F->getLinkage(), "", CombinedGlobalElemPtr, &M);
FAlias->setVisibility(F->getVisibility());
FAlias->takeName(F);
if (FAlias->hasName())
Expand Down Expand Up @@ -1353,15 +1415,37 @@ bool LowerTypeTestsModule::runForTesting(Module &M) {
bool LowerTypeTestsModule::lower() {
Function *TypeTestFunc =
M.getFunction(Intrinsic::getName(Intrinsic::type_test));
if ((!TypeTestFunc || TypeTestFunc->use_empty()) && !ExportSummary)
if ((!TypeTestFunc || TypeTestFunc->use_empty()) && !ExportSummary &&
!ImportSummary)
return false;

if (ImportSummary) {
for (auto UI = TypeTestFunc->use_begin(), UE = TypeTestFunc->use_end();
UI != UE;) {
auto *CI = cast<CallInst>((*UI++).getUser());
importTypeTest(CI);
if (TypeTestFunc) {
for (auto UI = TypeTestFunc->use_begin(), UE = TypeTestFunc->use_end();
UI != UE;) {
auto *CI = cast<CallInst>((*UI++).getUser());
importTypeTest(CI);
}
}

SmallVector<Function *, 8> Defs;
SmallVector<Function *, 8> Decls;
for (auto &F : M) {
// CFI functions are either external, or promoted. A local function may
// have the same name, but it's not the one we are looking for.
if (F.hasLocalLinkage())
continue;
if (ImportSummary->cfiFunctionDefs().count(F.getName()))
Defs.push_back(&F);
else if (ImportSummary->cfiFunctionDecls().count(F.getName()))
Decls.push_back(&F);
}

for (auto F : Defs)
importFunction(F, /*isDefinition*/ true);
for (auto F : Decls)
importFunction(F, /*isDefinition*/ false);

return true;
}

Expand All @@ -1387,6 +1471,58 @@ bool LowerTypeTestsModule::lower() {
llvm::DenseMap<Metadata *, TIInfo> TypeIdInfo;
unsigned I = 0;
SmallVector<MDNode *, 2> Types;

struct ExportedFunctionInfo {
CfiFunctionLinkage Linkage;
MDNode *FuncMD; // {name, linkage, type[, type...]}
};
DenseMap<StringRef, ExportedFunctionInfo> ExportedFunctions;
if (ExportSummary) {
NamedMDNode *CfiFunctionsMD = M.getNamedMetadata("cfi.functions");
if (CfiFunctionsMD) {
for (auto FuncMD : CfiFunctionsMD->operands()) {
assert(FuncMD->getNumOperands() >= 2);
StringRef FunctionName =
cast<MDString>(FuncMD->getOperand(0))->getString();
if (!ExportSummary->isGUIDLive(GlobalValue::getGUID(
GlobalValue::dropLLVMManglingEscape(FunctionName))))
continue;
CfiFunctionLinkage Linkage = static_cast<CfiFunctionLinkage>(
cast<ConstantAsMetadata>(FuncMD->getOperand(1))
->getValue()
->getUniqueInteger()
.getZExtValue());
auto P = ExportedFunctions.insert({FunctionName, {Linkage, FuncMD}});
if (!P.second && P.first->second.Linkage != CFL_Definition)
P.first->second = {Linkage, FuncMD};
}

for (const auto &P : ExportedFunctions) {
StringRef FunctionName = P.first;
CfiFunctionLinkage Linkage = P.second.Linkage;
MDNode *FuncMD = P.second.FuncMD;
Function *F = M.getFunction(FunctionName);
if (!F)
F = Function::Create(
FunctionType::get(Type::getVoidTy(M.getContext()), false),
GlobalVariable::ExternalLinkage, FunctionName, &M);

if (Linkage == CFL_Definition)
F->eraseMetadata(LLVMContext::MD_type);

if (F->isDeclaration()) {
if (Linkage == CFL_WeakDeclaration)
F->setLinkage(GlobalValue::ExternalWeakLinkage);

SmallVector<MDNode *, 2> Types;
for (unsigned I = 2; I < FuncMD->getNumOperands(); ++I)
F->addMetadata(LLVMContext::MD_type,
*cast<MDNode>(FuncMD->getOperand(I).get()));
}
}
}
}

for (GlobalObject &GO : M.global_objects()) {
if (isa<GlobalVariable>(GO) && GO.isDeclarationForLinker())
continue;
Expand All @@ -1396,7 +1532,15 @@ bool LowerTypeTestsModule::lower() {
if (Types.empty())
continue;

auto *GTM = GlobalTypeMember::create(Alloc, &GO, Types);
bool IsDefinition = !GO.isDeclarationForLinker();
bool IsExported = false;
if (isa<Function>(GO) && ExportedFunctions.count(GO.getName())) {
IsDefinition |= ExportedFunctions[GO.getName()].Linkage == CFL_Definition;
IsExported = true;
}

auto *GTM =
GlobalTypeMember::create(Alloc, &GO, IsDefinition, IsExported, Types);
for (MDNode *Type : Types) {
verifyTypeMDNode(&GO, Type);
auto &Info = TypeIdInfo[cast<MDNode>(Type)->getOperand(1)];
Expand Down
50 changes: 43 additions & 7 deletions llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,16 @@ namespace {

// Promote each local-linkage entity defined by ExportM and used by ImportM by
// changing visibility and appending the given ModuleId.
void promoteInternals(Module &ExportM, Module &ImportM, StringRef ModuleId) {
void promoteInternals(Module &ExportM, Module &ImportM, StringRef ModuleId,
SetVector<GlobalValue *> &PromoteExtra) {
DenseMap<const Comdat *, Comdat *> RenamedComdats;
for (auto &ExportGV : ExportM.global_values()) {
if (!ExportGV.hasLocalLinkage())
continue;

auto Name = ExportGV.getName();
GlobalValue *ImportGV = ImportM.getNamedValue(Name);
if (!ImportGV || ImportGV->use_empty())
if ((!ImportGV || ImportGV->use_empty()) && !PromoteExtra.count(&ExportGV))
continue;

std::string NewName = (Name + ModuleId).str();
Expand All @@ -53,8 +54,10 @@ void promoteInternals(Module &ExportM, Module &ImportM, StringRef ModuleId) {
ExportGV.setLinkage(GlobalValue::ExternalLinkage);
ExportGV.setVisibility(GlobalValue::HiddenVisibility);

ImportGV->setName(NewName);
ImportGV->setVisibility(GlobalValue::HiddenVisibility);
if (ImportGV) {
ImportGV->setName(NewName);
ImportGV->setVisibility(GlobalValue::HiddenVisibility);
}
}

if (!RenamedComdats.empty())
Expand Down Expand Up @@ -296,6 +299,11 @@ void splitAndWriteThinLTOBitcode(
F.setComdat(nullptr);
}

SetVector<GlobalValue *> CfiFunctions;
for (auto &F : M)
if ((!F.hasLocalLinkage() || F.hasAddressTaken()) && HasTypeMetadata(&F))
CfiFunctions.insert(&F);

// Remove all globals with type metadata, globals with comdats that live in
// MergedM, and aliases pointing to such globals from the thin LTO module.
filterModule(&M, [&](const GlobalValue *GV) {
Expand All @@ -308,11 +316,39 @@ void splitAndWriteThinLTOBitcode(
return true;
});

promoteInternals(*MergedM, M, ModuleId);
promoteInternals(M, *MergedM, ModuleId);
promoteInternals(*MergedM, M, ModuleId, CfiFunctions);
promoteInternals(M, *MergedM, ModuleId, CfiFunctions);

SmallVector<MDNode *, 8> CfiFunctionMDs;
for (auto V : CfiFunctions) {
Function &F = *cast<Function>(V);
SmallVector<MDNode *, 2> Types;
F.getMetadata(LLVMContext::MD_type, Types);

auto &Ctx = MergedM->getContext();
SmallVector<Metadata *, 4> Elts;
Elts.push_back(MDString::get(Ctx, F.getName()));
CfiFunctionLinkage Linkage;
if (!F.isDeclarationForLinker())
Linkage = CFL_Definition;
else if (F.isWeakForLinker())
Linkage = CFL_WeakDeclaration;
else
Linkage = CFL_Declaration;
Elts.push_back(ConstantAsMetadata::get(
llvm::ConstantInt::get(Type::getInt8Ty(Ctx), Linkage)));
for (auto Type : Types)
Elts.push_back(Type);
CfiFunctionMDs.push_back(MDTuple::get(Ctx, Elts));
}

simplifyExternals(*MergedM);
if(!CfiFunctionMDs.empty()) {
NamedMDNode *NMD = MergedM->getOrInsertNamedMetadata("cfi.functions");
for (auto MD : CfiFunctionMDs)
NMD->addOperand(MD);
}

simplifyExternals(*MergedM);

// FIXME: Try to re-use BSI and PFI from the original module here.
ProfileSummaryInfo PSI(M);
Expand Down
29 changes: 29 additions & 0 deletions llvm/test/ThinLTO/X86/cfi-icall.ll
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
; RUN: opt -thinlto-bc %s -o %t1.bc
; RUN: llvm-lto2 run -thinlto-distributed-indexes %t1.bc -o %t.out -save-temps \
; RUN: -r %t1.bc,foo,plx \
; RUN: -r %t1.bc,bar,x
; RUN: llvm-bcanalyzer -dump %t.out.index.bc | FileCheck %s --check-prefix=COMBINED

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

define i1 @foo(i8* %p) !type !0 {
entry:
%x = call i1 @llvm.type.test(i8* %p, metadata !"typeid1")
ret i1 %x
}

declare !type !0 void @bar()

declare i1 @llvm.type.test(i8* %ptr, metadata %type) nounwind readnone

!0 = !{i64 0, !"typeid1"}

; COMBINED: <GLOBALVAL_SUMMARY_BLOCK
; COMBINED: <CFI_FUNCTION_DEFS op0=0 op1=3/>
; COMBINED: <CFI_FUNCTION_DECLS op0=3 op1=3/>
; COMBINED: </GLOBALVAL_SUMMARY_BLOCK>

; COMBINED: <STRTAB_BLOCK
; COMBINED-NEXT: <BLOB abbrevid=4/> blob data = 'foobar'
; COMBINED-NEXT: </STRTAB_BLOCK>
23 changes: 23 additions & 0 deletions llvm/test/Transforms/CrossDSOCFI/cfi_functions.ll
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
; Test that types referenced in ThinLTO-style !cfi.functions are known to __cfi_check.
; RUN: opt -S -cross-dso-cfi < %s | FileCheck %s
; RUN: opt -S -passes=cross-dso-cfi < %s | FileCheck %s

; CHECK: define void @__cfi_check(
; CHECK: switch i64
; CHECK-NEXT: i64 1234, label
; CHECK-NEXT: i64 5678, label
; CHECK-NEXT: ]

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

!cfi.functions = !{!0, !1}
!llvm.module.flags = !{!6}

!0 = !{!"f", i8 0, !2, !4}
!1 = !{!"g", i8 1, !3, !5}
!2 = !{i64 0, !"typeid1"}
!3 = !{i64 0, !"typeid2"}
!4 = !{i64 0, i64 1234}
!5 = !{i64 0, i64 5678}
!6 = !{i32 4, !"Cross-DSO CFI", i32 1}
19 changes: 19 additions & 0 deletions llvm/test/Transforms/LowerTypeTests/Inputs/import-icall.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
---
TypeIdMap:
typeid1:
TTRes:
Kind: AllOnes
SizeM1BitWidth: 7
typeid2:
TTRes:
Kind: Single
SizeM1BitWidth: 0
WithGlobalValueDeadStripping: false
CfiFunctionDefs:
- local_a
- local_b
- does_not_exist
CfiFunctionDecls:
- external
- external_weak
...
70 changes: 70 additions & 0 deletions llvm/test/Transforms/LowerTypeTests/export-icall.ll
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
; RUN: opt -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t < %s | FileCheck %s
; RUN: FileCheck --check-prefix=SUMMARY %s < %t

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

define void @h(i8 %x) !type !2 {
ret void
}

declare !type !8 void @f(i32 %x)

!cfi.functions = !{!0, !1, !3, !4, !5, !6}

; declaration of @h with a different type is ignored
!0 = !{!"h", i8 1, !7}

; extern_weak declaration of @h with a different type is ignored as well
!1 = !{!"h", i8 2, !8}
!2 = !{i64 0, !"typeid1"}

; definition of @f replaces types on the IR declaration above
!3 = !{!"f", i8 0, !2}
!4 = !{!"external", i8 1, !2}
!5 = !{!"external_weak", i8 2, !2}
!6 = !{!"g", i8 0, !7}
!7 = !{i64 0, !"typeid2"}
!8 = !{i64 0, !"typeid3"}


; CHECK-DAG: @__typeid_typeid1_global_addr = hidden alias i8, bitcast (void ()* [[JT1:.*]] to i8*)
; CHECK-DAG: @__typeid_typeid1_align = hidden alias i8, inttoptr (i8 3 to i8*)
; CHECK-DAG: @__typeid_typeid1_size_m1 = hidden alias i8, inttoptr (i64 3 to i8*)

; CHECK-DAG: @h = alias void (i8), bitcast (void ()* [[JT1]] to void (i8)*)
; CHECK-DAG: @f = alias void (i32), {{.*}}getelementptr {{.*}}void ()* [[JT1]]
; CHECK-DAG: @external.cfi_jt = hidden alias void (), {{.*}}getelementptr {{.*}}void ()* [[JT1]]
; CHECK-DAG: @external_weak.cfi_jt = hidden alias void (), {{.*}}getelementptr {{.*}}void ()* [[JT1]]

; CHECK-DAG: @__typeid_typeid2_global_addr = hidden alias i8, bitcast (void ()* [[JT2:.*]] to i8*)

; CHECK-DAG: @g = alias void (), void ()* [[JT2]]

; CHECK-DAG: define internal void @h.cfi(i8 {{.*}}) !type !{{.*}}
; CHECK-DAG: declare !type !{{.*}} void @external()
; CHECK-DAG: declare !type !{{.*}} void @external_weak()
; CHECK-DAG: declare !type !{{.*}} void @f.cfi(i32)
; CHECK-DAG: declare !type !{{.*}} void @g.cfi()


; SUMMARY: TypeIdMap:
; SUMMARY-NEXT: typeid1:
; SUMMARY-NEXT: TTRes:
; SUMMARY-NEXT: Kind: AllOnes
; SUMMARY-NEXT: SizeM1BitWidth: 7
; SUMMARY-NEXT: WPDRes:
; SUMMARY-NEXT: typeid2:
; SUMMARY-NEXT: TTRes:
; SUMMARY-NEXT: Kind: Single
; SUMMARY-NEXT: SizeM1BitWidth: 0
; SUMMARY-NEXT: WPDRes:

; SUMMARY: CfiFunctionDefs:
; SUMMARY-NEXT: - f
; SUMMARY-NEXT: - g
; SUMMARY-NEXT: - h
; SUMMARY-NEXT: CfiFunctionDecls:
; SUMMARY-NEXT: - external
; SUMMARY-NEXT: - external_weak
; SUMMARY-NEXT: ...
40 changes: 40 additions & 0 deletions llvm/test/Transforms/LowerTypeTests/import-icall.ll
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
; RUN: opt -S -lowertypetests -lowertypetests-summary-action=import -lowertypetests-read-summary=%S/Inputs/import-icall.yaml < %s | FileCheck %s

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

define i8 @local_a() {
call void @external()
call void @external_weak()
ret i8 1
}

define internal i8 @local_b() {
%x = call i8 @local_a()
ret i8 %x
}

define i8 @use_b() {
%x = call i8 @local_b()
ret i8 %x
}


declare void @external()
declare extern_weak void @external_weak()

; CHECK: define hidden i8 @local_a.cfi() {
; CHECK-NEXT: call void @external.cfi_jt()
; CHECK-NEXT: call void select (i1 icmp ne (void ()* @external_weak, void ()* null), void ()* @external_weak.cfi_jt, void ()* null)()
; CHECK-NEXT: ret i8 1
; CHECK-NEXT: }

; internal @local_b is not the same function as "local_b" in the summary.
; CHECK: define internal i8 @local_b() {
; CHECK-NEXT: call i8 @local_a()

; CHECK: declare void @external()
; CHECK: declare extern_weak void @external_weak()
; CHECK: declare i8 @local_a()
; CHECK: declare hidden void @external.cfi_jt()
; CHECK: declare hidden void @external_weak.cfi_jt()
2 changes: 2 additions & 0 deletions llvm/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,8 @@ static const char *GetCodeName(unsigned CodeID, unsigned BlockID,
STRINGIFY_CODE(FS, TYPE_TEST_ASSUME_CONST_VCALL)
STRINGIFY_CODE(FS, TYPE_CHECKED_LOAD_CONST_VCALL)
STRINGIFY_CODE(FS, VALUE_GUID)
STRINGIFY_CODE(FS, CFI_FUNCTION_DEFS)
STRINGIFY_CODE(FS, CFI_FUNCTION_DECLS)
}
case bitc::METADATA_ATTACHMENT_ID:
switch(CodeID) {
Expand Down