47 changes: 32 additions & 15 deletions clang/test/Analysis/uninit-asm-goto.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ int test1(int x) {
return -1;
}

// test2: Expect no diagnostics
int test2(int x) {
int y; // expected-warning {{variable 'y' is used uninitialized whenever its declaration is reached}}
// expected-note@-1 {{initialize the variable}}
int y;
if (x < 42)
asm goto("" : "+S"(x), "+D"(y) : "r"(x) :: indirect_1, indirect_2);
else
Expand All @@ -20,29 +20,29 @@ int test2(int x) {
indirect_1:
return -42;
indirect_2:
return y; // expected-note {{uninitialized use occurs here}}
return y;
}

// test3: Expect no diagnostics
int test3(int x) {
int y; // expected-warning {{variable 'y' is used uninitialized whenever its declaration is reached}}
// expected-note@-1 {{initialize the variable}}
int y;
asm goto("" : "=&r"(y) : "r"(x) : : fail);
normal:
y += x;
return y;
if (x) {
fail:
return y; // expected-note {{uninitialized use occurs here}}
return y;
}
return 0;
}

// test4: Expect no diagnostics
int test4(int x) {
int y; // expected-warning {{variable 'y' is used uninitialized whenever its declaration is reached}}
// expected-note@-1 {{initialize the variable}}
int y;
goto forward;
backward:
return y; // expected-note {{uninitialized use occurs here}}
return y;
forward:
asm goto("" : "=r"(y) : "r"(x) : : backward);
return y;
Expand Down Expand Up @@ -70,23 +70,40 @@ int test6(unsigned int *x) {
return -1;
}

// test7: Expect no diagnostics.
int test7(int z) {
int x; // expected-warning {{variable 'x' is used uninitialized whenever its declaration is reached}}
// expected-note@-1 {{initialize the variable 'x' to silence this warning}}
int x;
if (z)
asm goto ("":"=r"(x):::A1,A2);
return 0;
A1:
A2:
return x; // expected-note {{uninitialized use occurs here}}
return x;
}

// test8: Expect no diagnostics
int test8() {
int x = 0; // expected-warning {{variable 'x' is used uninitialized whenever its declaration is reached}}
// expected-note@-1 {{variable 'x' is declared here}}
int x = 0;
asm goto ("":"=r"(x):::A1,A2);
return 0;
A1:
A2:
return x; // expected-note {{uninitialized use occurs here}}
return x;
}

// test9: Expect no diagnostics
int test9 (int x) {
int y;
asm goto("": "=r"(y) :::out);
return 42;
out:
return y;
}

int test10() {
int y; // expected-note {{initialize the variable 'y' to silence this warning}}
asm goto(""::::out);
return 42;
out:
return y; // expected-warning {{variable 'y' is uninitialized when used here}}
}
24 changes: 12 additions & 12 deletions clang/test/CodeGen/asm-goto.c
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@ int test1(int cond) {
int test2(int cond) {
// CHECK-LABEL: define{{.*}} i32 @test2(
// CHECK: callbr i32 asm sideeffect
// CHECK: to label %asm.fallthrough [label %label_true, label %loop]
// CHECK: to label %asm.fallthrough [label %label_true.split, label %loop.split]
// CHECK-LABEL: asm.fallthrough:
asm volatile goto("testl %0, %0; jne %l2;" : "=r"(cond) : "r"(cond) :: label_true, loop);
asm volatile goto("testl %0, %0; jne %l3;" : "=r"(cond) : "r"(cond) :: label_true, loop);
// CHECK: callbr i32 asm sideeffect
// CHECK: to label %asm.fallthrough1 [label %label_true, label %loop]
// CHECK: to label %asm.fallthrough1 [label %label_true.split2, label %loop.split3]
// CHECK-LABEL: asm.fallthrough1:
return 0;
loop:
Expand All @@ -39,13 +39,13 @@ int test2(int cond) {
int test3(int out1, int out2) {
// CHECK-LABEL: define{{.*}} i32 @test3(
// CHECK: callbr { i32, i32 } asm sideeffect
// CHECK: to label %asm.fallthrough [label %label_true, label %loop]
// CHECK: to label %asm.fallthrough [label %label_true.split, label %loop.split]
// CHECK-LABEL: asm.fallthrough:
asm volatile goto("testl %0, %0; jne %l3;" : "=r"(out1), "=r"(out2) : "r"(out1) :: label_true, loop);
asm volatile goto("testl %0, %0; jne %l4;" : "=r"(out1), "=r"(out2) : "r"(out1) :: label_true, loop);
// CHECK: callbr { i32, i32 } asm sideeffect
// CHECK: to label %asm.fallthrough2 [label %label_true, label %loop]
// CHECK-LABEL: asm.fallthrough2:
// CHECK: to label %asm.fallthrough6 [label %label_true.split11, label %loop.split14]
// CHECK-LABEL: asm.fallthrough6:
return 0;
loop:
return 0;
Expand All @@ -56,15 +56,15 @@ int test3(int out1, int out2) {
int test4(int out1, int out2) {
// CHECK-LABEL: define{{.*}} i32 @test4(
// CHECK: callbr { i32, i32 } asm sideeffect "jne ${5:l}", "={si},={di},r,0,1,!i,!i
// CHECK: to label %asm.fallthrough [label %label_true, label %loop]
// CHECK: to label %asm.fallthrough [label %label_true.split, label %loop.split]
// CHECK-LABEL: asm.fallthrough:
if (out1 < out2)
asm volatile goto("jne %l5" : "+S"(out1), "+D"(out2) : "r"(out1) :: label_true, loop);
else
asm volatile goto("jne %l7" : "+S"(out1), "+D"(out2) : "r"(out1), "r"(out2) :: label_true, loop);
// CHECK: callbr { i32, i32 } asm sideeffect "jne ${7:l}", "={si},={di},r,r,0,1,!i,!i
// CHECK: to label %asm.fallthrough2 [label %label_true, label %loop]
// CHECK-LABEL: asm.fallthrough2:
// CHECK: to label %asm.fallthrough6 [label %label_true.split11, label %loop.split14]
// CHECK-LABEL: asm.fallthrough6:
return out1 + out2;
loop:
return -1;
Expand All @@ -75,7 +75,7 @@ int test4(int out1, int out2) {
int test5(int addr, int size, int limit) {
// CHECK-LABEL: define{{.*}} i32 @test5(
// CHECK: callbr i32 asm "add $1,$0 ; jc ${4:l} ; cmp $2,$0 ; ja ${4:l} ; ", "=r,imr,imr,0,!i
// CHECK: to label %asm.fallthrough [label %t_err]
// CHECK: to label %asm.fallthrough [label %t_err.split]
// CHECK-LABEL: asm.fallthrough:
asm goto(
"add %1,%0 ; "
Expand All @@ -93,7 +93,7 @@ int test5(int addr, int size, int limit) {
int test6(int out1) {
// CHECK-LABEL: define{{.*}} i32 @test6(
// CHECK: callbr i32 asm sideeffect "testl $0, $0; testl $1, $1; jne ${3:l}", "={si},r,0,!i,!i,{{.*}}
// CHECK: to label %asm.fallthrough [label %label_true, label %landing]
// CHECK: to label %asm.fallthrough [label %label_true.split, label %landing.split]
// CHECK-LABEL: asm.fallthrough:
// CHECK-LABEL: landing:
int out2 = 42;
Expand All @@ -111,7 +111,7 @@ int test6(int out1) {
void *test7(void) {
// CHECK-LABEL: define{{.*}} ptr @test7(
// CHECK: %1 = callbr ptr asm "# $0\0A\09# ${2:l}", "=r,0,!i,~{dirflag},~{fpsr},~{flags}"(ptr %0)
// CHECK-NEXT: to label %asm.fallthrough [label %foo]
// CHECK-NEXT: to label %asm.fallthrough [label %foo.split]
void *p = &&foo;
asm goto ("# %0\n\t# %l2":"+r"(p):::foo);
foo:
Expand All @@ -123,7 +123,7 @@ void *test7(void) {
void *test8(void) {
// CHECK-LABEL: define{{.*}} ptr @test8(
// CHECK: %1 = callbr ptr asm "# $0\0A\09# ${2:l}", "=r,0,!i,~{dirflag},~{fpsr},~{flags}"(ptr %0)
// CHECK-NEXT: to label %asm.fallthrough [label %foo]
// CHECK-NEXT: to label %asm.fallthrough [label %foo.split]
void *p = &&foo;
asm goto ("# %0\n\t# %l[foo]":"+r"(p):::foo);
foo:
Expand Down
156 changes: 156 additions & 0 deletions clang/test/CodeGen/asm-goto2.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
// REQUIRES: x86-registered-target
// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -O0 -emit-llvm %s -o - | FileCheck %s

// CHECK-LABEL: @test0(
// CHECK-NEXT: entry:
// CHECK-NEXT: [[RET:%.*]] = alloca i32, align 4
// CHECK-NEXT: [[TMP0:%.*]] = callbr i32 asm "", "=r,!i,~{dirflag},~{fpsr},~{flags}"() #[[ATTR1:[0-9]+]]
// CHECK-NEXT: to label [[ASM_FALLTHROUGH:%.*]] [label %z.split], !srcloc !2
// CHECK: asm.fallthrough:
// CHECK-NEXT: store i32 [[TMP0]], ptr [[RET]], align 4
// CHECK-NEXT: store i32 42, ptr [[RET]], align 4
// CHECK-NEXT: br label [[Z:%.*]]
// CHECK: z:
// CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[RET]], align 4
// CHECK-NEXT: ret i32 [[TMP1]]
// CHECK: z.split:
// CHECK-NEXT: store i32 [[TMP0]], ptr [[RET]], align 4
// CHECK-NEXT: br label [[Z]]
//
int test0 (void) {
int ret;
asm goto ("" : "=r"(ret):::z);
ret = 42;
z:
return ret;
}

// CHECK-LABEL: @test1(
// CHECK-NEXT: entry:
// CHECK-NEXT: [[RET:%.*]] = alloca i32, align 4
// CHECK-NEXT: [[B:%.*]] = alloca i32, align 4
// CHECK-NEXT: [[TMP0:%.*]] = callbr { i32, i32 } asm "", "=r,=r,!i,~{dirflag},~{fpsr},~{flags}"() #[[ATTR1]]
// CHECK-NEXT: to label [[ASM_FALLTHROUGH:%.*]] [label %z.split], !srcloc !3
// CHECK: asm.fallthrough:
// CHECK-NEXT: [[ASMRESULT:%.*]] = extractvalue { i32, i32 } [[TMP0]], 0
// CHECK-NEXT: [[ASMRESULT1:%.*]] = extractvalue { i32, i32 } [[TMP0]], 1
// CHECK-NEXT: store i32 [[ASMRESULT]], ptr [[RET]], align 4
// CHECK-NEXT: store i32 [[ASMRESULT1]], ptr [[B]], align 4
// CHECK-NEXT: store i32 42, ptr [[RET]], align 4
// CHECK-NEXT: br label [[Z:%.*]]
// CHECK: z:
// CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[RET]], align 4
// CHECK-NEXT: ret i32 [[TMP1]]
// CHECK: z.split:
// CHECK-NEXT: [[ASMRESULT2:%.*]] = extractvalue { i32, i32 } [[TMP0]], 0
// CHECK-NEXT: [[ASMRESULT3:%.*]] = extractvalue { i32, i32 } [[TMP0]], 1
// CHECK-NEXT: store i32 [[ASMRESULT2]], ptr [[RET]], align 4
// CHECK-NEXT: store i32 [[ASMRESULT3]], ptr [[B]], align 4
// CHECK-NEXT: br label [[Z]]
//
int test1 (void) {
int ret, b;
asm goto ("" : "=r"(ret), "=r"(b):::z);
ret = 42;
z:
return ret;
}

// CHECK-LABEL: @test2(
// CHECK-NEXT: entry:
// CHECK-NEXT: [[RET:%.*]] = alloca i32, align 4
// CHECK-NEXT: [[B:%.*]] = alloca i32, align 4
// CHECK-NEXT: [[TMP0:%.*]] = callbr { i32, i32 } asm "", "=r,=r,!i,~{dirflag},~{fpsr},~{flags}"() #[[ATTR1]]
// CHECK-NEXT: to label [[ASM_FALLTHROUGH:%.*]] [label %z.split], !srcloc !4
// CHECK: asm.fallthrough:
// CHECK-NEXT: [[ASMRESULT:%.*]] = extractvalue { i32, i32 } [[TMP0]], 0
// CHECK-NEXT: [[ASMRESULT1:%.*]] = extractvalue { i32, i32 } [[TMP0]], 1
// CHECK-NEXT: store i32 [[ASMRESULT]], ptr [[RET]], align 4
// CHECK-NEXT: store i32 [[ASMRESULT1]], ptr [[B]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = callbr { i32, i32 } asm "", "=r,=r,!i,~{dirflag},~{fpsr},~{flags}"() #[[ATTR1]]
// CHECK-NEXT: to label [[ASM_FALLTHROUGH4:%.*]] [label %z.split9], !srcloc !5
// CHECK: asm.fallthrough4:
// CHECK-NEXT: [[ASMRESULT5:%.*]] = extractvalue { i32, i32 } [[TMP1]], 0
// CHECK-NEXT: [[ASMRESULT6:%.*]] = extractvalue { i32, i32 } [[TMP1]], 1
// CHECK-NEXT: store i32 [[ASMRESULT5]], ptr [[RET]], align 4
// CHECK-NEXT: store i32 [[ASMRESULT6]], ptr [[B]], align 4
// CHECK-NEXT: br label [[Z:%.*]]
// CHECK: z:
// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[RET]], align 4
// CHECK-NEXT: ret i32 [[TMP2]]
// CHECK: z.split:
// CHECK-NEXT: [[ASMRESULT2:%.*]] = extractvalue { i32, i32 } [[TMP0]], 0
// CHECK-NEXT: [[ASMRESULT3:%.*]] = extractvalue { i32, i32 } [[TMP0]], 1
// CHECK-NEXT: store i32 [[ASMRESULT2]], ptr [[RET]], align 4
// CHECK-NEXT: store i32 [[ASMRESULT3]], ptr [[B]], align 4
// CHECK-NEXT: br label [[Z]]
// CHECK: z.split9:
// CHECK-NEXT: [[ASMRESULT7:%.*]] = extractvalue { i32, i32 } [[TMP1]], 0
// CHECK-NEXT: [[ASMRESULT8:%.*]] = extractvalue { i32, i32 } [[TMP1]], 1
// CHECK-NEXT: store i32 [[ASMRESULT7]], ptr [[RET]], align 4
// CHECK-NEXT: store i32 [[ASMRESULT8]], ptr [[B]], align 4
// CHECK-NEXT: br label [[Z]]
//
int test2 (void) {
int ret, b;
asm goto ("" : "=r"(ret), "=r"(b):::z);
asm goto ("" : "=r"(ret), "=r"(b):::z);
z:
return ret;
}
// CHECK-LABEL: @test3(
// CHECK-NEXT: entry:
// CHECK-NEXT: [[RETVAL:%.*]] = alloca i32, align 4
// CHECK-NEXT: [[OUT1_ADDR:%.*]] = alloca i32, align 4
// CHECK-NEXT: store i32 [[OUT1:%.*]], ptr [[OUT1_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = callbr i32 asm "", "=r,!i,!i,~{dirflag},~{fpsr},~{flags}"() #[[ATTR1]]
// CHECK-NEXT: to label [[ASM_FALLTHROUGH:%.*]] [label [[LABEL_TRUE_SPLIT:%.*]], label %loop.split], !srcloc !6
// CHECK: asm.fallthrough:
// CHECK-NEXT: store i32 [[TMP0]], ptr [[OUT1_ADDR]], align 4
// CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4
// CHECK-NEXT: br label [[RETURN:%.*]]
// CHECK: label_true.split:
// CHECK-NEXT: store i32 [[TMP0]], ptr [[OUT1_ADDR]], align 4
// CHECK-NEXT: br label [[LABEL_TRUE:%.*]]
// CHECK: loop.split:
// CHECK-NEXT: store i32 [[TMP0]], ptr [[OUT1_ADDR]], align 4
// CHECK-NEXT: br label [[LOOP:%.*]]
// CHECK: loop:
// CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4
// CHECK-NEXT: br label [[RETURN]]
// CHECK: label_true:
// CHECK-NEXT: store i32 1, ptr [[RETVAL]], align 4
// CHECK-NEXT: br label [[RETURN]]
// CHECK: return:
// CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[RETVAL]], align 4
// CHECK-NEXT: ret i32 [[TMP1]]
//
int test3 (int out1) {
asm goto("" : "=r"(out1)::: label_true, loop);
return 0;
loop:
return 0;
label_true:
return 1;
}

// CHECK-LABEL: @test4(
// CHECK-NEXT: entry:
// CHECK-NEXT: [[X:%.*]] = alloca i32, align 4
// CHECK-NEXT: br label [[FOO:%.*]]
// CHECK: foo:
// CHECK-NEXT: [[TMP0:%.*]] = callbr i32 asm "", "=r,!i,~{dirflag},~{fpsr},~{flags}"() #[[ATTR1]]
// CHECK-NEXT: to label [[ASM_FALLTHROUGH:%.*]] [label %foo.split], !srcloc !7
// CHECK: asm.fallthrough:
// CHECK-NEXT: store i32 [[TMP0]], ptr [[X]], align 4
// CHECK-NEXT: ret void
// CHECK: foo.split:
// CHECK-NEXT: store i32 [[TMP0]], ptr [[X]], align 4
// CHECK-NEXT: br label [[FOO]]
//
void test4 (void) {
int x;
foo:
asm goto ("" : "=r"(x):::foo);
}
2 changes: 1 addition & 1 deletion clang/test/Modules/asm-goto.c
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

// CHECK-LABEL: define {{.*}} @foo(
// CHECK: callbr {{.*}} "=r,!i{{.*}}()
// CHECK-NEXT: to label %asm.fallthrough [label %indirect]
// CHECK-NEXT: to label %asm.fallthrough [label %indirect.split]

int bar(void) {
return foo();
Expand Down
3 changes: 3 additions & 0 deletions clang/test/Parser/asm-goto.c
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@
#if !__has_extension(gnu_asm_goto_with_outputs)
#error Extension 'gnu_asm_goto_with_outputs' should be available by default
#endif
#if !__has_extension(gnu_asm_goto_with_outputs_full)
#error Extension 'gnu_asm_goto_with_outputs_full' should be available by default
#endif

int a, b, c, d, e, f, g, h, i, j, k, l;

Expand Down
8 changes: 8 additions & 0 deletions llvm/docs/LangRef.rst
Original file line number Diff line number Diff line change
Expand Up @@ -8540,6 +8540,14 @@ function, with the possibility of control flow transfer to either the
This instruction should only be used to implement the "goto" feature of gcc
style inline assembly. Any other usage is an error in the IR verifier.

Note that in order to support outputs along indirect edges, LLVM may need to
split critical edges, which may require synthesizing a replacement block for
the ``indirect labels``. Therefore, the address of a label as seen by another
``callbr`` instruction, or for a :ref:`blockaddress <blockaddress>` constant,
may not be equal to the address provided for the same block to this
instruction's ``indirect labels`` operand. The assembly code may only transfer
control to addresses provided via this instruction's ``indirect labels``.

Arguments:
""""""""""

Expand Down
1 change: 1 addition & 0 deletions llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
Original file line number Diff line number Diff line change
Expand Up @@ -714,6 +714,7 @@ template <typename Derived>
void CodeGenPassBuilder<Derived>::addISelPrepare(AddIRPass &addPass) const {
derived().addPreISel(addPass);

addPass(CallBrPrepare());
// Add both the safe stack and the stack protection passes: each of them will
// only protect functions that have corresponding attributes.
addPass(SafeStackPass());
Expand Down
1 change: 1 addition & 0 deletions llvm/include/llvm/CodeGen/MachinePassRegistry.def
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ DUMMY_FUNCTION_PASS("cfguard-dispatch", CFGuardDispatchPass, ())
DUMMY_FUNCTION_PASS("cfguard-check", CFGuardCheckPass, ())
DUMMY_FUNCTION_PASS("gc-info-printer", GCInfoPrinterPass, ())
DUMMY_FUNCTION_PASS("select-optimize", SelectOptimizePass, ())
DUMMY_FUNCTION_PASS("callbrprepare", CallBrPrepare, ())
#undef DUMMY_FUNCTION_PASS

#ifndef DUMMY_MODULE_PASS
Expand Down
2 changes: 2 additions & 0 deletions llvm/include/llvm/CodeGen/Passes.h
Original file line number Diff line number Diff line change
Expand Up @@ -597,6 +597,8 @@ namespace llvm {

/// This pass converts conditional moves to conditional jumps when profitable.
FunctionPass *createSelectOptimizePass();

FunctionPass *createCallBrPass();
} // End llvm namespace

#endif
2 changes: 1 addition & 1 deletion llvm/include/llvm/IR/Dominators.h
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ class DominatorTree : public DominatorTreeBase<BasicBlock, false> {
/// * Non-instruction Defs dominate everything.
/// * Def does not dominate a use in Def itself (outside of degenerate cases
/// like unreachable code or trivial phi cycles).
/// * Invoke/callbr Defs only dominate uses in their default destination.
/// * Invoke Defs only dominate uses in their default destination.
bool dominates(const Value *Def, const Use &U) const;
/// Return true if value Def dominates all possible uses inside instruction
/// User. Same comments as for the Use-based API apply.
Expand Down
5 changes: 5 additions & 0 deletions llvm/include/llvm/IR/Intrinsics.td
Original file line number Diff line number Diff line change
Expand Up @@ -620,6 +620,11 @@ def int_call_preallocated_setup : DefaultAttrsIntrinsic<[llvm_token_ty], [llvm_i
def int_call_preallocated_arg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_i32_ty]>;
def int_call_preallocated_teardown : DefaultAttrsIntrinsic<[], [llvm_token_ty]>;

// This intrinsic is intentionally undocumented and users shouldn't call it;
// it's produced then quickly consumed during codegen.
def int_callbr_landingpad : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>],
[IntrNoMerge]>;

//===------------------- Standard C Library Intrinsics --------------------===//
//

Expand Down
1 change: 1 addition & 0 deletions llvm/include/llvm/InitializePasses.h
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ void initializeCFGuardLongjmpPass(PassRegistry&);
void initializeCFGViewerLegacyPassPass(PassRegistry&);
void initializeCFIFixupPass(PassRegistry&);
void initializeCFIInstrInserterPass(PassRegistry&);
void initializeCallBrPreparePass(PassRegistry &);
void initializeCallGraphDOTPrinterPass(PassRegistry&);
void initializeCallGraphPrinterLegacyPassPass(PassRegistry&);
void initializeCallGraphViewerPass(PassRegistry&);
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/CodeGen/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ add_llvm_component_library(LLVMCodeGen
BasicBlockSections.cpp
BasicBlockSectionsProfileReader.cpp
CalcSpillWeights.cpp
CallBrPrepare.cpp
CallingConvLower.cpp
CFGuardLongjmp.cpp
CFIFixup.cpp
Expand Down
230 changes: 230 additions & 0 deletions llvm/lib/CodeGen/CallBrPrepare.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,230 @@
//===-- CallBrPrepare - Prepare callbr for code generation ----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This pass lowers callbrs in LLVM IR in order to to assist SelectionDAG's
// codegen.
//
// In particular, this pass assists in inserting register copies for the output
// values of a callbr along the edges leading to the indirect target blocks.
// Though the output SSA value is defined by the callbr instruction itself in
// the IR representation, the value cannot be copied to the appropriate virtual
// registers prior to jumping to an indirect label, since the jump occurs
// within the user-provided assembly blob.
//
// Instead, those copies must occur separately at the beginning of each
// indirect target. That requires that we create a separate SSA definition in
// each of them (via llvm.callbr.landingpad), and may require splitting
// critical edges so we have a location to place the intrinsic. Finally, we
// remap users of the original callbr output SSA value to instead point to the
// appropriate llvm.callbr.landingpad value.
//
// Ideally, this could be done inside SelectionDAG, or in the
// MachineInstruction representation, without the use of an IR-level intrinsic.
// But, within the current framework, it’s simpler to implement as an IR pass.
// (If support for callbr in GlobalISel is implemented, it’s worth considering
// whether this is still required.)
//
//===----------------------------------------------------------------------===//

#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/iterator.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/SSAUpdater.h"

using namespace llvm;

#define DEBUG_TYPE "callbrprepare"

namespace {

class CallBrPrepare : public FunctionPass {
bool SplitCriticalEdges(ArrayRef<CallBrInst *> CBRs, DominatorTree &DT);
bool InsertIntrinsicCalls(ArrayRef<CallBrInst *> CBRs,
DominatorTree &DT) const;
void UpdateSSA(DominatorTree &DT, CallBrInst *CBR, CallInst *Intrinsic,
SSAUpdater &SSAUpdate) const;

public:
CallBrPrepare() : FunctionPass(ID) {}
void getAnalysisUsage(AnalysisUsage &AU) const override;
bool runOnFunction(Function &Fn) override;
static char ID;
};

} // end anonymous namespace

char CallBrPrepare::ID = 0;
INITIALIZE_PASS_BEGIN(CallBrPrepare, DEBUG_TYPE, "Prepare callbr", false, false)
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
INITIALIZE_PASS_END(CallBrPrepare, DEBUG_TYPE, "Prepare callbr", false, false)

FunctionPass *llvm::createCallBrPass() { return new CallBrPrepare(); }

void CallBrPrepare::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addPreserved<DominatorTreeWrapperPass>();
}

static SmallVector<CallBrInst *, 2> FindCallBrs(Function &Fn) {
SmallVector<CallBrInst *, 2> CBRs;
for (BasicBlock &BB : Fn)
if (auto *CBR = dyn_cast<CallBrInst>(BB.getTerminator()))
if (!CBR->getType()->isVoidTy() && !CBR->use_empty())
CBRs.push_back(CBR);
return CBRs;
}

bool CallBrPrepare::SplitCriticalEdges(ArrayRef<CallBrInst *> CBRs,
DominatorTree &DT) {
bool Changed = false;
CriticalEdgeSplittingOptions Options(&DT);
Options.setMergeIdenticalEdges();

// The indirect destination might be duplicated between another parameter...
// %0 = callbr ... [label %x, label %x]
// ...hence MergeIdenticalEdges and AllowIndentical edges, but we don't need
// to split the default destination if it's duplicated between an indirect
// destination...
// %1 = callbr ... to label %x [label %x]
// ...hence starting at 1 and checking against successor 0 (aka the default
// destination).
for (CallBrInst *CBR : CBRs)
for (unsigned i = 1, e = CBR->getNumSuccessors(); i != e; ++i)
if (CBR->getSuccessor(i) == CBR->getSuccessor(0) ||
isCriticalEdge(CBR, i, /*AllowIdenticalEdges*/ true))
if (SplitKnownCriticalEdge(CBR, i, Options))
Changed = true;
return Changed;
}

bool CallBrPrepare::InsertIntrinsicCalls(ArrayRef<CallBrInst *> CBRs,
DominatorTree &DT) const {
bool Changed = false;
SmallPtrSet<const BasicBlock *, 4> Visited;
IRBuilder<> Builder(CBRs[0]->getContext());
for (CallBrInst *CBR : CBRs) {
if (!CBR->getNumIndirectDests())
continue;

SSAUpdater SSAUpdate;
SSAUpdate.Initialize(CBR->getType(), CBR->getName());
SSAUpdate.AddAvailableValue(CBR->getParent(), CBR);
SSAUpdate.AddAvailableValue(CBR->getDefaultDest(), CBR);

for (BasicBlock *IndDest : CBR->getIndirectDests()) {
if (!Visited.insert(IndDest).second)
continue;
Builder.SetInsertPoint(&*IndDest->begin());
CallInst *Intrinsic = Builder.CreateIntrinsic(
CBR->getType(), Intrinsic::callbr_landingpad, {CBR});
SSAUpdate.AddAvailableValue(IndDest, Intrinsic);
UpdateSSA(DT, CBR, Intrinsic, SSAUpdate);
Changed = true;
}
}
return Changed;
}

static bool IsInSameBasicBlock(const Use &U, const BasicBlock *BB) {
const auto *I = dyn_cast<Instruction>(U.getUser());
return I && I->getParent() == BB;
}

static void PrintDebugDomInfo(const DominatorTree &DT, const Use &U,
const BasicBlock *BB, bool IsDefaultDest) {
if (!isa<Instruction>(U.getUser()))
return;
const bool IsDominated = DT.dominates(BB, U);
LLVM_DEBUG(dbgs() << "Use: " << *U.getUser() << ", in block "
<< cast<Instruction>(U.getUser())->getParent()->getName()
<< ", is " << (IsDominated ? "" : "NOT ") << "dominated by "
<< BB->getName() << " (" << (IsDefaultDest ? "in" : "")
<< "direct)\n");
}

void CallBrPrepare::UpdateSSA(DominatorTree &DT, CallBrInst *CBR,
CallInst *Intrinsic,
SSAUpdater &SSAUpdate) const {

SmallPtrSet<Use *, 4> Visited;
BasicBlock *DefaultDest = CBR->getDefaultDest();
BasicBlock *LandingPad = Intrinsic->getParent();

SmallVector<Use *, 4> Uses(make_pointer_range(CBR->uses()));
for (Use *U : Uses) {
if (!Visited.insert(U).second)
continue;

#ifndef NDEBUG
PrintDebugDomInfo(DT, *U, LandingPad, /*IsDefaultDest*/ false);
PrintDebugDomInfo(DT, *U, DefaultDest, /*IsDefaultDest*/ true);
#endif

// Don't rewrite the use in the newly inserted intrinsic.
if (const auto *II = dyn_cast<IntrinsicInst>(U->getUser()))
if (II->getIntrinsicID() == Intrinsic::callbr_landingpad)
continue;

// If the Use is in the same BasicBlock as the Intrinsic call, replace
// the Use with the value of the Intrinsic call.
if (IsInSameBasicBlock(*U, LandingPad)) {
U->set(Intrinsic);
continue;
}

// If the Use is dominated by the default dest, do not touch it.
if (DT.dominates(DefaultDest, *U))
continue;

SSAUpdate.RewriteUse(*U);
}
}

bool CallBrPrepare::runOnFunction(Function &Fn) {
bool Changed = false;
SmallVector<CallBrInst *, 2> CBRs = FindCallBrs(Fn);

if (CBRs.empty())
return Changed;

// It's highly likely that most programs do not contain CallBrInsts. Follow a
// similar pattern from SafeStackLegacyPass::runOnFunction to reuse previous
// domtree analysis if available, otherwise compute it lazily. This avoids
// forcing Dominator Tree Construction at -O0 for programs that likely do not
// contain CallBrInsts. It does pessimize programs with callbr at higher
// optimization levels, as the DominatorTree created here is not reused by
// subsequent passes.
DominatorTree *DT;
std::optional<DominatorTree> LazilyComputedDomTree;
if (auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>())
DT = &DTWP->getDomTree();
else {
LazilyComputedDomTree.emplace(Fn);
DT = &*LazilyComputedDomTree;
}

if (SplitCriticalEdges(CBRs, *DT))
Changed = true;

if (InsertIntrinsicCalls(CBRs, *DT))
Changed = true;

return Changed;
}
1 change: 1 addition & 0 deletions llvm/lib/CodeGen/CodeGen.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
initializeBasicBlockSectionsPass(Registry);
initializeBranchFolderPassPass(Registry);
initializeBranchRelaxationPass(Registry);
initializeCallBrPreparePass(Registry);
initializeCFGuardLongjmpPass(Registry);
initializeCFIFixupPass(Registry);
initializeCFIInstrInserterPass(Registry);
Expand Down
7 changes: 5 additions & 2 deletions llvm/lib/CodeGen/MachineVerifier.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -627,8 +627,11 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
// it is an entry block or landing pad.
for (const auto &LI : MBB->liveins()) {
if (isAllocatable(LI.PhysReg) && !MBB->isEHPad() &&
MBB->getIterator() != MBB->getParent()->begin()) {
report("MBB has allocatable live-in, but isn't entry or landing-pad.", MBB);
MBB->getIterator() != MBB->getParent()->begin() &&
!MBB->isInlineAsmBrIndirectTarget()) {
report("MBB has allocatable live-in, but isn't entry, landing-pad, or "
"inlineasm-br-indirect-target.",
MBB);
report_context(LI.PhysReg);
}
}
Expand Down
116 changes: 115 additions & 1 deletion llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2123,7 +2123,8 @@ void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {

DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V);
if (VMI != FuncInfo.ValueMap.end()) {
assert(!V->use_empty() && "Unused value assigned virtual registers!");
assert((!V->use_empty() || isa<CallBrInst>(V)) &&
"Unused value assigned virtual registers!");
CopyValueToVirtualRegister(V, VMI->second);
}
}
Expand Down Expand Up @@ -7317,6 +7318,9 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
case Intrinsic::experimental_vector_splice:
visitVectorSplice(I);
return;
case Intrinsic::callbr_landingpad:
visitCallBrLandingPad(I);
return;
}
}

Expand Down Expand Up @@ -11592,3 +11596,113 @@ void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
Mask.push_back(Idx + i);
setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
}

// Consider the following MIR after SelectionDAG, which produces output in
// phyregs in the first case or virtregs in the second case.
//
// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
// %5:gr32 = COPY $ebx
// %6:gr32 = COPY $edx
// %1:gr32 = COPY %6:gr32
// %0:gr32 = COPY %5:gr32
//
// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
// %1:gr32 = COPY %6:gr32
// %0:gr32 = COPY %5:gr32
//
// Given %0, we'd like to return $ebx in the first case and %5 in the second.
// Given %1, we'd like to return $edx in the first case and %6 in the second.
//
// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
// to a single virtreg (such as %0). The remaining outputs monotonically
// increase in virtreg number from there. If a callbr has no outputs, then it
// should not have a corresponding callbr landingpad; in fact, the callbr
// landingpad would not even be able to refer to such a callbr.
static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) {
MachineInstr *MI = MRI.def_begin(Reg)->getParent();
// There is definitely at least one copy.
assert(MI->getOpcode() == TargetOpcode::COPY &&
"start of copy chain MUST be COPY");
Reg = MI->getOperand(1).getReg();
MI = MRI.def_begin(Reg)->getParent();
// There may be an optional second copy.
if (MI->getOpcode() == TargetOpcode::COPY) {
assert(Reg.isVirtual() && "expected COPY of virtual register");
Reg = MI->getOperand(1).getReg();
assert(Reg.isPhysical() && "expected COPY of physical register");
MI = MRI.def_begin(Reg)->getParent();
}
// The start of the chain must be an INLINEASM_BR.
assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
"end of copy chain MUST be INLINEASM_BR");
return Reg;
}

// We must do this walk rather than the simpler
// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
// otherwise we will end up with copies of virtregs only valid along direct
// edges.
void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
SmallVector<EVT, 8> ResultVTs;
SmallVector<SDValue, 8> ResultValues;
const auto *CBR =
cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator());

const TargetLowering &TLI = DAG.getTargetLoweringInfo();
const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();

unsigned InitialDef = FuncInfo.ValueMap[CBR];
SDValue Chain = DAG.getRoot();

// Re-parse the asm constraints string.
TargetLowering::AsmOperandInfoVector TargetConstraints =
TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR);
for (auto &T : TargetConstraints) {
SDISelAsmOperandInfo OpInfo(T);
if (OpInfo.Type != InlineAsm::isOutput)
continue;

// Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
// individual constraint.
TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);

switch (OpInfo.ConstraintType) {
case TargetLowering::C_Register:
case TargetLowering::C_RegisterClass: {
// Fill in OpInfo.AssignedRegs.Regs.
getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo);

// getRegistersForValue may produce 1 to many registers based on whether
// the OpInfo.ConstraintVT is legal on the target or not.
for (size_t i = 0, e = OpInfo.AssignedRegs.Regs.size(); i != e; ++i) {
Register OriginalDef = FollowCopyChain(MRI, InitialDef++);
if (Register::isPhysicalRegister(OriginalDef))
FuncInfo.MBB->addLiveIn(OriginalDef);
// Update the assigned registers to use the original defs.
OpInfo.AssignedRegs.Regs[i] = OriginalDef;
}

SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR);
ResultValues.push_back(V);
ResultVTs.push_back(OpInfo.ConstraintVT);
break;
}
case TargetLowering::C_Other: {
SDValue Flag;
SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(),
OpInfo, DAG);
++InitialDef;
ResultValues.push_back(V);
ResultVTs.push_back(OpInfo.ConstraintVT);
break;
}
default:
break;
}
}
SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
DAG.getVTList(ResultVTs), ResultValues);
setValue(&I, V);
}
1 change: 1 addition & 0 deletions llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
Original file line number Diff line number Diff line change
Expand Up @@ -534,6 +534,7 @@ class SelectionDAGBuilder {
// These all get lowered before this pass.
void visitInvoke(const InvokeInst &I);
void visitCallBr(const CallBrInst &I);
void visitCallBrLandingPad(const CallInst &I);
void visitResume(const ResumeInst &I);

void visitUnary(const User &I, unsigned Opcode);
Expand Down
2 changes: 2 additions & 0 deletions llvm/lib/CodeGen/TargetPassConfig.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -978,6 +978,8 @@ void TargetPassConfig::addISelPrepare() {
if (requiresCodeGenSCCOrder())
addPass(new DummyCGSCCPass);

addPass(createCallBrPass());

// Add both the safe stack and the stack protection passes: each of them will
// only protect functions that have corresponding attributes.
addPass(createSafeStackPass());
Expand Down
14 changes: 0 additions & 14 deletions llvm/lib/IR/Dominators.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -194,13 +194,6 @@ bool DominatorTree::dominates(const Instruction *Def,
return dominates(E, UseBB);
}

// Callbr results are similarly only usable in the default destination.
if (const auto *CBI = dyn_cast<CallBrInst>(Def)) {
BasicBlock *NormalDest = CBI->getDefaultDest();
BasicBlockEdge E(DefBB, NormalDest);
return dominates(E, UseBB);
}

return dominates(DefBB, UseBB);
}

Expand Down Expand Up @@ -311,13 +304,6 @@ bool DominatorTree::dominates(const Value *DefV, const Use &U) const {
return dominates(E, U);
}

// Callbr results are similarly only usable in the default destination.
if (const auto *CBI = dyn_cast<CallBrInst>(Def)) {
BasicBlock *NormalDest = CBI->getDefaultDest();
BasicBlockEdge E(DefBB, NormalDest);
return dominates(E, U);
}

// If the def and use are in different blocks, do a simple CFG dominator
// tree query.
if (DefBB != UseBB)
Expand Down
5 changes: 3 additions & 2 deletions llvm/lib/IR/Instruction.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,9 @@ Instruction *Instruction::getInsertionPointAfterDef() {
InsertBB = II->getNormalDest();
InsertPt = InsertBB->getFirstInsertionPt();
} else if (auto *CB = dyn_cast<CallBrInst>(this)) {
InsertBB = CB->getDefaultDest();
InsertPt = InsertBB->getFirstInsertionPt();
// Def is available in multiple successors, there's no single dominating
// insertion point.
return nullptr;
} else {
assert(!isTerminator() && "Only invoke/callbr terminators return value");
InsertBB = getParent();
Expand Down
29 changes: 29 additions & 0 deletions llvm/lib/IR/Verifier.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5824,6 +5824,35 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
"isdata argument to llvm.aarch64.prefetch must be 0 or 1", Call);
break;
}
case Intrinsic::callbr_landingpad: {
const auto *CBR = dyn_cast<CallBrInst>(Call.getOperand(0));
Check(CBR, "intrinstic requires callbr operand", &Call);
if (!CBR)
break;

const BasicBlock *LandingPadBB = Call.getParent();
const BasicBlock *PredBB = LandingPadBB->getUniquePredecessor();
if (!PredBB) {
CheckFailed("Intrinsic in block must have 1 unique predecessor", &Call);
break;
}
if (!isa<CallBrInst>(PredBB->getTerminator())) {
CheckFailed("Intrinsic must have corresponding callbr in predecessor",
&Call);
break;
}
Check(llvm::any_of(CBR->getIndirectDests(),
[LandingPadBB](const BasicBlock *IndDest) {
return IndDest == LandingPadBB;
}),
"Intrinsic's corresponding callbr must have intrinsic's parent basic "
"block in indirect destination list",
&Call);
const Instruction &First = *LandingPadBB->begin();
Check(&First == &Call, "No other instructions may proceed intrinsic",
&Call);
break;
}
};

// Verify that there aren't any unmediated control transfers between funclets.
Expand Down
13 changes: 6 additions & 7 deletions llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3477,13 +3477,17 @@ Instruction *InstCombinerImpl::visitCallBase(CallBase &Call) {
}

/// If the callee is a constexpr cast of a function, attempt to move the cast to
/// the arguments of the call/callbr/invoke.
/// the arguments of the call/invoke.
/// CallBrInst is not supported.
bool InstCombinerImpl::transformConstExprCastCall(CallBase &Call) {
auto *Callee =
dyn_cast<Function>(Call.getCalledOperand()->stripPointerCasts());
if (!Callee)
return false;

assert(!isa<CallBrInst>(Call) &&
"CallBr's don't have a single point after a def to insert at");

// If this is a call to a thunk function, don't remove the cast. Thunks are
// used to transparently forward all incoming parameters and outgoing return
// values, so it's important to leave the cast in place.
Expand Down Expand Up @@ -3529,16 +3533,14 @@ bool InstCombinerImpl::transformConstExprCastCall(CallBase &Call) {
return false; // Attribute not compatible with transformed value.
}

// If the callbase is an invoke/callbr instruction, and the return value is
// If the callbase is an invoke instruction, and the return value is
// used by a PHI node in a successor, we cannot change the return type of
// the call because there is no place to put the cast instruction (without
// breaking the critical edge). Bail out in this case.
if (!Caller->use_empty()) {
BasicBlock *PhisNotSupportedBlock = nullptr;
if (auto *II = dyn_cast<InvokeInst>(Caller))
PhisNotSupportedBlock = II->getNormalDest();
if (auto *CB = dyn_cast<CallBrInst>(Caller))
PhisNotSupportedBlock = CB->getDefaultDest();
if (PhisNotSupportedBlock)
for (User *U : Caller->users())
if (PHINode *PN = dyn_cast<PHINode>(U))
Expand Down Expand Up @@ -3722,9 +3724,6 @@ bool InstCombinerImpl::transformConstExprCastCall(CallBase &Call) {
if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
NewCall = Builder.CreateInvoke(Callee, II->getNormalDest(),
II->getUnwindDest(), Args, OpBundles);
} else if (CallBrInst *CBI = dyn_cast<CallBrInst>(Caller)) {
NewCall = Builder.CreateCallBr(Callee, CBI->getDefaultDest(),
CBI->getIndirectDests(), Args, OpBundles);
} else {
NewCall = Builder.CreateCall(Callee, Args, OpBundles);
cast<CallInst>(NewCall)->setTailCallKind(
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/AArch64/O0-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
; CHECK-NEXT: AArch64 Stack Tagging
; CHECK-NEXT: SME ABI Pass
; CHECK-NEXT: Exception handling preparation
; CHECK-NEXT: Prepare callbr
; CHECK-NEXT: Safe Stack instrumentation pass
; CHECK-NEXT: Insert stack protectors
; CHECK-NEXT: Module Verifier
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/AArch64/O3-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@
; CHECK-NEXT: Dominator Tree Construction
; CHECK-NEXT: FunctionPass Manager
; CHECK-NEXT: Merge internal globals
; CHECK-NEXT: Prepare callbr
; CHECK-NEXT: Safe Stack instrumentation pass
; CHECK-NEXT: Insert stack protectors
; CHECK-NEXT: Module Verifier
Expand Down
559 changes: 559 additions & 0 deletions llvm/test/CodeGen/AArch64/callbr-asm-outputs-indirect-isel.ll

Large diffs are not rendered by default.

432 changes: 432 additions & 0 deletions llvm/test/CodeGen/AArch64/callbr-prepare.ll

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@
; GCN-O0-NEXT: Loop-Closed SSA Form Pass
; GCN-O0-NEXT: DummyCGSCCPass
; GCN-O0-NEXT: FunctionPass Manager
; GCN-O0-NEXT: Prepare callbr
; GCN-O0-NEXT: Safe Stack instrumentation pass
; GCN-O0-NEXT: Insert stack protectors
; GCN-O0-NEXT: Dominator Tree Construction
Expand Down Expand Up @@ -285,6 +286,7 @@
; GCN-O1-NEXT: Loop-Closed SSA Form Pass
; GCN-O1-NEXT: DummyCGSCCPass
; GCN-O1-NEXT: FunctionPass Manager
; GCN-O1-NEXT: Prepare callbr
; GCN-O1-NEXT: Safe Stack instrumentation pass
; GCN-O1-NEXT: Insert stack protectors
; GCN-O1-NEXT: Dominator Tree Construction
Expand Down Expand Up @@ -581,6 +583,7 @@
; GCN-O1-OPTS-NEXT: Loop-Closed SSA Form Pass
; GCN-O1-OPTS-NEXT: DummyCGSCCPass
; GCN-O1-OPTS-NEXT: FunctionPass Manager
; GCN-O1-OPTS-NEXT: Prepare callbr
; GCN-O1-OPTS-NEXT: Safe Stack instrumentation pass
; GCN-O1-OPTS-NEXT: Insert stack protectors
; GCN-O1-OPTS-NEXT: Dominator Tree Construction
Expand Down Expand Up @@ -886,6 +889,7 @@
; GCN-O2-NEXT: Analysis if a function is memory bound
; GCN-O2-NEXT: DummyCGSCCPass
; GCN-O2-NEXT: FunctionPass Manager
; GCN-O2-NEXT: Prepare callbr
; GCN-O2-NEXT: Safe Stack instrumentation pass
; GCN-O2-NEXT: Insert stack protectors
; GCN-O2-NEXT: Dominator Tree Construction
Expand Down Expand Up @@ -1203,6 +1207,7 @@
; GCN-O3-NEXT: Analysis if a function is memory bound
; GCN-O3-NEXT: DummyCGSCCPass
; GCN-O3-NEXT: FunctionPass Manager
; GCN-O3-NEXT: Prepare callbr
; GCN-O3-NEXT: Safe Stack instrumentation pass
; GCN-O3-NEXT: Insert stack protectors
; GCN-O3-NEXT: Dominator Tree Construction
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/ARM/O3-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
; CHECK-NEXT: Transform predicated vector loops to use MVE tail predication
; CHECK-NEXT: A No-Op Barrier Pass
; CHECK-NEXT: FunctionPass Manager
; CHECK-NEXT: Prepare callbr
; CHECK-NEXT: Safe Stack instrumentation pass
; CHECK-NEXT: Insert stack protectors
; CHECK-NEXT: Module Verifier
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/LoongArch/O0-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
; CHECK-NEXT: Exception handling preparation
; CHECK-NEXT: Prepare callbr
; CHECK-NEXT: Safe Stack instrumentation pass
; CHECK-NEXT: Insert stack protectors
; CHECK-NEXT: Module Verifier
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/LoongArch/opt-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
; CHECK-NEXT: CodeGen Prepare
; CHECK-NEXT: Dominator Tree Construction
; CHECK-NEXT: Exception handling preparation
; CHECK-NEXT: Prepare callbr
; CHECK-NEXT: Safe Stack instrumentation pass
; CHECK-NEXT: Insert stack protectors
; CHECK-NEXT: Module Verifier
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/PowerPC/O0-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
; CHECK-NEXT: Exception handling preparation
; CHECK-NEXT: Prepare callbr
; CHECK-NEXT: Safe Stack instrumentation pass
; CHECK-NEXT: Insert stack protectors
; CHECK-NEXT: Module Verifier
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/PowerPC/O3-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@
; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Optimization Remark Emitter
; CHECK-NEXT: Hardware Loop Insertion
; CHECK-NEXT: Prepare callbr
; CHECK-NEXT: Safe Stack instrumentation pass
; CHECK-NEXT: Insert stack protectors
; CHECK-NEXT: Module Verifier
Expand Down
45 changes: 45 additions & 0 deletions llvm/test/CodeGen/PowerPC/callbr-asm-outputs-indirect-isel.ll
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc %s -o - -stop-after=finalize-isel -start-before=ppc-isel | FileCheck %s

target datalayout = "e-m:e-i64:64-n32:64-S128-v256:256:256-v512:512:512"
target triple = "powerpc64le-unknown-linux-gnu"

define void @strncpy_from_kernel_nofault_count() {
; CHECK-LABEL: name: strncpy_from_kernel_nofault_count
; CHECK: bb.0.entry:
; CHECK-NEXT: successors: %bb.1(0x80000000), %bb.3(0x00000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 131082 /* regdef:GPRC */, def %1, 13 /* imm */, %bb.3
; CHECK-NEXT: [[COPY:%[0-9]+]]:gprc = COPY %1
; CHECK-NEXT: B %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.asm.fallthrough:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: STB [[COPY]], 0, $zero8 :: (store (s8) into `ptr null`)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.Efault:
; CHECK-NEXT: BLR8 implicit $lr8, implicit $rm
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3.Efault.split (machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: STB %1, 0, $zero8 :: (store (s8) into `ptr null`)
; CHECK-NEXT: B %bb.2
entry:
%0 = callbr i8 asm "", "=r,!i"()
to label %asm.fallthrough [label %Efault.split]

asm.fallthrough:
store i8 %0, ptr null, align 1
br label %Efault

Efault:
ret void

Efault.split:
%1 = call i8 @llvm.callbr.landingpad.i8(i8 %0)
store i8 %1, ptr null, align 1
br label %Efault
}
declare i8 @llvm.callbr.landingpad.i8(i8)
1 change: 1 addition & 0 deletions llvm/test/CodeGen/RISCV/O0-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
; CHECK-NEXT: Exception handling preparation
; CHECK-NEXT: Prepare callbr
; CHECK-NEXT: Safe Stack instrumentation pass
; CHECK-NEXT: Insert stack protectors
; CHECK-NEXT: Module Verifier
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/RISCV/O3-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
; CHECK-NEXT: Exception handling preparation
; CHECK-NEXT: A No-Op Barrier Pass
; CHECK-NEXT: FunctionPass Manager
; CHECK-NEXT: Prepare callbr
; CHECK-NEXT: Safe Stack instrumentation pass
; CHECK-NEXT: Insert stack protectors
; CHECK-NEXT: Module Verifier
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/X86/O0-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
; CHECK-NEXT: Expand reduction intrinsics
; CHECK-NEXT: Expand indirectbr instructions
; CHECK-NEXT: Exception handling preparation
; CHECK-NEXT: Prepare callbr
; CHECK-NEXT: Safe Stack instrumentation pass
; CHECK-NEXT: Insert stack protectors
; CHECK-NEXT: Module Verifier
Expand Down
115 changes: 115 additions & 0 deletions llvm/test/CodeGen/X86/callbr-asm-outputs-indirect-isel-m32.ll
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -mtriple=i386-linux-gnu %s -o - -stop-after=finalize-isel \
; RUN: -start-before=x86-isel | FileCheck %s

define i8 @emulator_cmpxchg_emulated() {
; CHECK-LABEL: name: emulator_cmpxchg_emulated
; CHECK: bb.0.entry:
; CHECK-NEXT: successors: %bb.1(0x80000000), %bb.2(0x00000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm $noreg, 1, $noreg, 0, $noreg :: (load (s32) from `ptr null`, align 8)
; CHECK-NEXT: INLINEASM_BR &"", 16 /* maystore attdialect */, 2359306 /* regdef:GR32 */, def %2, 2359306 /* regdef:GR32 */, def %3, 2147549193 /* reguse tiedto:$1 */, [[MOV32rm]](tied-def 5), 13 /* imm */, %bb.2
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eflags
; CHECK-NEXT: $eflags = COPY [[COPY]]
; CHECK-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY %3
; CHECK-NEXT: JMP_1 %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.asm.fallthrough:
; CHECK-NEXT: $al = COPY [[SETCCr]]
; CHECK-NEXT: RET 0, $al
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.efaultu64.split (machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK-NEXT: [[SETCCr1:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
; CHECK-NEXT: $al = COPY [[SETCCr1]]
; CHECK-NEXT: RET 0, $al
entry:
%0 = load i32, ptr null, align 8
%1 = callbr { i8, i32 } asm "", "={@ccz},=r,1,!i"(i32 %0)
to label %asm.fallthrough [label %efaultu64.split]

asm.fallthrough:
%asmresult = extractvalue { i8, i32 } %1, 0
%asmresult1 = extractvalue { i8, i32 } %1, 1
ret i8 %asmresult

efaultu64.split:
%2 = call { i8, i32 } @llvm.callbr.landingpad.sl_i8i32s({ i8, i32 } %1)
%asmresult2 = extractvalue { i8, i32 } %2, 0
%asmresult3 = extractvalue { i8, i32 } %2, 1
ret i8 %asmresult2
}

; Same test but return second value
define i32 @emulator_cmpxchg_emulated2() {
; CHECK-LABEL: name: emulator_cmpxchg_emulated2
; CHECK: bb.0.entry:
; CHECK-NEXT: successors: %bb.1(0x80000000), %bb.2(0x00000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm $noreg, 1, $noreg, 0, $noreg :: (load (s32) from `ptr null`, align 8)
; CHECK-NEXT: INLINEASM_BR &"", 16 /* maystore attdialect */, 2359306 /* regdef:GR32 */, def %2, 2359306 /* regdef:GR32 */, def %3, 2147549193 /* reguse tiedto:$1 */, [[MOV32rm]](tied-def 5), 13 /* imm */, %bb.2
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eflags
; CHECK-NEXT: $eflags = COPY [[COPY]]
; CHECK-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY %3
; CHECK-NEXT: JMP_1 %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.asm.fallthrough:
; CHECK-NEXT: $eax = COPY [[COPY1]]
; CHECK-NEXT: RET 0, $eax
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.efaultu64.split (machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK-NEXT: $eax = COPY %3
; CHECK-NEXT: RET 0, $eax
entry:
%0 = load i32, ptr null, align 8
%1 = callbr { i8, i32 } asm "", "={@ccz},=r,1,!i"(i32 %0)
to label %asm.fallthrough [label %efaultu64.split]

asm.fallthrough:
%asmresult = extractvalue { i8, i32 } %1, 0
%asmresult1 = extractvalue { i8, i32 } %1, 1
ret i32 %asmresult1

efaultu64.split:
%2 = call { i8, i32 } @llvm.callbr.landingpad.sl_i8i32s({ i8, i32 } %1)
%asmresult2 = extractvalue { i8, i32 } %2, 0
%asmresult3 = extractvalue { i8, i32 } %2, 1
ret i32 %asmresult3
}

define i64 @multireg() {
; CHECK-LABEL: name: multireg
; CHECK: bb.0.entry:
; CHECK-NEXT: successors: %bb.1(0x80000000), %bb.2(0x00000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 18 /* regdef */, implicit-def $eax, implicit-def $edx, 13 /* imm */, %bb.2
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $edx
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY [[COPY1]]
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr32 = COPY [[COPY]]
; CHECK-NEXT: JMP_1 %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.ft:
; CHECK-NEXT: $eax = COPY [[COPY3]]
; CHECK-NEXT: $edx = COPY [[COPY2]]
; CHECK-NEXT: RET 0, $eax, $edx
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.split (machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK-NEXT: liveins: $eax, $edx
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr32 = COPY $eax
; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr32 = COPY $edx
; CHECK-NEXT: $eax = COPY [[COPY4]]
; CHECK-NEXT: $edx = COPY [[COPY5]]
; CHECK-NEXT: RET 0, $eax, $edx
entry:
%0 = callbr i64 asm "", "=A,!i"() to label %ft [label %split]
ft:
ret i64 %0
split:
%1 = call i64 @llvm.callbr.landingpad.i64(i64 %0)
ret i64 %1
}
declare i64 @llvm.callbr.landingpad.i64(i64)
declare { i8, i32 } @llvm.callbr.landingpad.sl_i8i32s({ i8, i32 })
368 changes: 368 additions & 0 deletions llvm/test/CodeGen/X86/callbr-asm-outputs-indirect-isel.ll
Original file line number Diff line number Diff line change
@@ -0,0 +1,368 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -mtriple=x86_64-linux-gnu %s -o - -stop-after=finalize-isel \
; RUN: -start-before=x86-isel | FileCheck %s

; One virtual register, w/o phi
define i32 @test0() {
; CHECK-LABEL: name: test0
; CHECK: bb.0 (%ir-block.0):
; CHECK-NEXT: successors: %bb.1(0x80000000), %bb.2(0x00000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 2359306 /* regdef:GR32 */, def %1, 13 /* imm */, %bb.2
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY %1
; CHECK-NEXT: JMP_1 %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.cleanup:
; CHECK-NEXT: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 42
; CHECK-NEXT: $eax = COPY [[MOV32ri]]
; CHECK-NEXT: RET 0, $eax
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.z.split (machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK-NEXT: $eax = COPY %1
; CHECK-NEXT: RET 0, $eax
%direct = callbr i32 asm "", "=r,!i"()
to label %cleanup [label %z.split]

cleanup:
ret i32 42
z.split:
%indirect = call i32 @llvm.callbr.landingpad.i32(i32 %direct)
ret i32 %indirect
}

; One virtual register, w/ phi
define i32 @test1() {
; CHECK-LABEL: name: test1
; CHECK: bb.0.entry:
; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.1(0x00000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 42
; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 2359306 /* regdef:GR32 */, def %4, 13 /* imm */, %bb.1
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY %4
; CHECK-NEXT: JMP_1 %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.z.split (machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY %4
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.cleanup:
; CHECK-NEXT: [[PHI:%[0-9]+]]:gr32 = PHI [[MOV32ri]], %bb.0, [[COPY1]], %bb.1
; CHECK-NEXT: $eax = COPY [[PHI]]
; CHECK-NEXT: RET 0, $eax
entry:
%direct = callbr i32 asm "", "=r,!i"()
to label %cleanup [label %z.split]

z.split:
%indirect = call i32 @llvm.callbr.landingpad.i32(i32 %direct)
br label %cleanup

cleanup:
%retval.0 = phi i32 [ %indirect, %z.split ], [ 42, %entry ]
ret i32 %retval.0
}

; Two virtual registers
define i32 @test2() {
; CHECK-LABEL: name: test2
; CHECK: bb.0.entry:
; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.1(0x00000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 42
; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 2359306 /* regdef:GR32 */, def %5, 2359306 /* regdef:GR32 */, def %6, 13 /* imm */, %bb.1
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY %6
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY %5
; CHECK-NEXT: JMP_1 %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.z.split (machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY %5
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.cleanup:
; CHECK-NEXT: [[PHI:%[0-9]+]]:gr32 = PHI [[MOV32ri]], %bb.0, [[COPY2]], %bb.1
; CHECK-NEXT: $eax = COPY [[PHI]]
; CHECK-NEXT: RET 0, $eax
entry:
%direct = callbr { i32, i32 } asm "", "=r,=r,!i"()
to label %cleanup [label %z.split]

z.split:
%indirect = call { i32, i32 } @llvm.callbr.landingpad.sl_i32i32s({ i32, i32 } %direct)
%asmresult2 = extractvalue { i32, i32 } %indirect, 0
br label %cleanup

cleanup:
%retval.0 = phi i32 [ %asmresult2, %z.split ], [ 42, %entry ]
ret i32 %retval.0
}

; One physical register
define i32 @test3() {
; CHECK-LABEL: name: test3
; CHECK: bb.0.entry:
; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.1(0x00000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 42
; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 10 /* regdef */, implicit-def $ebx, 13 /* imm */, %bb.1
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY [[COPY]]
; CHECK-NEXT: JMP_1 %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.z.split (machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: liveins: $ebx
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY $ebx
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr32 = COPY [[COPY2]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.cleanup:
; CHECK-NEXT: [[PHI:%[0-9]+]]:gr32 = PHI [[MOV32ri]], %bb.0, [[COPY3]], %bb.1
; CHECK-NEXT: $eax = COPY [[PHI]]
; CHECK-NEXT: RET 0, $eax
entry:
%direct = callbr i32 asm "", "={bx},!i"()
to label %cleanup [label %z.split]

z.split:
%indirect = call i32 @llvm.callbr.landingpad.i32(i32 %direct)
br label %cleanup

cleanup:
%retval.0 = phi i32 [ %indirect, %z.split ], [ 42, %entry ]
ret i32 %retval.0
}

; Two physical registers
define i32 @test4() {
; CHECK-LABEL: name: test4
; CHECK: bb.0.entry:
; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.1(0x00000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 42
; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 10 /* regdef */, implicit-def $ebx, 10 /* regdef */, implicit-def $edx, 13 /* imm */, %bb.1
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $edx
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY [[COPY1]]
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr32 = COPY [[COPY]]
; CHECK-NEXT: JMP_1 %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.z.split (machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: liveins: $ebx, $edx
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr32 = COPY $ebx
; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr32 = COPY [[COPY4]]
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.cleanup:
; CHECK-NEXT: [[PHI:%[0-9]+]]:gr32 = PHI [[MOV32ri]], %bb.0, [[COPY5]], %bb.1
; CHECK-NEXT: $eax = COPY [[PHI]]
; CHECK-NEXT: RET 0, $eax
entry:
%direct = callbr { i32, i32 } asm "", "={bx},={dx},!i"()
to label %cleanup [label %z.split]

z.split:
%indirect = call { i32, i32 } @llvm.callbr.landingpad.sl_i32i32s({ i32, i32 } %direct)
%asmresult2 = extractvalue { i32, i32 } %indirect, 0
br label %cleanup

cleanup:
%retval.0 = phi i32 [ %asmresult2, %z.split ], [ 42, %entry ]
ret i32 %retval.0
}

; Test the same destination appearing in the direct/fallthrough branch as the
; indirect branch. Physreg.
define i32 @test5() {
; CHECK-LABEL: name: test5
; CHECK: bb.0.entry:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: INLINEASM_BR &"# $0", 0 /* attdialect */, 10 /* regdef */, implicit-def $ebx, 13 /* imm */, %bb.1
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY [[COPY]]
; CHECK-NEXT: JMP_1 %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.cleanup (machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK-NEXT: liveins: $ebx
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY $ebx
; CHECK-NEXT: $eax = COPY [[COPY2]]
; CHECK-NEXT: RET 0, $eax
entry:
%direct = callbr i32 asm "# $0", "={bx},!i"()
to label %cleanup [label %cleanup]

cleanup:
%indirect = call i32 @llvm.callbr.landingpad.i32(i32 %direct)
ret i32 %indirect
}

; "The Devil's cross" (i.e. two asm goto with conflicting physreg constraints
; going to the same destination) as expressed by clang.
define i64 @test6() {
; CHECK-LABEL: name: test6
; CHECK: bb.0.entry:
; CHECK-NEXT: successors: %bb.1(0x80000000), %bb.3(0x00000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 10 /* regdef */, implicit-def $rdx, 13 /* imm */, %bb.3
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdx
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY [[COPY]]
; CHECK-NEXT: JMP_1 %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.asm.fallthrough:
; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.4(0x00000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 10 /* regdef */, implicit-def $rbx, 13 /* imm */, %bb.4
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY $rbx
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY [[COPY2]]
; CHECK-NEXT: JMP_1 %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.foo:
; CHECK-NEXT: [[PHI:%[0-9]+]]:gr64 = PHI %3, %bb.3, [[COPY3]], %bb.1, %4, %bb.4
; CHECK-NEXT: $rax = COPY [[PHI]]
; CHECK-NEXT: RET 0, $rax
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3.foo.split (machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: liveins: $rdx
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $rdx
; CHECK-NEXT: %3:gr64 = COPY [[COPY4]]
; CHECK-NEXT: JMP_1 %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.4.foo.split2 (machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: liveins: $rbx
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY6:%[0-9]+]]:gr64 = COPY $rbx
; CHECK-NEXT: %4:gr64 = COPY [[COPY6]]
; CHECK-NEXT: JMP_1 %bb.2
entry:
%0 = callbr i64 asm "", "={dx},!i"()
to label %asm.fallthrough [label %foo.split]

asm.fallthrough:
%1 = callbr i64 asm "", "={bx},!i"()
to label %foo [label %foo.split2]

foo:
%x.0 = phi i64 [ %3, %foo.split2 ], [ %2, %foo.split ], [ %1, %asm.fallthrough ]
ret i64 %x.0

foo.split:
%2 = call i64 @llvm.callbr.landingpad.i64(i64 %0)
br label %foo

foo.split2:
%3 = call i64 @llvm.callbr.landingpad.i64(i64 %1)
br label %foo
}


; Test a callbr looping back on itself.
define i32 @test7() {
; CHECK-LABEL: name: test7
; CHECK: bb.0.entry:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[DEF:%[0-9]+]]:gr32 = IMPLICIT_DEF
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.retry:
; CHECK-NEXT: successors: %bb.2(0x80000000), %bb.3(0x00000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[PHI:%[0-9]+]]:gr32 = PHI [[DEF]], %bb.0, %2, %bb.3
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY [[PHI]]
; CHECK-NEXT: INLINEASM_BR &"", 0 /* attdialect */, 10 /* regdef */, implicit-def $edx, 2147483657 /* reguse tiedto:$0 */, [[COPY]](tied-def 3), 13 /* imm */, %bb.3
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $edx
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY [[COPY1]]
; CHECK-NEXT: JMP_1 %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.asm.fallthrough:
; CHECK-NEXT: $eax = COPY [[COPY2]]
; CHECK-NEXT: RET 0, $eax
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3.retry.split (machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: liveins: $edx
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr32 = COPY $edx
; CHECK-NEXT: %2:gr32 = COPY [[COPY3]]
; CHECK-NEXT: JMP_1 %bb.1
entry:
br label %retry

retry:
%x.0 = phi i32 [ undef, %entry ], [ %1, %retry.split ]
%0 = callbr i32 asm "", "={dx},0,!i"(i32 %x.0)
to label %asm.fallthrough [label %retry.split]

asm.fallthrough:
ret i32 %0

retry.split:
%1 = call i32 @llvm.callbr.landingpad.i32(i32 %0)
br label %retry
}

; Test the same destination appearing in the direct/fallthrough branch as the
; indirect branch. Same as test5 but with a virtreg rather than a physreg
; constraint.
define i32 @test8() {
; CHECK-LABEL: name: test8
; CHECK: bb.0.entry:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: INLINEASM_BR &"# $0", 0 /* attdialect */, 2359306 /* regdef:GR32 */, def %1, 13 /* imm */, %bb.1
; CHECK-NEXT: %0:gr32 = COPY %1
; CHECK-NEXT: JMP_1 %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.cleanup (machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK-NEXT: $eax = COPY %1
; CHECK-NEXT: RET 0, $eax
entry:
%direct = callbr i32 asm "# $0", "=r,!i"()
to label %cleanup [label %cleanup]

cleanup:
%indirect = call i32 @llvm.callbr.landingpad.i32(i32 %direct)
ret i32 %indirect
}

define i64 @condition_code() {
; CHECK-LABEL: name: condition_code
; CHECK: bb.0 (%ir-block.0):
; CHECK-NEXT: successors: %bb.1(0x80000000), %bb.2(0x00000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: INLINEASM_BR &"", 16 /* maystore attdialect */, 2359306 /* regdef:GR32 */, def %1, 13 /* imm */, %bb.2
; CHECK-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
; CHECK-NEXT: [[MOVZX32rr8_:%[0-9]+]]:gr32 = MOVZX32rr8 killed [[SETCCr]]
; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gr64 = SUBREG_TO_REG 0, killed [[MOVZX32rr8_]], %subreg.sub_32bit
; CHECK-NEXT: JMP_1 %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.b:
; CHECK-NEXT: $rax = COPY [[SUBREG_TO_REG]]
; CHECK-NEXT: RET 0, $rax
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.c (machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK-NEXT: [[SETCCr1:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
; CHECK-NEXT: [[MOVZX32rr8_1:%[0-9]+]]:gr32 = MOVZX32rr8 killed [[SETCCr1]]
; CHECK-NEXT: [[SUBREG_TO_REG1:%[0-9]+]]:gr64 = SUBREG_TO_REG 0, killed [[MOVZX32rr8_1]], %subreg.sub_32bit
; CHECK-NEXT: $rax = COPY [[SUBREG_TO_REG1]]
; CHECK-NEXT: RET 0, $rax
%a = callbr i64 asm "", "={@ccz},!i"()
to label %b [label %c]

b:
ret i64 %a

c:
%1 = call i64 @llvm.callbr.landingpad.i64(i64 %a)
ret i64 %1
}

declare i64 @llvm.callbr.landingpad.i64(i64)
declare i32 @llvm.callbr.landingpad.i32(i32)
declare { i32, i32 } @llvm.callbr.landingpad.sl_i32i32s({ i32, i32 })
2 changes: 1 addition & 1 deletion llvm/test/CodeGen/X86/callbr-asm-outputs-pred-succ.ll
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

; Check the first INLINEASM_BR target block is predecessed by the block with
; the first INLINEASM_BR.
; CHECK: bb.4 (%ir-block.11, machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK: bb.4 (%ir-block.12, machine-block-address-taken, inlineasm-br-indirect-target):
; CHECK-NEXT: predecessors: %bb.0

@.str = private unnamed_addr constant [26 x i8] c"inline asm#1 returned %d\0A\00", align 1
Expand Down
81 changes: 51 additions & 30 deletions llvm/test/CodeGen/X86/callbr-asm-outputs.ll
Original file line number Diff line number Diff line change
Expand Up @@ -38,36 +38,46 @@ define i32 @test2(i32 %out1, i32 %out2) nounwind {
; CHECK-NEXT: pushl %esi
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edi
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %esi
; CHECK-NEXT: movl $-1, %eax
; CHECK-NEXT: cmpl %edi, %esi
; CHECK-NEXT: jge .LBB1_2
; CHECK-NEXT: jge .LBB1_3
; CHECK-NEXT: # %bb.1: # %if.then
; CHECK-NEXT: #APP
; CHECK-NEXT: testl %esi, %esi
; CHECK-NEXT: testl %edi, %esi
; CHECK-NEXT: jne .LBB1_4
; CHECK-NEXT: jne .LBB1_2
; CHECK-NEXT: #NO_APP
; CHECK-NEXT: jmp .LBB1_3
; CHECK-NEXT: .LBB1_2: # %if.else
; CHECK-NEXT: jmp .LBB1_4
; CHECK-NEXT: .LBB1_2: # Block address taken
; CHECK-NEXT: # %if.then.label_true_crit_edge
; CHECK-NEXT: # Label of block must be emitted
; CHECK-NEXT: jmp .LBB1_8
; CHECK-NEXT: .LBB1_3: # %if.else
; CHECK-NEXT: #APP
; CHECK-NEXT: testl %esi, %edi
; CHECK-NEXT: testl %esi, %edi
; CHECK-NEXT: jne .LBB1_5
; CHECK-NEXT: jne .LBB1_9
; CHECK-NEXT: #NO_APP
; CHECK-NEXT: .LBB1_3:
; CHECK-NEXT: .LBB1_4:
; CHECK-NEXT: movl %esi, %eax
; CHECK-NEXT: addl %edi, %eax
; CHECK-NEXT: .LBB1_5: # Block address taken
; CHECK-NEXT: # %return
; CHECK-NEXT: # Label of block must be emitted
; CHECK-NEXT: .LBB1_5: # %return
; CHECK-NEXT: popl %esi
; CHECK-NEXT: popl %edi
; CHECK-NEXT: retl
; CHECK-NEXT: .LBB1_4: # Block address taken
; CHECK-NEXT: # %label_true
; CHECK-NEXT: .LBB1_7: # Block address taken
; CHECK-NEXT: # %if.else.label_true_crit_edge
; CHECK-NEXT: # Label of block must be emitted
; CHECK-NEXT: .LBB1_8: # %label_true
; CHECK-NEXT: movl $-2, %eax
; CHECK-NEXT: jmp .LBB1_5
; CHECK-NEXT: .LBB1_9: # Block address taken
; CHECK-NEXT: # %if.else.return_crit_edge
; CHECK-NEXT: # Label of block must be emitted
; CHECK-NEXT: .LBB1_6: # Block address taken
; CHECK-NEXT: # %if.then.return_crit_edge
; CHECK-NEXT: # Label of block must be emitted
; CHECK-NEXT: movl $-1, %eax
; CHECK-NEXT: jmp .LBB1_5
entry:
%cmp = icmp slt i32 %out1, %out2
br i1 %cmp, label %if.then, label %if.else
Expand Down Expand Up @@ -109,23 +119,25 @@ define i32 @test3(i1 %cmp) nounwind {
; CHECK-NEXT: #NO_APP
; CHECK-NEXT: # %bb.2:
; CHECK-NEXT: movl %edi, %eax
; CHECK-NEXT: jmp .LBB2_5
; CHECK-NEXT: jmp .LBB2_4
; CHECK-NEXT: .LBB2_3: # %false
; CHECK-NEXT: #APP
; CHECK-NEXT: .short %eax
; CHECK-NEXT: .short %edx
; CHECK-NEXT: #NO_APP
; CHECK-NEXT: # %bb.4:
; CHECK-NEXT: movl %edx, %eax
; CHECK-NEXT: .LBB2_5: # %asm.fallthrough
; CHECK-NEXT: .LBB2_4: # %asm.fallthrough
; CHECK-NEXT: popl %esi
; CHECK-NEXT: popl %edi
; CHECK-NEXT: retl
; CHECK-NEXT: .LBB2_5: # Block address taken
; CHECK-NEXT: # %true.indirect_crit_edge
; CHECK-NEXT: # Label of block must be emitted
; CHECK-NEXT: .LBB2_6: # Block address taken
; CHECK-NEXT: # %indirect
; CHECK-NEXT: # %false.indirect_crit_edge
; CHECK-NEXT: # Label of block must be emitted
; CHECK-NEXT: movl $42, %eax
; CHECK-NEXT: jmp .LBB2_5
; CHECK-NEXT: jmp .LBB2_4
entry:
br i1 %cmp, label %true, label %false

Expand All @@ -148,31 +160,37 @@ indirect:
define i32 @test4(i32 %out1, i32 %out2) {
; CHECK-LABEL: test4:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: movl $-1, %eax
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
; CHECK-NEXT: #APP
; CHECK-NEXT: testl %ecx, %ecx
; CHECK-NEXT: testl %edx, %ecx
; CHECK-NEXT: testl %eax, %eax
; CHECK-NEXT: testl %ecx, %eax
; CHECK-NEXT: jne .LBB3_3
; CHECK-NEXT: #NO_APP
; CHECK-NEXT: # %bb.1: # %asm.fallthrough
; CHECK-NEXT: #APP
; CHECK-NEXT: testl %ecx, %edx
; CHECK-NEXT: testl %ecx, %edx
; CHECK-NEXT: jne .LBB3_4
; CHECK-NEXT: testl %eax, %ecx
; CHECK-NEXT: testl %eax, %ecx
; CHECK-NEXT: jne .LBB3_5
; CHECK-NEXT: #NO_APP
; CHECK-NEXT: # %bb.2: # %asm.fallthrough2
; CHECK-NEXT: addl %edx, %ecx
; CHECK-NEXT: movl %ecx, %eax
; CHECK-NEXT: addl %ecx, %eax
; CHECK-NEXT: retl
; CHECK-NEXT: .LBB3_4: # Block address taken
; CHECK-NEXT: # %return
; CHECK-NEXT: # %entry.return_crit_edge
; CHECK-NEXT: # Label of block must be emitted
; CHECK-NEXT: .LBB3_5: # Block address taken
; CHECK-NEXT: # %asm.fallthrough.return_crit_edge
; CHECK-NEXT: # Label of block must be emitted
; CHECK-NEXT: movl $-1, %eax
; CHECK-NEXT: retl
; CHECK-NEXT: .LBB3_6: # Block address taken
; CHECK-NEXT: # %asm.fallthrough.label_true_crit_edge
; CHECK-NEXT: # Label of block must be emitted
; CHECK-NEXT: .LBB3_3: # Block address taken
; CHECK-NEXT: # %label_true
; CHECK-NEXT: # %entry.label_true_crit_edge
; CHECK-NEXT: # Label of block must be emitted
; CHECK-NEXT: movl $-2, %eax
; CHECK-NEXT: jmp .LBB3_4
; CHECK-NEXT: retl
entry:
%0 = callbr { i32, i32 } asm sideeffect "testl $0, $0; testl $1, $2; jne ${3:l}", "=r,=r,r,!i,!i"(i32 %out1)
to label %asm.fallthrough [label %label_true, label %return]
Expand Down Expand Up @@ -206,7 +224,10 @@ define dso_local void @test5() {
; CHECK: # %bb.0:
; CHECK-NEXT: #APP
; CHECK-NEXT: #NO_APP
; CHECK-NEXT: .LBB4_1: # Block address taken
; CHECK-NEXT: # %bb.1:
; CHECK-NEXT: retl
; CHECK-NEXT: .LBB4_2: # Block address taken
; CHECK-NEXT: # %._crit_edge
; CHECK-NEXT: # Label of block must be emitted
; CHECK-NEXT: retl
%1 = call i32 @llvm.read_register.i32(metadata !3)
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/X86/opt-pipeline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@
; CHECK-NEXT: CodeGen Prepare
; CHECK-NEXT: Dominator Tree Construction
; CHECK-NEXT: Exception handling preparation
; CHECK-NEXT: Prepare callbr
; CHECK-NEXT: Safe Stack instrumentation pass
; CHECK-NEXT: Insert stack protectors
; CHECK-NEXT: Module Verifier
Expand Down
3 changes: 2 additions & 1 deletion llvm/test/Transforms/Coroutines/coro-debug.ll
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,8 @@ attributes #7 = { noduplicate }
; CHECK: %[[CALLBR_RES:.+]] = callbr i32 asm
; CHECK-NEXT: to label %[[DEFAULT_DEST:.+]] [label
; CHECK: [[DEFAULT_DEST]]:
; CHECK-NEXT: call void @llvm.dbg.declare(metadata i32 %[[CALLBR_RES]]
; CHECK-NOT: {{.*}}:
; CHECK: call void @llvm.dbg.declare(metadata i32 %[[CALLBR_RES]]
; CHECK: define internal fastcc void @f.destroy(%f.Frame* noundef nonnull align 8 dereferenceable(40) %FramePtr) #0 personality i32 0 !dbg ![[DESTROY:[0-9]+]]
; CHECK: define internal fastcc void @f.cleanup(%f.Frame* noundef nonnull align 8 dereferenceable(40) %FramePtr) #0 personality i32 0 !dbg ![[CLEANUP:[0-9]+]]

Expand Down
2 changes: 1 addition & 1 deletion llvm/test/Transforms/InstCombine/freeze.ll
Original file line number Diff line number Diff line change
Expand Up @@ -453,9 +453,9 @@ define i32 @freeze_callbr_use_after_phi(i1 %c) {
; CHECK-NEXT: to label [[CALLBR_CONT:%.*]] []
; CHECK: callbr.cont:
; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[X]], [[ENTRY:%.*]] ], [ 0, [[CALLBR_CONT]] ]
; CHECK-NEXT: call void @use_i32(i32 [[X]])
; CHECK-NEXT: [[FR:%.*]] = freeze i32 [[X]]
; CHECK-NEXT: call void @use_i32(i32 [[FR]])
; CHECK-NEXT: call void @use_i32(i32 [[FR]])
; CHECK-NEXT: call void @use_i32(i32 [[PHI]])
; CHECK-NEXT: br label [[CALLBR_CONT]]
;
Expand Down
6 changes: 4 additions & 2 deletions llvm/test/Transforms/Reassociate/callbr.ll
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@ define i32 @test(i1 %b) {
; CHECK-NEXT: [[RES:%.*]] = callbr i32 asm "", "=r,!i"()
; CHECK-NEXT: to label [[NORMAL:%.*]] [label %abnormal]
; CHECK: normal:
; CHECK-NEXT: [[FACTOR:%.*]] = mul i32 [[RES]], -2
; CHECK-NEXT: [[SUB2:%.*]] = add i32 [[FACTOR]], 5
; CHECK-NEXT: [[RES_NEG:%.*]] = sub i32 0, [[RES]]
; CHECK-NEXT: [[SUB1:%.*]] = add i32 [[RES_NEG]], 5
; CHECK-NEXT: [[RES_NEG1:%.*]] = sub i32 0, [[RES]]
; CHECK-NEXT: [[SUB2:%.*]] = add i32 [[SUB1]], [[RES_NEG1]]
; CHECK-NEXT: ret i32 [[SUB2]]
; CHECK: abnormal:
; CHECK-NEXT: ret i32 0
Expand Down
49 changes: 48 additions & 1 deletion llvm/test/Transforms/SimplifyCFG/callbr-destinations.ll
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -S -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 | FileCheck %s
; RUN: opt < %s -S -passes="simplifycfg<sink-common-insts>" \
; RUN: -simplifycfg-require-and-preserve-domtree=1 | FileCheck %s

define void @callbr_duplicate_dest() {
; CHECK-LABEL: @callbr_duplicate_dest(
Expand Down Expand Up @@ -57,3 +58,49 @@ bb2:
bb3:
ret void
}

; Validate that callbr landingpad intrinsics do not get merged (via the
; IntrNoMerge attribute).
define i32 @callbr_landingpad_nomerge() {
; CHECK-LABEL: @callbr_landingpad_nomerge(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[OUT:%.*]] = callbr i32 asm "# $0", "=r,!i"()
; CHECK-NEXT: to label [[DIRECT:%.*]] [label %entry.indirect_crit_edge]
; CHECK: entry.indirect_crit_edge:
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[OUT]])
; CHECK-NEXT: br label [[COMMON_RET:%.*]]
; CHECK: direct:
; CHECK-NEXT: [[OUT2:%.*]] = callbr i32 asm "# $0", "=r,!i"()
; CHECK-NEXT: to label [[COMMON_RET]] [label %direct.indirect_crit_edge]
; CHECK: direct.indirect_crit_edge:
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.callbr.landingpad.i32(i32 [[OUT2]])
; CHECK-NEXT: br label [[COMMON_RET]]
; CHECK: common.ret:
; CHECK-NEXT: [[COMMON_RET_OP:%.*]] = phi i32 [ 0, [[DIRECT]] ], [ [[TMP0]], [[ENTRY_INDIRECT_CRIT_EDGE:%.*]] ], [ [[TMP1]], [[DIRECT_INDIRECT_CRIT_EDGE:%.*]] ]
; CHECK-NEXT: ret i32 [[COMMON_RET_OP]]
;
entry:
%out = callbr i32 asm "# $0", "=r,!i"()
to label %direct [label %entry.indirect_crit_edge]

entry.indirect_crit_edge:
%0 = call i32 @llvm.callbr.landingpad.i32(i32 %out)
br label %indirect

direct:
%out2 = callbr i32 asm "# $0", "=r,!i"()
to label %direct2 [label %direct.indirect_crit_edge]

direct.indirect_crit_edge:
%1 = call i32 @llvm.callbr.landingpad.i32(i32 %out2)
br label %indirect

direct2:
ret i32 0

indirect:
%out3 = phi i32 [ %0, %entry.indirect_crit_edge ], [ %1, %direct.indirect_crit_edge ]
ret i32 %out3
}

declare i32 @llvm.callbr.landingpad.i32(i32)
57 changes: 54 additions & 3 deletions llvm/test/Verifier/callbr.ll
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,8 @@ define void @callbr_without_label_constraint() {
ret void
}

;; Ensure you cannot use the return value of a callbr in indirect targets.
; CHECK: Instruction does not dominate all uses!
; CHECK-NEXT: #test4
;; Ensure you can use the return value of a callbr in indirect targets.
;; No issue!
define i32 @test4(i1 %var) {
entry:
%ret = callbr i32 asm sideeffect "#test4", "=r,!i"() to label %normal [label %abnormal]
Expand All @@ -69,3 +68,55 @@ normal:
abnormal:
ret i32 %ret
}

;; Tests of the callbr.landingpad intrinsic function.
declare i32 @llvm.callbr.landingpad.i64(i64)
define void @callbrpad_bad_type() {
entry:
; CHECK: Intrinsic has incorrect argument type!
; CHECK-NEXT: ptr @llvm.callbr.landingpad.i64
%foo = call i32 @llvm.callbr.landingpad.i64(i64 42)
ret void
}

declare i32 @llvm.callbr.landingpad.i32(i32)
define i32 @callbrpad_multi_preds() {
entry:
%foo = callbr i32 asm "", "=r,!i"() to label %direct [label %indirect]
direct:
br label %indirect
indirect:
; CHECK-NEXT: Intrinsic in block must have 1 unique predecessor
; CHECK-NEXT: %out = call i32 @llvm.callbr.landingpad.i32(i32 %foo)
%out = call i32 @llvm.callbr.landingpad.i32(i32 %foo)
ret i32 %out
}

define void @callbrpad_wrong_callbr() {
entry:
%foo = callbr i32 asm "", "=r,!i"() to label %direct [label %indirect]
direct:
; CHECK-NEXT: Intrinsic's corresponding callbr must have intrinsic's parent basic block in indirect destination list
; CHECK-NEXT: %x = call i32 @llvm.callbr.landingpad.i32(i32 %foo)
%x = call i32 @llvm.callbr.landingpad.i32(i32 %foo)
ret void
indirect:
ret void
}

declare i32 @foo(i32)
define i32 @test_callbr_landingpad_not_first_inst() {
entry:
%0 = callbr i32 asm "", "=r,!i"()
to label %asm.fallthrough [label %landingpad]

asm.fallthrough:
ret i32 42

landingpad:
%foo = call i32 @foo(i32 42)
; CHECK-NEXT: No other instructions may proceed intrinsic
; CHECK-NEXT: %out = call i32 @llvm.callbr.landingpad.i32(i32 %0)
%out = call i32 @llvm.callbr.landingpad.i32(i32 %0)
ret i32 %out
}
5 changes: 1 addition & 4 deletions llvm/test/Verifier/dominates.ll
Original file line number Diff line number Diff line change
Expand Up @@ -69,15 +69,12 @@ next:
; CHECK-NEXT: %x = phi i32 [ %y, %entry ]
}

;; No issue!
define i32 @f6(i32 %x) {
bb0:
%y1 = callbr i32 asm "", "=r,!i"() to label %bb1 [label %bb2]
bb1:
ret i32 0
bb2:
ret i32 %y1
; CHECK: Instruction does not dominate all uses!
; CHECK-NEXT: %y1 = callbr i32 asm "", "=r,!i"()
; CHECK-NEXT: to label %bb1 [label %bb2]
; CHECK-NEXT: ret i32 %y1
}
4 changes: 3 additions & 1 deletion llvm/tools/opt/opt.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,8 @@ static bool shouldPinPassToLegacyPM(StringRef Pass) {
"expand-large-div-rem",
"structurizecfg",
"fix-irreducible",
"expand-large-fp-convert"
"expand-large-fp-convert",
"callbrprepare",
};
for (const auto &P : PassNamePrefix)
if (Pass.startswith(P))
Expand Down Expand Up @@ -444,6 +445,7 @@ int main(int argc, char **argv) {
initializeExpandMemCmpPassPass(Registry);
initializeScalarizeMaskedMemIntrinLegacyPassPass(Registry);
initializeSelectOptimizePass(Registry);
initializeCallBrPreparePass(Registry);
initializeCodeGenPreparePass(Registry);
initializeAtomicExpandPass(Registry);
initializeRewriteSymbolsLegacyPassPass(Registry);
Expand Down
43 changes: 43 additions & 0 deletions llvm/unittests/IR/DominatorTreeTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1100,3 +1100,46 @@ TEST(DominatorTree, ValueDomination) {
EXPECT_TRUE(DT->dominates(C, U));
});
}
TEST(DominatorTree, CallBrDomination) {
StringRef ModuleString = R"(
define void @x() {
%y = alloca i32
%w = callbr i32 asm "", "=r,!i"()
to label %asm.fallthrough [label %z]

asm.fallthrough:
br label %cleanup

z:
store i32 %w, ptr %y
br label %cleanup

cleanup:
ret void
})";

LLVMContext Context;
std::unique_ptr<Module> M = makeLLVMModule(Context, ModuleString);

runWithDomTree(
*M, "x", [&](Function &F, DominatorTree *DT, PostDominatorTree *PDT) {
Function::iterator FI = F.begin();

BasicBlock *Entry = &*FI++;
BasicBlock *ASMFallthrough = &*FI++;
BasicBlock *Z = &*FI++;

EXPECT_TRUE(DT->dominates(Entry, ASMFallthrough));
EXPECT_TRUE(DT->dominates(Entry, Z));

BasicBlock::iterator BBI = Entry->begin();
++BBI;
Instruction &I = *BBI;
EXPECT_TRUE(isa<CallBrInst>(I));
EXPECT_TRUE(isa<Value>(I));
for (const User *U : I.users()) {
EXPECT_TRUE(isa<Instruction>(U));
EXPECT_TRUE(DT->dominates(cast<Value>(&I), cast<Instruction>(U)));
}
});
}
1 change: 1 addition & 0 deletions llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ static_library("CodeGen") {
"CFIFixup.cpp",
"CFIInstrInserter.cpp",
"CalcSpillWeights.cpp",
"CallBrPrepare.cpp",
"CallingConvLower.cpp",
"CodeGen.cpp",
"CodeGenCommonISel.cpp",
Expand Down