1,076 changes: 414 additions & 662 deletions llvm/lib/Transforms/IPO/FunctionSpecialization.cpp

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion llvm/lib/Transforms/IPO/IPO.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ void llvm::initializeIPO(PassRegistry &Registry) {
initializeDAEPass(Registry);
initializeDAHPass(Registry);
initializeForceFunctionAttrsLegacyPassPass(Registry);
initializeFunctionSpecializationLegacyPassPass(Registry);
initializeGlobalDCELegacyPassPass(Registry);
initializeGlobalOptLegacyPassPass(Registry);
initializeGlobalSplitPass(Registry);
Expand Down
453 changes: 358 additions & 95 deletions llvm/lib/Transforms/IPO/SCCP.cpp

Large diffs are not rendered by default.

591 changes: 0 additions & 591 deletions llvm/lib/Transforms/Scalar/SCCP.cpp

Large diffs are not rendered by default.

278 changes: 275 additions & 3 deletions llvm/lib/Transforms/Utils/SCCPSolver.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,13 @@
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/ValueLattice.h"
#include "llvm/Analysis/ValueLatticeUtils.h"
#include "llvm/IR/InstVisitor.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/Local.h"
#include <cassert>
#include <utility>
#include <vector>
Expand All @@ -39,7 +41,7 @@ static ValueLatticeElement::MergeOptions getMaxWidenStepsOpts() {
MaxNumRangeExtensions);
}

namespace {
namespace llvm {

// Helper to check if \p LV is either a constant or a constant
// range with a single element. This should cover exactly the same cases as the
Expand All @@ -58,9 +60,247 @@ bool isOverdefined(const ValueLatticeElement &LV) {
return !LV.isUnknownOrUndef() && !isConstant(LV);
}

} // namespace
static bool canRemoveInstruction(Instruction *I) {
if (wouldInstructionBeTriviallyDead(I))
return true;

namespace llvm {
// Some instructions can be handled but are rejected above. Catch
// those cases by falling through to here.
// TODO: Mark globals as being constant earlier, so
// TODO: wouldInstructionBeTriviallyDead() knows that atomic loads
// TODO: are safe to remove.
return isa<LoadInst>(I);
}

bool tryToReplaceWithConstant(SCCPSolver &Solver, Value *V) {
Constant *Const = nullptr;
if (V->getType()->isStructTy()) {
std::vector<ValueLatticeElement> IVs = Solver.getStructLatticeValueFor(V);
if (llvm::any_of(IVs, isOverdefined))
return false;
std::vector<Constant *> ConstVals;
auto *ST = cast<StructType>(V->getType());
for (unsigned i = 0, e = ST->getNumElements(); i != e; ++i) {
ValueLatticeElement V = IVs[i];
ConstVals.push_back(isConstant(V)
? Solver.getConstant(V)
: UndefValue::get(ST->getElementType(i)));
}
Const = ConstantStruct::get(ST, ConstVals);
} else {
const ValueLatticeElement &IV = Solver.getLatticeValueFor(V);
if (isOverdefined(IV))
return false;

Const =
isConstant(IV) ? Solver.getConstant(IV) : UndefValue::get(V->getType());
}
assert(Const && "Constant is nullptr here!");

// Replacing `musttail` instructions with constant breaks `musttail` invariant
// unless the call itself can be removed.
// Calls with "clang.arc.attachedcall" implicitly use the return value and
// those uses cannot be updated with a constant.
CallBase *CB = dyn_cast<CallBase>(V);
if (CB && ((CB->isMustTailCall() &&
!canRemoveInstruction(CB)) ||
CB->getOperandBundle(LLVMContext::OB_clang_arc_attachedcall))) {
Function *F = CB->getCalledFunction();

// Don't zap returns of the callee
if (F)
Solver.addToMustPreserveReturnsInFunctions(F);

LLVM_DEBUG(dbgs() << " Can\'t treat the result of call " << *CB
<< " as a constant\n");
return false;
}

LLVM_DEBUG(dbgs() << " Constant: " << *Const << " = " << *V << '\n');

// Replaces all of the uses of a variable with uses of the constant.
V->replaceAllUsesWith(Const);
return true;
}

/// Try to replace signed instructions with their unsigned equivalent.
static bool replaceSignedInst(SCCPSolver &Solver,
SmallPtrSetImpl<Value *> &InsertedValues,
Instruction &Inst) {
// Determine if a signed value is known to be >= 0.
auto isNonNegative = [&Solver](Value *V) {
// If this value was constant-folded, it may not have a solver entry.
// Handle integers. Otherwise, return false.
if (auto *C = dyn_cast<Constant>(V)) {
auto *CInt = dyn_cast<ConstantInt>(C);
return CInt && !CInt->isNegative();
}
const ValueLatticeElement &IV = Solver.getLatticeValueFor(V);
return IV.isConstantRange(/*UndefAllowed=*/false) &&
IV.getConstantRange().isAllNonNegative();
};

Instruction *NewInst = nullptr;
switch (Inst.getOpcode()) {
// Note: We do not fold sitofp -> uitofp here because that could be more
// expensive in codegen and may not be reversible in the backend.
case Instruction::SExt: {
// If the source value is not negative, this is a zext.
Value *Op0 = Inst.getOperand(0);
if (InsertedValues.count(Op0) || !isNonNegative(Op0))
return false;
NewInst = new ZExtInst(Op0, Inst.getType(), "", &Inst);
break;
}
case Instruction::AShr: {
// If the shifted value is not negative, this is a logical shift right.
Value *Op0 = Inst.getOperand(0);
if (InsertedValues.count(Op0) || !isNonNegative(Op0))
return false;
NewInst = BinaryOperator::CreateLShr(Op0, Inst.getOperand(1), "", &Inst);
break;
}
case Instruction::SDiv:
case Instruction::SRem: {
// If both operands are not negative, this is the same as udiv/urem.
Value *Op0 = Inst.getOperand(0), *Op1 = Inst.getOperand(1);
if (InsertedValues.count(Op0) || InsertedValues.count(Op1) ||
!isNonNegative(Op0) || !isNonNegative(Op1))
return false;
auto NewOpcode = Inst.getOpcode() == Instruction::SDiv ? Instruction::UDiv
: Instruction::URem;
NewInst = BinaryOperator::Create(NewOpcode, Op0, Op1, "", &Inst);
break;
}
default:
return false;
}

// Wire up the new instruction and update state.
assert(NewInst && "Expected replacement instruction");
NewInst->takeName(&Inst);
InsertedValues.insert(NewInst);
Inst.replaceAllUsesWith(NewInst);
Solver.removeLatticeValueFor(&Inst);
Inst.eraseFromParent();
return true;
}

bool simplifyInstsInBlock(SCCPSolver &Solver, BasicBlock &BB,
SmallPtrSetImpl<Value *> &InsertedValues,
Statistic &InstRemovedStat,
Statistic &InstReplacedStat) {
bool MadeChanges = false;
for (Instruction &Inst : make_early_inc_range(BB)) {
if (Inst.getType()->isVoidTy())
continue;
if (tryToReplaceWithConstant(Solver, &Inst)) {
if (canRemoveInstruction(&Inst))
Inst.eraseFromParent();

MadeChanges = true;
++InstRemovedStat;
} else if (replaceSignedInst(Solver, InsertedValues, Inst)) {
MadeChanges = true;
++InstReplacedStat;
}
}
return MadeChanges;
}

bool removeNonFeasibleEdges(const SCCPSolver &Solver, BasicBlock *BB,
DomTreeUpdater &DTU,
BasicBlock *&NewUnreachableBB) {
SmallPtrSet<BasicBlock *, 8> FeasibleSuccessors;
bool HasNonFeasibleEdges = false;
for (BasicBlock *Succ : successors(BB)) {
if (Solver.isEdgeFeasible(BB, Succ))
FeasibleSuccessors.insert(Succ);
else
HasNonFeasibleEdges = true;
}

// All edges feasible, nothing to do.
if (!HasNonFeasibleEdges)
return false;

// SCCP can only determine non-feasible edges for br, switch and indirectbr.
Instruction *TI = BB->getTerminator();
assert((isa<BranchInst>(TI) || isa<SwitchInst>(TI) ||
isa<IndirectBrInst>(TI)) &&
"Terminator must be a br, switch or indirectbr");

if (FeasibleSuccessors.size() == 0) {
// Branch on undef/poison, replace with unreachable.
SmallPtrSet<BasicBlock *, 8> SeenSuccs;
SmallVector<DominatorTree::UpdateType, 8> Updates;
for (BasicBlock *Succ : successors(BB)) {
Succ->removePredecessor(BB);
if (SeenSuccs.insert(Succ).second)
Updates.push_back({DominatorTree::Delete, BB, Succ});
}
TI->eraseFromParent();
new UnreachableInst(BB->getContext(), BB);
DTU.applyUpdatesPermissive(Updates);
} else if (FeasibleSuccessors.size() == 1) {
// Replace with an unconditional branch to the only feasible successor.
BasicBlock *OnlyFeasibleSuccessor = *FeasibleSuccessors.begin();
SmallVector<DominatorTree::UpdateType, 8> Updates;
bool HaveSeenOnlyFeasibleSuccessor = false;
for (BasicBlock *Succ : successors(BB)) {
if (Succ == OnlyFeasibleSuccessor && !HaveSeenOnlyFeasibleSuccessor) {
// Don't remove the edge to the only feasible successor the first time
// we see it. We still do need to remove any multi-edges to it though.
HaveSeenOnlyFeasibleSuccessor = true;
continue;
}

Succ->removePredecessor(BB);
Updates.push_back({DominatorTree::Delete, BB, Succ});
}

BranchInst::Create(OnlyFeasibleSuccessor, BB);
TI->eraseFromParent();
DTU.applyUpdatesPermissive(Updates);
} else if (FeasibleSuccessors.size() > 1) {
SwitchInstProfUpdateWrapper SI(*cast<SwitchInst>(TI));
SmallVector<DominatorTree::UpdateType, 8> Updates;

// If the default destination is unfeasible it will never be taken. Replace
// it with a new block with a single Unreachable instruction.
BasicBlock *DefaultDest = SI->getDefaultDest();
if (!FeasibleSuccessors.contains(DefaultDest)) {
if (!NewUnreachableBB) {
NewUnreachableBB =
BasicBlock::Create(DefaultDest->getContext(), "default.unreachable",
DefaultDest->getParent(), DefaultDest);
new UnreachableInst(DefaultDest->getContext(), NewUnreachableBB);
}

SI->setDefaultDest(NewUnreachableBB);
Updates.push_back({DominatorTree::Delete, BB, DefaultDest});
Updates.push_back({DominatorTree::Insert, BB, NewUnreachableBB});
}

for (auto CI = SI->case_begin(); CI != SI->case_end();) {
if (FeasibleSuccessors.contains(CI->getCaseSuccessor())) {
++CI;
continue;
}

BasicBlock *Succ = CI->getCaseSuccessor();
Succ->removePredecessor(BB);
Updates.push_back({DominatorTree::Delete, BB, Succ});
SI.removeCase(CI);
// Don't increment CI, as we removed a case.
}

DTU.applyUpdatesPermissive(Updates);
} else {
llvm_unreachable("Must have at least one feasible successor");
}
return true;
}

/// Helper class for SCCPSolver. This implements the instruction visitor and
/// holds all the state.
Expand Down Expand Up @@ -464,6 +704,26 @@ class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
for (auto &BB : *F)
BBExecutable.erase(&BB);
}

void solveWhileResolvedUndefsIn(Module &M) {
bool ResolvedUndefs = true;
while (ResolvedUndefs) {
solve();
ResolvedUndefs = false;
for (Function &F : M)
ResolvedUndefs |= resolvedUndefsIn(F);
}
}

void solveWhileResolvedUndefsIn(SmallVectorImpl<Function *> &WorkList) {
bool ResolvedUndefs = true;
while (ResolvedUndefs) {
solve();
ResolvedUndefs = false;
for (Function *F : WorkList)
ResolvedUndefs |= resolvedUndefsIn(*F);
}
}
};

} // namespace llvm
Expand Down Expand Up @@ -1531,6 +1791,9 @@ bool SCCPInstVisitor::resolvedUndefsIn(Function &F) {
}
}

LLVM_DEBUG(if (MadeChange) dbgs()
<< "\nResolved undefs in " << F.getName() << '\n');

return MadeChange;
}

Expand Down Expand Up @@ -1594,6 +1857,15 @@ bool SCCPSolver::resolvedUndefsIn(Function &F) {
return Visitor->resolvedUndefsIn(F);
}

void SCCPSolver::solveWhileResolvedUndefsIn(Module &M) {
Visitor->solveWhileResolvedUndefsIn(M);
}

void
SCCPSolver::solveWhileResolvedUndefsIn(SmallVectorImpl<Function *> &WorkList) {
Visitor->solveWhileResolvedUndefsIn(WorkList);
}

bool SCCPSolver::isBlockExecutable(BasicBlock *BB) const {
return Visitor->isBlockExecutable(BB);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes=function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -S < %s | FileCheck %s

%mystruct = type { i32, [2 x i64] }

Expand All @@ -8,17 +8,11 @@ define internal ptr @myfunc(ptr %arg) {
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[FOR_COND:%.*]]
; CHECK: for.cond:
; CHECK-NEXT: br i1 true, label [[FOR_COND2:%.*]], label [[FOR_BODY:%.*]]
; CHECK: for.body:
; CHECK-NEXT: call void @callee(ptr nonnull null)
; CHECK-NEXT: br label [[FOR_COND]]
; CHECK-NEXT: br label [[FOR_COND2:%.*]]
; CHECK: for.cond2:
; CHECK-NEXT: br i1 false, label [[FOR_END:%.*]], label [[FOR_BODY2:%.*]]
; CHECK-NEXT: br label [[FOR_BODY2:%.*]]
; CHECK: for.body2:
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [[MYSTRUCT:%.*]], ptr null, i64 0, i32 1, i64 3
; CHECK-NEXT: br label [[FOR_COND2]]
; CHECK: for.end:
; CHECK-NEXT: ret ptr [[ARG:%.*]]
;
entry:
br label %for.cond
Expand Down Expand Up @@ -48,11 +42,12 @@ define ptr @caller() {
; CHECK-LABEL: @caller(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[CALL:%.*]] = call ptr @myfunc(ptr undef)
; CHECK-NEXT: ret ptr [[CALL]]
; CHECK-NEXT: ret ptr undef
;
entry:
%call = call ptr @myfunc(ptr undef)
ret ptr %call
}

declare void @callee(ptr)

Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -func-specialization-max-iters=2 -func-specialization-size-threshold=20 -func-specialization-avg-iters-cost=20 -function-specialization-for-literal-constant=true -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -func-specialization-max-iters=2 -func-specialization-max-clones=1 -function-specialization-for-literal-constant=true -S < %s | FileCheck %s

declare hidden i1 @compare(ptr) align 2
declare hidden { i8, ptr } @getType(ptr) align 2
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -S --passes='default<O3>' -enable-function-specialization < %s | FileCheck %s
; RUN: opt -S --passes='default<O3>' -specialize-functions < %s | FileCheck %s

define dso_local i32 @g0(i32 noundef %x) local_unnamed_addr {
entry:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -func-specialization-avg-iters-cost=3 -func-specialization-size-threshold=10 -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -func-specialization-avg-iters-cost=3 -func-specialization-size-threshold=10 -S < %s | FileCheck %s

; CHECK-NOT: foo.{{[0-9]+}}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
; Note that this test case shows that function specialization pass would
; transform the function even if no specialization happened.

; RUN: opt -passes=function-specialization -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -S < %s | FileCheck %s

%struct = type { i8, i16, i32, i64, i64}
@Global = internal constant %struct {i8 0, i16 1, i32 2, i64 3, i64 4}
Expand All @@ -18,8 +18,7 @@ entry:
define internal i64 @func(ptr %x, ptr %binop) {
; CHECK-LABEL: @func(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 [[BINOP:%.*]](ptr [[X:%.*]])
; CHECK-NEXT: ret i64 [[TMP0]]
; CHECK-NEXT: unreachable
;
entry:
%tmp0 = call i64 %binop(ptr %x)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes=function-specialization -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -S < %s | FileCheck %s

; Check that we don't crash and specialise on a constant expression.

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes=function-specialization -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -S < %s | FileCheck %s

define i32 @main() {
; CHECK-LABEL: @main(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -S < %s | FileCheck %s

; Check that we don't crash and specialise on a function call with byval attribute.

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes=function-specialization -force-function-specialization -func-specialization-on-address -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -func-specialization-on-address -S < %s | FileCheck %s

; Check that we don't crash and specialise on a scalar global variable with byval attribute.

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -function-specialization-for-literal-constant=true -func-specialization-size-threshold=10 -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -function-specialization-for-literal-constant=true -func-specialization-size-threshold=10 -S < %s | FileCheck %s

; Check that the literal constant parameter could be specialized.
; CHECK: @foo.1(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -func-specialization-avg-iters-cost=5 -func-specialization-size-threshold=10 -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -func-specialization-avg-iters-cost=5 -func-specialization-size-threshold=10 -S < %s | FileCheck %s

; Check that the loop depth results in a larger specialization bonus.
; CHECK: @foo.1(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -S < %s | FileCheck %s

; CHECK-NOT: @compute.1
; CHECK-NOT: @compute.2
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -func-specialization-size-threshold=3 -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -func-specialization-size-threshold=3 -S < %s | FileCheck %s

; Checks for callsites that have been annotated with MinSize. No specialisation
; expected here:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -func-specialization-size-threshold=3 -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -func-specialization-size-threshold=3 -S < %s | FileCheck %s

; Checks for callsites that have been annotated with MinSize. We only expect
; specialisation for the call that does not have the attribute:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -S < %s | FileCheck %s

; Function @foo has function attribute 'noduplicate', so check that we don't
; specialize it:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -S < %s | FileCheck %s

; Check that function foo does not gets specialised as it contains an intrinsic
; that is marked as NoDuplicate.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -S < %s | FileCheck %s

; The if.then block is not executed, so check that we don't specialise here.

Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py

; RUN: opt -passes=function-specialization -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=function-specialization -force-function-specialization -func-specialization-on-address=0 -S < %s | FileCheck %s
; RUN: opt -passes=function-specialization -force-function-specialization -func-specialization-on-address=1 -S < %s | FileCheck %s --check-prefix=ON-ADDRESS
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -func-specialization-on-address=0 -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -func-specialization-on-address=1 -S < %s | FileCheck %s --check-prefix=ON-ADDRESS

; Global B is not constant. We do not specialise on addresses unless we
; enable that:
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes=function-specialization -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -S < %s | FileCheck %s

; Check that we don't crash and specialise on a poison value.

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
; RUN: opt -passes=function-specialization,inline,instcombine -force-function-specialization -func-specialization-max-iters=2 -S < %s | FileCheck %s --check-prefix=ITERS2
; RUN: opt -passes=function-specialization,inline,instcombine -force-function-specialization -func-specialization-max-iters=3 -S < %s | FileCheck %s --check-prefix=ITERS3
; RUN: opt -passes=function-specialization,inline,instcombine -force-function-specialization -func-specialization-max-iters=4 -S < %s | FileCheck %s --check-prefix=ITERS4
; RUN: opt -passes=ipsccp,inline,instcombine -specialize-functions -force-function-specialization -func-specialization-max-iters=2 -S < %s | FileCheck %s --check-prefix=ITERS2
; RUN: opt -passes=ipsccp,inline,instcombine -specialize-functions -force-function-specialization -func-specialization-max-iters=3 -S < %s | FileCheck %s --check-prefix=ITERS3
; RUN: opt -passes=ipsccp,inline,instcombine -specialize-functions -force-function-specialization -func-specialization-max-iters=4 -S < %s | FileCheck %s --check-prefix=ITERS4

@low = internal constant i32 0, align 4
@high = internal constant i32 6, align 4
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -force-function-specialization -func-specialization-max-iters=2 -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -func-specialization-max-iters=2 -S < %s | FileCheck %s

; Volatile store preventing recursive specialisation:
;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -force-function-specialization -func-specialization-max-iters=2 -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -func-specialization-max-iters=2 -S < %s | FileCheck %s

; Duplicate store preventing recursive specialisation:
;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -force-function-specialization -func-specialization-max-iters=2 -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -func-specialization-max-iters=2 -S < %s | FileCheck %s

; Alloca is not an integer type:
;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
; REQUIRES: asserts
; RUN: opt -stats -passes=function-specialization -S -force-function-specialization < %s 2>&1 | FileCheck %s
; RUN: opt -stats -passes=ipsccp -specialize-functions -S -force-function-specialization < %s 2>&1 | FileCheck %s

; CHECK: 2 function-specialization - Number of functions specialized

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -func-specialization-size-threshold=3 -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -func-specialization-size-threshold=3 -S < %s | FileCheck %s

define i64 @main(i64 %x, i1 %flag) {
;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes=function-specialization,deadargelim -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=function-specialization,deadargelim -func-specialization-max-iters=1 -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=function-specialization,deadargelim -func-specialization-max-iters=0 -force-function-specialization -S < %s | FileCheck %s --check-prefix=DISABLED
; RUN: opt -passes=function-specialization,deadargelim -func-specialization-avg-iters-cost=1 -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp,deadargelim -specialize-functions -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp,deadargelim -specialize-functions -func-specialization-max-iters=1 -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp,deadargelim -specialize-functions -func-specialization-max-iters=0 -force-function-specialization -S < %s | FileCheck %s --check-prefix=DISABLED
; RUN: opt -passes=ipsccp,deadargelim -specialize-functions -func-specialization-avg-iters-cost=1 -force-function-specialization -S < %s | FileCheck %s

; DISABLED-NOT: @func.1(
; DISABLED-NOT: @func.2(
Expand Down Expand Up @@ -43,10 +43,11 @@ define internal void @decrement(ptr nocapture %0) {
}

define i32 @main(ptr %0, i32 %1) {
; CHECK: [[TMP3:%.*]] = call i32 @func.2(ptr [[TMP0:%.*]], i32 [[TMP1:%.*]])
; CHECK: call void @func.2(ptr [[TMP0:%.*]], i32 [[TMP1:%.*]])
%3 = call i32 @func(ptr %0, i32 %1, ptr nonnull @increment)
; CHECK: [[TMP4:%.*]] = call i32 @func.1(ptr [[TMP0]], i32 [[TMP3]])
; CHECK: call void @func.1(ptr [[TMP0]], i32 0)
%4 = call i32 @func(ptr %0, i32 %3, ptr nonnull @decrement)
; CHECK: ret i32 0
ret i32 %4
}

Expand All @@ -63,10 +64,10 @@ define i32 @main(ptr %0, i32 %1) {
; CHECK: call void @decrement(ptr [[TMP9]])
; CHECK: [[TMP10:%.*]] = load i32, ptr [[TMP3]], align 4
; CHECK: [[TMP11:%.*]] = add nsw i32 [[TMP10]], -1
; CHECK: [[TMP12:%.*]] = call i32 @func.1(ptr [[TMP0]], i32 [[TMP11]])
; CHECK: br label [[TMP13]]
; CHECK: 13:
; CHECK: ret i32 0
; CHECK: call void @func.1(ptr [[TMP0]], i32 [[TMP11]])
; CHECK: br label [[TMP12:%.*]]
; CHECK: 12:
; CHECK: ret void
;
;
; CHECK: @func.2(
Expand All @@ -82,6 +83,7 @@ define i32 @main(ptr %0, i32 %1) {
; CHECK: call void @increment(ptr [[TMP9]])
; CHECK: [[TMP10:%.*]] = load i32, ptr [[TMP3]], align 4
; CHECK: [[TMP11:%.*]] = add nsw i32 [[TMP10]], -1
; CHECK: [[TMP12:%.*]] = call i32 @func.2(ptr [[TMP0]], i32 [[TMP11]])
; CHECK: br label [[TMP13]]
; CHECK: ret i32 0
; CHECK: call void @func.2(ptr [[TMP0]], i32 [[TMP11]])
; CHECK: br label [[TMP12:%.*]]
; CHECK: 12:
; CHECK: ret void
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
; RUN: opt -passes=function-specialization -func-specialization-avg-iters-cost=3 -S < %s | \
; RUN: opt -passes=ipsccp -specialize-functions -func-specialization-avg-iters-cost=3 -S < %s | \
; RUN: FileCheck %s --check-prefixes=COMMON,DISABLED
; RUN: opt -passes=function-specialization -force-function-specialization -S < %s | \
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -S < %s | \
; RUN: FileCheck %s --check-prefixes=COMMON,FORCE
; RUN: opt -passes=function-specialization -func-specialization-avg-iters-cost=3 -force-function-specialization -S < %s | \
; RUN: opt -passes=ipsccp -specialize-functions -func-specialization-avg-iters-cost=3 -force-function-specialization -S < %s | \
; RUN: FileCheck %s --check-prefixes=COMMON,FORCE

; Test for specializing a constant global.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
; RUN: opt -passes=function-specialization -force-function-specialization \
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization \
; RUN: -func-specialization-max-clones=2 -S < %s | FileCheck %s

; RUN: opt -passes=function-specialization -force-function-specialization \
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization \
; RUN: -func-specialization-max-clones=1 -S < %s | FileCheck %s --check-prefix=CONST1

target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -S < %s | FileCheck %s

; There's nothing to specialize here as both calls are the same, so check that:
;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -S --passes=function-specialization < %s | FileCheck %s
; RUN: opt -S --passes=ipsccp -specialize-functions < %s | FileCheck %s
define dso_local i32 @p0(i32 noundef %x) {
entry:
%add = add nsw i32 %x, 1
Expand All @@ -11,6 +11,37 @@ entry:
ret i32 %sub
}

; CHECK-LABEL: define dso_local i32 @f0
; CHECK: tail call fastcc i32 @g.[[#A:]]({{.*}}@p0)
;
define dso_local i32 @f0(i32 noundef %x) {
entry:
%call = tail call fastcc i32 @g(i32 noundef %x, ptr noundef nonnull @p0)
ret i32 %call
}

; CHECK-LABEL: define dso_local i32 @f1
; CHECK: tail call fastcc i32 @g.[[#B:]]({{.*}}@p1)
;
define dso_local i32 @f1(i32 noundef %x) {
entry:
%call = tail call fastcc i32 @g(i32 noundef %x, ptr noundef nonnull @p1)
ret i32 %call
}

; @g gets fully specialized
; CHECK-NOT: define internal fastcc i32 @g(

define internal fastcc i32 @g(i32 noundef %x, ptr nocapture noundef readonly %p) noinline {
entry:
%pcall = tail call i32 %p(i32 noundef %x)
%fcall = tail call fastcc i32 @f(i32 noundef %pcall, ptr noundef nonnull %p)
ret i32 %fcall
}

; CHECK-LABEL: define dso_local i32 @g0
; CHECK: tail call fastcc i32 @f.[[#C:]]({{.*}}@p0)
;
define dso_local i32 @g0(i32 noundef %x) {
entry:
%call = tail call fastcc i32 @f(i32 noundef %x, ptr noundef nonnull @p0)
Expand All @@ -24,6 +55,9 @@ entry:
ret i32 %add
}

; CHECK-LABEL: define dso_local i32 @g1
; CHECK: tail call fastcc i32 @f.[[#D:]]({{.*}}@p1)
;
define dso_local i32 @g1(i32 noundef %x) {
entry:
%call = tail call fastcc i32 @f(i32 noundef %x, ptr noundef nonnull @p1)
Expand All @@ -38,5 +72,11 @@ entry:

; Check that a single argument, that cannot be used for specialisation, does not
; prevent specialisation based on other arguments.
; CHECK: @f.1
; CHECK: @f.2
;
; Also check that for callsites which reside in the body of newly created
; (specialized) functions, the lattice value of the arguments is known.
;
; CHECK-DAG: define internal fastcc i32 @g.[[#A]]
; CHECK-DAG: define internal fastcc i32 @g.[[#B]]
; CHECK-DAG: define internal fastcc i32 @f.[[#C]]
; CHECK-DAG: define internal fastcc i32 @f.[[#D]]
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes=function-specialization -force-function-specialization -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -force-function-specialization -S < %s | FileCheck %s

define i64 @main(i64 %x, i64 %y, i1 %flag) {
; CHECK-LABEL: @main(
Expand Down Expand Up @@ -70,7 +70,7 @@ entry:
; CHECK-NEXT: entry:
; CHECK-NEXT: [[CMP0:%.*]] = call i64 @minus(i64 [[X:%.*]], i64 [[Y:%.*]])
; CHECK-NEXT: [[CMP1:%.*]] = call i64 @plus(i64 [[X]], i64 [[Y]])
; CHECK-NEXT: [[CMP2:%.*]] = call i64 @compute(i64 [[X]], i64 [[Y]], ptr @minus, ptr @plus)
; CHECK-NEXT: [[CMP2:%.*]] = call i64 @compute.2(i64 [[X]], i64 [[Y]], ptr @minus, ptr @plus)

; CHECK-LABEL: @compute.3
; CHECK-NEXT: entry:
Expand Down
4 changes: 2 additions & 2 deletions llvm/test/Transforms/FunctionSpecialization/literal-const.ll
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
; RUN: opt -S --passes=function-specialization \
; RUN: opt -S --passes=ipsccp -specialize-functions \
; RUN: -force-function-specialization < %s | FileCheck %s -check-prefix CHECK-NOLIT
; RUN: opt -S --passes=function-specialization \
; RUN: opt -S --passes=ipsccp -specialize-functions \
; RUN: -function-specialization-for-literal-constant \
; RUN: -force-function-specialization < %s | FileCheck %s -check-prefix CHECK-LIT

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -S --passes=function-specialization -force-function-specialization -function-specialization-for-literal-constant < %s | FileCheck %s
; RUN: opt -S --passes=ipsccp -specialize-functions -force-function-specialization -function-specialization-for-literal-constant < %s | FileCheck %s
define internal i32 @f(i32 %x, i32 %y) noinline {
ret i32 %x
}
Expand All @@ -17,4 +17,4 @@ define i32 @g1() {
; to be a constant without the need for function specialisation and
; the second parameter is unused.

; CHECK-NOT: @f.
; CHECK-NOT: @f.
2 changes: 1 addition & 1 deletion llvm/test/Transforms/FunctionSpecialization/noinline.ll
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -S --passes=function-specialization < %s | FileCheck %s
; RUN: opt -S --passes=ipsccp -specialize-functions < %s | FileCheck %s
define dso_local i32 @p0(i32 noundef %x) {
entry:
%add = add nsw i32 %x, 1
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -passes=function-specialization -func-specialization-size-threshold=3 -S < %s | FileCheck %s
; RUN: opt -passes=ipsccp -specialize-functions -func-specialization-size-threshold=3 -S < %s | FileCheck %s

define i64 @main(i64 %x, i1 %flag) {
entry:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; RUN: opt -S --passes=function-specialization,deadargelim -force-function-specialization < %s | FileCheck %s
; RUN: opt -S --passes=ipsccp,deadargelim -specialize-functions -force-function-specialization < %s | FileCheck %s
define dso_local i32 @add(i32 %x, i32 %y) {
entry:
%add = add nsw i32 %y, %x
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes=function-specialization -func-specialization-max-clones=0 -func-specialization-size-threshold=14 -S < %s | FileCheck %s --check-prefix=NONE
; RUN: opt -passes=function-specialization -func-specialization-max-clones=1 -func-specialization-size-threshold=14 -S < %s | FileCheck %s --check-prefix=ONE
; RUN: opt -passes=function-specialization -func-specialization-max-clones=2 -func-specialization-size-threshold=14 -S < %s | FileCheck %s --check-prefix=TWO
; RUN: opt -passes=function-specialization -func-specialization-max-clones=3 -func-specialization-size-threshold=14 -S < %s | FileCheck %s --check-prefix=THREE
; RUN: opt -passes=ipsccp -specialize-functions -func-specialization-max-clones=0 -func-specialization-size-threshold=14 -S < %s | FileCheck %s --check-prefix=NONE
; RUN: opt -passes=ipsccp -specialize-functions -func-specialization-max-clones=1 -func-specialization-size-threshold=14 -S < %s | FileCheck %s --check-prefix=ONE
; RUN: opt -passes=ipsccp -specialize-functions -func-specialization-max-clones=2 -func-specialization-size-threshold=14 -S < %s | FileCheck %s --check-prefix=TWO
; RUN: opt -passes=ipsccp -specialize-functions -func-specialization-max-clones=3 -func-specialization-size-threshold=14 -S < %s | FileCheck %s --check-prefix=THREE

; Make sure that we iterate correctly after sorting the specializations:
; FnSpecialization: Specializations for function compute
Expand Down
1 change: 0 additions & 1 deletion llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ static_library("IPO") {
"//llvm/lib/Transforms/AggressiveInstCombine",
"//llvm/lib/Transforms/InstCombine",
"//llvm/lib/Transforms/Instrumentation",
"//llvm/lib/Transforms/Scalar",
"//llvm/lib/Transforms/Utils",
"//llvm/lib/Transforms/Vectorize",
]
Expand Down