Skip to content

Commit

Permalink
[coroutines] Part 4[ab]: Coroutine Devirtualization: Lower coro.resum…
Browse files Browse the repository at this point in the history
…e and coro.destroy.

This is the forth patch in the coroutine series. CoroEaly pass now lowers coro.resume
and coro.destroy intrinsics by replacing them with an indirect call to an address
returned by coro.subfn.addr intrinsic. This is done so that CGPassManager recognizes
devirtualization when CoroElide replaces a call to coro.subfn.addr with an appropriate
function address.

Patch by Gor Nishanov!

Differential Revision: https://reviews.llvm.org/D22998

llvm-svn: 277765
  • Loading branch information
majnemer committed Aug 4, 2016
1 parent c696467 commit f93082e
Show file tree
Hide file tree
Showing 8 changed files with 278 additions and 7 deletions.
11 changes: 11 additions & 0 deletions llvm/include/llvm/IR/CallSite.h
Expand Up @@ -109,6 +109,17 @@ class CallSiteBase {
*getCallee() = V;
}

/// Return the intrinsic ID of the intrinsic called by this CallSite,
/// or Intrinsic::not_intrinsic if the called function is not an
/// intrinsic, or if this CallSite is an indirect call.
Intrinsic::ID getIntrinsicID() const {
if (auto *F = getCalledFunction())
return F->getIntrinsicID();
// Don't use Intrinsic::not_intrinsic, as it will require pulling
// Intrinsics.h into every header that uses CallSite.
return static_cast<Intrinsic::ID>(0);
}

/// isCallee - Determine whether the passed iterator points to the
/// callee operand's Use.
bool isCallee(Value::const_user_iterator UI) const {
Expand Down
6 changes: 4 additions & 2 deletions llvm/lib/IR/Verifier.cpp
Expand Up @@ -3681,11 +3681,13 @@ void Verifier::visitInstruction(Instruction &I) {
Assert(
!F->isIntrinsic() || isa<CallInst>(I) ||
F->getIntrinsicID() == Intrinsic::donothing ||
F->getIntrinsicID() == Intrinsic::coro_resume ||
F->getIntrinsicID() == Intrinsic::coro_destroy ||
F->getIntrinsicID() == Intrinsic::experimental_patchpoint_void ||
F->getIntrinsicID() == Intrinsic::experimental_patchpoint_i64 ||
F->getIntrinsicID() == Intrinsic::experimental_gc_statepoint,
"Cannot invoke an intrinsic other than donothing, patchpoint or "
"statepoint",
"Cannot invoke an intrinsic other than donothing, patchpoint, "
"statepoint, coro_resume or coro_destroy",
&I);
Assert(F->getParent() == &M, "Referencing function in another module!",
&I, &M, F, F->getParent());
Expand Down
77 changes: 74 additions & 3 deletions llvm/lib/Transforms/Coroutines/CoroEarly.cpp
Expand Up @@ -12,25 +12,96 @@
//===----------------------------------------------------------------------===//

#include "CoroInternal.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/Module.h"
#include "llvm/Pass.h"

using namespace llvm;

#define DEBUG_TYPE "coro-early"

namespace {
// Created on demand if CoroEarly pass has work to do.
class Lowerer : public coro::LowererBase {
void lowerResumeOrDestroy(CallSite CS, CoroSubFnInst::ResumeKind);

public:
Lowerer(Module &M) : LowererBase(M) {}
static std::unique_ptr<Lowerer> createIfNeeded(Module &M);
bool lowerEarlyIntrinsics(Function &F);
};
}

// Replace a direct call to coro.resume or coro.destroy with an indirect call to
// an address returned by coro.subfn.addr intrinsic. This is done so that
// CGPassManager recognizes devirtualization when CoroElide pass replaces a call
// to coro.subfn.addr with an appropriate function address.
void Lowerer::lowerResumeOrDestroy(CallSite CS,
CoroSubFnInst::ResumeKind Index) {
Value *ResumeAddr =
makeSubFnCall(CS.getArgOperand(0), Index, CS.getInstruction());
CS.setCalledFunction(ResumeAddr);
CS.setCallingConv(CallingConv::Fast);
}

bool Lowerer::lowerEarlyIntrinsics(Function &F) {
bool Changed = false;
for (auto IB = inst_begin(F), IE = inst_end(F); IB != IE;) {
Instruction &I = *IB++;
if (auto CS = CallSite(&I)) {
switch (CS.getIntrinsicID()) {
default:
continue;
case Intrinsic::coro_resume:
lowerResumeOrDestroy(CS, CoroSubFnInst::ResumeIndex);
break;
case Intrinsic::coro_destroy:
lowerResumeOrDestroy(CS, CoroSubFnInst::DestroyIndex);
break;
}
Changed = true;
continue;
}
}
return Changed;
}

// This pass has work to do only if we find intrinsics we are going to lower in
// the module.
std::unique_ptr<Lowerer> Lowerer::createIfNeeded(Module &M) {
if (declaresIntrinsics(M, {"llvm.coro.resume", "llvm.coro.destroy"}))
return llvm::make_unique<Lowerer>(M);

return {};
}

//===----------------------------------------------------------------------===//
// Top Level Driver
//===----------------------------------------------------------------------===//

namespace {

struct CoroEarly : public FunctionPass {
static char ID; // Pass identification, replacement for typeid
static char ID; // Pass identification, replacement for typeid.
CoroEarly() : FunctionPass(ID) {}

bool runOnFunction(Function &F) override { return false; }
std::unique_ptr<Lowerer> L;

bool doInitialization(Module &M) override {
L = Lowerer::createIfNeeded(M);
return false;
}

bool runOnFunction(Function &F) override {
if (!L)
return false;

return L->lowerEarlyIntrinsics(F);
}

void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesAll();
AU.setPreservesCFG();
}
};

Expand Down
64 changes: 64 additions & 0 deletions llvm/lib/Transforms/Coroutines/CoroInstr.h
@@ -0,0 +1,64 @@
//===-- CoroInstr.h - Coroutine Intrinsics Instruction Wrappers -*- C++ -*-===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
// This file defines classes that make it really easy to deal with intrinsic
// functions with the isa/dyncast family of functions. In particular, this
// allows you to do things like:
//
// if (auto *SF = dyn_cast<CoroSubFnInst>(Inst))
// ... SF->getFrame() ... SF->getAlloc() ...
//
// All intrinsic function calls are instances of the call instruction, so these
// are all subclasses of the CallInst class. Note that none of these classes
// has state or virtual methods, which is an important part of this gross/neat
// hack working.
//
// The helpful comment above is borrowed from llvm/IntrinsicInst.h, we keep
// coroutine intrinsic wrappers here since they are only used by the passes in
// the Coroutine library.
//===----------------------------------------------------------------------===//

#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IntrinsicInst.h"

namespace llvm {

/// This class represents the llvm.coro.subfn.addr instruction.
class LLVM_LIBRARY_VISIBILITY CoroSubFnInst : public IntrinsicInst {
enum { FrameArg, IndexArg };

public:
enum ResumeKind {
ResumeIndex,
DestroyIndex,
IndexLast,
IndexFirst = ResumeIndex
};

Value *getFrame() const { return getArgOperand(FrameArg); }
ResumeKind getIndex() const {
int64_t Index = getRawIndex()->getValue().getSExtValue();
assert(Index >= IndexFirst && Index < IndexLast &&
"unexpected CoroSubFnInst index argument");
return static_cast<ResumeKind>(Index);
}

ConstantInt *getRawIndex() const {
return cast<ConstantInt>(getArgOperand(IndexArg));
}

// Methods to support type inquiry through isa, cast, and dyn_cast:
static inline bool classof(const IntrinsicInst *I) {
return I->getIntrinsicID() == Intrinsic::coro_subfn_addr;
}
static inline bool classof(const Value *V) {
return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
}
};

} // End namespace llvm.
20 changes: 19 additions & 1 deletion llvm/lib/Transforms/Coroutines/CoroInternal.h
Expand Up @@ -12,17 +12,35 @@
#ifndef LLVM_LIB_TRANSFORMS_COROUTINES_COROINTERNAL_H
#define LLVM_LIB_TRANSFORMS_COROUTINES_COROINTERNAL_H

#include "CoroInstr.h"
#include "llvm/Transforms/Coroutines.h"

namespace llvm {

class FunctionType;
class LLVMContext;
class Module;
class PassRegistry;

void initializeCoroEarlyPass(PassRegistry &);
void initializeCoroSplitPass(PassRegistry &);
void initializeCoroElidePass(PassRegistry &);
void initializeCoroCleanupPass(PassRegistry &);

}
namespace coro {

// Keeps data and helper functions for lowering coroutine intrinsics.
struct LowererBase {
Module &TheModule;
LLVMContext &Context;
FunctionType *const ResumeFnType;

LowererBase(Module &M);
Value *makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt);
static bool declaresIntrinsics(Module &M, std::initializer_list<StringRef>);
};

} // End namespace coro.
} // End namespace llvm

#endif
64 changes: 64 additions & 0 deletions llvm/lib/Transforms/Coroutines/Coroutines.cpp
Expand Up @@ -66,3 +66,67 @@ void llvm::addCoroutinePassesToExtensionPoints(PassManagerBuilder &Builder) {
Builder.addExtension(PassManagerBuilder::EP_OptimizerLast,
addCoroutineOptimizerLastPasses);
}

// Construct the lowerer base class and initialize its members.
coro::LowererBase::LowererBase(Module &M)
: TheModule(M), Context(M.getContext()),
ResumeFnType(FunctionType::get(Type::getVoidTy(Context),
Type::getInt8PtrTy(Context),
/*isVarArg=*/false)) {}

// Creates a sequence of instructions to obtain a resume function address using
// llvm.coro.subfn.addr. It generates the following sequence:
//
// call i8* @llvm.coro.subfn.addr(i8* %Arg, i8 %index)
// bitcast i8* %2 to void(i8*)*

Value *coro::LowererBase::makeSubFnCall(Value *Arg, int Index,
Instruction *InsertPt) {
auto *IndexVal = ConstantInt::get(Type::getInt8Ty(Context), Index);
auto *Fn = Intrinsic::getDeclaration(&TheModule, Intrinsic::coro_subfn_addr);

assert(Index >= CoroSubFnInst::IndexFirst &&
Index < CoroSubFnInst::IndexLast &&
"makeSubFnCall: Index value out of range");
auto *Call = CallInst::Create(Fn, {Arg, IndexVal}, "", InsertPt);

auto *Bitcast =
new BitCastInst(Call, ResumeFnType->getPointerTo(), "", InsertPt);
return Bitcast;
}

#ifndef NDEBUG
static bool isCoroutineIntrinsicName(StringRef Name) {
// NOTE: Must be sorted!
static const char *const CoroIntrinsics[] = {
"llvm.coro.alloc",
"llvm.coro.begin",
"llvm.coro.destroy",
"llvm.coro.done",
"llvm.coro.end",
"llvm.coro.frame",
"llvm.coro.free",
"llvm.coro.param",
"llvm.coro.promise",
"llvm.coro.resume",
"llvm.coro.save",
"llvm.coro.size",
"llvm.coro.suspend",
};
return Intrinsic::lookupLLVMIntrinsicByName(CoroIntrinsics, Name) != -1;
}
#endif

// Verifies if a module has named values listed. Also, in debug mode verifies
// that names are intrinsic names.
bool coro::LowererBase::declaresIntrinsics(
Module &M, std::initializer_list<StringRef> List) {

for (StringRef Name : List) {
assert(isCoroutineIntrinsicName(Name) && "not a coroutine intrinsic");
if (M.getNamedValue(Name))
return true;
}

return false;
}
41 changes: 41 additions & 0 deletions llvm/test/Transforms/Coroutines/coro-early.ll
@@ -0,0 +1,41 @@
; Tests that CoroEarly pass correctly lowers coro.resume and coro.destroy
; intrinsics.
; RUN: opt < %s -S -coro-early | FileCheck %s

; CHECK-LABEL: @callResume
define void @callResume(i8* %hdl) {
; CHECK-NEXT: entry
entry:
; CHECK-NEXT: %0 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
; CHECK-NEXT: %1 = bitcast i8* %0 to void (i8*)*
; CHECK-NEXT: call fastcc void %1(i8* %hdl)
call void @llvm.coro.resume(i8* %hdl)

; CHECK-NEXT: %2 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 1)
; CHECK-NEXT: %3 = bitcast i8* %2 to void (i8*)*
; CHECK-NEXT: call fastcc void %3(i8* %hdl)
call void @llvm.coro.destroy(i8* %hdl)

ret void
; CHECK-NEXT: ret void
}

; CHECK-LABEL: @eh
define void @eh(i8* %hdl) personality i8* null {
; CHECK-NEXT: entry
entry:
; CHECK-NEXT: %0 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
; CHECK-NEXT: %1 = bitcast i8* %0 to void (i8*)*
; CHECK-NEXT: invoke fastcc void %1(i8* %hdl)
invoke void @llvm.coro.resume(i8* %hdl)
to label %cont unwind label %ehcleanup
cont:
ret void

ehcleanup:
%0 = cleanuppad within none []
cleanupret from %0 unwind to caller
}

declare void @llvm.coro.resume(i8*)
declare void @llvm.coro.destroy(i8*)
2 changes: 1 addition & 1 deletion llvm/test/Verifier/invoke.ll
Expand Up @@ -46,7 +46,7 @@ contb:

define i8 @f2() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
entry:
; CHECK: Cannot invoke an intrinsic other than donothing, patchpoint or statepoint
; CHECK: Cannot invoke an intrinsic other than donothing, patchpoint, statepoint, coro_resume or coro_destroy
invoke void @llvm.trap()
to label %cont unwind label %lpad

Expand Down

0 comments on commit f93082e

Please sign in to comment.