diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td index f20ba262d6480..d2b47876f2bef 100644 --- a/clang/include/clang/CIR/Dialect/IR/CIROps.td +++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td @@ -954,9 +954,9 @@ def CIR_ConditionOp : CIR_Op<"condition", [ //===----------------------------------------------------------------------===// defvar CIR_YieldableScopes = [ - "ArrayCtor", "ArrayDtor", "AwaitOp", "CaseOp", "CleanupScopeOp", "DoWhileOp", - "ForOp", "GlobalOp", "IfOp", "ScopeOp", "SwitchOp", "TernaryOp", "TryOp", - "WhileOp" + "ArrayCtor", "ArrayDtor", "AwaitOp", "CaseOp", "CleanupScopeOp", "CoroBodyOp", + "DoWhileOp", "ForOp", "GlobalOp", "IfOp", "ScopeOp", "SwitchOp", "TernaryOp", + "TryOp", "WhileOp" ]; def CIR_YieldOp : CIR_Op<"yield", [ @@ -4156,6 +4156,70 @@ def CIR_AwaitOp : CIR_Op<"await",[ let hasVerifier = 1; } +//===----------------------------------------------------------------------===// +// CoroBody +//===----------------------------------------------------------------------===// +def CIR_CoroBodyOp : CIR_Op<"coro.body", [ + DeclareOpInterfaceMethods, + RecursivelySpeculatable, AutomaticAllocationScope, NoRegionArguments, + RecursiveMemoryEffects +]> { + let summary = "Region containing the user-authored coroutine body"; + let description = [{ + The `cir.coro.body` operation models the region where the user-authored + coroutine code is emitted. + + This operation serves as a structural boundary separating the coroutine + setup and teardown logic (e.g. initial suspend, final suspend, and cleanup) + from the user-provided statements inside the coroutine. + + The body region contains the code corresponding to the original function + body, including `co_await` and `co_return` expressions. In particular, + `cir.co_return` operations inside this region mark coroutine exit points + and introduce structured control flow that transfers execution to the + final suspend point of the coroutine. + }]; + + let regions = (region AnyRegion:$body); + + let skipDefaultBuilders = 1; + + let builders = [ + OpBuilder<(ins "BuilderCallbackRef":$bodyBuilder)> + ]; + + let assemblyFormat = [{ + $body attr-dict + }]; + + let hasLLVMLowering = false; + let hasVerifier = 1; +} + +//===----------------------------------------------------------------------===// +// CoReturnOp +//===----------------------------------------------------------------------===// + +def CIR_CoReturnOp : CIR_Op<"co_return", [ + ReturnLike, Pure, Terminator +]> { + let summary = "Coroutine return operation"; + let description = [{ + The `cir.co_return` operation models a coroutine return point inside a + `cir.coro.body` region. + This operation is expected to appear only within a `cir.coro.body` region, + but it may be nested within other operations or regions inside that body. + }]; + + let assemblyFormat = [{ + attr-dict + }]; + + let hasVerifier = 1; + + let hasLLVMLowering = false; +} + //===----------------------------------------------------------------------===// // CopyOp diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp index 1f03b1da0bbef..dacc5f8c099ce 100644 --- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp @@ -1271,6 +1271,7 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID, return emitCoroutineFrame(); } case Builtin::BI__builtin_coro_free: + return RValue::get(emitCoroFreeBuiltin(e).getResult()); case Builtin::BI__builtin_coro_size: { GlobalDecl gd{fd}; mlir::Type ty = cgm.getTypes().getFunctionType( diff --git a/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp b/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp index 036f1b1cfe637..12408b7c59458 100644 --- a/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenCoroutine.cpp @@ -33,17 +33,16 @@ struct clang::CIRGen::CGCoroData { // Stores the result of __builtin_coro_begin call. mlir::Value coroBegin = nullptr; - // Stores the insertion point for final suspend, this happens after the - // promise call (return_xxx promise member) but before a cir.br to the return - // block. - mlir::Operation *finalSuspendInsPoint; - // How many co_return statements are in the coroutine. Used to decide whether // we need to add co_return; equivalent at the end of the user authored body. unsigned coreturnCount = 0; // The promise type's 'unhandled_exception' handler, if it defines one. Stmt *exceptionHandler = nullptr; + + // Stores the last emitted coro.free for the deallocate expressions, we use it + // to wrap dealloc code with if(auto mem = coro.free) dealloc(mem). + cir::CallOp lastCoroFree = nullptr; }; // Defining these here allows to keep CGCoroData private to this file. @@ -110,6 +109,63 @@ struct ParamReferenceReplacerRAII { }; } // namespace +namespace { +// Make sure to call coro.delete on scope exit. +struct CallCoroDelete final : public EHScopeStack::Cleanup { + Stmt *deallocate; + + // Emit "if (coro.free(CoroId, CoroBegin)) Deallocate;" + + // Note: That deallocation will be emitted twice: once for a normal exit and + // once for exceptional exit. This usage is safe because Deallocate does not + // contain any declarations. The SubStmtBuilder::makeNewAndDeleteExpr() + // builds a single call to a deallocation function which is safe to emit + // multiple times. + void emit(CIRGenFunction &cgf, Flags) override { + // Remember the current point, as we are going to emit deallocation code + // first to get to coro.free instruction that is an argument to a delete + // call. + + if (cgf.emitStmt(deallocate, /*useCurrentScope=*/true).failed()) { + cgf.cgm.error(deallocate->getBeginLoc(), + "failed to emit coroutine deallocation expression"); + return; + } + + CIRGenBuilderTy &builder = cgf.getBuilder(); + cir::CallOp coroFree = cgf.curCoro.data->lastCoroFree; + + if (!coroFree) { + cgf.cgm.error(deallocate->getBeginLoc(), + "Deallocation expression does not refer to coro.free"); + return; + } + + builder.setInsertionPointAfter(coroFree); + mlir::Value isPtrNotNull = builder.createPtrIsNotNull(coroFree.getResult()); + + llvm::SmallVector opsToMove; + mlir::Block *block = builder.getInsertionBlock(); + mlir::Block::iterator it(isPtrNotNull.getDefiningOp()); + + for (++it; it != block->end(); ++it) + opsToMove.push_back(&*it); + + auto ifOp = + cir::IfOp::create(builder, cgf.getLoc(deallocate->getSourceRange()), + isPtrNotNull, /*withElseRegion*/ false, + [&](mlir::OpBuilder &builder, mlir::Location loc) { + cir::YieldOp::create(builder, loc); + }); + + mlir::Operation *yieldOp = ifOp.getThenRegion().back().getTerminator(); + for (auto *op : opsToMove) + op->moveBefore(yieldOp); + } + explicit CallCoroDelete(Stmt *deallocStmt) : deallocate(deallocStmt) {} +}; +} // namespace + RValue CIRGenFunction::emitCoroutineFrame() { if (curCoro.data && curCoro.data->coroBegin) { return RValue::get(curCoro.data->coroBegin); @@ -235,6 +291,28 @@ cir::CallOp CIRGenFunction::emitCoroEndBuiltinCall(mlir::Location loc, loc, fnOp, mlir::ValueRange{nullPtr, builder.getBool(false, loc)}); } +cir::CallOp CIRGenFunction::emitCoroFreeBuiltin(const CallExpr *e) { + mlir::Operation *builtin = cgm.getGlobalValue(cgm.builtinCoroFree); + mlir::Location loc = getLoc(e->getBeginLoc()); + cir::FuncOp fnOp; + if (!builtin) { + fnOp = cgm.createCIRBuiltinFunction( + loc, cgm.builtinCoroFree, + cir::FuncType::get({uInt32Ty, voidPtrTy}, voidPtrTy), + /*fd=*/nullptr); + assert(fnOp && "should always succeed"); + } else { + fnOp = cast(builtin); + } + cir::CallOp coroFree = + builder.createCallOp(loc, fnOp, + mlir::ValueRange{curCoro.data->coroId.getResult(), + curCoro.data->coroBegin}); + + curCoro.data->lastCoroFree = coroFree; + return coroFree; +} + mlir::LogicalResult CIRGenFunction::emitCoroutineBody(const CoroutineBodyStmt &s) { mlir::Location openCurlyLoc = getLoc(s.getBeginLoc()); @@ -280,6 +358,8 @@ CIRGenFunction::emitCoroutineBody(const CoroutineBodyStmt &s) { { assert(!cir::MissingFeatures::generateDebugInfo()); ParamReferenceReplacerRAII paramReplacer(localDeclMap); + RunCleanupsScope resumeScope(*this); + ehStack.pushCleanup(NormalAndEHCleanup, s.getDeallocate()); // Create mapping between parameters and copy-params for coroutine // function. llvm::ArrayRef paramMoves = s.getParamMoves(); @@ -326,11 +406,31 @@ CIRGenFunction::emitCoroutineBody(const CoroutineBodyStmt &s) { curCoro.data->currentAwaitKind = cir::AwaitKind::User; - // FIXME(cir): wrap emitBodyAndFallthrough with try/catch bits. - if (s.getExceptionHandler()) - assert(!cir::MissingFeatures::coroutineExceptions()); - if (emitBodyAndFallthrough(*this, s, s.getBody(), curLexScope).failed()) - return mlir::failure(); + mlir::OpBuilder::InsertPoint userBody; + auto coroBodyOp = + cir::CoroBodyOp::create(builder, openCurlyLoc, /*scopeBuilder=*/ + [&](mlir::OpBuilder &b, mlir::Location loc) { + userBody = b.saveInsertionPoint(); + }); + { + mlir::OpBuilder::InsertionGuard guard(builder); + builder.restoreInsertionPoint(userBody); + // FIXME(cir): wrap emitBodyAndFallthrough with try/catch bits. + if (s.getExceptionHandler()) { + assert(!cir::MissingFeatures::coroutineExceptions()); + cgm.errorNYI("exceptions in coroutines are not yet supported in CIR"); + } + if (emitBodyAndFallthrough(*this, s, s.getBody(), curLexScope).failed()) { + return mlir::failure(); + } + } + + mlir::Block &coroBodyBlock = coroBodyOp.getBody().back(); + if (!coroBodyBlock.mightHaveTerminator()) { + mlir::OpBuilder::InsertionGuard guard(builder); + builder.setInsertionPointToEnd(&coroBodyBlock); + cir::YieldOp::create(builder, openCurlyLoc); + } // Note that LLVM checks CanFallthrough by looking into the availability // of the insert block which is kinda brittle and unintuitive, seems to be @@ -346,13 +446,26 @@ CIRGenFunction::emitCoroutineBody(const CoroutineBodyStmt &s) { curCoro.data->currentAwaitKind = cir::AwaitKind::Final; { mlir::OpBuilder::InsertionGuard guard(builder); - builder.setInsertionPoint(curCoro.data->finalSuspendInsPoint); if (emitStmt(s.getFinalSuspendStmt(), /*useCurrentScope=*/true) .failed()) return mlir::failure(); } } } + + emitCoroEndBuiltinCall( + openCurlyLoc, builder.getNullPtr(builder.getVoidPtrTy(), openCurlyLoc)); + if (auto *ret = cast_or_null(s.getReturnStmt())) { + // Since we already emitted the return value above, so we shouldn't + // emit it again here. + Expr *previousRetValue = ret->getRetValue(); + ret->setRetValue(nullptr); + if (emitStmt(ret, /*useCurrentScope=*/true).failed()) + return mlir::failure(); + // Set the return value back. The code generator, as the AST **Consumer**, + // shouldn't change the AST. + ret->setRetValue(previousRetValue); + } return mlir::success(); } @@ -538,13 +651,7 @@ mlir::LogicalResult CIRGenFunction::emitCoreturnStmt(CoreturnStmt const &s) { // it. The actual return instruction is only inserted during current // scope cleanup handling. mlir::Location loc = getLoc(s.getSourceRange()); - mlir::Block *retBlock = curLexScope->getOrCreateRetBlock(*this, loc); - curCoro.data->finalSuspendInsPoint = - cir::BrOp::create(builder, loc, retBlock); - - // Insert the new block to continue codegen after branch to ret block, - // this will likely be an empty block. - builder.createBlock(builder.getBlock()->getParent()); + cir::CoReturnOp::create(builder, loc); return mlir::success(); } diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp index 057e2f9c0f70a..a71d594684da9 100644 --- a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp @@ -318,10 +318,6 @@ cir::ReturnOp CIRGenFunction::LexicalScope::emitReturn(mlir::Location loc) { auto fn = dyn_cast(cgf.curFn); assert(fn && "emitReturn from non-function"); - // If we are on a coroutine, add the coro_end builtin call. - if (fn.getCoroutine()) - cgf.emitCoroEndBuiltinCall(loc, - builder.getNullPtr(builder.getVoidPtrTy(), loc)); if (!fn.getFunctionType().hasVoidReturn()) { // Load the value from `__retval` and return it via the `cir.return` op. auto value = cir::LoadOp::create( diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h index c798833d877ea..0353b5f3bece6 100644 --- a/clang/lib/CIR/CodeGen/CIRGenFunction.h +++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h @@ -1677,6 +1677,8 @@ class CIRGenFunction : public CIRGenTypeCache { cir::CallOp emitCoroAllocBuiltinCall(mlir::Location loc); cir::CallOp emitCoroBeginBuiltinCall(mlir::Location loc, mlir::Value coroframeAddr); + + cir::CallOp emitCoroFreeBuiltin(const CallExpr *e); RValue emitCoroutineFrame(); void emitDestroy(Address addr, QualType type, Destroyer *destroyer); diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.h b/clang/lib/CIR/CodeGen/CIRGenModule.h index 1b999abd4caea..3e56f2fe6b987 100644 --- a/clang/lib/CIR/CodeGen/CIRGenModule.h +++ b/clang/lib/CIR/CodeGen/CIRGenModule.h @@ -704,6 +704,7 @@ class CIRGenModule : public CIRGenTypeCache { static constexpr const char *builtinCoroAlloc = "__builtin_coro_alloc"; static constexpr const char *builtinCoroBegin = "__builtin_coro_begin"; static constexpr const char *builtinCoroEnd = "__builtin_coro_end"; + static constexpr const char *builtinCoroFree = "__builtin_coro_free"; /// Given a builtin id for a function like "__builtin_fabsf", return a /// Function* for "fabsf". diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp index 35a31b0dbda63..b3e818ffe853f 100644 --- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp +++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp @@ -2538,15 +2538,24 @@ mlir::LogicalResult cir::FuncOp::verify() { if (!isDeclaration() && getCoroutine()) { bool foundAwait = false; + int coroBodyCount = 0; this->walk([&](Operation *op) { if (auto await = dyn_cast(op)) { foundAwait = true; - return; + } else if (isa(op)) { + coroBodyCount++; + if (coroBodyCount > 1) { + return mlir::WalkResult::interrupt(); + } } + return mlir::WalkResult::advance(); }); if (!foundAwait) return emitOpError() << "coroutine body must use at least one cir.await op"; + if (coroBodyCount != 1) + return emitOpError() + << "coroutine function must have exactly one cir.body op"; } llvm::SmallSet labels; @@ -2959,6 +2968,48 @@ LogicalResult cir::AwaitOp::verify() { return success(); } +LogicalResult cir::CoReturnOp::verify() { + if (!getOperation()->getParentOfType()) + return emitOpError("must be inside a cir.coro.body"); + return success(); +} + +//===----------------------------------------------------------------------===// +// CoroBody +//===----------------------------------------------------------------------===// + +void cir::CoroBodyOp::getSuccessorRegions( + mlir::RegionBranchPoint point, SmallVectorImpl ®ions) { + if (!point.isParent()) { + regions.push_back(RegionSuccessor::parent()); + return; + } + + regions.push_back(RegionSuccessor(&getBody())); +} + +mlir::ValueRange +cir::CoroBodyOp::getSuccessorInputs(RegionSuccessor successor) { + return ValueRange(); +} + +LogicalResult cir::CoroBodyOp::verify() { + if (!getOperation()->getParentOfType().getCoroutine()) + return emitOpError("enclosing function must be a coroutine"); + return success(); +} + +void cir::CoroBodyOp::build(OpBuilder &builder, OperationState &result, + BuilderCallbackRef bodyBuilder) { + assert(bodyBuilder && + "the builder callback for 'CoroBodyOp' must be present"); + OpBuilder::InsertionGuard guard(builder); + + Region *bodyRegion = result.addRegion(); + builder.createBlock(bodyRegion); + bodyBuilder(builder, result.location); +} + //===----------------------------------------------------------------------===// // CopyOp Definitions //===----------------------------------------------------------------------===// diff --git a/clang/test/CIR/CodeGen/coro-task.cpp b/clang/test/CIR/CodeGen/coro-task.cpp index a87cc2af6a14b..24a25695b38dd 100644 --- a/clang/test/CIR/CodeGen/coro-task.cpp +++ b/clang/test/CIR/CodeGen/coro-task.cpp @@ -196,8 +196,11 @@ VoidTask silly_task() { // Call promise.get_return_object() to retrieve the task object. -// CIR: %[[RetObj:.*]] = cir.call @_ZN5folly4coro4TaskIvE12promise_type17get_return_objectEv(%[[VoidPromisseAddr]]) nothrow : {{.*}} -> ![[VoidTask]] -// CIR: cir.store{{.*}} %[[RetObj]], %[[VoidTaskAddr]] : ![[VoidTask]] + +// CIR: cir.cleanup.scope { + +// CIR: %[[RetObj:.*]] = cir.call @_ZN5folly4coro4TaskIvE12promise_type17get_return_objectEv(%[[VoidPromisseAddr]]) nothrow : {{.*}} -> ![[VoidTask]] +// CIR: cir.store{{.*}} %[[RetObj]], %[[VoidTaskAddr]] : ![[VoidTask]] // OGCG: call void @llvm.lifetime.start.p0(ptr %[[VoidPromisseAddr]]) // OGCG: call void @_ZN5folly4coro4TaskIvE12promise_type17get_return_objectEv(ptr noundef nonnull align 1 dereferenceable(1) %[[VoidPromisseAddr]]) @@ -210,8 +213,8 @@ VoidTask silly_task() { // the suspend_always struct to use for cir.await. Note that we return by-value since we defer ABI lowering // to later passes, same is done elsewhere. -// CIR: %[[Tmp0:.*]] = cir.call @_ZN5folly4coro4TaskIvE12promise_type15initial_suspendEv(%[[VoidPromisseAddr]]) -// CIR: cir.store{{.*}} %[[Tmp0:.*]], %[[SuspendAlwaysAddr]] +// CIR: %[[Tmp0:.*]] = cir.call @_ZN5folly4coro4TaskIvE12promise_type15initial_suspendEv(%[[VoidPromisseAddr]]) +// CIR: cir.store{{.*}} %[[Tmp0:.*]], %[[SuspendAlwaysAddr]] // OGCG: call void @_ZN5folly4coro4TaskIvE12promise_type15initial_suspendEv(ptr noundef nonnull align 1 dereferenceable(1) %[[VoidPromisseAddr]]) @@ -221,9 +224,9 @@ VoidTask silly_task() { // First regions `ready` has a special cir.yield code to veto suspension. -// CIR: cir.await(init, ready : { -// CIR: %[[ReadyVeto:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[SuspendAlwaysAddr]]) -// CIR: cir.condition(%[[ReadyVeto]]) +// CIR: cir.await(init, ready : { +// CIR: %[[ReadyVeto:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[SuspendAlwaysAddr]]) +// CIR: cir.condition(%[[ReadyVeto]]) // OGCG: %[[Tmp0:.*]] = call noundef zeroext i1 @_ZNSt14suspend_always11await_readyEv(ptr noundef nonnull align 1 dereferenceable(1) %[[SuspendAlwaysAddr]]) // OGCG: br i1 %[[Tmp0]], label %init.ready, label %init.suspend @@ -237,14 +240,14 @@ VoidTask silly_task() { // // FIXME: add veto support for non-void await_suspends. -// CIR: }, suspend : { -// CIR: %[[FromAddrRes:.*]] = cir.call @_ZNSt16coroutine_handleIN5folly4coro4TaskIvE12promise_typeEE12from_addressEPv(%[[CoroFrameAddr]]) -// CIR: cir.store{{.*}} %[[FromAddrRes]], %[[CoroHandlePromiseAddr]] : ![[CoroHandlePromiseVoid]] -// CIR: %[[CoroHandlePromiseReload:.*]] = cir.load{{.*}} %[[CoroHandlePromiseAddr]] -// CIR: cir.call @_ZNSt16coroutine_handleIvEC1IN5folly4coro4TaskIvE12promise_typeEEES_IT_E(%[[CoroHandleVoidAddr]], %[[CoroHandlePromiseReload]]) -// CIR: %[[CoroHandleVoidReload:.*]] = cir.load{{.*}} %[[CoroHandleVoidAddr]] : !cir.ptr, ![[CoroHandleVoid]] -// CIR: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%[[SuspendAlwaysAddr]], %[[CoroHandleVoidReload]]) -// CIR: cir.yield +// CIR: }, suspend : { +// CIR: %[[FromAddrRes:.*]] = cir.call @_ZNSt16coroutine_handleIN5folly4coro4TaskIvE12promise_typeEE12from_addressEPv(%[[CoroFrameAddr]]) +// CIR: cir.store{{.*}} %[[FromAddrRes]], %[[CoroHandlePromiseAddr]] : ![[CoroHandlePromiseVoid]] +// CIR: %[[CoroHandlePromiseReload:.*]] = cir.load{{.*}} %[[CoroHandlePromiseAddr]] +// CIR: cir.call @_ZNSt16coroutine_handleIvEC1IN5folly4coro4TaskIvE12promise_typeEEES_IT_E(%[[CoroHandleVoidAddr]], %[[CoroHandlePromiseReload]]) +// CIR: %[[CoroHandleVoidReload:.*]] = cir.load{{.*}} %[[CoroHandleVoidAddr]] : !cir.ptr, ![[CoroHandleVoid]] +// CIR: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%[[SuspendAlwaysAddr]], %[[CoroHandleVoidReload]]) +// CIR: cir.yield // OGCG: init.suspend: // OGCG: %[[Save:.*]] = call token @llvm.coro.save(ptr null) @@ -257,10 +260,10 @@ VoidTask silly_task() { // Third region `resume` handles coroutine resuming logic. -// CIR: }, resume : { -// CIR: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[SuspendAlwaysAddr]]) -// CIR: cir.yield -// CIR: },) +// CIR: }, resume : { +// CIR: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[SuspendAlwaysAddr]]) +// CIR: cir.yield +// CIR: },) // OGCG: init.ready: // OGCG: call void @_ZNSt14suspend_always12await_resumeEv(ptr noundef nonnull align 1 dereferenceable(1) %[[SuspendAlwaysAddr]] @@ -272,31 +275,63 @@ VoidTask silly_task() { // - The final suspend co_await // - Return +// CIR: cir.coro.body { + // The actual user written co_await -// CIR: cir.await(user, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: },) +// CIR: cir.await(user, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) // OGCG: cleanup.cont // OGCG: await.suspend: // OGCG: await.ready: // The promise call -// CHECK: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(%[[VoidPromisseAddr]]) +// CIR: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(%[[VoidPromisseAddr]]) +// CIR: cir.co_return +// CIR: } // OGCG: call void @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(ptr noundef nonnull align 1 dereferenceable(1) %[[VoidPromisseAddr]]) // The final suspend co_await -// CIR: cir.await(final, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: },) +// CIR: cir.await(final, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) +// CIR: cir.yield // OGCG: coro.final: // OGCG: final.suspend: // OGCG: final.ready: +// Cleanup of coroutine frame. +// +// `__builtin_coro_free` returns the frame pointer or null. +// If null, no dynamic allocation happened, so nothing to free. +// The `if` ensures we only call delete on non-null. + +// CIR: } cleanup normal { +// CIR: %[[FreeMem:.*]] = cir.call @__builtin_coro_free(%[[CoroId]], %[[CoroFrameAddr]]) +// CIR: %[[NullPtr2:.*]] = cir.const #cir.ptr +// CIR: %[[Cond:.*]] = cir.cmp ne %[[FreeMem]], %[[NullPtr2]] +// CIR: cir.if %[[Cond]] { +// CIR: %[[Size:.*]] = cir.call @__builtin_coro_size() +// CIR: cir.call @_ZdlPvm(%[[FreeMem]], %[[Size]]) +// CIR: } +// CIR: cir.yield +// CIR: } + +// OGCG: %[[FreeMem:.*]] = call ptr @llvm.coro.free(token %[[CoroId]], ptr %[[CoroFrameAddr]]) +// OGCG: %[[Cond:.*]] = icmp ne ptr %[[FreeMem]], null +// OGCG: br i1 %[[Cond]], label %coro.free, label %after.coro.free + +// OGCG: coro.free: +// OGCG: %[[Size:.*]] = call i64 @llvm.coro.size.i64() +// OGCG: call void @_ZdlPvm(ptr {{.*}} %[[FreeMem]], i64 {{.*}} %[[Size]]) +// OGCG: br label %after.coro.free +// OGCG: after.coro.free: + // Call builtin coro end and return // CIR: %[[CoroEndArg0:.*]] = cir.const #cir.ptr : !cir.ptr @@ -326,38 +361,46 @@ folly::coro::Task byRef(const std::string& s) { // CIR: %[[CoroHandlePromiseAddr:.*]] = cir.alloca ![[CoroHandlePromiseInt]], {{.*}} ["agg.tmp1"] {alignment = 1 : i64} // CIR: cir.store %[[ARG]], %[[AllocaParam]] : !cir.ptr, {{.*}} +// CIR: cir.cleanup.scope { // Call promise.get_return_object() to retrieve the task object. -// CIR: %[[LOAD:.*]] = cir.load %[[AllocaParam]] : !cir.ptr>, !cir.ptr -// CIR: cir.store {{.*}} %[[LOAD]], %[[AllocaFnUse]] : !cir.ptr, !cir.ptr> -// CIR: %[[RetObj:.*]] = cir.call @_ZN5folly4coro4TaskIiE12promise_type17get_return_objectEv(%[[IntPromisseAddr]]) nothrow : {{.*}} -> ![[IntTask]] -// CIR: cir.store {{.*}} %[[RetObj]], %[[IntTaskAddr]] : ![[IntTask]] -// CIR: %[[Tmp0:.*]] = cir.call @_ZN5folly4coro4TaskIiE12promise_type15initial_suspendEv(%[[IntPromisseAddr]]) -// CIR: cir.store{{.*}} %[[Tmp0]], %[[SuspendAlwaysAddr]] -// CIR: cir.await(init, ready : { -// CIR: %[[TmpCallRes:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[SuspendAlwaysAddr]]) -// CIR: cir.condition(%[[TmpCallRes]]) -// CIR: }, suspend : { -// CIR: %[[FromAddrRes:.*]] = cir.call @_ZNSt16coroutine_handleIN5folly4coro4TaskIiE12promise_typeEE12from_addressEPv(%[[CoroFrameAddr:.*]]) -// CIR: cir.store{{.*}} %[[FromAddrRes]], %[[CoroHandlePromiseAddr]] : ![[CoroHandlePromiseInt]] -// CIR: %[[CoroHandlePromiseReload:.*]] = cir.load{{.*}} %[[CoroHandlePromiseAddr]] -// CIR: cir.call @_ZNSt16coroutine_handleIvEC1IN5folly4coro4TaskIiE12promise_typeEEES_IT_E(%[[CoroHandleVoidAddr]], %[[CoroHandlePromiseReload]]) -// CIR: %[[CoroHandleVoidReload:.*]] = cir.load{{.*}} %[[CoroHandleVoidAddr]] : !cir.ptr, ![[CoroHandleVoid]] -// CIR: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%[[SuspendAlwaysAddr]], %[[CoroHandleVoidReload]]) -// CIR: cir.yield -// CIR: }, resume : { -// CIR: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[SuspendAlwaysAddr]]) -// CIR: cir.yield -// CIR: },) - -// can't fallthrough +// CIR: %[[LOAD:.*]] = cir.load %[[AllocaParam]] : !cir.ptr>, !cir.ptr +// CIR: cir.store {{.*}} %[[LOAD]], %[[AllocaFnUse]] : !cir.ptr, !cir.ptr> +// CIR: %[[RetObj:.*]] = cir.call @_ZN5folly4coro4TaskIiE12promise_type17get_return_objectEv(%[[IntPromisseAddr]]) nothrow : {{.*}} -> ![[IntTask]] +// CIR: cir.store {{.*}} %[[RetObj]], %[[IntTaskAddr]] : ![[IntTask]] +// CIR: %[[Tmp0:.*]] = cir.call @_ZN5folly4coro4TaskIiE12promise_type15initial_suspendEv(%[[IntPromisseAddr]]) +// CIR: cir.store{{.*}} %[[Tmp0]], %[[SuspendAlwaysAddr]] +// CIR: cir.await(init, ready : { +// CIR: %[[TmpCallRes:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[SuspendAlwaysAddr]]) +// CIR: cir.condition(%[[TmpCallRes]]) +// CIR: }, suspend : { +// CIR: %[[FromAddrRes:.*]] = cir.call @_ZNSt16coroutine_handleIN5folly4coro4TaskIiE12promise_typeEE12from_addressEPv(%[[CoroFrameAddr:.*]]) +// CIR: cir.store{{.*}} %[[FromAddrRes]], %[[CoroHandlePromiseAddr]] : ![[CoroHandlePromiseInt]] +// CIR: %[[CoroHandlePromiseReload:.*]] = cir.load{{.*}} %[[CoroHandlePromiseAddr]] +// CIR: cir.call @_ZNSt16coroutine_handleIvEC1IN5folly4coro4TaskIiE12promise_typeEEES_IT_E(%[[CoroHandleVoidAddr]], %[[CoroHandlePromiseReload]]) +// CIR: %[[CoroHandleVoidReload:.*]] = cir.load{{.*}} %[[CoroHandleVoidAddr]] : !cir.ptr, ![[CoroHandleVoid]] +// CIR: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%[[SuspendAlwaysAddr]], %[[CoroHandleVoidReload]]) +// CIR: cir.yield +// CIR: }, resume : { +// CIR: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[SuspendAlwaysAddr]]) +// CIR: cir.yield +// CIR: },) +// CIR: cir.coro.body { + // can't fallthrough // CIR-NOT: cir.await(user +// CIR: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[IntPromisseAddr]], %[[STRING_SIZE:.*]]) +//CIR: cir.co_return +// CIR: } + // The final suspend co_await -// CIR: cir.await(final, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: },) +// CIR: cir.await(final, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) +// CIR: cir.yield +// CIR: } cleanup normal { +// CIR: } folly::coro::Task silly_coro() { std::optional> task; @@ -373,16 +416,22 @@ folly::coro::Task silly_coro() { // check there are not multiple co_returns emitted. // CIR: cir.func coroutine {{.*}} @_Z10silly_corov() {{.*}} ![[VoidTask]] -// CIR: cir.await(init, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: },) -// CIR: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv -// CIR-NOT: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv -// CIR: cir.await(final, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: },) +// CIR: cir.cleanup.scope { +// CIR: cir.await(init, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) +// CIR: cir.coro.body { +// CIR: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv +// CIR: cir.co_return +// CIR: } +// CIR: cir.await(final, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) +// CIR: cir.yield +// CIR: } cleanup normal { +// CIR: } folly::coro::Task yield(); folly::coro::Task yield1() { @@ -408,66 +457,73 @@ folly::coro::Task yield1() { // CIR-DAG: %[[CH_VOID2:.*]] = cir.alloca ![[CoroHandleVoid]], {{.*}} ["agg.tmp5"] // CIR-DAG: %[[CH_PROM2:.*]] = cir.alloca ![[CoroHandlePromiseVoid]], {{.*}} ["agg.tmp6"] +// CIR: cir.cleanup.scope { // initial_suspend + await(init) -// CIR: %[[INIT_SUSP:.*]] = cir.call @_ZN5folly4coro4TaskIvE12promise_type15initial_suspendEv(%[[PROMISE]]){{.*}} -// CIR: cir.store{{.*}} %[[INIT_SUSP]], %[[SUSP0]] -// CIR: cir.await(init, ready : { -// CIR: %[[READY0:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[SUSP0]]){{.*}} -// CIR: cir.condition(%[[READY0]]) -// CIR: }, suspend : { -// CIR: %[[FROMADDR0:.*]] = cir.call @_ZNSt16coroutine_handleIN5folly4coro4TaskIvE12promise_typeEE12from_addressEPv(%{{.*}}){{.*}} -// CIR: cir.store{{.*}} %[[FROMADDR0]], %[[CH_PROM0]] -// CIR: %[[PROM_RELOAD0:.*]] = cir.load{{.*}} %[[CH_PROM0]] -// CIR: cir.call @_ZNSt16coroutine_handleIvEC1IN5folly4coro4TaskIvE12promise_typeEEES_IT_E(%[[CH_VOID0]], %[[PROM_RELOAD0]]){{.*}} -// CIR: %[[VOID_RELOAD0:.*]] = cir.load{{.*}} %[[CH_VOID0]] -// CIR: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%[[SUSP0]], %[[VOID_RELOAD0]]){{.*}} -// CIR: cir.yield -// CIR: }, resume : { -// CIR: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[SUSP0]]){{.*}} -// CIR: cir.yield -// CIR: },) +// CIR: %[[INIT_SUSP:.*]] = cir.call @_ZN5folly4coro4TaskIvE12promise_type15initial_suspendEv(%[[PROMISE]]){{.*}} +// CIR: cir.store{{.*}} %[[INIT_SUSP]], %[[SUSP0]] +// CIR: cir.await(init, ready : { +// CIR: %[[READY0:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[SUSP0]]){{.*}} +// CIR: cir.condition(%[[READY0]]) +// CIR: }, suspend : { +// CIR: %[[FROMADDR0:.*]] = cir.call @_ZNSt16coroutine_handleIN5folly4coro4TaskIvE12promise_typeEE12from_addressEPv(%{{.*}}){{.*}} +// CIR: cir.store{{.*}} %[[FROMADDR0]], %[[CH_PROM0]] +// CIR: %[[PROM_RELOAD0:.*]] = cir.load{{.*}} %[[CH_PROM0]] +// CIR: cir.call @_ZNSt16coroutine_handleIvEC1IN5folly4coro4TaskIvE12promise_typeEEES_IT_E(%[[CH_VOID0]], %[[PROM_RELOAD0]]){{.*}} +// CIR: %[[VOID_RELOAD0:.*]] = cir.load{{.*}} %[[CH_VOID0]] +// CIR: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%[[SUSP0]], %[[VOID_RELOAD0]]){{.*}} +// CIR: cir.yield +// CIR: }, resume : { +// CIR: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[SUSP0]]){{.*}} +// CIR: cir.yield +// CIR: },) // yield_value + await(yield) -// CIR: %[[YIELD_TASK:.*]] = cir.call @_Z5yieldv(){{.*}} -// CIR: cir.store{{.*}} %[[YIELD_TASK]], %[[T_ADDR]] -// CIR: %[[AWAITER:.*]] = cir.load{{.*}} %[[AWAITER_COPY_ADDR]] -// CIR: %[[YIELD_SUSP:.*]] = cir.call @_ZN5folly4coro4TaskIvE12promise_type11yield_valueES2_(%[[PROMISE]], %[[AWAITER]]){{.*}} -// CIR: cir.store{{.*}} %[[YIELD_SUSP]], %[[SUSP1]] -// CIR: cir.await(yield, ready : { -// CIR: %[[READY1:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[SUSP1]]){{.*}} -// CIR: cir.condition(%[[READY1]]) -// CIR: }, suspend : { -// CIR: %[[FROMADDR1:.*]] = cir.call @_ZNSt16coroutine_handleIN5folly4coro4TaskIvE12promise_typeEE12from_addressEPv(%{{.*}}){{.*}} -// CIR: cir.store{{.*}} %[[FROMADDR1]], %[[CH_PROM1]] -// CIR: %[[PROM_RELOAD1:.*]] = cir.load{{.*}} %[[CH_PROM1]] -// CIR: cir.call @_ZNSt16coroutine_handleIvEC1IN5folly4coro4TaskIvE12promise_typeEEES_IT_E(%[[CH_VOID1]], %[[PROM_RELOAD1]]){{.*}} -// CIR: %[[VOID_RELOAD1:.*]] = cir.load{{.*}} %[[CH_VOID1]] -// CIR: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%[[SUSP1]], %[[VOID_RELOAD1]]){{.*}} -// CIR: cir.yield -// CIR: }, resume : { -// CIR: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[SUSP1]]){{.*}} -// CIR: cir.yield -// CIR: },) +// CIR: cir.coro.body { +// CIR: %[[YIELD_TASK:.*]] = cir.call @_Z5yieldv(){{.*}} +// CIR: cir.store{{.*}} %[[YIELD_TASK]], %[[T_ADDR]] +// CIR: %[[AWAITER:.*]] = cir.load{{.*}} %[[AWAITER_COPY_ADDR]] +// CIR: %[[YIELD_SUSP:.*]] = cir.call @_ZN5folly4coro4TaskIvE12promise_type11yield_valueES2_(%[[PROMISE]], %[[AWAITER]]){{.*}} +// CIR: cir.store{{.*}} %[[YIELD_SUSP]], %[[SUSP1]] +// CIR: cir.await(yield, ready : { +// CIR: %[[READY1:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[SUSP1]]){{.*}} +// CIR: cir.condition(%[[READY1]]) +// CIR: }, suspend : { +// CIR: %[[FROMADDR1:.*]] = cir.call @_ZNSt16coroutine_handleIN5folly4coro4TaskIvE12promise_typeEE12from_addressEPv(%{{.*}}){{.*}} +// CIR: cir.store{{.*}} %[[FROMADDR1]], %[[CH_PROM1]] +// CIR: %[[PROM_RELOAD1:.*]] = cir.load{{.*}} %[[CH_PROM1]] +// CIR: cir.call @_ZNSt16coroutine_handleIvEC1IN5folly4coro4TaskIvE12promise_typeEEES_IT_E(%[[CH_VOID1]], %[[PROM_RELOAD1]]){{.*}} +// CIR: %[[VOID_RELOAD1:.*]] = cir.load{{.*}} %[[CH_VOID1]] +// CIR: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%[[SUSP1]], %[[VOID_RELOAD1]]){{.*}} +// CIR: cir.yield +// CIR: }, resume : { +// CIR: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[SUSP1]]){{.*}} +// CIR: cir.yield +// CIR: },) +// CIR: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(%[[PROMISE]]) +// CIR: cir.co_return +// CIR: } // return_void + await(final) -// CIR: cir.call @_ZN5folly4coro4TaskIvE12promise_type11return_voidEv(%[[PROMISE]]){{.*}} -// CIR: %[[FINAL_SUSP:.*]] = cir.call @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv(%[[PROMISE]]){{.*}} -// CIR: cir.store{{.*}} %[[FINAL_SUSP]], %[[SUSP2]] -// CIR: cir.await(final, ready : { -// CIR: %[[READY2:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[SUSP2]]){{.*}} -// CIR: cir.condition(%[[READY2]]) -// CIR: }, suspend : { -// CIR: %[[FROMADDR2:.*]] = cir.call @_ZNSt16coroutine_handleIN5folly4coro4TaskIvE12promise_typeEE12from_addressEPv(%{{.*}}){{.*}} -// CIR: cir.store{{.*}} %[[FROMADDR2]], %[[CH_PROM2]] -// CIR: %[[PROM_RELOAD2:.*]] = cir.load{{.*}} %[[CH_PROM2]] -// CIR: cir.call @_ZNSt16coroutine_handleIvEC1IN5folly4coro4TaskIvE12promise_typeEEES_IT_E(%[[CH_VOID2]], %[[PROM_RELOAD2]]){{.*}} -// CIR: %[[VOID_RELOAD2:.*]] = cir.load{{.*}} %[[CH_VOID2]] -// CIR: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%[[SUSP2]], %[[VOID_RELOAD2]]){{.*}} +// CIR: %[[FINAL_SUSP:.*]] = cir.call @_ZN5folly4coro4TaskIvE12promise_type13final_suspendEv(%[[PROMISE]]){{.*}} +// CIR: cir.store{{.*}} %[[FINAL_SUSP]], %[[SUSP2]] +// CIR: cir.await(final, ready : { +// CIR: %[[READY2:.*]] = cir.call @_ZNSt14suspend_always11await_readyEv(%[[SUSP2]]){{.*}} +// CIR: cir.condition(%[[READY2]]) +// CIR: }, suspend : { +// CIR: %[[FROMADDR2:.*]] = cir.call @_ZNSt16coroutine_handleIN5folly4coro4TaskIvE12promise_typeEE12from_addressEPv(%{{.*}}){{.*}} +// CIR: cir.store{{.*}} %[[FROMADDR2]], %[[CH_PROM2]] +// CIR: %[[PROM_RELOAD2:.*]] = cir.load{{.*}} %[[CH_PROM2]] +// CIR: cir.call @_ZNSt16coroutine_handleIvEC1IN5folly4coro4TaskIvE12promise_typeEEES_IT_E(%[[CH_VOID2]], %[[PROM_RELOAD2]]){{.*}} +// CIR: %[[VOID_RELOAD2:.*]] = cir.load{{.*}} %[[CH_VOID2]] +// CIR: cir.call @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(%[[SUSP2]], %[[VOID_RELOAD2]]){{.*}} +// CIR: cir.yield +// CIR: }, resume : { +// CIR: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[SUSP2]]){{.*}} +// CIR: cir.yield +// CIR: },) // CIR: cir.yield -// CIR: }, resume : { -// CIR: cir.call @_ZNSt14suspend_always12await_resumeEv(%[[SUSP2]]){{.*}} -// CIR: cir.yield -// CIR: },) +// CIR: } cleanup normal { +// CIR: } // CIR: = cir.call @__builtin_coro_end(%{{.*}}, %{{.*}}){{.*}} // CIR: %[[RETLOAD:.*]] = cir.load{{.*}} %[[RETVAL]] // CIR: cir.return %[[RETLOAD]] @@ -485,30 +541,36 @@ folly::coro::Task go1() { // CIR: %[[IntTaskAddr:.*]] = cir.alloca ![[IntTask]], !cir.ptr, ["task", init] // CIR: %[[OneAddr:.*]] = cir.alloca !s32i, !cir.ptr, ["ref.tmp1", init] {alignment = 4 : i64} -// CIR: cir.await(init, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: },) - -// CIR: %[[One:.*]] = cir.const #cir.int<1> : !s32i -// CIR: cir.store{{.*}} %[[One]], %[[OneAddr]] : !s32i, !cir.ptr -// CIR: %[[IntTaskTmp:.*]] = cir.call @_Z2goRKi(%[[OneAddr]]) : (!cir.ptr{{.*}}) -> ![[IntTask]] -// CIR: cir.store{{.*}} %[[IntTaskTmp]], %[[IntTaskAddr]] : ![[IntTask]], !cir.ptr - -// CIR: cir.await(user, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: %[[ResumeVal:.*]] = cir.call @_ZN5folly4coro4TaskIiE12await_resumeEv(%[[IntTaskAddr]]) -// CIR: cir.store{{.*}} %[[ResumeVal]], %[[CoReturnValAddr:.*]] : !s32i, !cir.ptr -// CIR: },) -// CIR: %[[V:.*]] = cir.load{{.*}} %[[CoReturnValAddr]] : !cir.ptr, !s32i -// CIR: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi({{.*}}, %[[V]]) - -// CIR: cir.await(final, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: },) - +// CIR: cir.cleanup.scope { +// CIR: cir.await(init, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) + +// CIR: cir.coro.body { +// CIR: %[[One:.*]] = cir.const #cir.int<1> : !s32i +// CIR: cir.store{{.*}} %[[One]], %[[OneAddr]] : !s32i, !cir.ptr +// CIR: %[[IntTaskTmp:.*]] = cir.call @_Z2goRKi(%[[OneAddr]]) : (!cir.ptr{{.*}}) -> ![[IntTask]] +// CIR: cir.store{{.*}} %[[IntTaskTmp]], %[[IntTaskAddr]] : ![[IntTask]], !cir.ptr + +// CIR: cir.await(user, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: %[[ResumeVal:.*]] = cir.call @_ZN5folly4coro4TaskIiE12await_resumeEv(%[[IntTaskAddr]]) +// CIR: cir.store{{.*}} %[[ResumeVal]], %[[CoReturnValAddr:.*]] : !s32i, !cir.ptr +// CIR: },) +// CIR: %[[V:.*]] = cir.load{{.*}} %[[CoReturnValAddr]] : !cir.ptr, !s32i +// CIR: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi({{.*}}, %[[V]]) +// CIR: cir.co_return +// CIR: } + +// CIR: cir.await(final, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) +// CIR: cir.yield +// CIR: } cleanup normal { +// CIR: } folly::coro::Task go1_lambda() { auto task = []() -> folly::coro::Task { @@ -518,27 +580,45 @@ folly::coro::Task go1_lambda() { } // CIR: cir.func coroutine {{.*}} @_ZZ10go1_lambdavENK3$_0clEv{{.*}} ![[IntTask]] -// CIR: cir.await(init, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: },) -// CIR: cir.await(final, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: },) +// CIR: cir.cleanup.scope { +// CIR: cir.await(init, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) +// CIR: cir.coro.body { +// CIR: %[[ONE:.*]] = cir.const #cir.int<1> +// CIR: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE:.*]], %[[ONE]]) +// CIR: cir.co_return +// CIR: } +// CIR: cir.await(final, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) +// CIR: } cleanup normal { + // CIR: cir.func coroutine {{.*}} @_Z10go1_lambdav() {{.*}} ![[IntTask]] -// CIR: cir.await(init, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: },) -// CIR: cir.await(user, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: },) -// CIR: cir.await(final, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: },) +// CIR: cir.cleanup.scope { +// CIR: cir.await(init, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) +// CIR: cir.coro.body { +// CIR: cir.call @_ZZ10go1_lambdavENK3$_0clEv +// CIR: cir.await(user, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: %[[RESUME_RES:.*]] = cir.call @_ZN5folly4coro4TaskIiE12await_resumeEv(%[[TASK:.*]]) +// CIR: cir.store %[[RESUME_RES]], %[[resume_rval:.*]] : !s32i, !cir.ptr +// CIR: },) +// CIR: %[[TMP1:.*]] = cir.load %[[resume_rval:.*]] : !cir.ptr +// CIR: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE:.*]], %[[TMP1]]) +// CIR: cir.co_return +// CIR: } +// CIR: cir.await(final, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) +// CIR: } cleanup normal { folly::coro::Task go4() { auto* fn = +[](int const& i) -> folly::coro::Task { co_return i; }; @@ -547,21 +627,28 @@ folly::coro::Task go4() { } // CIR: cir.func coroutine{{.*}} @_ZZ3go4vENK3$_0clERKi( -// CIR: cir.await(init, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: },) -// CIR: cir.await(final, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: },) +// CIR: cir.cleanup.scope { +// CIR: cir.await(init, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) +// CIR: cir.coro.body { +// CIR: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE:.*]], %[[I:.*]]) +// CIR: cir.co_return +// CIR: } +// CIR: cir.await(final, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) +// CIR: } cleanup normal { // CIR: cir.func coroutine {{.*}} @_Z3go4v() {{.*}} ![[IntTask]] -// CIR: cir.await(init, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: },) +// CIR: cir.cleanup.scope { +// CIR: cir.await(init, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) // Get the lambda invoker ptr via `lambda operator folly::coro::Task (*)(int const&)()` // CIR: %[[INVOKER:.*]] = cir.call @_ZZ3go4vENK3$_0cvPFN5folly4coro4TaskIiEERKiEEv(%{{.*}}) nothrow : {{.*}} -> (!cir.ptr) -> ![[IntTask]]>> {llvm.noundef}) @@ -574,19 +661,23 @@ folly::coro::Task go4() { // CIR: %[[CALLRES:.*]] = cir.call %[[FN]](%[[ARG]]) : (!cir.ptr) -> ![[IntTask]]>>, !cir.ptr {{.*}}) -> ![[IntTask]] // CIR: cir.store{{.*}} %[[CALLRES]], %[[TASK_ADDR:.*]] : ![[IntTask]], !cir.ptr -// CIR: cir.await(user, ready : { -// CIR: = cir.call @_ZN5folly4coro4TaskIiE11await_readyEv(%[[TASK_ADDR]]) -// CIR: cir.condition( -// CIR: }, suspend : { -// CIR: cir.yield -// CIR: }, resume : { -// CIR: cir.yield -// CIR: },) +// CIR: cir.await(user, ready : { +// CIR: = cir.call @_ZN5folly4coro4TaskIiE11await_readyEv(%[[TASK_ADDR]]) +// CIR: cir.condition( +// CIR: }, suspend : { +// CIR: cir.yield +// CIR: }, resume : { +// CIR: cir.yield +// CIR: },) +// CIR: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi +// CIR: cir.co_return +// CIR: } -// CIR: cir.await(final, ready : { -// CIR: }, suspend : { -// CIR: }, resume : { -// CIR: },) +// CIR: cir.await(final, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) +// CIR: } cleanup normal { // OGCG: define {{.*}}__await_suspend_wrapper__init(ptr noundef nonnull %[[Awaiter:.*]], ptr noundef %[[Handle:.*]]) // OGCG: entry: @@ -602,3 +693,149 @@ folly::coro::Task go4() { // OGCG: call void @_ZNSt14suspend_always13await_suspendESt16coroutine_handleIvE(ptr noundef nonnull align 1 dereferenceable(1) %[[AwaiterReload]]) // OGCG: ret void // OGCG: } + +folly::coro::Task co_returns(int x) { + if (x < 0) + co_return -1; + else if (x == 1) + co_return -2; + + co_await std::suspend_always(); + + co_return x * 2; +} + +// CIR: cir.func coroutine {{.*}} @_Z10co_returnsi +// CIR: cir.cleanup.scope { +// CIR: cir.await(init, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) +// CIR: cir.coro.body { +// CIR: cir.scope { +// CIR: cir.if {{.*}} { +// CIR: %[[MINUS_ONE:.*]] = cir.const #cir.int<-1> +// CIR: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE:.*]], %[[MINUS_ONE]]) +// CIR: cir.co_return +// CIR: } else { +// CIR: cir.if {{.*}} { +// CIR: %[[MINUS_TWO:.*]] = cir.const #cir.int<-2> + // CIR: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[MINUS_TWO]]) + // CIR: cir.co_return +// CIR: } +// CIR: } +// CIR: } +// CIR: cir.await(user, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) +// CIR: %[[X_LOAD:.*]] = cir.load {{.*}} %[[X:.*]] +// CIR: %[[TWO:.*]] = cir.const #cir.int<2> +// CIR: %[[RES:.*]] = cir.mul nsw %[[X_LOAD]], %[[TWO]] +// CIR: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[PROMISE]], %[[RES]]) +// CIR: cir.co_return +// CIR: } +// CIR: cir.await(final, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) +// CIR: cir.yield +// CIR: } cleanup normal { + + +// OGCG: define {{.*}} @_Z10co_returnsi +// OGCG: coro.init: +// OGCG: init.suspend: +// OGCG: init.ready: +// OGCG: if.then: +// OGCG: call void @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi({{.*}} %[[promise:.*]], {{.*}} -1) +// OGCG: br label %coro.final +// OGCG: if.else: +// OGCG: if.then5: +// OGCG: call void @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi({{.*}} %[[promise:.*]], {{.*}} -2) +// OGCG: br label %coro.final +// OGCG: await.suspend: +// OGCG: await.ready: +// OGCG: coro.final: +// OGCG: final.suspend: +// OGCG: final.ready: + +folly::coro::Task implicit_yield_terminator(int x) { + if (x < 0) + co_return -1; + else + co_return 2; +} +// CIR: cir.func coroutine {{.*}} @_Z25implicit_yield_terminatori +// CIR: cir.coro.body { +// CIR: cir.if %{{.*}} { +// CIR: cir.co_return +// CIR: } else { +// CIR: cir.co_return +// CIR: } +// CIR: cir.yield +// CIR: } + +struct HasDtor { + ~ HasDtor(); +}; + +folly::coro::Task co_return_with_dtor(int flag) { + HasDtor local; + if (flag) + co_return 1; // local dtor must run here + co_return 2; +} + +// CIR: cir.func coroutine {{.*}} @_Z19co_return_with_dtori + +// CIR: %[[LOCAL:.*]] = cir.alloca !rec_HasDtor, !cir.ptr, ["local"] +// CIR: cir.cleanup.scope { +// CIR: cir.await(init, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) + +// CIR: cir.coro.body { +// CIR: cir.cleanup.scope { +// CIR: cir.scope { +// CIR: %[[CAST_FLAG:.*]] = cir.cast int_to_bool %[[FLAG:.*]] +// CIR: cir.if %[[CAST_FLAG]] { +// CIR: %[[ONE:.*]] = cir.const #cir.int<1> +// CIR: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[promise:.*]], %[[ONE]]) +// CIR: cir.co_return +// CIR: } +// CIR: } +// CIR: %[[TWO:.*]] = cir.const #cir.int<2> +// CIR: cir.call @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi(%[[promise]], %[[TWO]]) +// CIR: cir.co_return +// CIR: } cleanup normal { +// CIR: cir.call @_ZN7HasDtorD1Ev(%[[LOCAL]]) +// CIR: cir.yield +// CIR: } +// CIR: cir.yield +// CIR: } + +// CIR: cir.await(final, ready : { +// CIR: }, suspend : { +// CIR: }, resume : { +// CIR: },) +// CIR: cir.yield +// CIR: } cleanup normal { +// CIR: cir.yield +// CIR: } + +// OGCG: define {{.*}} void @_Z19co_return_with_dtori +// OGCG: %[[LOCAL:.*]] = alloca %struct.HasDtor +// OGCG: coro.init: +// OGCG: init.suspend: +// OGCG: init.ready: +// OGCG: if.then: +// OGCG: call void @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi({{.*}} %[[promise:.*]], {{.*}} 1) +// OGCG: br label %cleanup4 +// OGCG: if.end: +// OGCG: call void @_ZN5folly4coro4TaskIiE12promise_type12return_valueEi({{.*}} %[[promise]], {{.*}} 2) +// OGCG: br label %cleanup4 +// OGCG: cleanup4: +// OGCG: call void @_ZN7HasDtorD1Ev({{.*}} %[[LOCAL]]) +// OGCG: br label %coro.final diff --git a/clang/test/CIR/IR/await.cir b/clang/test/CIR/IR/await.cir index c1fb0d6d7c57c..06b925bcc07b3 100644 --- a/clang/test/CIR/IR/await.cir +++ b/clang/test/CIR/IR/await.cir @@ -1,13 +1,16 @@ // RUN: cir-opt %s --verify-roundtrip | FileCheck %s cir.func coroutine @checkPrintParse(%arg0 : !cir.bool) { - cir.await(user, ready : { - cir.condition(%arg0) - }, suspend : { + cir.coro.body { + cir.await(user, ready : { + cir.condition(%arg0) + }, suspend : { + cir.yield + }, resume : { + cir.yield + },) cir.yield - }, resume : { - cir.yield - },) + } cir.return } diff --git a/clang/test/CIR/IR/co-return.cir b/clang/test/CIR/IR/co-return.cir new file mode 100644 index 0000000000000..613399b142650 --- /dev/null +++ b/clang/test/CIR/IR/co-return.cir @@ -0,0 +1,21 @@ +// RUN: cir-opt %s --verify-roundtrip | FileCheck %s +cir.func coroutine @coro_co_return(%arg0 : !cir.bool) { + cir.coro.body { + cir.await(user, ready : { + cir.condition(%arg0) + }, suspend : { + cir.yield + }, resume : { + cir.yield + },) + cir.co_return + } + cir.return +} + +// CHECK: cir.func coroutine @coro_co_return +// CHECK: cir.coro.body { +// CHECK: cir.co_return +// CHECK: } + + diff --git a/clang/test/CIR/IR/coro-body.cir b/clang/test/CIR/IR/coro-body.cir new file mode 100644 index 0000000000000..1c0dae384691a --- /dev/null +++ b/clang/test/CIR/IR/coro-body.cir @@ -0,0 +1,19 @@ +// RUN: cir-opt %s --verify-roundtrip | FileCheck %s + +cir.func coroutine @coro_body(%arg0 : !cir.bool) { + cir.coro.body { + cir.await(user, ready : { + cir.condition(%arg0) + }, suspend : { + cir.yield + }, resume : { + cir.yield + },) + cir.co_return + } + cir.return +} + +// CHECK: cir.func coroutine @coro_body +// CHECK: cir.coro.body { +// CHECK: } diff --git a/clang/test/CIR/IR/func.cir b/clang/test/CIR/IR/func.cir index 52589c8a5e39e..b8e497fbddb10 100644 --- a/clang/test/CIR/IR/func.cir +++ b/clang/test/CIR/IR/func.cir @@ -109,15 +109,18 @@ cir.func @ullfunc() -> !u64i { // CHECK: } cir.func coroutine @coro() { - cir.await(init, ready : { - %0 = cir.alloca !cir.bool, !cir.ptr, [""] {alignment = 1 : i64} - %1 = cir.load align(1) %0 : !cir.ptr, !cir.bool - cir.condition(%1) - }, suspend : { + cir.coro.body { + cir.await(init, ready : { + %0 = cir.alloca !cir.bool, !cir.ptr, [""] {alignment = 1 : i64} + %1 = cir.load align(1) %0 : !cir.ptr, !cir.bool + cir.condition(%1) + }, suspend : { + cir.yield + }, resume : { + cir.yield + },) cir.yield - }, resume : { - cir.yield - },) + } cir.return } // CHECK: cir.func{{.*}} coroutine @coro() diff --git a/clang/test/CIR/IR/invalid-await.cir b/clang/test/CIR/IR/invalid-await.cir index 5f422ca7e954e..fe0fd76769e44 100644 --- a/clang/test/CIR/IR/invalid-await.cir +++ b/clang/test/CIR/IR/invalid-await.cir @@ -15,5 +15,7 @@ cir.func coroutine @missing_condition() { cir.yield },) } + cir.coro.body { + } cir.return } diff --git a/clang/test/CIR/IR/invalid-co-return.cir b/clang/test/CIR/IR/invalid-co-return.cir new file mode 100644 index 0000000000000..6085bcfe3a408 --- /dev/null +++ b/clang/test/CIR/IR/invalid-co-return.cir @@ -0,0 +1,5 @@ +// RUN: cir-opt %s -verify-diagnostics -split-input-file + +cir.func @must_be_inside_coro_body() { + cir.co_return // expected-error {{must be inside a cir.coro.body}} +} diff --git a/clang/test/CIR/IR/invalid-coro-body.cir b/clang/test/CIR/IR/invalid-coro-body.cir new file mode 100644 index 0000000000000..e08c96192e5b5 --- /dev/null +++ b/clang/test/CIR/IR/invalid-coro-body.cir @@ -0,0 +1,21 @@ +// RUN: cir-opt %s -verify-diagnostics -split-input-file + +cir.func @must_be_inside_coroutine() { + cir.coro.body { // expected-error {{enclosing function must be a coroutine}} + + } +} + +cir.func coroutine @must_have_one_coro_body(%arg0 : !cir.bool) { // expected-error {{coroutine function must have exactly one cir.body op}} + cir.coro.body { + cir.await(user, ready : { + cir.condition(%arg0) + }, suspend : { + cir.yield + }, resume : { + cir.yield + },) + } + cir.coro.body { + } +}