[CIR] Implement static lambda invoker

[andykaylor]

This adds support for handling static lambda invokers.

[llvmbot]

@llvm/pr-subscribers-clangir

@llvm/pr-subscribers-clang

Author: Andy Kaylor (andykaylor)

Changes

This adds support for handling static lambda invokers.

---

Patch is 20.33 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/160137.diff

8 Files Affected:

• (modified) clang/include/clang/CIR/MissingFeatures.h (+1)
• (modified) clang/lib/CIR/CodeGen/CIRGenCall.h (+1)
• (modified) clang/lib/CIR/CodeGen/CIRGenClass.cpp (+80)
• (modified) clang/lib/CIR/CodeGen/CIRGenFunction.cpp (+4-1)
• (modified) clang/lib/CIR/CodeGen/CIRGenFunction.h (+8)
• (modified) clang/lib/CIR/CodeGen/CIRGenStmt.cpp (+21)
• (modified) clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp (+8-2)
• (added) clang/test/CIR/CodeGen/lambda-static-invoker.cpp (+199)

diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
index 9d2cf03b24c0c..0fac1b211239a 100644
--- a/clang/include/clang/CIR/MissingFeatures.h
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -214,6 +214,7 @@ struct MissingFeatures {
   static bool ehCleanupScopeRequiresEHCleanup() { return false; }
   static bool ehCleanupBranchFixups() { return false; }
   static bool ehstackBranches() { return false; }
+  static bool emitBranchThroughCleanup() { return false; }
   static bool emitCheckedInBoundsGEP() { return false; }
   static bool emitCondLikelihoodViaExpectIntrinsic() { return false; }
   static bool emitLifetimeMarkers() { return false; }
diff --git a/clang/lib/CIR/CodeGen/CIRGenCall.h b/clang/lib/CIR/CodeGen/CIRGenCall.h
index 81cbb854f3b7d..52d541f2b09b5 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCall.h
+++ b/clang/lib/CIR/CodeGen/CIRGenCall.h
@@ -256,6 +256,7 @@ class ReturnValueSlot {
   ReturnValueSlot() = default;
   ReturnValueSlot(Address addr) : addr(addr) {}
 
+  bool isNull() const { return !addr.isValid(); }
   Address getValue() const { return addr; }
 };
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenClass.cpp b/clang/lib/CIR/CodeGen/CIRGenClass.cpp
index 18e62f0213dd6..cb8fe6c8862dc 100644
--- a/clang/lib/CIR/CodeGen/CIRGenClass.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenClass.cpp
@@ -778,6 +778,86 @@ void CIRGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &args) {
                        s->getStmtClassName());
 }
 
+void CIRGenFunction::emitForwardingCallToLambda(
+    const CXXMethodDecl *callOperator, CallArgList &callArgs) {
+  // Get the address of the call operator.
+  const CIRGenFunctionInfo &calleeFnInfo =
+      cgm.getTypes().arrangeCXXMethodDeclaration(callOperator);
+  cir::FuncOp calleePtr = cgm.getAddrOfFunction(
+      GlobalDecl(callOperator), cgm.getTypes().getFunctionType(calleeFnInfo));
+
+  // Prepare the return slot.
+  const FunctionProtoType *fpt =
+      callOperator->getType()->castAs<FunctionProtoType>();
+  QualType resultType = fpt->getReturnType();
+  ReturnValueSlot returnSlot;
+
+  // We don't need to separately arrange the call arguments because
+  // the call can't be variadic anyway --- it's impossible to forward
+  // variadic arguments.
+
+  // Now emit our call.
+  CIRGenCallee callee =
+      CIRGenCallee::forDirect(calleePtr, GlobalDecl(callOperator));
+  RValue rv = emitCall(calleeFnInfo, callee, returnSlot, callArgs);
+
+  // If necessary, copy the returned value into the slot.
+  if (!resultType->isVoidType() && returnSlot.isNull()) {
+    if (getLangOpts().ObjCAutoRefCount && resultType->isObjCRetainableType())
+      cgm.errorNYI(callOperator->getSourceRange(),
+                   "emitForwardingCallToLambda: ObjCAutoRefCount");
+    emitReturnOfRValue(*currSrcLoc, rv, resultType);
+  } else {
+    cgm.errorNYI(callOperator->getSourceRange(),
+                 "emitForwardingCallToLambda: return slot is not null");
+  }
+}
+
+void CIRGenFunction::emitLambdaDelegatingInvokeBody(const CXXMethodDecl *md) {
+  const CXXRecordDecl *lambda = md->getParent();
+
+  // Start building arguments for forwarding call
+  CallArgList callArgs;
+
+  QualType lambdaType = getContext().getCanonicalTagType(lambda);
+  QualType thisType = getContext().getPointerType(lambdaType);
+  Address thisPtr =
+      createMemTemp(lambdaType, getLoc(md->getSourceRange()), "unused.capture");
+  callArgs.add(RValue::get(thisPtr.getPointer()), thisType);
+
+  // Add the rest of the parameters.
+  for (auto *param : md->parameters())
+    emitDelegateCallArg(callArgs, param, param->getBeginLoc());
+
+  const CXXMethodDecl *callOp = lambda->getLambdaCallOperator();
+  // For a generic lambda, find the corresponding call operator specialization
+  // to which the call to the static-invoker shall be forwarded.
+  if (lambda->isGenericLambda()) {
+    assert(md->isFunctionTemplateSpecialization());
+    const TemplateArgumentList *tal = md->getTemplateSpecializationArgs();
+    FunctionTemplateDecl *callOpTemplate =
+        callOp->getDescribedFunctionTemplate();
+    void *InsertPos = nullptr;
+    FunctionDecl *correspondingCallOpSpecialization =
+        callOpTemplate->findSpecialization(tal->asArray(), InsertPos);
+    assert(correspondingCallOpSpecialization);
+    callOp = cast<CXXMethodDecl>(correspondingCallOpSpecialization);
+  }
+  emitForwardingCallToLambda(callOp, callArgs);
+}
+
+void CIRGenFunction::emitLambdaStaticInvokeBody(const CXXMethodDecl *md) {
+  if (md->isVariadic()) {
+    // Codgen for LLVM doesn't emit code for this as well, it says:
+    // FIXME: Making this work correctly is nasty because it requires either
+    // cloning the body of the call operator or making the call operator
+    // forward.
+    cgm.errorNYI(md->getSourceRange(), "emitLambdaStaticInvokeBody: variadic");
+  }
+
+  emitLambdaDelegatingInvokeBody(md);
+}
+
 void CIRGenFunction::destroyCXXObject(CIRGenFunction &cgf, Address addr,
                                       QualType type) {
   const auto *record = type->castAsCXXRecordDecl();
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
index f43a0e60c9f5b..0abb21a670719 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
@@ -577,7 +577,10 @@ cir::FuncOp CIRGenFunction::generateCode(clang::GlobalDecl gd, cir::FuncOp fn,
       getCIRGenModule().errorNYI(bodyRange, "CUDA kernel");
     } else if (isa<CXXMethodDecl>(funcDecl) &&
                cast<CXXMethodDecl>(funcDecl)->isLambdaStaticInvoker()) {
-      getCIRGenModule().errorNYI(bodyRange, "Lambda static invoker");
+      // The lambda static invoker function is special, because it forwards or
+      // clones the body of the function call operator (but is actually
+      // static).
+      emitLambdaStaticInvokeBody(cast<CXXMethodDecl>(funcDecl));
     } else if (funcDecl->isDefaulted() && isa<CXXMethodDecl>(funcDecl) &&
                (cast<CXXMethodDecl>(funcDecl)->isCopyAssignmentOperator() ||
                 cast<CXXMethodDecl>(funcDecl)->isMoveAssignmentOperator())) {
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index a0c571a544322..b91bb1567f257 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -1274,6 +1274,8 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   mlir::Value emitPromotedValue(mlir::Value result, QualType promotionType);
 
+  void emitReturnOfRValue(mlir::Location loc, RValue rv, QualType ty);
+
   /// Emit the computation of the specified expression of scalar type.
   mlir::Value emitScalarExpr(const clang::Expr *e);
 
@@ -1293,6 +1295,9 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   mlir::LogicalResult emitForStmt(const clang::ForStmt &s);
 
+  void emitForwardingCallToLambda(const CXXMethodDecl *lambdaCallOperator,
+                                  CallArgList &callArgs);
+
   /// Emit the computation of the specified expression of complex type,
   /// returning the result.
   mlir::Value emitComplexExpr(const Expr *e);
@@ -1355,6 +1360,9 @@ class CIRGenFunction : public CIRGenTypeCache {
   mlir::LogicalResult emitLabel(const clang::LabelDecl &d);
   mlir::LogicalResult emitLabelStmt(const clang::LabelStmt &s);
 
+  void emitLambdaDelegatingInvokeBody(const CXXMethodDecl *md);
+  void emitLambdaStaticInvokeBody(const CXXMethodDecl *md);
+
   mlir::LogicalResult emitIfStmt(const clang::IfStmt &s);
 
   /// Emit code to compute the specified expression,
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
index f116efc202061..e842892d085d2 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
@@ -488,8 +488,11 @@ mlir::LogicalResult CIRGenFunction::emitReturnStmt(const ReturnStmt &s) {
   auto *retBlock = curLexScope->getOrCreateRetBlock(*this, loc);
   // This should emit a branch through the cleanup block if one exists.
   builder.create<cir::BrOp>(loc, retBlock);
+  assert(!cir::MissingFeatures::emitBranchThroughCleanup());
   if (ehStack.stable_begin() != currentCleanupStackDepth)
     cgm.errorNYI(s.getSourceRange(), "return with cleanup stack");
+
+  // Insert the new block to continue codegen after branch to ret block.
   builder.createBlock(builder.getBlock()->getParent());
 
   return mlir::success();
@@ -1041,3 +1044,21 @@ mlir::LogicalResult CIRGenFunction::emitSwitchStmt(const clang::SwitchStmt &s) {
 
   return res;
 }
+
+void CIRGenFunction::emitReturnOfRValue(mlir::Location loc, RValue rv,
+                                        QualType ty) {
+  if (rv.isScalar()) {
+    builder.createStore(loc, rv.getValue(), returnValue);
+  } else if (rv.isAggregate()) {
+    LValue dest = makeAddrLValue(returnValue, ty);
+    LValue src = makeAddrLValue(rv.getAggregateAddress(), ty);
+    emitAggregateCopy(dest, src, ty, getOverlapForReturnValue());
+  } else {
+    cgm.errorNYI(loc, "emitReturnOfRValue: complex return type");
+  }
+  mlir::Block *retBlock = curLexScope->getOrCreateRetBlock(*this, loc);
+  assert(!cir::MissingFeatures::emitBranchThroughCleanup());
+  builder.create<cir::BrOp>(loc, retBlock);
+  if (ehStack.stable_begin() != currentCleanupStackDepth)
+    cgm.errorNYI(loc, "return with cleanup stack");
+}
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index 1865698838134..b574095dde826 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -1941,8 +1941,14 @@ mlir::LogicalResult CIRToLLVMUnaryOpLowering::matchAndRewrite(
   // Pointer unary operations: + only.  (++ and -- of pointers are implemented
   // with cir.ptr_stride, not cir.unary.)
   if (mlir::isa<cir::PointerType>(elementType)) {
-    return op.emitError()
-           << "Unary operation on pointer types is not yet implemented";
+    switch (op.getKind()) {
+    case cir::UnaryOpKind::Plus:
+      rewriter.replaceOp(op, adaptor.getInput());
+      return mlir::success();
+    default:
+      op.emitError() << "Unknown pointer unary operation during CIR lowering";
+      return mlir::failure();
+    }
   }
 
   return op.emitError() << "Unary operation has unsupported type: "
diff --git a/clang/test/CIR/CodeGen/lambda-static-invoker.cpp b/clang/test/CIR/CodeGen/lambda-static-invoker.cpp
new file mode 100644
index 0000000000000..15d768ef21b03
--- /dev/null
+++ b/clang/test/CIR/CodeGen/lambda-static-invoker.cpp
@@ -0,0 +1,199 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefix=OGCG --input-file=%t.ll %s
+
+// We declare anonymous record types to represent lambdas. Rather than trying to
+// to match the declarations, we establish variables for these when they are used.
+
+int g3() {
+  auto* fn = +[](int const& i) -> int { return i; };
+  auto task = fn(3);
+  return task;
+}
+
+// The order of these functions is different in OGCG.
+
+// OGCG: define dso_local noundef i32 @_Z2g3v()
+// OGCG:   %[[FN_PTR:.*]] = alloca ptr
+// OGCG:   %[[REF_TMP:.*]] = alloca %[[REC_LAM_G3:.*]]
+// OGCG:   %[[TASK:.*]] = alloca i32
+// OGCG:   %[[REF_TMP1:.*]] = alloca i32
+// OGCG:   %[[CALL:.*]] = call {{.*}} ptr @"_ZZ2g3vENK3$_0cvPFiRKiEEv"(ptr {{.*}} %[[REF_TMP]])
+// OGCG:   store ptr %[[CALL]], ptr %[[FN_PTR]]
+// OGCG:   %[[FN:.*]] = load ptr, ptr %[[FN_PTR]]
+// OGCG:   store i32 3, ptr %[[REF_TMP1]]
+// OGCG:   %[[CALL2:.*]] = call {{.*}} i32 %[[FN]](ptr {{.*}} %[[REF_TMP1]])
+// OGCG:   store i32 %[[CALL2]], ptr %[[TASK]]
+// OGCG:   %[[RESULT:.*]] = load i32, ptr %[[TASK]]
+// OGCG:   ret i32 %[[RESULT]]
+
+// OGCG: define internal noundef ptr @"_ZZ2g3vENK3$_0cvPFiRKiEEv"(ptr {{.*}} %[[THIS_ARG:.*]])
+// OGCG:   %[[THIS_ADDR:.*]] = alloca ptr
+// OGCG:   store ptr %[[THIS_ARG]], ptr %[[THIS_ADDR]]
+// OGCG:   %[[THIS:.*]] = load ptr, ptr %[[THIS_ADDR]]
+// OGCG:   ret ptr @"_ZZ2g3vEN3$_08__invokeERKi"
+
+// lambda operator()
+// CIR: cir.func lambda internal private dso_local @_ZZ2g3vENK3$_0clERKi(%[[THIS_ARG:.*]]: !cir.ptr<![[REC_LAM_G3:.*]]> {{.*}}, %[[REF_I_ARG:.*]]: !cir.ptr<!s32i> {{.*}})
+// CIR:   %[[THIS_ALLOCA:.*]] = cir.alloca !cir.ptr<![[REC_LAM_G3]]>, !cir.ptr<!cir.ptr<![[REC_LAM_G3]]>>, ["this", init]
+// CIR:   %[[REF_I_ALLOCA:.*]] = cir.alloca {{.*}} ["i", init, const]
+// CIR:   %[[RETVAL:.*]] = cir.alloca {{.*}} ["__retval"]
+// CIR:   cir.store %[[THIS_ARG]], %[[THIS_ALLOCA]]
+// CIR:   cir.store %[[REF_I_ARG]], %[[REF_I_ALLOCA]]
+// CIR:   %[[THIS:.*]] = cir.load %[[THIS_ALLOCA]]
+// CIR:   %[[REF_I:.*]] = cir.load %[[REF_I_ALLOCA]]
+// CIR:   %[[I:.*]] = cir.load{{.*}} %[[REF_I]]
+// CIR:   cir.store %[[I]], %[[RETVAL]]
+// CIR:   %[[RET:.*]] = cir.load %[[RETVAL]]
+// CIR:   cir.return %[[RET]]
+
+// LLVM: define internal i32 @"_ZZ2g3vENK3$_0clERKi"(ptr %[[THIS_ARG:.*]], ptr %[[REF_I_ARG:.*]]) {
+// LLVM:   %[[THIS_ALLOCA:.*]] = alloca ptr
+// LLVM:   %[[REF_I_ALLOCA:.*]] = alloca ptr
+// LLVM:   %[[RETVAL:.*]] = alloca i32
+// LLVM:   store ptr %[[THIS_ARG]], ptr %[[THIS_ALLOCA]]
+// LLVM:   store ptr %[[REF_I_ARG]], ptr %[[REF_I_ALLOCA]]
+// LLVM:   %[[THIS:.*]] = load ptr, ptr %[[THIS_ALLOCA]]
+// LLVM:   %[[REF_I:.*]] = load ptr, ptr %[[REF_I_ALLOCA]]
+// LLVM:   %[[I:.*]] = load i32, ptr %[[REF_I]]
+// LLVM:   store i32 %[[I]], ptr %[[RETVAL]]
+// LLVM:   %[[RET:.*]] = load i32, ptr %[[RETVAL]]
+// LLVM:   ret i32 %[[RET]]
+
+// In OGCG, the _ZZ2g3vENK3$_0clERKi function is emitted after _ZZ2g3vEN3$_08__invokeERKi, see below.
+
+// lambda invoker
+// CIR: cir.func internal private dso_local @_ZZ2g3vEN3$_08__invokeERKi(%[[REF_I_ARG:.*]]: !cir.ptr<!s32i> {{.*}}) -> !s32i {
+// CIR:   %[[REF_I_ALLOCA:.*]] = cir.alloca {{.*}} ["i", init, const]
+// CIR:   %[[RETVAL:.*]] = cir.alloca {{.*}} ["__retval"]
+// CIR:   %[[LAM_ALLOCA:.*]] = cir.alloca ![[REC_LAM_G3]], !cir.ptr<![[REC_LAM_G3]]>, ["unused.capture"]
+// CIR:   cir.store %[[REF_I_ARG]], %[[REF_I_ALLOCA]]
+// CIR:   %[[REF_I:.*]] = cir.load{{.*}} %[[REF_I_ALLOCA]]
+// CIR:   %[[LAM_RESULT:.*]] = cir.call @_ZZ2g3vENK3$_0clERKi(%2, %3) : (!cir.ptr<![[REC_LAM_G3]]>, !cir.ptr<!s32i>) -> !s32i
+// CIR:   cir.store{{.*}} %[[LAM_RESULT]], %[[RETVAL]]
+// CIR:   %[[RET:.*]] = cir.load %[[RETVAL]]
+// CIR:   cir.return %[[RET]]
+
+// LLVM: define internal i32 @"_ZZ2g3vEN3$_08__invokeERKi"(ptr %[[REF_I_ARG:.*]]) {
+// LLVM:   %[[REF_I_ALLOCA:.*]] = alloca ptr
+// LLVM:   %[[RETVAL:.*]] = alloca i32
+// LLVM:   %[[LAM_ALLOCA:.*]] = alloca %[[REC_LAM_G3:.*]],
+// LLVM:   store ptr %[[REF_I_ARG]], ptr %[[REF_I_ALLOCA]]
+// LLVM:   %[[REF_I:.*]] = load ptr, ptr %[[REF_I_ALLOCA]]
+// LLVM:   %[[LAM_RESULT:.*]] = call i32 @"_ZZ2g3vENK3$_0clERKi"(ptr %[[LAM_ALLOCA]], ptr %[[REF_I]])
+// LLVM:   store i32 %[[LAM_RESULT]], ptr %[[RETVAL]]
+// LLVM:   %[[RET:.*]] = load i32, ptr %[[RETVAL]]
+// LLVM:   ret i32 %[[RET]]
+
+// In OGCG, the _ZZ2g3vEN3$_08__invokeERKi function is emitted after _ZN1A3barEv, see below.
+
+// lambda operator int (*)(int const&)()
+// CIR:   cir.func internal private dso_local @_ZZ2g3vENK3$_0cvPFiRKiEEv(%[[THIS_ARG:.*]]: !cir.ptr<![[REC_LAM_G3]]> {{.*}}) -> !cir.ptr<!cir.func<(!cir.ptr<!s32i>) -> !s32i>> {
+// CIR:   %[[THIS_ALLOCA:.*]] = cir.alloca !cir.ptr<![[REC_LAM_G3]]>, !cir.ptr<!cir.ptr<![[REC_LAM_G3]]>>, ["this", init]
+// CIR:   %[[RETVAL:.*]] = cir.alloca !cir.ptr<!cir.func<(!cir.ptr<!s32i>) -> !s32i>>, !cir.ptr<!cir.ptr<!cir.func<(!cir.ptr<!s32i>) -> !s32i>>>, ["__retval"]
+// CIR:   cir.store %[[THIS_ARG]], %[[THIS_ALLOCA]]
+// CIR:   %[[THIS:.*]] = cir.load %[[THIS_ALLOCA]]
+// CIR:   %[[INVOKER:.*]] = cir.get_global @_ZZ2g3vEN3$_08__invokeERKi : !cir.ptr<!cir.func<(!cir.ptr<!s32i>) -> !s32i>>
+// CIR:   cir.store %[[INVOKER]], %[[RETVAL]]
+// CIR:   %[[RET:.*]] = cir.load %[[RETVAL]]
+// CIR:   cir.return %[[RET]]
+
+// LLVM: define internal ptr @"_ZZ2g3vENK3$_0cvPFiRKiEEv"(ptr %[[THIS_ARG:.*]]) {
+// LLVM:  %[[THIS_ALLOCA:.*]] = alloca ptr
+// LLVM:  %[[RETVAL:.*]] = alloca ptr
+// LLVM:  store ptr %[[THIS_ARG]], ptr %[[THIS_ALLOCA]]
+// LLVM:  %[[THIS:.*]] = load ptr, ptr %[[THIS_ALLOCA]]
+// LLVM:  store ptr @"_ZZ2g3vEN3$_08__invokeERKi", ptr %[[RETVAL]]
+// LLVM:  %[[RET:.*]] = load ptr, ptr %[[RETVAL]]
+// LLVM:  ret ptr %[[RET]]
+
+// In OGCG, the _ZZ2g3vENK3$_0cvPFiRKiEEv function is emitted just after the _Z2g3v function, see above.
+
+// CIR: cir.func{{.*}} @_Z2g3v() -> !s32i {
+// CIR:   %[[RETVAL:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR:   %[[FN_ADDR:.*]] = cir.alloca !cir.ptr<!cir.func<(!cir.ptr<!s32i>) -> !s32i>>, !cir.ptr<!cir.ptr<!cir.func<(!cir.ptr<!s32i>) -> !s32i>>>, ["fn", init]
+// CIR:   %[[TASK:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["task", init]
+
+// 1. Use `operator int (*)(int const&)()` to retrieve the fnptr to `__invoke()`.
+// CIR:     %[[SCOPE_RET:.*]] = cir.scope {
+// CIR:       %[[LAM_ALLOCA:.*]] = cir.alloca ![[REC_LAM_G3]], !cir.ptr<![[REC_LAM_G3]]>, ["ref.tmp0"]
+// CIR:       %[[OPERATOR_RESULT:.*]] = cir.call @_ZZ2g3vENK3$_0cvPFiRKiEEv(%[[LAM_ALLOCA]]){{.*}}
+// CIR:       %[[PLUS:.*]] = cir.unary(plus, %[[OPERATOR_RESULT]])
+// CIR:       cir.yield %[[PLUS]]
+// CIR:     }
+
+// 2. Load ptr to `__invoke()`.
+// CIR:     cir.store{{.*}} %[[SCOPE_RET]], %[[FN_ADDR]]
+// CIR:     %[[SCOPE_RET2:.*]] = cir.scope {
+// CIR:       %[[REF_TMP1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["ref.tmp1", init]
+// CIR:       %[[FN:.*]] = cir.load{{.*}} %[[FN_ADDR]]
+// CIR:       %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
+// CIR:       cir.store{{.*}} %[[THREE]], %[[REF_TMP1]]
+
+// 3. Call `__invoke()`, which effectively executes `operator()`.
+// CIR:       %[[RESULT:.*]] = cir.call %[[FN]](%[[REF_TMP1]])
+// CIR:       cir.yield %[[RESULT]]
+// CIR:     }
+
+// CIR:     cir.store{{.*}} %[[SCOPE_RET2]], %[[TASK]]
+// CIR:     %[[TASK_RET:.*]] = cir.load{{.*}} %[[TASK]]
+// CIR:     cir.store{{.*}} %[[TASK_RET]], %[[RETVAL]]
+// CIR:     %[[RET:.*]] = cir.load{{.*}} %[[RETVAL]]
+// CIR:     cir.return %[[RET]]
+// CIR:   }
+
+// LLVM: define dso_local i32 @_Z2g3v() {
+// LLVM:   %[[LAM_ALLOCA:.*]] = alloca %[[REC_LAM_G3]]
+// LLVM:   %[[REF_TMP1:.*]] = alloca i32
+// LLVM:   %[[RETVAL:.*]] = alloca i32
+// LLVM:   %[[FN_PTR:.*]] = alloca ptr
+// LLVM:   %[[TASK:.*]] = alloca i32
+// LLVM:   br label %[[SCOPE_BB0:.*]]
+
+// LLVM: [[SCOPE_BB0]]:
+// LLVM:   %[[OPERATOR_RESULT:.*]] = call ptr @"_ZZ2g3vENK3$_0cvPFiRKiEEv"(ptr %[[LAM_ALLOCA]])
+// LLVM:   br label %[[SCOPE_BB1:.*]]
+
+// LLVM: [[SCOPE_BB1]]:
+// LLVM:   %[[TMP0:.*]] = phi ptr [ %[[OPERATOR_RESULT]], %[[SCOPE_BB0]] ]
+// LLVM:   store ptr %[[TMP0]], ptr %[[FN_PTR]]
+// LLVM:   br label %[[SCOPE_BB2:.*]]
+
+// LLVM: [[SCOPE_BB2]]:
+// LLVM:   %[[FN:.*]] = load ptr, ptr %[[FN_PTR]]
+// LLVM:   store i32 3, ptr %[[REF_TMP1]]
+// LLVM:   %[[RESULT:.*]] = call i32 %[[FN]](ptr %[[REF_TMP1]])
+// LLVM:   br label %[[RET_BB:.*]]
+
+// LLVM: [[RET_BB]]:
+// LLVM:   %[[TMP1:.*]] = phi i32 [ %[[RESULT]], %[[SCOPE_BB2]] ]
+// LLVM:   store i32 %[[TMP1]], ptr %[[TASK]]
+// LLVM:   %[[TMP2:.*]] = load i32, ptr %[[TASK]]
+// LLVM:   store i32 %[[TMP2]], ptr %[[RETVAL]]
+// LLVM:   %[[RET:.*]] = load i32, ptr %[[RETVAL]]
+// LLVM:   ret i32 %[[RET]]
+
+// The definition for _Z2g3v in OGCG is first among the functions for the g3 test, see above.
+
+// The functions below are emitted later in OGCG, see above for the corresponding LLVM checks.
+
+// OGCG: define internal noundef i32 @"_ZZ2g3vEN3$_08__invokeERKi"(ptr {{.*}} %[[I_ARG:.*]])
+// OGCG:   %[[I_ADDR:.*]] = alloca ptr
+// OGCG:   %[[UNUSED_CAPTURE:.*]] = alloca %[[REC_LAM_G3:.*]]
+// OGCG:   store ptr %[[I_ARG]], ptr %[[I_ADDR]]
+// OGCG:   %[[I_PTR:.*]] = load ptr, ptr %[[I_ADDR]]
+// OGCG:   %[[CALL:.*]] = call {{.*}} i32 @"_ZZ2g3vENK3$_0clERKi"(ptr {{.*}} %[[UNUSED_CAPTURE]], ptr {{.*}} %[[I_PTR]])
+// OGCG:   ret i32 %[[CALL]]
+
+// OGCG: define internal noundef i32 @"_ZZ2g3vENK3$_0clERKi"(ptr {{.*}} %[[THIS_ARG:.*]], ptr {{.*}} %[[I_ARG:.*]])
+// OGCG:   %[[THIS_ADDR:.*]] = al...
[truncated]

[bcardosolopes]

LGTM (lambas will be a cool one to raise at some point)