1073 lines · cpp
1//===----- CGCoroutine.cpp - Emit LLVM Code for C++ coroutines ------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This contains code dealing with C++ code generation of coroutines.10//11//===----------------------------------------------------------------------===//12 13#include "CGCleanup.h"14#include "CGDebugInfo.h"15#include "CodeGenFunction.h"16#include "clang/AST/StmtCXX.h"17#include "clang/AST/StmtVisitor.h"18#include "llvm/ADT/ScopeExit.h"19 20using namespace clang;21using namespace CodeGen;22 23using llvm::Value;24using llvm::BasicBlock;25 26namespace {27enum class AwaitKind { Init, Normal, Yield, Final };28static constexpr llvm::StringLiteral AwaitKindStr[] = {"init", "await", "yield",29 "final"};30}31 32struct clang::CodeGen::CGCoroData {33 // What is the current await expression kind and how many34 // await/yield expressions were encountered so far.35 // These are used to generate pretty labels for await expressions in LLVM IR.36 AwaitKind CurrentAwaitKind = AwaitKind::Init;37 unsigned AwaitNum = 0;38 unsigned YieldNum = 0;39 40 // How many co_return statements are in the coroutine. Used to decide whether41 // we need to add co_return; equivalent at the end of the user authored body.42 unsigned CoreturnCount = 0;43 44 // A branch to this block is emitted when coroutine needs to suspend.45 llvm::BasicBlock *SuspendBB = nullptr;46 47 // The promise type's 'unhandled_exception' handler, if it defines one.48 Stmt *ExceptionHandler = nullptr;49 50 // A temporary i1 alloca that stores whether 'await_resume' threw an51 // exception. If it did, 'true' is stored in this variable, and the coroutine52 // body must be skipped. If the promise type does not define an exception53 // handler, this is null.54 llvm::Value *ResumeEHVar = nullptr;55 56 // Stores the jump destination just before the coroutine memory is freed.57 // This is the destination that every suspend point jumps to for the cleanup58 // branch.59 CodeGenFunction::JumpDest CleanupJD;60 61 // Stores the jump destination just before the final suspend. The co_return62 // statements jumps to this point after calling return_xxx promise member.63 CodeGenFunction::JumpDest FinalJD;64 65 // Stores the llvm.coro.id emitted in the function so that we can supply it66 // as the first argument to coro.begin, coro.alloc and coro.free intrinsics.67 // Note: llvm.coro.id returns a token that cannot be directly expressed in a68 // builtin.69 llvm::CallInst *CoroId = nullptr;70 71 // Stores the llvm.coro.begin emitted in the function so that we can replace72 // all coro.frame intrinsics with direct SSA value of coro.begin that returns73 // the address of the coroutine frame of the current coroutine.74 llvm::CallInst *CoroBegin = nullptr;75 76 // Stores the last emitted coro.free for the deallocate expressions, we use it77 // to wrap dealloc code with if(auto mem = coro.free) dealloc(mem).78 llvm::CallInst *LastCoroFree = nullptr;79 80 // If coro.id came from the builtin, remember the expression to give better81 // diagnostic. If CoroIdExpr is nullptr, the coro.id was created by82 // EmitCoroutineBody.83 CallExpr const *CoroIdExpr = nullptr;84};85 86// Defining these here allows to keep CGCoroData private to this file.87clang::CodeGen::CodeGenFunction::CGCoroInfo::CGCoroInfo() {}88CodeGenFunction::CGCoroInfo::~CGCoroInfo() {}89 90static void createCoroData(CodeGenFunction &CGF,91 CodeGenFunction::CGCoroInfo &CurCoro,92 llvm::CallInst *CoroId,93 CallExpr const *CoroIdExpr = nullptr) {94 if (CurCoro.Data) {95 if (CurCoro.Data->CoroIdExpr)96 CGF.CGM.Error(CoroIdExpr->getBeginLoc(),97 "only one __builtin_coro_id can be used in a function");98 else if (CoroIdExpr)99 CGF.CGM.Error(CoroIdExpr->getBeginLoc(),100 "__builtin_coro_id shall not be used in a C++ coroutine");101 else102 llvm_unreachable("EmitCoroutineBodyStatement called twice?");103 104 return;105 }106 107 CurCoro.Data = std::make_unique<CGCoroData>();108 CurCoro.Data->CoroId = CoroId;109 CurCoro.Data->CoroIdExpr = CoroIdExpr;110}111 112// Synthesize a pretty name for a suspend point.113static SmallString<32> buildSuspendPrefixStr(CGCoroData &Coro, AwaitKind Kind) {114 unsigned No = 0;115 switch (Kind) {116 case AwaitKind::Init:117 case AwaitKind::Final:118 break;119 case AwaitKind::Normal:120 No = ++Coro.AwaitNum;121 break;122 case AwaitKind::Yield:123 No = ++Coro.YieldNum;124 break;125 }126 SmallString<32> Prefix(AwaitKindStr[static_cast<unsigned>(Kind)]);127 if (No > 1) {128 Twine(No).toVector(Prefix);129 }130 return Prefix;131}132 133// Check if function can throw based on prototype noexcept, also works for134// destructors which are implicitly noexcept but can be marked noexcept(false).135static bool FunctionCanThrow(const FunctionDecl *D) {136 const auto *Proto = D->getType()->getAs<FunctionProtoType>();137 if (!Proto) {138 // Function proto is not found, we conservatively assume throwing.139 return true;140 }141 return !isNoexceptExceptionSpec(Proto->getExceptionSpecType()) ||142 Proto->canThrow() != CT_Cannot;143}144 145static bool StmtCanThrow(const Stmt *S) {146 if (const auto *CE = dyn_cast<CallExpr>(S)) {147 const auto *Callee = CE->getDirectCallee();148 if (!Callee)149 // We don't have direct callee. Conservatively assume throwing.150 return true;151 152 if (FunctionCanThrow(Callee))153 return true;154 155 // Fall through to visit the children.156 }157 158 if (const auto *TE = dyn_cast<CXXBindTemporaryExpr>(S)) {159 // Special handling of CXXBindTemporaryExpr here as calling of Dtor of the160 // temporary is not part of `children()` as covered in the fall through.161 // We need to mark entire statement as throwing if the destructor of the162 // temporary throws.163 const auto *Dtor = TE->getTemporary()->getDestructor();164 if (FunctionCanThrow(Dtor))165 return true;166 167 // Fall through to visit the children.168 }169 170 for (const auto *child : S->children())171 if (StmtCanThrow(child))172 return true;173 174 return false;175}176 177// Emit suspend expression which roughly looks like:178//179// auto && x = CommonExpr();180// if (!x.await_ready()) {181// llvm_coro_save();182// llvm_coro_await_suspend(&x, frame, wrapper) (*) (**)183// llvm_coro_suspend(); (***)184// }185// x.await_resume();186//187// where the result of the entire expression is the result of x.await_resume()188//189// (*) llvm_coro_await_suspend_{void, bool, handle} is lowered to190// wrapper(&x, frame) when it's certain not to interfere with191// coroutine transform. await_suspend expression is192// asynchronous to the coroutine body and not all analyses193// and transformations can handle it correctly at the moment.194//195// Wrapper function encapsulates x.await_suspend(...) call and looks like:196//197// auto __await_suspend_wrapper(auto& awaiter, void* frame) {198// std::coroutine_handle<> handle(frame);199// return awaiter.await_suspend(handle);200// }201//202// (**) If x.await_suspend return type is bool, it allows to veto a suspend:203// if (x.await_suspend(...))204// llvm_coro_suspend();205//206// (***) llvm_coro_suspend() encodes three possible continuations as207// a switch instruction:208//209// %where-to = call i8 @llvm.coro.suspend(...)210// switch i8 %where-to, label %coro.ret [ ; jump to epilogue to suspend211// i8 0, label %yield.ready ; go here when resumed212// i8 1, label %yield.cleanup ; go here when destroyed213// ]214//215// See llvm's docs/Coroutines.rst for more details.216//217namespace {218 struct LValueOrRValue {219 LValue LV;220 RValue RV;221 };222}223static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Coro,224 CoroutineSuspendExpr const &S,225 AwaitKind Kind, AggValueSlot aggSlot,226 bool ignoreResult, bool forLValue) {227 auto *E = S.getCommonExpr();228 229 auto CommonBinder =230 CodeGenFunction::OpaqueValueMappingData::bind(CGF, S.getOpaqueValue(), E);231 auto UnbindCommonOnExit =232 llvm::make_scope_exit([&] { CommonBinder.unbind(CGF); });233 234 auto Prefix = buildSuspendPrefixStr(Coro, Kind);235 BasicBlock *ReadyBlock = CGF.createBasicBlock(Prefix + Twine(".ready"));236 BasicBlock *SuspendBlock = CGF.createBasicBlock(Prefix + Twine(".suspend"));237 BasicBlock *CleanupBlock = CGF.createBasicBlock(Prefix + Twine(".cleanup"));238 239 // If expression is ready, no need to suspend.240 CGF.EmitBranchOnBoolExpr(S.getReadyExpr(), ReadyBlock, SuspendBlock, 0);241 242 // Otherwise, emit suspend logic.243 CGF.EmitBlock(SuspendBlock);244 245 auto &Builder = CGF.Builder;246 llvm::Function *CoroSave = CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_save);247 auto *NullPtr = llvm::ConstantPointerNull::get(CGF.CGM.Int8PtrTy);248 auto *SaveCall = Builder.CreateCall(CoroSave, {NullPtr});249 250 auto SuspendWrapper = CodeGenFunction(CGF.CGM).generateAwaitSuspendWrapper(251 CGF.CurFn->getName(), Prefix, S);252 253 CGF.CurCoro.InSuspendBlock = true;254 255 assert(CGF.CurCoro.Data && CGF.CurCoro.Data->CoroBegin &&256 "expected to be called in coroutine context");257 258 SmallVector<llvm::Value *, 3> SuspendIntrinsicCallArgs;259 SuspendIntrinsicCallArgs.push_back(260 CGF.getOrCreateOpaqueLValueMapping(S.getOpaqueValue()).getPointer(CGF));261 262 SuspendIntrinsicCallArgs.push_back(CGF.CurCoro.Data->CoroBegin);263 SuspendIntrinsicCallArgs.push_back(SuspendWrapper);264 265 const auto SuspendReturnType = S.getSuspendReturnType();266 llvm::Intrinsic::ID AwaitSuspendIID;267 268 switch (SuspendReturnType) {269 case CoroutineSuspendExpr::SuspendReturnType::SuspendVoid:270 AwaitSuspendIID = llvm::Intrinsic::coro_await_suspend_void;271 break;272 case CoroutineSuspendExpr::SuspendReturnType::SuspendBool:273 AwaitSuspendIID = llvm::Intrinsic::coro_await_suspend_bool;274 break;275 case CoroutineSuspendExpr::SuspendReturnType::SuspendHandle:276 AwaitSuspendIID = llvm::Intrinsic::coro_await_suspend_handle;277 break;278 }279 280 llvm::Function *AwaitSuspendIntrinsic = CGF.CGM.getIntrinsic(AwaitSuspendIID);281 282 // SuspendHandle might throw since it also resumes the returned handle.283 const bool AwaitSuspendCanThrow =284 SuspendReturnType ==285 CoroutineSuspendExpr::SuspendReturnType::SuspendHandle ||286 StmtCanThrow(S.getSuspendExpr());287 288 llvm::CallBase *SuspendRet = nullptr;289 // FIXME: add call attributes?290 if (AwaitSuspendCanThrow)291 SuspendRet =292 CGF.EmitCallOrInvoke(AwaitSuspendIntrinsic, SuspendIntrinsicCallArgs);293 else294 SuspendRet = CGF.EmitNounwindRuntimeCall(AwaitSuspendIntrinsic,295 SuspendIntrinsicCallArgs);296 297 assert(SuspendRet);298 CGF.CurCoro.InSuspendBlock = false;299 300 switch (SuspendReturnType) {301 case CoroutineSuspendExpr::SuspendReturnType::SuspendVoid:302 assert(SuspendRet->getType()->isVoidTy());303 break;304 case CoroutineSuspendExpr::SuspendReturnType::SuspendBool: {305 assert(SuspendRet->getType()->isIntegerTy());306 307 // Veto suspension if requested by bool returning await_suspend.308 BasicBlock *RealSuspendBlock =309 CGF.createBasicBlock(Prefix + Twine(".suspend.bool"));310 CGF.Builder.CreateCondBr(SuspendRet, RealSuspendBlock, ReadyBlock);311 CGF.EmitBlock(RealSuspendBlock);312 break;313 }314 case CoroutineSuspendExpr::SuspendReturnType::SuspendHandle: {315 assert(SuspendRet->getType()->isVoidTy());316 break;317 }318 }319 320 // Emit the suspend point.321 const bool IsFinalSuspend = (Kind == AwaitKind::Final);322 llvm::Function *CoroSuspend =323 CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_suspend);324 auto *SuspendResult = Builder.CreateCall(325 CoroSuspend, {SaveCall, Builder.getInt1(IsFinalSuspend)});326 327 // Create a switch capturing three possible continuations.328 auto *Switch = Builder.CreateSwitch(SuspendResult, Coro.SuspendBB, 2);329 Switch->addCase(Builder.getInt8(0), ReadyBlock);330 Switch->addCase(Builder.getInt8(1), CleanupBlock);331 332 // Emit cleanup for this suspend point.333 CGF.EmitBlock(CleanupBlock);334 CGF.EmitBranchThroughCleanup(Coro.CleanupJD);335 336 // Emit await_resume expression.337 CGF.EmitBlock(ReadyBlock);338 339 // Exception handling requires additional IR. If the 'await_resume' function340 // is marked as 'noexcept', we avoid generating this additional IR.341 CXXTryStmt *TryStmt = nullptr;342 if (Coro.ExceptionHandler && Kind == AwaitKind::Init &&343 StmtCanThrow(S.getResumeExpr())) {344 Coro.ResumeEHVar =345 CGF.CreateTempAlloca(Builder.getInt1Ty(), Prefix + Twine("resume.eh"));346 Builder.CreateFlagStore(true, Coro.ResumeEHVar);347 348 auto Loc = S.getResumeExpr()->getExprLoc();349 auto *Catch = new (CGF.getContext())350 CXXCatchStmt(Loc, /*exDecl=*/nullptr, Coro.ExceptionHandler);351 auto *TryBody = CompoundStmt::Create(CGF.getContext(), S.getResumeExpr(),352 FPOptionsOverride(), Loc, Loc);353 TryStmt = CXXTryStmt::Create(CGF.getContext(), Loc, TryBody, Catch);354 CGF.EnterCXXTryStmt(*TryStmt);355 CGF.EmitStmt(TryBody);356 // We don't use EmitCXXTryStmt here. We need to store to ResumeEHVar that357 // doesn't exist in the body.358 Builder.CreateFlagStore(false, Coro.ResumeEHVar);359 CGF.ExitCXXTryStmt(*TryStmt);360 LValueOrRValue Res;361 // We are not supposed to obtain the value from init suspend await_resume().362 Res.RV = RValue::getIgnored();363 return Res;364 }365 366 LValueOrRValue Res;367 if (forLValue)368 Res.LV = CGF.EmitLValue(S.getResumeExpr());369 else370 Res.RV = CGF.EmitAnyExpr(S.getResumeExpr(), aggSlot, ignoreResult);371 372 return Res;373}374 375RValue CodeGenFunction::EmitCoawaitExpr(const CoawaitExpr &E,376 AggValueSlot aggSlot,377 bool ignoreResult) {378 return emitSuspendExpression(*this, *CurCoro.Data, E,379 CurCoro.Data->CurrentAwaitKind, aggSlot,380 ignoreResult, /*forLValue*/false).RV;381}382RValue CodeGenFunction::EmitCoyieldExpr(const CoyieldExpr &E,383 AggValueSlot aggSlot,384 bool ignoreResult) {385 return emitSuspendExpression(*this, *CurCoro.Data, E, AwaitKind::Yield,386 aggSlot, ignoreResult, /*forLValue*/false).RV;387}388 389void CodeGenFunction::EmitCoreturnStmt(CoreturnStmt const &S) {390 ++CurCoro.Data->CoreturnCount;391 const Expr *RV = S.getOperand();392 if (RV && RV->getType()->isVoidType() && !isa<InitListExpr>(RV)) {393 // Make sure to evaluate the non initlist expression of a co_return394 // with a void expression for side effects.395 RunCleanupsScope cleanupScope(*this);396 EmitIgnoredExpr(RV);397 }398 EmitStmt(S.getPromiseCall());399 EmitBranchThroughCleanup(CurCoro.Data->FinalJD);400}401 402 403#ifndef NDEBUG404static QualType getCoroutineSuspendExprReturnType(const ASTContext &Ctx,405 const CoroutineSuspendExpr *E) {406 const auto *RE = E->getResumeExpr();407 // Is it possible for RE to be a CXXBindTemporaryExpr wrapping408 // a MemberCallExpr?409 assert(isa<CallExpr>(RE) && "unexpected suspend expression type");410 return cast<CallExpr>(RE)->getCallReturnType(Ctx);411}412#endif413 414llvm::Function *415CodeGenFunction::generateAwaitSuspendWrapper(Twine const &CoroName,416 Twine const &SuspendPointName,417 CoroutineSuspendExpr const &S) {418 std::string FuncName =419 (CoroName + ".__await_suspend_wrapper__" + SuspendPointName).str();420 421 ASTContext &C = getContext();422 423 FunctionArgList args;424 425 ImplicitParamDecl AwaiterDecl(C, C.VoidPtrTy, ImplicitParamKind::Other);426 ImplicitParamDecl FrameDecl(C, C.VoidPtrTy, ImplicitParamKind::Other);427 QualType ReturnTy = S.getSuspendExpr()->getType();428 429 args.push_back(&AwaiterDecl);430 args.push_back(&FrameDecl);431 432 const CGFunctionInfo &FI =433 CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, args);434 435 llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI);436 437 llvm::Function *Fn = llvm::Function::Create(438 LTy, llvm::GlobalValue::InternalLinkage, FuncName, &CGM.getModule());439 440 Fn->addParamAttr(0, llvm::Attribute::AttrKind::NonNull);441 Fn->addParamAttr(0, llvm::Attribute::AttrKind::NoUndef);442 443 Fn->addParamAttr(1, llvm::Attribute::AttrKind::NoUndef);444 445 Fn->setMustProgress();446 Fn->addFnAttr(llvm::Attribute::AttrKind::AlwaysInline);447 448 StartFunction(GlobalDecl(), ReturnTy, Fn, FI, args);449 450 // FIXME: add TBAA metadata to the loads451 llvm::Value *AwaiterPtr = Builder.CreateLoad(GetAddrOfLocalVar(&AwaiterDecl));452 auto AwaiterLValue =453 MakeNaturalAlignAddrLValue(AwaiterPtr, AwaiterDecl.getType());454 455 CurAwaitSuspendWrapper.FramePtr =456 Builder.CreateLoad(GetAddrOfLocalVar(&FrameDecl));457 458 auto AwaiterBinder = CodeGenFunction::OpaqueValueMappingData::bind(459 *this, S.getOpaqueValue(), AwaiterLValue);460 461 auto *SuspendRet = EmitScalarExpr(S.getSuspendExpr());462 463 auto UnbindCommonOnExit =464 llvm::make_scope_exit([&] { AwaiterBinder.unbind(*this); });465 if (SuspendRet != nullptr) {466 Fn->addRetAttr(llvm::Attribute::AttrKind::NoUndef);467 Builder.CreateStore(SuspendRet, ReturnValue);468 }469 470 CurAwaitSuspendWrapper.FramePtr = nullptr;471 FinishFunction();472 return Fn;473}474 475LValue476CodeGenFunction::EmitCoawaitLValue(const CoawaitExpr *E) {477 assert(getCoroutineSuspendExprReturnType(getContext(), E)->isReferenceType() &&478 "Can't have a scalar return unless the return type is a "479 "reference type!");480 return emitSuspendExpression(*this, *CurCoro.Data, *E,481 CurCoro.Data->CurrentAwaitKind, AggValueSlot::ignored(),482 /*ignoreResult*/false, /*forLValue*/true).LV;483}484 485LValue486CodeGenFunction::EmitCoyieldLValue(const CoyieldExpr *E) {487 assert(getCoroutineSuspendExprReturnType(getContext(), E)->isReferenceType() &&488 "Can't have a scalar return unless the return type is a "489 "reference type!");490 return emitSuspendExpression(*this, *CurCoro.Data, *E,491 AwaitKind::Yield, AggValueSlot::ignored(),492 /*ignoreResult*/false, /*forLValue*/true).LV;493}494 495// Hunts for the parameter reference in the parameter copy/move declaration.496namespace {497struct GetParamRef : public StmtVisitor<GetParamRef> {498public:499 DeclRefExpr *Expr = nullptr;500 GetParamRef() {}501 void VisitDeclRefExpr(DeclRefExpr *E) {502 assert(Expr == nullptr && "multilple declref in param move");503 Expr = E;504 }505 void VisitStmt(Stmt *S) {506 for (auto *C : S->children()) {507 if (C)508 Visit(C);509 }510 }511};512}513 514// This class replaces references to parameters to their copies by changing515// the addresses in CGF.LocalDeclMap and restoring back the original values in516// its destructor.517 518namespace {519 struct ParamReferenceReplacerRAII {520 CodeGenFunction::DeclMapTy SavedLocals;521 CodeGenFunction::DeclMapTy& LocalDeclMap;522 523 ParamReferenceReplacerRAII(CodeGenFunction::DeclMapTy &LocalDeclMap)524 : LocalDeclMap(LocalDeclMap) {}525 526 void addCopy(DeclStmt const *PM) {527 // Figure out what param it refers to.528 529 assert(PM->isSingleDecl());530 VarDecl const*VD = static_cast<VarDecl const*>(PM->getSingleDecl());531 Expr const *InitExpr = VD->getInit();532 GetParamRef Visitor;533 Visitor.Visit(const_cast<Expr*>(InitExpr));534 assert(Visitor.Expr);535 DeclRefExpr *DREOrig = Visitor.Expr;536 auto *PD = DREOrig->getDecl();537 538 auto it = LocalDeclMap.find(PD);539 assert(it != LocalDeclMap.end() && "parameter is not found");540 SavedLocals.insert({ PD, it->second });541 542 auto copyIt = LocalDeclMap.find(VD);543 assert(copyIt != LocalDeclMap.end() && "parameter copy is not found");544 it->second = copyIt->getSecond();545 }546 547 ~ParamReferenceReplacerRAII() {548 for (auto&& SavedLocal : SavedLocals) {549 LocalDeclMap.insert({SavedLocal.first, SavedLocal.second});550 }551 }552 };553}554 555// For WinEH exception representation backend needs to know what funclet coro.end556// belongs to. That information is passed in a funclet bundle.557static SmallVector<llvm::OperandBundleDef, 1>558getBundlesForCoroEnd(CodeGenFunction &CGF) {559 SmallVector<llvm::OperandBundleDef, 1> BundleList;560 561 if (llvm::Instruction *EHPad = CGF.CurrentFuncletPad)562 BundleList.emplace_back("funclet", EHPad);563 564 return BundleList;565}566 567namespace {568// We will insert coro.end to cut any of the destructors for objects that569// do not need to be destroyed once the coroutine is resumed.570// See llvm/docs/Coroutines.rst for more details about coro.end.571struct CallCoroEnd final : public EHScopeStack::Cleanup {572 void Emit(CodeGenFunction &CGF, Flags flags) override {573 auto &CGM = CGF.CGM;574 auto *NullPtr = llvm::ConstantPointerNull::get(CGF.Int8PtrTy);575 llvm::Function *CoroEndFn = CGM.getIntrinsic(llvm::Intrinsic::coro_end);576 // See if we have a funclet bundle to associate coro.end with. (WinEH)577 auto Bundles = getBundlesForCoroEnd(CGF);578 CGF.Builder.CreateCall(579 CoroEndFn,580 {NullPtr, CGF.Builder.getTrue(),581 llvm::ConstantTokenNone::get(CoroEndFn->getContext())},582 Bundles);583 if (Bundles.empty()) {584 // Otherwise, (landingpad model), create a conditional branch that leads585 // either to a cleanup block or a block with EH resume instruction.586 auto *ResumeBB = CGF.getEHResumeBlock(/*isCleanup=*/true);587 auto *CleanupContBB = CGF.createBasicBlock("cleanup.cont");588 auto *CoroIsInRampFn = CGM.getIntrinsic(llvm::Intrinsic::coro_is_in_ramp);589 auto *CoroIsInRamp = CGF.Builder.CreateCall(CoroIsInRampFn);590 CGF.Builder.CreateCondBr(CoroIsInRamp, CleanupContBB, ResumeBB);591 CGF.EmitBlock(CleanupContBB);592 }593 }594};595}596 597namespace {598// Make sure to call coro.delete on scope exit.599struct CallCoroDelete final : public EHScopeStack::Cleanup {600 Stmt *Deallocate;601 602 // Emit "if (coro.free(CoroId, CoroBegin)) Deallocate;"603 604 // Note: That deallocation will be emitted twice: once for a normal exit and605 // once for exceptional exit. This usage is safe because Deallocate does not606 // contain any declarations. The SubStmtBuilder::makeNewAndDeleteExpr()607 // builds a single call to a deallocation function which is safe to emit608 // multiple times.609 void Emit(CodeGenFunction &CGF, Flags) override {610 // Remember the current point, as we are going to emit deallocation code611 // first to get to coro.free instruction that is an argument to a delete612 // call.613 BasicBlock *SaveInsertBlock = CGF.Builder.GetInsertBlock();614 615 auto *FreeBB = CGF.createBasicBlock("coro.free");616 CGF.EmitBlock(FreeBB);617 CGF.EmitStmt(Deallocate);618 619 auto *AfterFreeBB = CGF.createBasicBlock("after.coro.free");620 CGF.EmitBlock(AfterFreeBB);621 622 // We should have captured coro.free from the emission of deallocate.623 auto *CoroFree = CGF.CurCoro.Data->LastCoroFree;624 if (!CoroFree) {625 CGF.CGM.Error(Deallocate->getBeginLoc(),626 "Deallocation expressoin does not refer to coro.free");627 return;628 }629 630 // Get back to the block we were originally and move coro.free there.631 auto *InsertPt = SaveInsertBlock->getTerminator();632 CoroFree->moveBefore(InsertPt->getIterator());633 CGF.Builder.SetInsertPoint(InsertPt);634 635 // Add if (auto *mem = coro.free) Deallocate;636 auto *NullPtr = llvm::ConstantPointerNull::get(CGF.Int8PtrTy);637 auto *Cond = CGF.Builder.CreateICmpNE(CoroFree, NullPtr);638 CGF.Builder.CreateCondBr(Cond, FreeBB, AfterFreeBB);639 640 // No longer need old terminator.641 InsertPt->eraseFromParent();642 CGF.Builder.SetInsertPoint(AfterFreeBB);643 }644 explicit CallCoroDelete(Stmt *DeallocStmt) : Deallocate(DeallocStmt) {}645};646}647 648namespace {649struct GetReturnObjectManager {650 CodeGenFunction &CGF;651 CGBuilderTy &Builder;652 const CoroutineBodyStmt &S;653 // When true, performs RVO for the return object.654 bool DirectEmit = false;655 656 Address GroActiveFlag;657 CodeGenFunction::AutoVarEmission GroEmission;658 659 GetReturnObjectManager(CodeGenFunction &CGF, const CoroutineBodyStmt &S)660 : CGF(CGF), Builder(CGF.Builder), S(S), GroActiveFlag(Address::invalid()),661 GroEmission(CodeGenFunction::AutoVarEmission::invalid()) {662 // The call to get_return_object is sequenced before the call to663 // initial_suspend and is invoked at most once, but there are caveats664 // regarding on whether the prvalue result object may be initialized665 // directly/eager or delayed, depending on the types involved.666 //667 // More info at https://github.com/cplusplus/papers/issues/1414668 //669 // The general cases:670 // 1. Same type of get_return_object and coroutine return type (direct671 // emission):672 // - Constructed in the return slot.673 // 2. Different types (delayed emission):674 // - Constructed temporary object prior to initial suspend initialized with675 // a call to get_return_object()676 // - When coroutine needs to to return to the caller and needs to construct677 // return value for the coroutine it is initialized with expiring value of678 // the temporary obtained above.679 //680 // Direct emission for void returning coroutines or GROs.681 DirectEmit = [&]() {682 auto *RVI = S.getReturnValueInit();683 assert(RVI && "expected RVI");684 auto GroType = RVI->getType();685 return CGF.getContext().hasSameType(GroType, CGF.FnRetTy);686 }();687 }688 689 // The gro variable has to outlive coroutine frame and coroutine promise, but,690 // it can only be initialized after coroutine promise was created, thus, we691 // split its emission in two parts. EmitGroAlloca emits an alloca and sets up692 // cleanups. Later when coroutine promise is available we initialize the gro693 // and sets the flag that the cleanup is now active.694 void EmitGroAlloca() {695 if (DirectEmit)696 return;697 698 auto *GroDeclStmt = dyn_cast_or_null<DeclStmt>(S.getResultDecl());699 if (!GroDeclStmt) {700 // If get_return_object returns void, no need to do an alloca.701 return;702 }703 704 auto *GroVarDecl = cast<VarDecl>(GroDeclStmt->getSingleDecl());705 706 // Set GRO flag that it is not initialized yet707 GroActiveFlag = CGF.CreateTempAlloca(Builder.getInt1Ty(), CharUnits::One(),708 "gro.active");709 Builder.CreateStore(Builder.getFalse(), GroActiveFlag);710 711 GroEmission = CGF.EmitAutoVarAlloca(*GroVarDecl);712 713 if (!GroVarDecl->isNRVOVariable()) {714 // NRVO variables don't have allocas and won't have the same issue.715 auto *GroAlloca = dyn_cast_or_null<llvm::AllocaInst>(716 GroEmission.getOriginalAllocatedAddress().getPointer());717 assert(GroAlloca && "expected alloca to be emitted");718 GroAlloca->setMetadata(llvm::LLVMContext::MD_coro_outside_frame,719 llvm::MDNode::get(CGF.CGM.getLLVMContext(), {}));720 }721 722 // Remember the top of EHStack before emitting the cleanup.723 auto old_top = CGF.EHStack.stable_begin();724 CGF.EmitAutoVarCleanups(GroEmission);725 auto top = CGF.EHStack.stable_begin();726 727 // Make the cleanup conditional on gro.active728 for (auto b = CGF.EHStack.find(top), e = CGF.EHStack.find(old_top); b != e;729 b++) {730 if (auto *Cleanup = dyn_cast<EHCleanupScope>(&*b)) {731 assert(!Cleanup->hasActiveFlag() && "cleanup already has active flag?");732 Cleanup->setActiveFlag(GroActiveFlag);733 Cleanup->setTestFlagInEHCleanup();734 Cleanup->setTestFlagInNormalCleanup();735 }736 }737 }738 739 void EmitGroInit() {740 if (DirectEmit) {741 // ReturnValue should be valid as long as the coroutine's return type742 // is not void. The assertion could help us to reduce the check later.743 assert(CGF.ReturnValue.isValid() == (bool)S.getReturnStmt());744 // Now we have the promise, initialize the GRO.745 // We need to emit `get_return_object` first. According to:746 // [dcl.fct.def.coroutine]p7747 // The call to get_return_object is sequenced before the call to748 // initial_suspend and is invoked at most once.749 //750 // So we couldn't emit return value when we emit return statment,751 // otherwise the call to get_return_object wouldn't be in front752 // of initial_suspend.753 if (CGF.ReturnValue.isValid()) {754 CGF.EmitAnyExprToMem(S.getReturnValue(), CGF.ReturnValue,755 S.getReturnValue()->getType().getQualifiers(),756 /*IsInit*/ true);757 }758 return;759 }760 761 if (!GroActiveFlag.isValid()) {762 // No Gro variable was allocated. Simply emit the call to763 // get_return_object.764 CGF.EmitStmt(S.getResultDecl());765 return;766 }767 768 CGF.EmitAutoVarInit(GroEmission);769 Builder.CreateStore(Builder.getTrue(), GroActiveFlag);770 }771};772} // namespace773 774static void emitBodyAndFallthrough(CodeGenFunction &CGF,775 const CoroutineBodyStmt &S, Stmt *Body) {776 CGF.EmitStmt(Body);777 const bool CanFallthrough = CGF.Builder.GetInsertBlock();778 if (CanFallthrough)779 if (Stmt *OnFallthrough = S.getFallthroughHandler())780 CGF.EmitStmt(OnFallthrough);781}782 783void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {784 auto *NullPtr = llvm::ConstantPointerNull::get(Builder.getPtrTy());785 auto &TI = CGM.getContext().getTargetInfo();786 unsigned NewAlign = TI.getNewAlign() / TI.getCharWidth();787 788 auto *EntryBB = Builder.GetInsertBlock();789 auto *AllocBB = createBasicBlock("coro.alloc");790 auto *InitBB = createBasicBlock("coro.init");791 auto *FinalBB = createBasicBlock("coro.final");792 auto *RetBB = createBasicBlock("coro.ret");793 794 auto *CoroId = Builder.CreateCall(795 CGM.getIntrinsic(llvm::Intrinsic::coro_id),796 {Builder.getInt32(NewAlign), NullPtr, NullPtr, NullPtr});797 createCoroData(*this, CurCoro, CoroId);798 CurCoro.Data->SuspendBB = RetBB;799 assert(ShouldEmitLifetimeMarkers &&800 "Must emit lifetime intrinsics for coroutines");801 802 // Backend is allowed to elide memory allocations, to help it, emit803 // auto mem = coro.alloc() ? 0 : ... allocation code ...;804 auto *CoroAlloc = Builder.CreateCall(805 CGM.getIntrinsic(llvm::Intrinsic::coro_alloc), {CoroId});806 807 Builder.CreateCondBr(CoroAlloc, AllocBB, InitBB);808 809 EmitBlock(AllocBB);810 auto *AllocateCall = EmitScalarExpr(S.getAllocate());811 auto *AllocOrInvokeContBB = Builder.GetInsertBlock();812 813 // Handle allocation failure if 'ReturnStmtOnAllocFailure' was provided.814 if (auto *RetOnAllocFailure = S.getReturnStmtOnAllocFailure()) {815 auto *RetOnFailureBB = createBasicBlock("coro.ret.on.failure");816 817 // See if allocation was successful.818 auto *NullPtr = llvm::ConstantPointerNull::get(Int8PtrTy);819 auto *Cond = Builder.CreateICmpNE(AllocateCall, NullPtr);820 // Expect the allocation to be successful.821 emitCondLikelihoodViaExpectIntrinsic(Cond, Stmt::LH_Likely);822 Builder.CreateCondBr(Cond, InitBB, RetOnFailureBB);823 824 // If not, return OnAllocFailure object.825 EmitBlock(RetOnFailureBB);826 EmitStmt(RetOnAllocFailure);827 }828 else {829 Builder.CreateBr(InitBB);830 }831 832 EmitBlock(InitBB);833 834 // Pass the result of the allocation to coro.begin.835 auto *Phi = Builder.CreatePHI(VoidPtrTy, 2);836 Phi->addIncoming(NullPtr, EntryBB);837 Phi->addIncoming(AllocateCall, AllocOrInvokeContBB);838 auto *CoroBegin = Builder.CreateCall(839 CGM.getIntrinsic(llvm::Intrinsic::coro_begin), {CoroId, Phi});840 CurCoro.Data->CoroBegin = CoroBegin;841 842 GetReturnObjectManager GroManager(*this, S);843 GroManager.EmitGroAlloca();844 845 CurCoro.Data->CleanupJD = getJumpDestInCurrentScope(RetBB);846 {847 CGDebugInfo *DI = getDebugInfo();848 ParamReferenceReplacerRAII ParamReplacer(LocalDeclMap);849 CodeGenFunction::RunCleanupsScope ResumeScope(*this);850 EHStack.pushCleanup<CallCoroDelete>(NormalAndEHCleanup, S.getDeallocate());851 852 // Create mapping between parameters and copy-params for coroutine function.853 llvm::ArrayRef<const Stmt *> ParamMoves = S.getParamMoves();854 assert(855 (ParamMoves.size() == 0 || (ParamMoves.size() == FnArgs.size())) &&856 "ParamMoves and FnArgs should be the same size for coroutine function");857 if (ParamMoves.size() == FnArgs.size() && DI)858 for (const auto Pair : llvm::zip(FnArgs, ParamMoves))859 DI->getCoroutineParameterMappings().insert(860 {std::get<0>(Pair), std::get<1>(Pair)});861 862 // Create parameter copies. We do it before creating a promise, since an863 // evolution of coroutine TS may allow promise constructor to observe864 // parameter copies.865 for (const ParmVarDecl *Parm : FnArgs) {866 // If the original param is in an alloca, exclude it from the coroutine867 // frame. The parameter copy will be part of the frame, but the original868 // parameter memory should remain on the stack. This is necessary to869 // ensure that parameters destroyed in callees, as with `trivial_abi` or870 // in the MSVC C++ ABI, are appropriately destroyed after setting up the871 // coroutine.872 Address ParmAddr = GetAddrOfLocalVar(Parm);873 if (auto *ParmAlloca =874 dyn_cast<llvm::AllocaInst>(ParmAddr.getBasePointer())) {875 ParmAlloca->setMetadata(llvm::LLVMContext::MD_coro_outside_frame,876 llvm::MDNode::get(CGM.getLLVMContext(), {}));877 }878 }879 for (auto *PM : S.getParamMoves()) {880 EmitStmt(PM);881 ParamReplacer.addCopy(cast<DeclStmt>(PM));882 // TODO: if(CoroParam(...)) need to surround ctor and dtor883 // for the copy, so that llvm can elide it if the copy is884 // not needed.885 }886 887 EmitStmt(S.getPromiseDeclStmt());888 889 Address PromiseAddr = GetAddrOfLocalVar(S.getPromiseDecl());890 auto *PromiseAddrVoidPtr =891 new llvm::BitCastInst(PromiseAddr.emitRawPointer(*this), VoidPtrTy, "",892 CoroId->getIterator());893 // Update CoroId to refer to the promise. We could not do it earlier because894 // promise local variable was not emitted yet.895 CoroId->setArgOperand(1, PromiseAddrVoidPtr);896 897 // Now we have the promise, initialize the GRO898 GroManager.EmitGroInit();899 900 EHStack.pushCleanup<CallCoroEnd>(EHCleanup);901 902 CurCoro.Data->CurrentAwaitKind = AwaitKind::Init;903 CurCoro.Data->ExceptionHandler = S.getExceptionHandler();904 EmitStmt(S.getInitSuspendStmt());905 CurCoro.Data->FinalJD = getJumpDestInCurrentScope(FinalBB);906 907 CurCoro.Data->CurrentAwaitKind = AwaitKind::Normal;908 909 if (CurCoro.Data->ExceptionHandler) {910 // If we generated IR to record whether an exception was thrown from911 // 'await_resume', then use that IR to determine whether the coroutine912 // body should be skipped.913 // If we didn't generate the IR (perhaps because 'await_resume' was marked914 // as 'noexcept'), then we skip this check.915 BasicBlock *ContBB = nullptr;916 if (CurCoro.Data->ResumeEHVar) {917 BasicBlock *BodyBB = createBasicBlock("coro.resumed.body");918 ContBB = createBasicBlock("coro.resumed.cont");919 Value *SkipBody = Builder.CreateFlagLoad(CurCoro.Data->ResumeEHVar,920 "coro.resumed.eh");921 Builder.CreateCondBr(SkipBody, ContBB, BodyBB);922 EmitBlock(BodyBB);923 }924 925 auto Loc = S.getBeginLoc();926 CXXCatchStmt Catch(Loc, /*exDecl=*/nullptr,927 CurCoro.Data->ExceptionHandler);928 auto *TryStmt =929 CXXTryStmt::Create(getContext(), Loc, S.getBody(), &Catch);930 931 EnterCXXTryStmt(*TryStmt);932 emitBodyAndFallthrough(*this, S, TryStmt->getTryBlock());933 ExitCXXTryStmt(*TryStmt);934 935 if (ContBB)936 EmitBlock(ContBB);937 }938 else {939 emitBodyAndFallthrough(*this, S, S.getBody());940 }941 942 // See if we need to generate final suspend.943 const bool CanFallthrough = Builder.GetInsertBlock();944 const bool HasCoreturns = CurCoro.Data->CoreturnCount > 0;945 if (CanFallthrough || HasCoreturns) {946 EmitBlock(FinalBB);947 CurCoro.Data->CurrentAwaitKind = AwaitKind::Final;948 EmitStmt(S.getFinalSuspendStmt());949 } else {950 // We don't need FinalBB. Emit it to make sure the block is deleted.951 EmitBlock(FinalBB, /*IsFinished=*/true);952 }953 }954 955 EmitBlock(RetBB);956 // Emit coro.end before getReturnStmt (and parameter destructors), since957 // resume and destroy parts of the coroutine should not include them.958 llvm::Function *CoroEnd = CGM.getIntrinsic(llvm::Intrinsic::coro_end);959 Builder.CreateCall(CoroEnd,960 {NullPtr, Builder.getFalse(),961 llvm::ConstantTokenNone::get(CoroEnd->getContext())});962 963 if (Stmt *Ret = S.getReturnStmt()) {964 // Since we already emitted the return value above, so we shouldn't965 // emit it again here.966 Expr *PreviousRetValue = nullptr;967 if (GroManager.DirectEmit) {968 PreviousRetValue = cast<ReturnStmt>(Ret)->getRetValue();969 cast<ReturnStmt>(Ret)->setRetValue(nullptr);970 }971 EmitStmt(Ret);972 // Set the return value back. The code generator, as the AST **Consumer**,973 // shouldn't change the AST.974 if (PreviousRetValue)975 cast<ReturnStmt>(Ret)->setRetValue(PreviousRetValue);976 }977 978 // LLVM require the frontend to mark the coroutine.979 CurFn->setPresplitCoroutine();980 981 if (CXXRecordDecl *RD = FnRetTy->getAsCXXRecordDecl();982 RD && RD->hasAttr<CoroOnlyDestroyWhenCompleteAttr>())983 CurFn->setCoroDestroyOnlyWhenComplete();984}985 986// Emit coroutine intrinsic and patch up arguments of the token type.987RValue CodeGenFunction::EmitCoroutineIntrinsic(const CallExpr *E,988 unsigned int IID) {989 SmallVector<llvm::Value *, 8> Args;990 switch (IID) {991 default:992 break;993 // The coro.frame builtin is replaced with an SSA value of the coro.begin994 // intrinsic.995 case llvm::Intrinsic::coro_frame: {996 if (CurCoro.Data && CurCoro.Data->CoroBegin) {997 return RValue::get(CurCoro.Data->CoroBegin);998 }999 1000 if (CurAwaitSuspendWrapper.FramePtr) {1001 return RValue::get(CurAwaitSuspendWrapper.FramePtr);1002 }1003 1004 CGM.Error(E->getBeginLoc(), "this builtin expect that __builtin_coro_begin "1005 "has been used earlier in this function");1006 auto *NullPtr = llvm::ConstantPointerNull::get(Builder.getPtrTy());1007 return RValue::get(NullPtr);1008 }1009 case llvm::Intrinsic::coro_size: {1010 auto &Context = getContext();1011 llvm::IntegerType *T =1012 Builder.getIntNTy(Context.getTypeSize(Context.getSizeType()));1013 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::coro_size, T);1014 return RValue::get(Builder.CreateCall(F));1015 }1016 case llvm::Intrinsic::coro_align: {1017 auto &Context = getContext();1018 llvm::IntegerType *T =1019 Builder.getIntNTy(Context.getTypeSize(Context.getSizeType()));1020 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::coro_align, T);1021 return RValue::get(Builder.CreateCall(F));1022 }1023 // The following three intrinsics take a token parameter referring to a token1024 // returned by earlier call to @llvm.coro.id. Since we cannot represent it in1025 // builtins, we patch it up here.1026 case llvm::Intrinsic::coro_alloc:1027 case llvm::Intrinsic::coro_begin:1028 case llvm::Intrinsic::coro_free: {1029 if (CurCoro.Data && CurCoro.Data->CoroId) {1030 Args.push_back(CurCoro.Data->CoroId);1031 break;1032 }1033 CGM.Error(E->getBeginLoc(), "this builtin expect that __builtin_coro_id has"1034 " been used earlier in this function");1035 // Fallthrough to the next case to add TokenNone as the first argument.1036 [[fallthrough]];1037 }1038 // @llvm.coro.suspend takes a token parameter. Add token 'none' as the first1039 // argument.1040 case llvm::Intrinsic::coro_suspend:1041 Args.push_back(llvm::ConstantTokenNone::get(getLLVMContext()));1042 break;1043 }1044 for (const Expr *Arg : E->arguments())1045 Args.push_back(EmitScalarExpr(Arg));1046 // @llvm.coro.end takes a token parameter. Add token 'none' as the last1047 // argument.1048 if (IID == llvm::Intrinsic::coro_end)1049 Args.push_back(llvm::ConstantTokenNone::get(getLLVMContext()));1050 1051 llvm::Function *F = CGM.getIntrinsic(IID);1052 llvm::CallInst *Call = Builder.CreateCall(F, Args);1053 1054 // Note: The following code is to enable to emit coro.id and coro.begin by1055 // hand to experiment with coroutines in C.1056 // If we see @llvm.coro.id remember it in the CoroData. We will update1057 // coro.alloc, coro.begin and coro.free intrinsics to refer to it.1058 if (IID == llvm::Intrinsic::coro_id) {1059 createCoroData(*this, CurCoro, Call, E);1060 }1061 else if (IID == llvm::Intrinsic::coro_begin) {1062 if (CurCoro.Data)1063 CurCoro.Data->CoroBegin = Call;1064 }1065 else if (IID == llvm::Intrinsic::coro_free) {1066 // Remember the last coro_free as we need it to build the conditional1067 // deletion of the coroutine frame.1068 if (CurCoro.Data)1069 CurCoro.Data->LastCoroFree = Call;1070 }1071 return RValue::get(Call);1072}1073