389 lines · cpp
1//===----- CGCoroutine.cpp - Emit CIR Code for C++ coroutines -------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This contains code dealing with C++ code generation of coroutines.10//11//===----------------------------------------------------------------------===//12 13#include "CIRGenFunction.h"14#include "mlir/Support/LLVM.h"15#include "clang/AST/StmtCXX.h"16#include "clang/AST/StmtVisitor.h"17#include "clang/Basic/TargetInfo.h"18#include "clang/CIR/Dialect/IR/CIRTypes.h"19#include "clang/CIR/MissingFeatures.h"20 21using namespace clang;22using namespace clang::CIRGen;23 24struct clang::CIRGen::CGCoroData {25 // What is the current await expression kind and how many26 // await/yield expressions were encountered so far.27 // These are used to generate pretty labels for await expressions in LLVM IR.28 cir::AwaitKind currentAwaitKind = cir::AwaitKind::Init;29 // Stores the __builtin_coro_id emitted in the function so that we can supply30 // it as the first argument to other builtins.31 cir::CallOp coroId = nullptr;32 33 // Stores the result of __builtin_coro_begin call.34 mlir::Value coroBegin = nullptr;35};36 37// Defining these here allows to keep CGCoroData private to this file.38CIRGenFunction::CGCoroInfo::CGCoroInfo() {}39CIRGenFunction::CGCoroInfo::~CGCoroInfo() {}40 41namespace {42// FIXME: both GetParamRef and ParamReferenceReplacerRAII are good template43// candidates to be shared among LLVM / CIR codegen.44 45// Hunts for the parameter reference in the parameter copy/move declaration.46struct GetParamRef : public StmtVisitor<GetParamRef> {47public:48 DeclRefExpr *expr = nullptr;49 GetParamRef() {}50 void VisitDeclRefExpr(DeclRefExpr *e) {51 assert(expr == nullptr && "multilple declref in param move");52 expr = e;53 }54 void VisitStmt(Stmt *s) {55 for (Stmt *c : s->children()) {56 if (c)57 Visit(c);58 }59 }60};61 62// This class replaces references to parameters to their copies by changing63// the addresses in CGF.LocalDeclMap and restoring back the original values in64// its destructor.65struct ParamReferenceReplacerRAII {66 CIRGenFunction::DeclMapTy savedLocals;67 CIRGenFunction::DeclMapTy &localDeclMap;68 69 ParamReferenceReplacerRAII(CIRGenFunction::DeclMapTy &localDeclMap)70 : localDeclMap(localDeclMap) {}71 72 void addCopy(const DeclStmt *pm) {73 // Figure out what param it refers to.74 75 assert(pm->isSingleDecl());76 const VarDecl *vd = static_cast<const VarDecl *>(pm->getSingleDecl());77 const Expr *initExpr = vd->getInit();78 GetParamRef visitor;79 visitor.Visit(const_cast<Expr *>(initExpr));80 assert(visitor.expr);81 DeclRefExpr *dreOrig = visitor.expr;82 auto *pd = dreOrig->getDecl();83 84 auto it = localDeclMap.find(pd);85 assert(it != localDeclMap.end() && "parameter is not found");86 savedLocals.insert({pd, it->second});87 88 auto copyIt = localDeclMap.find(vd);89 assert(copyIt != localDeclMap.end() && "parameter copy is not found");90 it->second = copyIt->getSecond();91 }92 93 ~ParamReferenceReplacerRAII() {94 for (auto &&savedLocal : savedLocals) {95 localDeclMap.insert({savedLocal.first, savedLocal.second});96 }97 }98};99} // namespace100 101RValue CIRGenFunction::emitCoroutineFrame() {102 if (curCoro.data && curCoro.data->coroBegin) {103 return RValue::get(curCoro.data->coroBegin);104 }105 cgm.errorNYI("NYI");106 return RValue();107}108 109static void createCoroData(CIRGenFunction &cgf,110 CIRGenFunction::CGCoroInfo &curCoro,111 cir::CallOp coroId) {112 assert(!curCoro.data && "EmitCoroutineBodyStatement called twice?");113 114 curCoro.data = std::make_unique<CGCoroData>();115 curCoro.data->coroId = coroId;116}117 118cir::CallOp CIRGenFunction::emitCoroIDBuiltinCall(mlir::Location loc,119 mlir::Value nullPtr) {120 cir::IntType int32Ty = builder.getUInt32Ty();121 122 const TargetInfo &ti = cgm.getASTContext().getTargetInfo();123 unsigned newAlign = ti.getNewAlign() / ti.getCharWidth();124 125 mlir::Operation *builtin = cgm.getGlobalValue(cgm.builtinCoroId);126 127 cir::FuncOp fnOp;128 if (!builtin) {129 fnOp = cgm.createCIRBuiltinFunction(130 loc, cgm.builtinCoroId,131 cir::FuncType::get({int32Ty, voidPtrTy, voidPtrTy, voidPtrTy}, int32Ty),132 /*FD=*/nullptr);133 assert(fnOp && "should always succeed");134 } else {135 fnOp = cast<cir::FuncOp>(builtin);136 }137 138 return builder.createCallOp(loc, fnOp,139 mlir::ValueRange{builder.getUInt32(newAlign, loc),140 nullPtr, nullPtr, nullPtr});141}142 143cir::CallOp CIRGenFunction::emitCoroAllocBuiltinCall(mlir::Location loc) {144 cir::BoolType boolTy = builder.getBoolTy();145 146 mlir::Operation *builtin = cgm.getGlobalValue(cgm.builtinCoroAlloc);147 148 cir::FuncOp fnOp;149 if (!builtin) {150 fnOp = cgm.createCIRBuiltinFunction(loc, cgm.builtinCoroAlloc,151 cir::FuncType::get({uInt32Ty}, boolTy),152 /*fd=*/nullptr);153 assert(fnOp && "should always succeed");154 } else {155 fnOp = cast<cir::FuncOp>(builtin);156 }157 158 return builder.createCallOp(159 loc, fnOp, mlir::ValueRange{curCoro.data->coroId.getResult()});160}161 162cir::CallOp163CIRGenFunction::emitCoroBeginBuiltinCall(mlir::Location loc,164 mlir::Value coroframeAddr) {165 mlir::Operation *builtin = cgm.getGlobalValue(cgm.builtinCoroBegin);166 167 cir::FuncOp fnOp;168 if (!builtin) {169 fnOp = cgm.createCIRBuiltinFunction(170 loc, cgm.builtinCoroBegin,171 cir::FuncType::get({uInt32Ty, voidPtrTy}, voidPtrTy),172 /*fd=*/nullptr);173 assert(fnOp && "should always succeed");174 } else {175 fnOp = cast<cir::FuncOp>(builtin);176 }177 178 return builder.createCallOp(179 loc, fnOp,180 mlir::ValueRange{curCoro.data->coroId.getResult(), coroframeAddr});181}182 183mlir::LogicalResult184CIRGenFunction::emitCoroutineBody(const CoroutineBodyStmt &s) {185 mlir::Location openCurlyLoc = getLoc(s.getBeginLoc());186 cir::ConstantOp nullPtrCst = builder.getNullPtr(voidPtrTy, openCurlyLoc);187 188 auto fn = mlir::cast<cir::FuncOp>(curFn);189 fn.setCoroutine(true);190 cir::CallOp coroId = emitCoroIDBuiltinCall(openCurlyLoc, nullPtrCst);191 createCoroData(*this, curCoro, coroId);192 193 // Backend is allowed to elide memory allocations, to help it, emit194 // auto mem = coro.alloc() ? 0 : ... allocation code ...;195 cir::CallOp coroAlloc = emitCoroAllocBuiltinCall(openCurlyLoc);196 197 // Initialize address of coroutine frame to null198 CanQualType astVoidPtrTy = cgm.getASTContext().VoidPtrTy;199 mlir::Type allocaTy = convertTypeForMem(astVoidPtrTy);200 Address coroFrame =201 createTempAlloca(allocaTy, getContext().getTypeAlignInChars(astVoidPtrTy),202 openCurlyLoc, "__coro_frame_addr",203 /*ArraySize=*/nullptr);204 205 mlir::Value storeAddr = coroFrame.getPointer();206 builder.CIRBaseBuilderTy::createStore(openCurlyLoc, nullPtrCst, storeAddr);207 cir::IfOp::create(208 builder, openCurlyLoc, coroAlloc.getResult(),209 /*withElseRegion=*/false,210 /*thenBuilder=*/[&](mlir::OpBuilder &b, mlir::Location loc) {211 builder.CIRBaseBuilderTy::createStore(212 loc, emitScalarExpr(s.getAllocate()), storeAddr);213 cir::YieldOp::create(builder, loc);214 });215 curCoro.data->coroBegin =216 emitCoroBeginBuiltinCall(217 openCurlyLoc,218 cir::LoadOp::create(builder, openCurlyLoc, allocaTy, storeAddr))219 .getResult();220 221 // Handle allocation failure if 'ReturnStmtOnAllocFailure' was provided.222 if (s.getReturnStmtOnAllocFailure())223 cgm.errorNYI("handle coroutine return alloc failure");224 225 {226 assert(!cir::MissingFeatures::generateDebugInfo());227 ParamReferenceReplacerRAII paramReplacer(localDeclMap);228 // Create mapping between parameters and copy-params for coroutine229 // function.230 llvm::ArrayRef<const Stmt *> paramMoves = s.getParamMoves();231 assert((paramMoves.size() == 0 || (paramMoves.size() == fnArgs.size())) &&232 "ParamMoves and FnArgs should be the same size for coroutine "233 "function");234 // For zipping the arg map into debug info.235 assert(!cir::MissingFeatures::generateDebugInfo());236 237 // Create parameter copies. We do it before creating a promise, since an238 // evolution of coroutine TS may allow promise constructor to observe239 // parameter copies.240 assert(!cir::MissingFeatures::coroOutsideFrameMD());241 for (auto *pm : paramMoves) {242 if (emitStmt(pm, /*useCurrentScope=*/true).failed())243 return mlir::failure();244 paramReplacer.addCopy(cast<DeclStmt>(pm));245 }246 247 if (emitStmt(s.getPromiseDeclStmt(), /*useCurrentScope=*/true).failed())248 return mlir::failure();249 // returnValue should be valid as long as the coroutine's return type250 // is not void. The assertion could help us to reduce the check later.251 assert(returnValue.isValid() == (bool)s.getReturnStmt());252 // Now we have the promise, initialize the GRO.253 // We need to emit `get_return_object` first. According to:254 // [dcl.fct.def.coroutine]p7255 // The call to get_return_object is sequenced before the call to256 // initial_suspend and is invoked at most once.257 //258 // So we couldn't emit return value when we emit return statment,259 // otherwise the call to get_return_object wouldn't be in front260 // of initial_suspend.261 if (returnValue.isValid())262 emitAnyExprToMem(s.getReturnValue(), returnValue,263 s.getReturnValue()->getType().getQualifiers(),264 /*isInit*/ true);265 266 assert(!cir::MissingFeatures::ehCleanupScope());267 // FIXME(cir): EHStack.pushCleanup<CallCoroEnd>(EHCleanup);268 curCoro.data->currentAwaitKind = cir::AwaitKind::Init;269 if (emitStmt(s.getInitSuspendStmt(), /*useCurrentScope=*/true).failed())270 return mlir::failure();271 assert(!cir::MissingFeatures::emitBodyAndFallthrough());272 }273 return mlir::success();274}275// Given a suspend expression which roughly looks like:276//277// auto && x = CommonExpr();278// if (!x.await_ready()) {279// x.await_suspend(...); (*)280// }281// x.await_resume();282//283// where the result of the entire expression is the result of x.await_resume()284//285// (*) If x.await_suspend return type is bool, it allows to veto a suspend:286// if (x.await_suspend(...))287// llvm_coro_suspend();288//289// This is more higher level than LLVM codegen, for that one see llvm's290// docs/Coroutines.rst for more details.291namespace {292struct LValueOrRValue {293 LValue lv;294 RValue rv;295};296} // namespace297 298static LValueOrRValue299emitSuspendExpression(CIRGenFunction &cgf, CGCoroData &coro,300 CoroutineSuspendExpr const &s, cir::AwaitKind kind,301 AggValueSlot aggSlot, bool ignoreResult,302 mlir::Block *scopeParentBlock,303 mlir::Value &tmpResumeRValAddr, bool forLValue) {304 [[maybe_unused]] mlir::LogicalResult awaitBuild = mlir::success();305 LValueOrRValue awaitRes;306 307 CIRGenFunction::OpaqueValueMapping binder =308 CIRGenFunction::OpaqueValueMapping(cgf, s.getOpaqueValue());309 CIRGenBuilderTy &builder = cgf.getBuilder();310 [[maybe_unused]] cir::AwaitOp awaitOp = cir::AwaitOp::create(311 builder, cgf.getLoc(s.getSourceRange()), kind,312 /*readyBuilder=*/313 [&](mlir::OpBuilder &b, mlir::Location loc) {314 Expr *condExpr = s.getReadyExpr()->IgnoreParens();315 builder.createCondition(cgf.evaluateExprAsBool(condExpr));316 },317 /*suspendBuilder=*/318 [&](mlir::OpBuilder &b, mlir::Location loc) {319 // Note that differently from LLVM codegen we do not emit coro.save320 // and coro.suspend here, that should be done as part of lowering this321 // to LLVM dialect (or some other MLIR dialect)322 323 // A invalid suspendRet indicates "void returning await_suspend"324 mlir::Value suspendRet = cgf.emitScalarExpr(s.getSuspendExpr());325 326 // Veto suspension if requested by bool returning await_suspend.327 if (suspendRet) {328 cgf.cgm.errorNYI("Veto await_suspend");329 }330 331 // Signals the parent that execution flows to next region.332 cir::YieldOp::create(builder, loc);333 },334 /*resumeBuilder=*/335 [&](mlir::OpBuilder &b, mlir::Location loc) {336 cir::YieldOp::create(builder, loc);337 });338 339 assert(awaitBuild.succeeded() && "Should know how to codegen");340 return awaitRes;341}342 343static RValue emitSuspendExpr(CIRGenFunction &cgf,344 const CoroutineSuspendExpr &e,345 cir::AwaitKind kind, AggValueSlot aggSlot,346 bool ignoreResult) {347 RValue rval;348 mlir::Location scopeLoc = cgf.getLoc(e.getSourceRange());349 350 // Since we model suspend / resume as an inner region, we must store351 // resume scalar results in a tmp alloca, and load it after we build the352 // suspend expression. An alternative way to do this would be to make353 // every region return a value when promise.return_value() is used, but354 // it's a bit awkward given that resume is the only region that actually355 // returns a value.356 mlir::Block *currEntryBlock = cgf.curLexScope->getEntryBlock();357 [[maybe_unused]] mlir::Value tmpResumeRValAddr;358 359 // No need to explicitly wrap this into a scope since the AST already uses a360 // ExprWithCleanups, which will wrap this into a cir.scope anyways.361 rval = emitSuspendExpression(cgf, *cgf.curCoro.data, e, kind, aggSlot,362 ignoreResult, currEntryBlock, tmpResumeRValAddr,363 /*forLValue*/ false)364 .rv;365 366 if (ignoreResult || rval.isIgnored())367 return rval;368 369 if (rval.isScalar()) {370 rval = RValue::get(cir::LoadOp::create(cgf.getBuilder(), scopeLoc,371 rval.getValue().getType(),372 tmpResumeRValAddr));373 } else if (rval.isAggregate()) {374 // This is probably already handled via AggSlot, remove this assertion375 // once we have a testcase and prove all pieces work.376 cgf.cgm.errorNYI("emitSuspendExpr Aggregate");377 } else { // complex378 cgf.cgm.errorNYI("emitSuspendExpr Complex");379 }380 return rval;381}382 383RValue CIRGenFunction::emitCoawaitExpr(const CoawaitExpr &e,384 AggValueSlot aggSlot,385 bool ignoreResult) {386 return emitSuspendExpr(*this, e, curCoro.data->currentAwaitKind, aggSlot,387 ignoreResult);388}389