467 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Emit OpenACC Stmt nodes as CIR code.10//11//===----------------------------------------------------------------------===//12 13#include "CIRGenBuilder.h"14#include "CIRGenFunction.h"15#include "mlir/Dialect/OpenACC/OpenACC.h"16#include "clang/AST/OpenACCClause.h"17#include "clang/AST/StmtOpenACC.h"18 19using namespace clang;20using namespace clang::CIRGen;21using namespace cir;22using namespace mlir::acc;23 24template <typename Op, typename TermOp>25mlir::LogicalResult CIRGenFunction::emitOpenACCOpAssociatedStmt(26 mlir::Location start, mlir::Location end, OpenACCDirectiveKind dirKind,27 llvm::ArrayRef<const OpenACCClause *> clauses, const Stmt *associatedStmt) {28 mlir::LogicalResult res = mlir::success();29 30 llvm::SmallVector<mlir::Type> retTy;31 llvm::SmallVector<mlir::Value> operands;32 auto op = Op::create(builder, start, retTy, operands);33 34 emitOpenACCClauses(op, dirKind, clauses);35 36 {37 mlir::Block &block = op.getRegion().emplaceBlock();38 mlir::OpBuilder::InsertionGuard guardCase(builder);39 builder.setInsertionPointToEnd(&block);40 41 LexicalScope ls{*this, start, builder.getInsertionBlock()};42 res = emitStmt(associatedStmt, /*useCurrentScope=*/true);43 44 TermOp::create(builder, end);45 }46 return res;47}48 49namespace {50template <typename Op> struct CombinedType;51template <> struct CombinedType<ParallelOp> {52 static constexpr mlir::acc::CombinedConstructsType value =53 mlir::acc::CombinedConstructsType::ParallelLoop;54};55template <> struct CombinedType<SerialOp> {56 static constexpr mlir::acc::CombinedConstructsType value =57 mlir::acc::CombinedConstructsType::SerialLoop;58};59template <> struct CombinedType<KernelsOp> {60 static constexpr mlir::acc::CombinedConstructsType value =61 mlir::acc::CombinedConstructsType::KernelsLoop;62};63} // namespace64 65template <typename Op, typename TermOp>66mlir::LogicalResult CIRGenFunction::emitOpenACCOpCombinedConstruct(67 mlir::Location start, mlir::Location end, OpenACCDirectiveKind dirKind,68 llvm::ArrayRef<const OpenACCClause *> clauses, const Stmt *loopStmt) {69 mlir::LogicalResult res = mlir::success();70 71 llvm::SmallVector<mlir::Type> retTy;72 llvm::SmallVector<mlir::Value> operands;73 74 auto computeOp = Op::create(builder, start, retTy, operands);75 computeOp.setCombinedAttr(builder.getUnitAttr());76 mlir::acc::LoopOp loopOp;77 78 // First, emit the bodies of both operations, with the loop inside the body of79 // the combined construct.80 {81 mlir::Block &block = computeOp.getRegion().emplaceBlock();82 mlir::OpBuilder::InsertionGuard guardCase(builder);83 builder.setInsertionPointToEnd(&block);84 85 LexicalScope ls{*this, start, builder.getInsertionBlock()};86 auto loopOp = LoopOp::create(builder, start, retTy, operands);87 loopOp.setCombinedAttr(mlir::acc::CombinedConstructsTypeAttr::get(88 builder.getContext(), CombinedType<Op>::value));89 90 {91 mlir::Block &innerBlock = loopOp.getRegion().emplaceBlock();92 mlir::OpBuilder::InsertionGuard guardCase(builder);93 builder.setInsertionPointToEnd(&innerBlock);94 95 LexicalScope ls{*this, start, builder.getInsertionBlock()};96 ActiveOpenACCLoopRAII activeLoop{*this, &loopOp};97 98 res = emitStmt(loopStmt, /*useCurrentScope=*/true);99 100 mlir::acc::YieldOp::create(builder, end);101 }102 103 emitOpenACCClauses(computeOp, loopOp, dirKind, clauses);104 105 updateLoopOpParallelism(loopOp, /*isOrphan=*/false, dirKind);106 107 TermOp::create(builder, end);108 }109 110 return res;111}112 113template <typename Op>114Op CIRGenFunction::emitOpenACCOp(115 mlir::Location start, OpenACCDirectiveKind dirKind,116 llvm::ArrayRef<const OpenACCClause *> clauses) {117 llvm::SmallVector<mlir::Type> retTy;118 llvm::SmallVector<mlir::Value> operands;119 auto op = Op::create(builder, start, retTy, operands);120 121 emitOpenACCClauses(op, dirKind, clauses);122 return op;123}124 125mlir::LogicalResult126CIRGenFunction::emitOpenACCComputeConstruct(const OpenACCComputeConstruct &s) {127 mlir::Location start = getLoc(s.getSourceRange().getBegin());128 mlir::Location end = getLoc(s.getSourceRange().getEnd());129 130 switch (s.getDirectiveKind()) {131 case OpenACCDirectiveKind::Parallel:132 return emitOpenACCOpAssociatedStmt<ParallelOp, mlir::acc::YieldOp>(133 start, end, s.getDirectiveKind(), s.clauses(), s.getStructuredBlock());134 case OpenACCDirectiveKind::Serial:135 return emitOpenACCOpAssociatedStmt<SerialOp, mlir::acc::YieldOp>(136 start, end, s.getDirectiveKind(), s.clauses(), s.getStructuredBlock());137 case OpenACCDirectiveKind::Kernels:138 return emitOpenACCOpAssociatedStmt<KernelsOp, mlir::acc::TerminatorOp>(139 start, end, s.getDirectiveKind(), s.clauses(), s.getStructuredBlock());140 default:141 llvm_unreachable("invalid compute construct kind");142 }143}144 145mlir::LogicalResult146CIRGenFunction::emitOpenACCDataConstruct(const OpenACCDataConstruct &s) {147 mlir::Location start = getLoc(s.getSourceRange().getBegin());148 mlir::Location end = getLoc(s.getSourceRange().getEnd());149 150 return emitOpenACCOpAssociatedStmt<DataOp, mlir::acc::TerminatorOp>(151 start, end, s.getDirectiveKind(), s.clauses(), s.getStructuredBlock());152}153 154mlir::LogicalResult155CIRGenFunction::emitOpenACCInitConstruct(const OpenACCInitConstruct &s) {156 mlir::Location start = getLoc(s.getSourceRange().getBegin());157 emitOpenACCOp<InitOp>(start, s.getDirectiveKind(), s.clauses());158 return mlir::success();159}160 161mlir::LogicalResult162CIRGenFunction::emitOpenACCSetConstruct(const OpenACCSetConstruct &s) {163 mlir::Location start = getLoc(s.getSourceRange().getBegin());164 emitOpenACCOp<SetOp>(start, s.getDirectiveKind(), s.clauses());165 return mlir::success();166}167 168mlir::LogicalResult CIRGenFunction::emitOpenACCShutdownConstruct(169 const OpenACCShutdownConstruct &s) {170 mlir::Location start = getLoc(s.getSourceRange().getBegin());171 emitOpenACCOp<ShutdownOp>(start, s.getDirectiveKind(), s.clauses());172 return mlir::success();173}174 175mlir::LogicalResult176CIRGenFunction::emitOpenACCWaitConstruct(const OpenACCWaitConstruct &s) {177 mlir::Location start = getLoc(s.getSourceRange().getBegin());178 auto waitOp = emitOpenACCOp<WaitOp>(start, s.getDirectiveKind(), s.clauses());179 180 auto createIntExpr = [this](const Expr *intExpr) {181 mlir::Value expr = emitScalarExpr(intExpr);182 mlir::Location exprLoc = cgm.getLoc(intExpr->getBeginLoc());183 184 mlir::IntegerType targetType = mlir::IntegerType::get(185 &getMLIRContext(), getContext().getIntWidth(intExpr->getType()),186 intExpr->getType()->isSignedIntegerOrEnumerationType()187 ? mlir::IntegerType::SignednessSemantics::Signed188 : mlir::IntegerType::SignednessSemantics::Unsigned);189 190 auto conversionOp = mlir::UnrealizedConversionCastOp::create(191 builder, exprLoc, targetType, expr);192 return conversionOp.getResult(0);193 };194 195 // Emit the correct 'wait' clauses.196 {197 mlir::OpBuilder::InsertionGuard guardCase(builder);198 builder.setInsertionPoint(waitOp);199 200 if (s.hasDevNumExpr())201 waitOp.getWaitDevnumMutable().append(createIntExpr(s.getDevNumExpr()));202 203 for (Expr *QueueExpr : s.getQueueIdExprs())204 waitOp.getWaitOperandsMutable().append(createIntExpr(QueueExpr));205 }206 207 return mlir::success();208}209 210mlir::LogicalResult CIRGenFunction::emitOpenACCCombinedConstruct(211 const OpenACCCombinedConstruct &s) {212 mlir::Location start = getLoc(s.getSourceRange().getBegin());213 mlir::Location end = getLoc(s.getSourceRange().getEnd());214 215 switch (s.getDirectiveKind()) {216 case OpenACCDirectiveKind::ParallelLoop:217 return emitOpenACCOpCombinedConstruct<ParallelOp, mlir::acc::YieldOp>(218 start, end, s.getDirectiveKind(), s.clauses(), s.getLoop());219 case OpenACCDirectiveKind::SerialLoop:220 return emitOpenACCOpCombinedConstruct<SerialOp, mlir::acc::YieldOp>(221 start, end, s.getDirectiveKind(), s.clauses(), s.getLoop());222 case OpenACCDirectiveKind::KernelsLoop:223 return emitOpenACCOpCombinedConstruct<KernelsOp, mlir::acc::TerminatorOp>(224 start, end, s.getDirectiveKind(), s.clauses(), s.getLoop());225 default:226 llvm_unreachable("invalid compute construct kind");227 }228}229 230mlir::LogicalResult CIRGenFunction::emitOpenACCHostDataConstruct(231 const OpenACCHostDataConstruct &s) {232 mlir::Location start = getLoc(s.getSourceRange().getBegin());233 mlir::Location end = getLoc(s.getSourceRange().getEnd());234 235 return emitOpenACCOpAssociatedStmt<HostDataOp, mlir::acc::TerminatorOp>(236 start, end, s.getDirectiveKind(), s.clauses(), s.getStructuredBlock());237}238 239mlir::LogicalResult CIRGenFunction::emitOpenACCEnterDataConstruct(240 const OpenACCEnterDataConstruct &s) {241 mlir::Location start = getLoc(s.getSourceRange().getBegin());242 emitOpenACCOp<EnterDataOp>(start, s.getDirectiveKind(), s.clauses());243 return mlir::success();244}245 246mlir::LogicalResult CIRGenFunction::emitOpenACCExitDataConstruct(247 const OpenACCExitDataConstruct &s) {248 mlir::Location start = getLoc(s.getSourceRange().getBegin());249 emitOpenACCOp<ExitDataOp>(start, s.getDirectiveKind(), s.clauses());250 return mlir::success();251}252 253mlir::LogicalResult254CIRGenFunction::emitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &s) {255 mlir::Location start = getLoc(s.getSourceRange().getBegin());256 emitOpenACCOp<UpdateOp>(start, s.getDirectiveKind(), s.clauses());257 return mlir::success();258}259 260mlir::LogicalResult261CIRGenFunction::emitOpenACCCacheConstruct(const OpenACCCacheConstruct &s) {262 // The 'cache' directive 'may' be at the top of a loop by standard, but263 // doesn't have to be. Additionally, there is nothing that requires this be a264 // loop affected by an OpenACC pragma. Sema doesn't do any level of265 // enforcement here, since it isn't particularly valuable to do so thanks to266 // that. Instead, we treat cache as a 'noop' if there is no acc.loop to apply267 // it to.268 if (!activeLoopOp)269 return mlir::success();270 271 mlir::acc::LoopOp loopOp = *activeLoopOp;272 273 mlir::OpBuilder::InsertionGuard guard(builder);274 builder.setInsertionPoint(loopOp);275 276 for (const Expr *var : s.getVarList()) {277 CIRGenFunction::OpenACCDataOperandInfo opInfo =278 getOpenACCDataOperandInfo(var);279 280 auto cacheOp = CacheOp::create(builder, opInfo.beginLoc, opInfo.varValue,281 /*structured=*/false, /*implicit=*/false,282 opInfo.name, opInfo.bounds);283 284 loopOp.getCacheOperandsMutable().append(cacheOp.getResult());285 }286 287 return mlir::success();288}289 290const VarDecl *getLValueDecl(const Expr *e) {291 // We are going to assume that after stripping implicit casts, that the LValue292 // is just a DRE around the var-decl.293 294 e = e->IgnoreImpCasts();295 296 const auto *dre = cast<DeclRefExpr>(e);297 return cast<VarDecl>(dre->getDecl());298}299 300static mlir::acc::AtomicReadOp301emitAtomicRead(CIRGenFunction &cgf, CIRGenBuilderTy &builder,302 mlir::Location start,303 const OpenACCAtomicConstruct::SingleStmtInfo &inf) {304 // Atomic 'read' only permits 'v = x', where v and x are both scalar L305 // values. The getAssociatedStmtInfo strips off implicit casts, which306 // includes implicit conversions and L-to-R-Value conversions, so we can307 // just emit it as an L value. The Flang implementation has no problem with308 // different types, so it appears that the dialect can handle the309 // conversions.310 mlir::Value v = cgf.emitLValue(inf.V).getPointer();311 mlir::Value x = cgf.emitLValue(inf.X).getPointer();312 mlir::Type resTy = cgf.convertType(inf.V->getType());313 return mlir::acc::AtomicReadOp::create(builder, start, x, v, resTy,314 /*ifCond=*/{});315}316 317static mlir::acc::AtomicWriteOp318emitAtomicWrite(CIRGenFunction &cgf, CIRGenBuilderTy &builder,319 mlir::Location start,320 const OpenACCAtomicConstruct::SingleStmtInfo &inf) {321 mlir::Value x = cgf.emitLValue(inf.X).getPointer();322 mlir::Value expr = cgf.emitAnyExpr(inf.RefExpr).getValue();323 return mlir::acc::AtomicWriteOp::create(builder, start, x, expr,324 /*ifCond=*/{});325}326 327static std::pair<mlir::LogicalResult, mlir::acc::AtomicUpdateOp>328emitAtomicUpdate(CIRGenFunction &cgf, CIRGenBuilderTy &builder,329 mlir::Location start, mlir::Location end,330 const OpenACCAtomicConstruct::SingleStmtInfo &inf) {331 mlir::Value x = cgf.emitLValue(inf.X).getPointer();332 auto op = mlir::acc::AtomicUpdateOp::create(builder, start, x, /*ifCond=*/{});333 334 mlir::LogicalResult res = mlir::success();335 {336 mlir::OpBuilder::InsertionGuard guardCase(builder);337 mlir::Type argTy = cast<cir::PointerType>(x.getType()).getPointee();338 std::array<mlir::Type, 1> recipeType{argTy};339 std::array<mlir::Location, 1> recipeLoc{start};340 auto *recipeBlock = builder.createBlock(341 &op.getRegion(), op.getRegion().end(), recipeType, recipeLoc);342 builder.setInsertionPointToEnd(recipeBlock);343 // Since we have an initial value that we know is a scalar type, we can344 // just emit the entire statement here after sneaking-in our 'alloca' in345 // the right place, then loading out of it. Flang does a lot less work346 // (probably does its own emitting!), but we have more complicated AST347 // nodes to worry about, so we can just count on opt to remove the extra348 // alloca/load/store set.349 auto alloca = cir::AllocaOp::create(350 builder, start, x.getType(), argTy, "x_var",351 cgf.cgm.getSize(352 cgf.getContext().getTypeAlignInChars(inf.X->getType())));353 354 alloca.setInitAttr(builder.getUnitAttr());355 builder.CIRBaseBuilderTy::createStore(start, recipeBlock->getArgument(0),356 alloca);357 358 const VarDecl *xval = getLValueDecl(inf.X);359 CIRGenFunction::DeclMapRevertingRAII declMapRAII{cgf, xval};360 cgf.replaceAddrOfLocalVar(361 xval, Address{alloca, argTy, cgf.getContext().getDeclAlign(xval)});362 363 res = cgf.emitStmt(inf.WholeExpr, /*useCurrentScope=*/true);364 365 auto load = cir::LoadOp::create(builder, start, {alloca});366 mlir::acc::YieldOp::create(builder, end, {load});367 }368 369 return {res, op};370}371 372mlir::LogicalResult373CIRGenFunction::emitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &s) {374 // While Atomic is an 'associated statement' construct, it 'steals' the375 // expression it is associated with rather than emitting it inside of it. So376 // it has custom emit logic.377 mlir::Location start = getLoc(s.getSourceRange().getBegin());378 mlir::Location end = getLoc(s.getSourceRange().getEnd());379 OpenACCAtomicConstruct::StmtInfo inf = s.getAssociatedStmtInfo();380 381 switch (s.getAtomicKind()) {382 case OpenACCAtomicKind::Read: {383 assert(inf.Form == OpenACCAtomicConstruct::StmtInfo::StmtForm::Read);384 mlir::acc::AtomicReadOp op =385 emitAtomicRead(*this, builder, start, inf.First);386 emitOpenACCClauses(op, s.getDirectiveKind(), s.clauses());387 return mlir::success();388 }389 case OpenACCAtomicKind::Write: {390 assert(inf.Form == OpenACCAtomicConstruct::StmtInfo::StmtForm::Write);391 auto op = emitAtomicWrite(*this, builder, start, inf.First);392 emitOpenACCClauses(op, s.getDirectiveKind(), s.clauses());393 return mlir::success();394 }395 case OpenACCAtomicKind::None:396 case OpenACCAtomicKind::Update: {397 assert(inf.Form == OpenACCAtomicConstruct::StmtInfo::StmtForm::Update);398 auto [res, op] = emitAtomicUpdate(*this, builder, start, end, inf.First);399 emitOpenACCClauses(op, s.getDirectiveKind(), s.clauses());400 return res;401 }402 case OpenACCAtomicKind::Capture: {403 // Atomic-capture is made up of two statements, either an update = read,404 // read + update, or read + write. As a result, the IR represents the405 // capture region as having those two 'inside' of it.406 auto op = mlir::acc::AtomicCaptureOp::create(builder, start, /*ifCond=*/{});407 emitOpenACCClauses(op, s.getDirectiveKind(), s.clauses());408 mlir::LogicalResult res = mlir::success();409 {410 mlir::OpBuilder::InsertionGuard guardCase(builder);411 412 mlir::Block *block =413 builder.createBlock(&op.getRegion(), op.getRegion().end(), {}, {});414 415 builder.setInsertionPointToStart(block);416 417 auto terminator = mlir::acc::TerminatorOp::create(builder, end);418 419 // The AtomicCaptureOp only permits the two acc.atomic.* operations inside420 // of it, so all other parts of the expression need to be emitted before421 // the AtomicCaptureOp, then moved into place.422 builder.setInsertionPoint(op);423 424 switch (inf.Form) {425 default:426 llvm_unreachable("invalid form for Capture");427 case OpenACCAtomicConstruct::StmtInfo::StmtForm::ReadWrite: {428 mlir::acc::AtomicReadOp first =429 emitAtomicRead(*this, builder, start, inf.First);430 mlir::acc::AtomicWriteOp second =431 emitAtomicWrite(*this, builder, start, inf.Second);432 433 first->moveBefore(terminator);434 second->moveBefore(terminator);435 break;436 }437 case OpenACCAtomicConstruct::StmtInfo::StmtForm::ReadUpdate: {438 mlir::acc::AtomicReadOp first =439 emitAtomicRead(*this, builder, start, inf.First);440 auto [this_res, second] =441 emitAtomicUpdate(*this, builder, start, end, inf.Second);442 res = this_res;443 444 first->moveBefore(terminator);445 second->moveBefore(terminator);446 break;447 }448 case OpenACCAtomicConstruct::StmtInfo::StmtForm::UpdateRead: {449 auto [this_res, first] =450 emitAtomicUpdate(*this, builder, start, end, inf.First);451 res = this_res;452 mlir::acc::AtomicReadOp second =453 emitAtomicRead(*this, builder, start, inf.Second);454 455 first->moveBefore(terminator);456 second->moveBefore(terminator);457 break;458 }459 }460 }461 return res;462 }463 }464 465 llvm_unreachable("unknown OpenACC atomic kind");466}467