8594 lines · cpp
1//===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This contains code to emit OpenMP nodes as LLVM code.10//11//===----------------------------------------------------------------------===//12 13#include "CGCleanup.h"14#include "CGDebugInfo.h"15#include "CGOpenMPRuntime.h"16#include "CodeGenFunction.h"17#include "CodeGenModule.h"18#include "CodeGenPGO.h"19#include "TargetInfo.h"20#include "clang/AST/ASTContext.h"21#include "clang/AST/Attr.h"22#include "clang/AST/DeclOpenMP.h"23#include "clang/AST/OpenMPClause.h"24#include "clang/AST/Stmt.h"25#include "clang/AST/StmtOpenMP.h"26#include "clang/AST/StmtVisitor.h"27#include "clang/Basic/OpenMPKinds.h"28#include "clang/Basic/PrettyStackTrace.h"29#include "clang/Basic/SourceManager.h"30#include "llvm/ADT/SmallSet.h"31#include "llvm/BinaryFormat/Dwarf.h"32#include "llvm/Frontend/OpenMP/OMPConstants.h"33#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"34#include "llvm/IR/Constants.h"35#include "llvm/IR/DebugInfoMetadata.h"36#include "llvm/IR/Instructions.h"37#include "llvm/IR/IntrinsicInst.h"38#include "llvm/IR/Metadata.h"39#include "llvm/Support/AtomicOrdering.h"40#include "llvm/Support/Debug.h"41#include <optional>42using namespace clang;43using namespace CodeGen;44using namespace llvm::omp;45 46#define TTL_CODEGEN_TYPE "target-teams-loop-codegen"47 48static const VarDecl *getBaseDecl(const Expr *Ref);49static OpenMPDirectiveKind50getEffectiveDirectiveKind(const OMPExecutableDirective &S);51 52namespace {53/// Lexical scope for OpenMP executable constructs, that handles correct codegen54/// for captured expressions.55class OMPLexicalScope : public CodeGenFunction::LexicalScope {56 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {57 for (const auto *C : S.clauses()) {58 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {59 if (const auto *PreInit =60 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {61 for (const auto *I : PreInit->decls()) {62 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {63 CGF.EmitVarDecl(cast<VarDecl>(*I));64 } else {65 CodeGenFunction::AutoVarEmission Emission =66 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));67 CGF.EmitAutoVarCleanups(Emission);68 }69 }70 }71 }72 }73 }74 CodeGenFunction::OMPPrivateScope InlinedShareds;75 76 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {77 return CGF.LambdaCaptureFields.lookup(VD) ||78 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||79 (isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl) &&80 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));81 }82 83public:84 OMPLexicalScope(85 CodeGenFunction &CGF, const OMPExecutableDirective &S,86 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,87 const bool EmitPreInitStmt = true)88 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),89 InlinedShareds(CGF) {90 if (EmitPreInitStmt)91 emitPreInitStmt(CGF, S);92 if (!CapturedRegion)93 return;94 assert(S.hasAssociatedStmt() &&95 "Expected associated statement for inlined directive.");96 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);97 for (const auto &C : CS->captures()) {98 if (C.capturesVariable() || C.capturesVariableByCopy()) {99 auto *VD = C.getCapturedVar();100 assert(VD == VD->getCanonicalDecl() &&101 "Canonical decl must be captured.");102 DeclRefExpr DRE(103 CGF.getContext(), const_cast<VarDecl *>(VD),104 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&105 InlinedShareds.isGlobalVarCaptured(VD)),106 VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());107 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());108 }109 }110 (void)InlinedShareds.Privatize();111 }112};113 114/// Lexical scope for OpenMP parallel construct, that handles correct codegen115/// for captured expressions.116class OMPParallelScope final : public OMPLexicalScope {117 bool EmitPreInitStmt(const OMPExecutableDirective &S) {118 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);119 return !(isOpenMPTargetExecutionDirective(EKind) ||120 isOpenMPLoopBoundSharingDirective(EKind)) &&121 isOpenMPParallelDirective(EKind);122 }123 124public:125 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)126 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,127 EmitPreInitStmt(S)) {}128};129 130/// Lexical scope for OpenMP teams construct, that handles correct codegen131/// for captured expressions.132class OMPTeamsScope final : public OMPLexicalScope {133 bool EmitPreInitStmt(const OMPExecutableDirective &S) {134 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);135 return !isOpenMPTargetExecutionDirective(EKind) &&136 isOpenMPTeamsDirective(EKind);137 }138 139public:140 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)141 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,142 EmitPreInitStmt(S)) {}143};144 145/// Private scope for OpenMP loop-based directives, that supports capturing146/// of used expression from loop statement.147class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {148 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) {149 const Stmt *PreInits;150 CodeGenFunction::OMPMapVars PreCondVars;151 if (auto *LD = dyn_cast<OMPLoopDirective>(&S)) {152 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;153 for (const auto *E : LD->counters()) {154 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());155 EmittedAsPrivate.insert(VD->getCanonicalDecl());156 (void)PreCondVars.setVarAddr(157 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));158 }159 // Mark private vars as undefs.160 for (const auto *C : LD->getClausesOfKind<OMPPrivateClause>()) {161 for (const Expr *IRef : C->varlist()) {162 const auto *OrigVD =163 cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());164 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {165 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();166 (void)PreCondVars.setVarAddr(167 CGF, OrigVD,168 Address(llvm::UndefValue::get(CGF.ConvertTypeForMem(169 CGF.getContext().getPointerType(OrigVDTy))),170 CGF.ConvertTypeForMem(OrigVDTy),171 CGF.getContext().getDeclAlign(OrigVD)));172 }173 }174 }175 (void)PreCondVars.apply(CGF);176 // Emit init, __range and __end variables for C++ range loops.177 (void)OMPLoopBasedDirective::doForAllLoops(178 LD->getInnermostCapturedStmt()->getCapturedStmt(),179 /*TryImperfectlyNestedLoops=*/true, LD->getLoopsNumber(),180 [&CGF](unsigned Cnt, const Stmt *CurStmt) {181 if (const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {182 if (const Stmt *Init = CXXFor->getInit())183 CGF.EmitStmt(Init);184 CGF.EmitStmt(CXXFor->getRangeStmt());185 CGF.EmitStmt(CXXFor->getEndStmt());186 }187 return false;188 });189 PreInits = LD->getPreInits();190 } else if (const auto *Tile = dyn_cast<OMPTileDirective>(&S)) {191 PreInits = Tile->getPreInits();192 } else if (const auto *Stripe = dyn_cast<OMPStripeDirective>(&S)) {193 PreInits = Stripe->getPreInits();194 } else if (const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {195 PreInits = Unroll->getPreInits();196 } else if (const auto *Reverse = dyn_cast<OMPReverseDirective>(&S)) {197 PreInits = Reverse->getPreInits();198 } else if (const auto *Interchange =199 dyn_cast<OMPInterchangeDirective>(&S)) {200 PreInits = Interchange->getPreInits();201 } else {202 llvm_unreachable("Unknown loop-based directive kind.");203 }204 doEmitPreinits(PreInits);205 PreCondVars.restore(CGF);206 }207 208 void209 emitPreInitStmt(CodeGenFunction &CGF,210 const OMPCanonicalLoopSequenceTransformationDirective &S) {211 const Stmt *PreInits;212 if (const auto *Fuse = dyn_cast<OMPFuseDirective>(&S)) {213 PreInits = Fuse->getPreInits();214 } else {215 llvm_unreachable(216 "Unknown canonical loop sequence transform directive kind.");217 }218 doEmitPreinits(PreInits);219 }220 221 void doEmitPreinits(const Stmt *PreInits) {222 if (PreInits) {223 // CompoundStmts and DeclStmts are used as lists of PreInit statements and224 // declarations. Since declarations must be visible in the the following225 // that they initialize, unpack the CompoundStmt they are nested in.226 SmallVector<const Stmt *> PreInitStmts;227 if (auto *PreInitCompound = dyn_cast<CompoundStmt>(PreInits))228 llvm::append_range(PreInitStmts, PreInitCompound->body());229 else230 PreInitStmts.push_back(PreInits);231 232 for (const Stmt *S : PreInitStmts) {233 // EmitStmt skips any OMPCapturedExprDecls, but needs to be emitted234 // here.235 if (auto *PreInitDecl = dyn_cast<DeclStmt>(S)) {236 for (Decl *I : PreInitDecl->decls())237 CGF.EmitVarDecl(cast<VarDecl>(*I));238 continue;239 }240 CGF.EmitStmt(S);241 }242 }243 }244 245public:246 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S)247 : CodeGenFunction::RunCleanupsScope(CGF) {248 emitPreInitStmt(CGF, S);249 }250 OMPLoopScope(CodeGenFunction &CGF,251 const OMPCanonicalLoopSequenceTransformationDirective &S)252 : CodeGenFunction::RunCleanupsScope(CGF) {253 emitPreInitStmt(CGF, S);254 }255};256 257class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {258 CodeGenFunction::OMPPrivateScope InlinedShareds;259 260 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {261 return CGF.LambdaCaptureFields.lookup(VD) ||262 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||263 (isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl) &&264 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));265 }266 267public:268 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)269 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),270 InlinedShareds(CGF) {271 for (const auto *C : S.clauses()) {272 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {273 if (const auto *PreInit =274 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {275 for (const auto *I : PreInit->decls()) {276 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {277 CGF.EmitVarDecl(cast<VarDecl>(*I));278 } else {279 CodeGenFunction::AutoVarEmission Emission =280 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));281 CGF.EmitAutoVarCleanups(Emission);282 }283 }284 }285 } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {286 for (const Expr *E : UDP->varlist()) {287 const Decl *D = cast<DeclRefExpr>(E)->getDecl();288 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))289 CGF.EmitVarDecl(*OED);290 }291 } else if (const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(C)) {292 for (const Expr *E : UDP->varlist()) {293 const Decl *D = getBaseDecl(E);294 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))295 CGF.EmitVarDecl(*OED);296 }297 }298 }299 if (!isOpenMPSimdDirective(getEffectiveDirectiveKind(S)))300 CGF.EmitOMPPrivateClause(S, InlinedShareds);301 if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {302 if (const Expr *E = TG->getReductionRef())303 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));304 }305 // Temp copy arrays for inscan reductions should not be emitted as they are306 // not used in simd only mode.307 llvm::DenseSet<CanonicalDeclPtr<const Decl>> CopyArrayTemps;308 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {309 if (C->getModifier() != OMPC_REDUCTION_inscan)310 continue;311 for (const Expr *E : C->copy_array_temps())312 CopyArrayTemps.insert(cast<DeclRefExpr>(E)->getDecl());313 }314 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());315 while (CS) {316 for (auto &C : CS->captures()) {317 if (C.capturesVariable() || C.capturesVariableByCopy()) {318 auto *VD = C.getCapturedVar();319 if (CopyArrayTemps.contains(VD))320 continue;321 assert(VD == VD->getCanonicalDecl() &&322 "Canonical decl must be captured.");323 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),324 isCapturedVar(CGF, VD) ||325 (CGF.CapturedStmtInfo &&326 InlinedShareds.isGlobalVarCaptured(VD)),327 VD->getType().getNonReferenceType(), VK_LValue,328 C.getLocation());329 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());330 }331 }332 CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());333 }334 (void)InlinedShareds.Privatize();335 }336};337 338} // namespace339 340// The loop directive with a bind clause will be mapped to a different341// directive with corresponding semantics.342static OpenMPDirectiveKind343getEffectiveDirectiveKind(const OMPExecutableDirective &S) {344 OpenMPDirectiveKind Kind = S.getDirectiveKind();345 if (Kind != OMPD_loop)346 return Kind;347 348 OpenMPBindClauseKind BindKind = OMPC_BIND_unknown;349 if (const auto *C = S.getSingleClause<OMPBindClause>())350 BindKind = C->getBindKind();351 352 switch (BindKind) {353 case OMPC_BIND_parallel:354 return OMPD_for;355 case OMPC_BIND_teams:356 return OMPD_distribute;357 case OMPC_BIND_thread:358 return OMPD_simd;359 default:360 return OMPD_loop;361 }362}363 364static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,365 const OMPExecutableDirective &S,366 const RegionCodeGenTy &CodeGen);367 368LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {369 if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {370 if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {371 OrigVD = OrigVD->getCanonicalDecl();372 bool IsCaptured =373 LambdaCaptureFields.lookup(OrigVD) ||374 (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||375 (isa_and_nonnull<BlockDecl>(CurCodeDecl));376 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,377 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());378 return EmitLValue(&DRE);379 }380 }381 return EmitLValue(E);382}383 384llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {385 ASTContext &C = getContext();386 llvm::Value *Size = nullptr;387 auto SizeInChars = C.getTypeSizeInChars(Ty);388 if (SizeInChars.isZero()) {389 // getTypeSizeInChars() returns 0 for a VLA.390 while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {391 VlaSizePair VlaSize = getVLASize(VAT);392 Ty = VlaSize.Type;393 Size =394 Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts) : VlaSize.NumElts;395 }396 SizeInChars = C.getTypeSizeInChars(Ty);397 if (SizeInChars.isZero())398 return llvm::ConstantInt::get(SizeTy, /*V=*/0);399 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));400 }401 return CGM.getSize(SizeInChars);402}403 404void CodeGenFunction::GenerateOpenMPCapturedVars(405 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {406 const RecordDecl *RD = S.getCapturedRecordDecl();407 auto CurField = RD->field_begin();408 auto CurCap = S.captures().begin();409 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),410 E = S.capture_init_end();411 I != E; ++I, ++CurField, ++CurCap) {412 if (CurField->hasCapturedVLAType()) {413 const VariableArrayType *VAT = CurField->getCapturedVLAType();414 llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];415 CapturedVars.push_back(Val);416 } else if (CurCap->capturesThis()) {417 CapturedVars.push_back(CXXThisValue);418 } else if (CurCap->capturesVariableByCopy()) {419 llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());420 421 // If the field is not a pointer, we need to save the actual value422 // and load it as a void pointer.423 if (!CurField->getType()->isAnyPointerType()) {424 ASTContext &Ctx = getContext();425 Address DstAddr = CreateMemTemp(426 Ctx.getUIntPtrType(),427 Twine(CurCap->getCapturedVar()->getName(), ".casted"));428 LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());429 430 llvm::Value *SrcAddrVal = EmitScalarConversion(431 DstAddr.emitRawPointer(*this),432 Ctx.getPointerType(Ctx.getUIntPtrType()),433 Ctx.getPointerType(CurField->getType()), CurCap->getLocation());434 LValue SrcLV =435 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());436 437 // Store the value using the source type pointer.438 EmitStoreThroughLValue(RValue::get(CV), SrcLV);439 440 // Load the value using the destination type pointer.441 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());442 }443 CapturedVars.push_back(CV);444 } else {445 assert(CurCap->capturesVariable() && "Expected capture by reference.");446 CapturedVars.push_back(EmitLValue(*I).getAddress().emitRawPointer(*this));447 }448 }449}450 451static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc,452 QualType DstType, StringRef Name,453 LValue AddrLV) {454 ASTContext &Ctx = CGF.getContext();455 456 llvm::Value *CastedPtr = CGF.EmitScalarConversion(457 AddrLV.getAddress().emitRawPointer(CGF), Ctx.getUIntPtrType(),458 Ctx.getPointerType(DstType), Loc);459 // FIXME: should the pointee type (DstType) be passed?460 Address TmpAddr =461 CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress();462 return TmpAddr;463}464 465static QualType getCanonicalParamType(ASTContext &C, QualType T) {466 if (T->isLValueReferenceType())467 return C.getLValueReferenceType(468 getCanonicalParamType(C, T.getNonReferenceType()),469 /*SpelledAsLValue=*/false);470 if (T->isPointerType())471 return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));472 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {473 if (const auto *VLA = dyn_cast<VariableArrayType>(A))474 return getCanonicalParamType(C, VLA->getElementType());475 if (!A->isVariablyModifiedType())476 return C.getCanonicalType(T);477 }478 return C.getCanonicalParamType(T);479}480 481namespace {482/// Contains required data for proper outlined function codegen.483struct FunctionOptions {484 /// Captured statement for which the function is generated.485 const CapturedStmt *S = nullptr;486 /// true if cast to/from UIntPtr is required for variables captured by487 /// value.488 const bool UIntPtrCastRequired = true;489 /// true if only casted arguments must be registered as local args or VLA490 /// sizes.491 const bool RegisterCastedArgsOnly = false;492 /// Name of the generated function.493 const StringRef FunctionName;494 /// Location of the non-debug version of the outlined function.495 SourceLocation Loc;496 const bool IsDeviceKernel = false;497 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,498 bool RegisterCastedArgsOnly, StringRef FunctionName,499 SourceLocation Loc, bool IsDeviceKernel)500 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),501 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),502 FunctionName(FunctionName), Loc(Loc), IsDeviceKernel(IsDeviceKernel) {}503};504} // namespace505 506static llvm::Function *emitOutlinedFunctionPrologue(507 CodeGenFunction &CGF, FunctionArgList &Args,508 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>509 &LocalAddrs,510 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>511 &VLASizes,512 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {513 const CapturedDecl *CD = FO.S->getCapturedDecl();514 const RecordDecl *RD = FO.S->getCapturedRecordDecl();515 assert(CD->hasBody() && "missing CapturedDecl body");516 517 CXXThisValue = nullptr;518 // Build the argument list.519 CodeGenModule &CGM = CGF.CGM;520 ASTContext &Ctx = CGM.getContext();521 FunctionArgList TargetArgs;522 Args.append(CD->param_begin(),523 std::next(CD->param_begin(), CD->getContextParamPosition()));524 TargetArgs.append(525 CD->param_begin(),526 std::next(CD->param_begin(), CD->getContextParamPosition()));527 auto I = FO.S->captures().begin();528 FunctionDecl *DebugFunctionDecl = nullptr;529 if (!FO.UIntPtrCastRequired) {530 FunctionProtoType::ExtProtoInfo EPI;531 QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, {}, EPI);532 DebugFunctionDecl = FunctionDecl::Create(533 Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),534 SourceLocation(), DeclarationName(), FunctionTy,535 Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,536 /*UsesFPIntrin=*/false, /*isInlineSpecified=*/false,537 /*hasWrittenPrototype=*/false);538 }539 for (const FieldDecl *FD : RD->fields()) {540 QualType ArgType = FD->getType();541 IdentifierInfo *II = nullptr;542 VarDecl *CapVar = nullptr;543 544 // If this is a capture by copy and the type is not a pointer, the outlined545 // function argument type should be uintptr and the value properly casted to546 // uintptr. This is necessary given that the runtime library is only able to547 // deal with pointers. We can pass in the same way the VLA type sizes to the548 // outlined function.549 if (FO.UIntPtrCastRequired &&550 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||551 I->capturesVariableArrayType()))552 ArgType = Ctx.getUIntPtrType();553 554 if (I->capturesVariable() || I->capturesVariableByCopy()) {555 CapVar = I->getCapturedVar();556 II = CapVar->getIdentifier();557 } else if (I->capturesThis()) {558 II = &Ctx.Idents.get("this");559 } else {560 assert(I->capturesVariableArrayType());561 II = &Ctx.Idents.get("vla");562 }563 if (ArgType->isVariablyModifiedType())564 ArgType = getCanonicalParamType(Ctx, ArgType);565 VarDecl *Arg;566 if (CapVar && (CapVar->getTLSKind() != clang::VarDecl::TLS_None)) {567 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),568 II, ArgType,569 ImplicitParamKind::ThreadPrivateVar);570 } else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {571 Arg = ParmVarDecl::Create(572 Ctx, DebugFunctionDecl,573 CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),574 CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,575 /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);576 } else {577 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),578 II, ArgType, ImplicitParamKind::Other);579 }580 Args.emplace_back(Arg);581 // Do not cast arguments if we emit function with non-original types.582 TargetArgs.emplace_back(583 FO.UIntPtrCastRequired584 ? Arg585 : CGM.getOpenMPRuntime().translateParameter(FD, Arg));586 ++I;587 }588 Args.append(std::next(CD->param_begin(), CD->getContextParamPosition() + 1),589 CD->param_end());590 TargetArgs.append(591 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),592 CD->param_end());593 594 // Create the function declaration.595 const CGFunctionInfo &FuncInfo =596 FO.IsDeviceKernel597 ? CGM.getTypes().arrangeDeviceKernelCallerDeclaration(Ctx.VoidTy,598 TargetArgs)599 : CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy,600 TargetArgs);601 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);602 603 auto *F =604 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,605 FO.FunctionName, &CGM.getModule());606 CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);607 if (CD->isNothrow())608 F->setDoesNotThrow();609 F->setDoesNotRecurse();610 611 // Always inline the outlined function if optimizations are enabled.612 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {613 F->removeFnAttr(llvm::Attribute::NoInline);614 F->addFnAttr(llvm::Attribute::AlwaysInline);615 }616 617 // Generate the function.618 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,619 FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),620 FO.UIntPtrCastRequired ? FO.Loc621 : CD->getBody()->getBeginLoc());622 unsigned Cnt = CD->getContextParamPosition();623 I = FO.S->captures().begin();624 for (const FieldDecl *FD : RD->fields()) {625 // Do not map arguments if we emit function with non-original types.626 Address LocalAddr(Address::invalid());627 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {628 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],629 TargetArgs[Cnt]);630 } else {631 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);632 }633 // If we are capturing a pointer by copy we don't need to do anything, just634 // use the value that we get from the arguments.635 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {636 const VarDecl *CurVD = I->getCapturedVar();637 if (!FO.RegisterCastedArgsOnly)638 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});639 ++Cnt;640 ++I;641 continue;642 }643 644 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),645 AlignmentSource::Decl);646 if (FD->hasCapturedVLAType()) {647 if (FO.UIntPtrCastRequired) {648 ArgLVal = CGF.MakeAddrLValue(649 castValueFromUintptr(CGF, I->getLocation(), FD->getType(),650 Args[Cnt]->getName(), ArgLVal),651 FD->getType(), AlignmentSource::Decl);652 }653 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());654 const VariableArrayType *VAT = FD->getCapturedVLAType();655 VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);656 } else if (I->capturesVariable()) {657 const VarDecl *Var = I->getCapturedVar();658 QualType VarTy = Var->getType();659 Address ArgAddr = ArgLVal.getAddress();660 if (ArgLVal.getType()->isLValueReferenceType()) {661 ArgAddr = CGF.EmitLoadOfReference(ArgLVal);662 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {663 assert(ArgLVal.getType()->isPointerType());664 ArgAddr = CGF.EmitLoadOfPointer(665 ArgAddr, ArgLVal.getType()->castAs<PointerType>());666 }667 if (!FO.RegisterCastedArgsOnly) {668 LocalAddrs.insert(669 {Args[Cnt], {Var, ArgAddr.withAlignment(Ctx.getDeclAlign(Var))}});670 }671 } else if (I->capturesVariableByCopy()) {672 assert(!FD->getType()->isAnyPointerType() &&673 "Not expecting a captured pointer.");674 const VarDecl *Var = I->getCapturedVar();675 LocalAddrs.insert({Args[Cnt],676 {Var, FO.UIntPtrCastRequired677 ? castValueFromUintptr(678 CGF, I->getLocation(), FD->getType(),679 Args[Cnt]->getName(), ArgLVal)680 : ArgLVal.getAddress()}});681 } else {682 // If 'this' is captured, load it into CXXThisValue.683 assert(I->capturesThis());684 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());685 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});686 }687 ++Cnt;688 ++I;689 }690 691 return F;692}693 694llvm::Function *CodeGenFunction::GenerateOpenMPCapturedStmtFunction(695 const CapturedStmt &S, const OMPExecutableDirective &D) {696 SourceLocation Loc = D.getBeginLoc();697 assert(698 CapturedStmtInfo &&699 "CapturedStmtInfo should be set when generating the captured function");700 const CapturedDecl *CD = S.getCapturedDecl();701 // Build the argument list.702 bool NeedWrapperFunction =703 getDebugInfo() && CGM.getCodeGenOpts().hasReducedDebugInfo();704 FunctionArgList Args, WrapperArgs;705 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs,706 WrapperLocalAddrs;707 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes,708 WrapperVLASizes;709 SmallString<256> Buffer;710 llvm::raw_svector_ostream Out(Buffer);711 Out << CapturedStmtInfo->getHelperName();712 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(D);713 bool IsDeviceKernel = CGM.getOpenMPRuntime().isGPU() &&714 isOpenMPTargetExecutionDirective(EKind) &&715 D.getCapturedStmt(OMPD_target) == &S;716 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);717 llvm::Function *WrapperF = nullptr;718 if (NeedWrapperFunction) {719 // Emit the final kernel early to allow attributes to be added by the720 // OpenMPI-IR-Builder.721 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,722 /*RegisterCastedArgsOnly=*/true,723 CapturedStmtInfo->getHelperName(), Loc,724 IsDeviceKernel);725 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;726 WrapperF =727 emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,728 WrapperCGF.CXXThisValue, WrapperFO);729 Out << "_debug__";730 }731 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,732 Out.str(), Loc, !NeedWrapperFunction && IsDeviceKernel);733 llvm::Function *F = emitOutlinedFunctionPrologue(734 *this, WrapperArgs, WrapperLocalAddrs, WrapperVLASizes, CXXThisValue, FO);735 CodeGenFunction::OMPPrivateScope LocalScope(*this);736 for (const auto &LocalAddrPair : WrapperLocalAddrs) {737 if (LocalAddrPair.second.first) {738 LocalScope.addPrivate(LocalAddrPair.second.first,739 LocalAddrPair.second.second);740 }741 }742 (void)LocalScope.Privatize();743 for (const auto &VLASizePair : WrapperVLASizes)744 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;745 PGO->assignRegionCounters(GlobalDecl(CD), F);746 CapturedStmtInfo->EmitBody(*this, CD->getBody());747 LocalScope.ForceCleanup();748 FinishFunction(CD->getBodyRBrace());749 if (!NeedWrapperFunction)750 return F;751 752 // Reverse the order.753 WrapperF->removeFromParent();754 F->getParent()->getFunctionList().insertAfter(F->getIterator(), WrapperF);755 756 llvm::SmallVector<llvm::Value *, 4> CallArgs;757 auto *PI = F->arg_begin();758 for (const auto *Arg : Args) {759 llvm::Value *CallArg;760 auto I = LocalAddrs.find(Arg);761 if (I != LocalAddrs.end()) {762 LValue LV = WrapperCGF.MakeAddrLValue(763 I->second.second,764 I->second.first ? I->second.first->getType() : Arg->getType(),765 AlignmentSource::Decl);766 if (LV.getType()->isAnyComplexType())767 LV.setAddress(LV.getAddress().withElementType(PI->getType()));768 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());769 } else {770 auto EI = VLASizes.find(Arg);771 if (EI != VLASizes.end()) {772 CallArg = EI->second.second;773 } else {774 LValue LV =775 WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),776 Arg->getType(), AlignmentSource::Decl);777 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());778 }779 }780 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));781 ++PI;782 }783 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, Loc, F, CallArgs);784 WrapperCGF.FinishFunction();785 return WrapperF;786}787 788//===----------------------------------------------------------------------===//789// OpenMP Directive Emission790//===----------------------------------------------------------------------===//791void CodeGenFunction::EmitOMPAggregateAssign(792 Address DestAddr, Address SrcAddr, QualType OriginalType,793 const llvm::function_ref<void(Address, Address)> CopyGen) {794 // Perform element-by-element initialization.795 QualType ElementTy;796 797 // Drill down to the base element type on both arrays.798 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();799 llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);800 SrcAddr = SrcAddr.withElementType(DestAddr.getElementType());801 802 llvm::Value *SrcBegin = SrcAddr.emitRawPointer(*this);803 llvm::Value *DestBegin = DestAddr.emitRawPointer(*this);804 // Cast from pointer to array type to pointer to single element.805 llvm::Value *DestEnd = Builder.CreateInBoundsGEP(DestAddr.getElementType(),806 DestBegin, NumElements);807 808 // The basic structure here is a while-do loop.809 llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");810 llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");811 llvm::Value *IsEmpty =812 Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");813 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);814 815 // Enter the loop body, making that address the current address.816 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();817 EmitBlock(BodyBB);818 819 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);820 821 llvm::PHINode *SrcElementPHI =822 Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");823 SrcElementPHI->addIncoming(SrcBegin, EntryBB);824 Address SrcElementCurrent =825 Address(SrcElementPHI, SrcAddr.getElementType(),826 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));827 828 llvm::PHINode *DestElementPHI = Builder.CreatePHI(829 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");830 DestElementPHI->addIncoming(DestBegin, EntryBB);831 Address DestElementCurrent =832 Address(DestElementPHI, DestAddr.getElementType(),833 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));834 835 // Emit copy.836 CopyGen(DestElementCurrent, SrcElementCurrent);837 838 // Shift the address forward by one element.839 llvm::Value *DestElementNext =840 Builder.CreateConstGEP1_32(DestAddr.getElementType(), DestElementPHI,841 /*Idx0=*/1, "omp.arraycpy.dest.element");842 llvm::Value *SrcElementNext =843 Builder.CreateConstGEP1_32(SrcAddr.getElementType(), SrcElementPHI,844 /*Idx0=*/1, "omp.arraycpy.src.element");845 // Check whether we've reached the end.846 llvm::Value *Done =847 Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");848 Builder.CreateCondBr(Done, DoneBB, BodyBB);849 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());850 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());851 852 // Done.853 EmitBlock(DoneBB, /*IsFinished=*/true);854}855 856void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,857 Address SrcAddr, const VarDecl *DestVD,858 const VarDecl *SrcVD, const Expr *Copy) {859 if (OriginalType->isArrayType()) {860 const auto *BO = dyn_cast<BinaryOperator>(Copy);861 if (BO && BO->getOpcode() == BO_Assign) {862 // Perform simple memcpy for simple copying.863 LValue Dest = MakeAddrLValue(DestAddr, OriginalType);864 LValue Src = MakeAddrLValue(SrcAddr, OriginalType);865 EmitAggregateAssign(Dest, Src, OriginalType);866 } else {867 // For arrays with complex element types perform element by element868 // copying.869 EmitOMPAggregateAssign(870 DestAddr, SrcAddr, OriginalType,871 [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {872 // Working with the single array element, so have to remap873 // destination and source variables to corresponding array874 // elements.875 CodeGenFunction::OMPPrivateScope Remap(*this);876 Remap.addPrivate(DestVD, DestElement);877 Remap.addPrivate(SrcVD, SrcElement);878 (void)Remap.Privatize();879 EmitIgnoredExpr(Copy);880 });881 }882 } else {883 // Remap pseudo source variable to private copy.884 CodeGenFunction::OMPPrivateScope Remap(*this);885 Remap.addPrivate(SrcVD, SrcAddr);886 Remap.addPrivate(DestVD, DestAddr);887 (void)Remap.Privatize();888 // Emit copying of the whole variable.889 EmitIgnoredExpr(Copy);890 }891}892 893bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D,894 OMPPrivateScope &PrivateScope) {895 if (!HaveInsertPoint())896 return false;897 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(D);898 bool DeviceConstTarget = getLangOpts().OpenMPIsTargetDevice &&899 isOpenMPTargetExecutionDirective(EKind);900 bool FirstprivateIsLastprivate = false;901 llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;902 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {903 for (const auto *D : C->varlist())904 Lastprivates.try_emplace(905 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl(),906 C->getKind());907 }908 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;909 llvm::SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;910 getOpenMPCaptureRegions(CaptureRegions, EKind);911 // Force emission of the firstprivate copy if the directive does not emit912 // outlined function, like omp for, omp simd, omp distribute etc.913 bool MustEmitFirstprivateCopy =914 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;915 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {916 const auto *IRef = C->varlist_begin();917 const auto *InitsRef = C->inits().begin();918 for (const Expr *IInit : C->private_copies()) {919 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());920 bool ThisFirstprivateIsLastprivate =921 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;922 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);923 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());924 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&925 !FD->getType()->isReferenceType() &&926 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {927 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());928 ++IRef;929 ++InitsRef;930 continue;931 }932 // Do not emit copy for firstprivate constant variables in target regions,933 // captured by reference.934 if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&935 FD && FD->getType()->isReferenceType() &&936 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {937 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());938 ++IRef;939 ++InitsRef;940 continue;941 }942 FirstprivateIsLastprivate =943 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;944 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {945 const auto *VDInit =946 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());947 bool IsRegistered;948 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),949 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,950 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());951 LValue OriginalLVal;952 if (!FD) {953 // Check if the firstprivate variable is just a constant value.954 ConstantEmission CE = tryEmitAsConstant(&DRE);955 if (CE && !CE.isReference()) {956 // Constant value, no need to create a copy.957 ++IRef;958 ++InitsRef;959 continue;960 }961 if (CE && CE.isReference()) {962 OriginalLVal = CE.getReferenceLValue(*this, &DRE);963 } else {964 assert(!CE && "Expected non-constant firstprivate.");965 OriginalLVal = EmitLValue(&DRE);966 }967 } else {968 OriginalLVal = EmitLValue(&DRE);969 }970 QualType Type = VD->getType();971 if (Type->isArrayType()) {972 // Emit VarDecl with copy init for arrays.973 // Get the address of the original variable captured in current974 // captured region.975 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);976 const Expr *Init = VD->getInit();977 if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {978 // Perform simple memcpy.979 LValue Dest = MakeAddrLValue(Emission.getAllocatedAddress(), Type);980 EmitAggregateAssign(Dest, OriginalLVal, Type);981 } else {982 EmitOMPAggregateAssign(983 Emission.getAllocatedAddress(), OriginalLVal.getAddress(), Type,984 [this, VDInit, Init](Address DestElement, Address SrcElement) {985 // Clean up any temporaries needed by the986 // initialization.987 RunCleanupsScope InitScope(*this);988 // Emit initialization for single element.989 setAddrOfLocalVar(VDInit, SrcElement);990 EmitAnyExprToMem(Init, DestElement,991 Init->getType().getQualifiers(),992 /*IsInitializer*/ false);993 LocalDeclMap.erase(VDInit);994 });995 }996 EmitAutoVarCleanups(Emission);997 IsRegistered =998 PrivateScope.addPrivate(OrigVD, Emission.getAllocatedAddress());999 } else {1000 Address OriginalAddr = OriginalLVal.getAddress();1001 // Emit private VarDecl with copy init.1002 // Remap temp VDInit variable to the address of the original1003 // variable (for proper handling of captured global variables).1004 setAddrOfLocalVar(VDInit, OriginalAddr);1005 EmitDecl(*VD);1006 LocalDeclMap.erase(VDInit);1007 Address VDAddr = GetAddrOfLocalVar(VD);1008 if (ThisFirstprivateIsLastprivate &&1009 Lastprivates[OrigVD->getCanonicalDecl()] ==1010 OMPC_LASTPRIVATE_conditional) {1011 // Create/init special variable for lastprivate conditionals.1012 llvm::Value *V =1013 EmitLoadOfScalar(MakeAddrLValue(VDAddr, (*IRef)->getType(),1014 AlignmentSource::Decl),1015 (*IRef)->getExprLoc());1016 VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(1017 *this, OrigVD);1018 EmitStoreOfScalar(V, MakeAddrLValue(VDAddr, (*IRef)->getType(),1019 AlignmentSource::Decl));1020 LocalDeclMap.erase(VD);1021 setAddrOfLocalVar(VD, VDAddr);1022 }1023 IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);1024 }1025 assert(IsRegistered &&1026 "firstprivate var already registered as private");1027 // Silence the warning about unused variable.1028 (void)IsRegistered;1029 }1030 ++IRef;1031 ++InitsRef;1032 }1033 }1034 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();1035}1036 1037void CodeGenFunction::EmitOMPPrivateClause(1038 const OMPExecutableDirective &D,1039 CodeGenFunction::OMPPrivateScope &PrivateScope) {1040 if (!HaveInsertPoint())1041 return;1042 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;1043 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {1044 auto IRef = C->varlist_begin();1045 for (const Expr *IInit : C->private_copies()) {1046 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());1047 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {1048 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());1049 EmitDecl(*VD);1050 // Emit private VarDecl with copy init.1051 bool IsRegistered =1052 PrivateScope.addPrivate(OrigVD, GetAddrOfLocalVar(VD));1053 assert(IsRegistered && "private var already registered as private");1054 // Silence the warning about unused variable.1055 (void)IsRegistered;1056 }1057 ++IRef;1058 }1059 }1060}1061 1062bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {1063 if (!HaveInsertPoint())1064 return false;1065 // threadprivate_var1 = master_threadprivate_var1;1066 // operator=(threadprivate_var2, master_threadprivate_var2);1067 // ...1068 // __kmpc_barrier(&loc, global_tid);1069 llvm::DenseSet<const VarDecl *> CopiedVars;1070 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;1071 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {1072 auto IRef = C->varlist_begin();1073 auto ISrcRef = C->source_exprs().begin();1074 auto IDestRef = C->destination_exprs().begin();1075 for (const Expr *AssignOp : C->assignment_ops()) {1076 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());1077 QualType Type = VD->getType();1078 if (CopiedVars.insert(VD->getCanonicalDecl()).second) {1079 // Get the address of the master variable. If we are emitting code with1080 // TLS support, the address is passed from the master as field in the1081 // captured declaration.1082 Address MasterAddr = Address::invalid();1083 if (getLangOpts().OpenMPUseTLS &&1084 getContext().getTargetInfo().isTLSSupported()) {1085 assert(CapturedStmtInfo->lookup(VD) &&1086 "Copyin threadprivates should have been captured!");1087 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,1088 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());1089 MasterAddr = EmitLValue(&DRE).getAddress();1090 LocalDeclMap.erase(VD);1091 } else {1092 MasterAddr =1093 Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)1094 : CGM.GetAddrOfGlobal(VD),1095 CGM.getTypes().ConvertTypeForMem(VD->getType()),1096 getContext().getDeclAlign(VD));1097 }1098 // Get the address of the threadprivate variable.1099 Address PrivateAddr = EmitLValue(*IRef).getAddress();1100 if (CopiedVars.size() == 1) {1101 // At first check if current thread is a master thread. If it is, no1102 // need to copy data.1103 CopyBegin = createBasicBlock("copyin.not.master");1104 CopyEnd = createBasicBlock("copyin.not.master.end");1105 // TODO: Avoid ptrtoint conversion.1106 auto *MasterAddrInt = Builder.CreatePtrToInt(1107 MasterAddr.emitRawPointer(*this), CGM.IntPtrTy);1108 auto *PrivateAddrInt = Builder.CreatePtrToInt(1109 PrivateAddr.emitRawPointer(*this), CGM.IntPtrTy);1110 Builder.CreateCondBr(1111 Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,1112 CopyEnd);1113 EmitBlock(CopyBegin);1114 }1115 const auto *SrcVD =1116 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());1117 const auto *DestVD =1118 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());1119 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);1120 }1121 ++IRef;1122 ++ISrcRef;1123 ++IDestRef;1124 }1125 }1126 if (CopyEnd) {1127 // Exit out of copying procedure for non-master thread.1128 EmitBlock(CopyEnd, /*IsFinished=*/true);1129 return true;1130 }1131 return false;1132}1133 1134bool CodeGenFunction::EmitOMPLastprivateClauseInit(1135 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {1136 if (!HaveInsertPoint())1137 return false;1138 bool HasAtLeastOneLastprivate = false;1139 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(D);1140 llvm::DenseSet<const VarDecl *> SIMDLCVs;1141 if (isOpenMPSimdDirective(EKind)) {1142 const auto *LoopDirective = cast<OMPLoopDirective>(&D);1143 for (const Expr *C : LoopDirective->counters()) {1144 SIMDLCVs.insert(1145 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());1146 }1147 }1148 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;1149 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {1150 HasAtLeastOneLastprivate = true;1151 if (isOpenMPTaskLoopDirective(EKind) && !getLangOpts().OpenMPSimd)1152 break;1153 const auto *IRef = C->varlist_begin();1154 const auto *IDestRef = C->destination_exprs().begin();1155 for (const Expr *IInit : C->private_copies()) {1156 // Keep the address of the original variable for future update at the end1157 // of the loop.1158 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());1159 // Taskloops do not require additional initialization, it is done in1160 // runtime support library.1161 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {1162 const auto *DestVD =1163 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());1164 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),1165 /*RefersToEnclosingVariableOrCapture=*/1166 CapturedStmtInfo->lookup(OrigVD) != nullptr,1167 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());1168 PrivateScope.addPrivate(DestVD, EmitLValue(&DRE).getAddress());1169 // Check if the variable is also a firstprivate: in this case IInit is1170 // not generated. Initialization of this variable will happen in codegen1171 // for 'firstprivate' clause.1172 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {1173 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());1174 Address VDAddr = Address::invalid();1175 if (C->getKind() == OMPC_LASTPRIVATE_conditional) {1176 VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(1177 *this, OrigVD);1178 setAddrOfLocalVar(VD, VDAddr);1179 } else {1180 // Emit private VarDecl with copy init.1181 EmitDecl(*VD);1182 VDAddr = GetAddrOfLocalVar(VD);1183 }1184 bool IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);1185 assert(IsRegistered &&1186 "lastprivate var already registered as private");1187 (void)IsRegistered;1188 }1189 }1190 ++IRef;1191 ++IDestRef;1192 }1193 }1194 return HasAtLeastOneLastprivate;1195}1196 1197void CodeGenFunction::EmitOMPLastprivateClauseFinal(1198 const OMPExecutableDirective &D, bool NoFinals,1199 llvm::Value *IsLastIterCond) {1200 if (!HaveInsertPoint())1201 return;1202 // Emit following code:1203 // if (<IsLastIterCond>) {1204 // orig_var1 = private_orig_var1;1205 // ...1206 // orig_varn = private_orig_varn;1207 // }1208 llvm::BasicBlock *ThenBB = nullptr;1209 llvm::BasicBlock *DoneBB = nullptr;1210 if (IsLastIterCond) {1211 // Emit implicit barrier if at least one lastprivate conditional is found1212 // and this is not a simd mode.1213 if (!getLangOpts().OpenMPSimd &&1214 llvm::any_of(D.getClausesOfKind<OMPLastprivateClause>(),1215 [](const OMPLastprivateClause *C) {1216 return C->getKind() == OMPC_LASTPRIVATE_conditional;1217 })) {1218 CGM.getOpenMPRuntime().emitBarrierCall(*this, D.getBeginLoc(),1219 OMPD_unknown,1220 /*EmitChecks=*/false,1221 /*ForceSimpleCall=*/true);1222 }1223 ThenBB = createBasicBlock(".omp.lastprivate.then");1224 DoneBB = createBasicBlock(".omp.lastprivate.done");1225 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);1226 EmitBlock(ThenBB);1227 }1228 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;1229 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;1230 if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {1231 auto IC = LoopDirective->counters().begin();1232 for (const Expr *F : LoopDirective->finals()) {1233 const auto *D =1234 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();1235 if (NoFinals)1236 AlreadyEmittedVars.insert(D);1237 else1238 LoopCountersAndUpdates[D] = F;1239 ++IC;1240 }1241 }1242 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {1243 auto IRef = C->varlist_begin();1244 auto ISrcRef = C->source_exprs().begin();1245 auto IDestRef = C->destination_exprs().begin();1246 for (const Expr *AssignOp : C->assignment_ops()) {1247 const auto *PrivateVD =1248 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());1249 QualType Type = PrivateVD->getType();1250 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();1251 if (AlreadyEmittedVars.insert(CanonicalVD).second) {1252 // If lastprivate variable is a loop control variable for loop-based1253 // directive, update its value before copyin back to original1254 // variable.1255 if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))1256 EmitIgnoredExpr(FinalExpr);1257 const auto *SrcVD =1258 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());1259 const auto *DestVD =1260 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());1261 // Get the address of the private variable.1262 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);1263 if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())1264 PrivateAddr = Address(1265 Builder.CreateLoad(PrivateAddr),1266 CGM.getTypes().ConvertTypeForMem(RefTy->getPointeeType()),1267 CGM.getNaturalTypeAlignment(RefTy->getPointeeType()));1268 // Store the last value to the private copy in the last iteration.1269 if (C->getKind() == OMPC_LASTPRIVATE_conditional)1270 CGM.getOpenMPRuntime().emitLastprivateConditionalFinalUpdate(1271 *this, MakeAddrLValue(PrivateAddr, (*IRef)->getType()), PrivateVD,1272 (*IRef)->getExprLoc());1273 // Get the address of the original variable.1274 Address OriginalAddr = GetAddrOfLocalVar(DestVD);1275 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);1276 }1277 ++IRef;1278 ++ISrcRef;1279 ++IDestRef;1280 }1281 if (const Expr *PostUpdate = C->getPostUpdateExpr())1282 EmitIgnoredExpr(PostUpdate);1283 }1284 if (IsLastIterCond)1285 EmitBlock(DoneBB, /*IsFinished=*/true);1286}1287 1288void CodeGenFunction::EmitOMPReductionClauseInit(1289 const OMPExecutableDirective &D,1290 CodeGenFunction::OMPPrivateScope &PrivateScope, bool ForInscan) {1291 if (!HaveInsertPoint())1292 return;1293 SmallVector<const Expr *, 4> Shareds;1294 SmallVector<const Expr *, 4> Privates;1295 SmallVector<const Expr *, 4> ReductionOps;1296 SmallVector<const Expr *, 4> LHSs;1297 SmallVector<const Expr *, 4> RHSs;1298 OMPTaskDataTy Data;1299 SmallVector<const Expr *, 4> TaskLHSs;1300 SmallVector<const Expr *, 4> TaskRHSs;1301 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {1302 if (ForInscan != (C->getModifier() == OMPC_REDUCTION_inscan))1303 continue;1304 Shareds.append(C->varlist_begin(), C->varlist_end());1305 Privates.append(C->privates().begin(), C->privates().end());1306 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());1307 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());1308 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());1309 if (C->getModifier() == OMPC_REDUCTION_task) {1310 Data.ReductionVars.append(C->privates().begin(), C->privates().end());1311 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());1312 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());1313 Data.ReductionOps.append(C->reduction_ops().begin(),1314 C->reduction_ops().end());1315 TaskLHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());1316 TaskRHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());1317 }1318 }1319 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);1320 unsigned Count = 0;1321 auto *ILHS = LHSs.begin();1322 auto *IRHS = RHSs.begin();1323 auto *IPriv = Privates.begin();1324 for (const Expr *IRef : Shareds) {1325 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());1326 // Emit private VarDecl with reduction init.1327 RedCG.emitSharedOrigLValue(*this, Count);1328 RedCG.emitAggregateType(*this, Count);1329 AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);1330 RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),1331 RedCG.getSharedLValue(Count).getAddress(),1332 [&Emission](CodeGenFunction &CGF) {1333 CGF.EmitAutoVarInit(Emission);1334 return true;1335 });1336 EmitAutoVarCleanups(Emission);1337 Address BaseAddr = RedCG.adjustPrivateAddress(1338 *this, Count, Emission.getAllocatedAddress());1339 bool IsRegistered =1340 PrivateScope.addPrivate(RedCG.getBaseDecl(Count), BaseAddr);1341 assert(IsRegistered && "private var already registered as private");1342 // Silence the warning about unused variable.1343 (void)IsRegistered;1344 1345 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());1346 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());1347 QualType Type = PrivateVD->getType();1348 bool isaOMPArraySectionExpr = isa<ArraySectionExpr>(IRef);1349 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {1350 // Store the address of the original variable associated with the LHS1351 // implicit variable.1352 PrivateScope.addPrivate(LHSVD, RedCG.getSharedLValue(Count).getAddress());1353 PrivateScope.addPrivate(RHSVD, GetAddrOfLocalVar(PrivateVD));1354 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||1355 isa<ArraySubscriptExpr>(IRef)) {1356 // Store the address of the original variable associated with the LHS1357 // implicit variable.1358 PrivateScope.addPrivate(LHSVD, RedCG.getSharedLValue(Count).getAddress());1359 PrivateScope.addPrivate(RHSVD,1360 GetAddrOfLocalVar(PrivateVD).withElementType(1361 ConvertTypeForMem(RHSVD->getType())));1362 } else {1363 QualType Type = PrivateVD->getType();1364 bool IsArray = getContext().getAsArrayType(Type) != nullptr;1365 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();1366 // Store the address of the original variable associated with the LHS1367 // implicit variable.1368 if (IsArray) {1369 OriginalAddr =1370 OriginalAddr.withElementType(ConvertTypeForMem(LHSVD->getType()));1371 }1372 PrivateScope.addPrivate(LHSVD, OriginalAddr);1373 PrivateScope.addPrivate(1374 RHSVD, IsArray ? GetAddrOfLocalVar(PrivateVD).withElementType(1375 ConvertTypeForMem(RHSVD->getType()))1376 : GetAddrOfLocalVar(PrivateVD));1377 }1378 ++ILHS;1379 ++IRHS;1380 ++IPriv;1381 ++Count;1382 }1383 if (!Data.ReductionVars.empty()) {1384 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(D);1385 Data.IsReductionWithTaskMod = true;1386 Data.IsWorksharingReduction = isOpenMPWorksharingDirective(EKind);1387 llvm::Value *ReductionDesc = CGM.getOpenMPRuntime().emitTaskReductionInit(1388 *this, D.getBeginLoc(), TaskLHSs, TaskRHSs, Data);1389 const Expr *TaskRedRef = nullptr;1390 switch (EKind) {1391 case OMPD_parallel:1392 TaskRedRef = cast<OMPParallelDirective>(D).getTaskReductionRefExpr();1393 break;1394 case OMPD_for:1395 TaskRedRef = cast<OMPForDirective>(D).getTaskReductionRefExpr();1396 break;1397 case OMPD_sections:1398 TaskRedRef = cast<OMPSectionsDirective>(D).getTaskReductionRefExpr();1399 break;1400 case OMPD_parallel_for:1401 TaskRedRef = cast<OMPParallelForDirective>(D).getTaskReductionRefExpr();1402 break;1403 case OMPD_parallel_master:1404 TaskRedRef =1405 cast<OMPParallelMasterDirective>(D).getTaskReductionRefExpr();1406 break;1407 case OMPD_parallel_sections:1408 TaskRedRef =1409 cast<OMPParallelSectionsDirective>(D).getTaskReductionRefExpr();1410 break;1411 case OMPD_target_parallel:1412 TaskRedRef =1413 cast<OMPTargetParallelDirective>(D).getTaskReductionRefExpr();1414 break;1415 case OMPD_target_parallel_for:1416 TaskRedRef =1417 cast<OMPTargetParallelForDirective>(D).getTaskReductionRefExpr();1418 break;1419 case OMPD_distribute_parallel_for:1420 TaskRedRef =1421 cast<OMPDistributeParallelForDirective>(D).getTaskReductionRefExpr();1422 break;1423 case OMPD_teams_distribute_parallel_for:1424 TaskRedRef = cast<OMPTeamsDistributeParallelForDirective>(D)1425 .getTaskReductionRefExpr();1426 break;1427 case OMPD_target_teams_distribute_parallel_for:1428 TaskRedRef = cast<OMPTargetTeamsDistributeParallelForDirective>(D)1429 .getTaskReductionRefExpr();1430 break;1431 case OMPD_simd:1432 case OMPD_for_simd:1433 case OMPD_section:1434 case OMPD_single:1435 case OMPD_master:1436 case OMPD_critical:1437 case OMPD_parallel_for_simd:1438 case OMPD_task:1439 case OMPD_taskyield:1440 case OMPD_error:1441 case OMPD_barrier:1442 case OMPD_taskwait:1443 case OMPD_taskgroup:1444 case OMPD_flush:1445 case OMPD_depobj:1446 case OMPD_scan:1447 case OMPD_ordered:1448 case OMPD_atomic:1449 case OMPD_teams:1450 case OMPD_target:1451 case OMPD_cancellation_point:1452 case OMPD_cancel:1453 case OMPD_target_data:1454 case OMPD_target_enter_data:1455 case OMPD_target_exit_data:1456 case OMPD_taskloop:1457 case OMPD_taskloop_simd:1458 case OMPD_master_taskloop:1459 case OMPD_master_taskloop_simd:1460 case OMPD_parallel_master_taskloop:1461 case OMPD_parallel_master_taskloop_simd:1462 case OMPD_distribute:1463 case OMPD_target_update:1464 case OMPD_distribute_parallel_for_simd:1465 case OMPD_distribute_simd:1466 case OMPD_target_parallel_for_simd:1467 case OMPD_target_simd:1468 case OMPD_teams_distribute:1469 case OMPD_teams_distribute_simd:1470 case OMPD_teams_distribute_parallel_for_simd:1471 case OMPD_target_teams:1472 case OMPD_target_teams_distribute:1473 case OMPD_target_teams_distribute_parallel_for_simd:1474 case OMPD_target_teams_distribute_simd:1475 case OMPD_declare_target:1476 case OMPD_end_declare_target:1477 case OMPD_threadprivate:1478 case OMPD_allocate:1479 case OMPD_declare_reduction:1480 case OMPD_declare_mapper:1481 case OMPD_declare_simd:1482 case OMPD_requires:1483 case OMPD_declare_variant:1484 case OMPD_begin_declare_variant:1485 case OMPD_end_declare_variant:1486 case OMPD_unknown:1487 default:1488 llvm_unreachable("Unexpected directive with task reductions.");1489 }1490 1491 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl());1492 EmitVarDecl(*VD);1493 EmitStoreOfScalar(ReductionDesc, GetAddrOfLocalVar(VD),1494 /*Volatile=*/false, TaskRedRef->getType());1495 }1496}1497 1498void CodeGenFunction::EmitOMPReductionClauseFinal(1499 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {1500 if (!HaveInsertPoint())1501 return;1502 llvm::SmallVector<const Expr *, 8> Privates;1503 llvm::SmallVector<const Expr *, 8> LHSExprs;1504 llvm::SmallVector<const Expr *, 8> RHSExprs;1505 llvm::SmallVector<const Expr *, 8> ReductionOps;1506 llvm::SmallVector<bool, 8> IsPrivateVarReduction;1507 bool HasAtLeastOneReduction = false;1508 bool IsReductionWithTaskMod = false;1509 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {1510 // Do not emit for inscan reductions.1511 if (C->getModifier() == OMPC_REDUCTION_inscan)1512 continue;1513 HasAtLeastOneReduction = true;1514 Privates.append(C->privates().begin(), C->privates().end());1515 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());1516 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());1517 IsPrivateVarReduction.append(C->private_var_reduction_flags().begin(),1518 C->private_var_reduction_flags().end());1519 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());1520 IsReductionWithTaskMod =1521 IsReductionWithTaskMod || C->getModifier() == OMPC_REDUCTION_task;1522 }1523 if (HasAtLeastOneReduction) {1524 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(D);1525 if (IsReductionWithTaskMod) {1526 CGM.getOpenMPRuntime().emitTaskReductionFini(1527 *this, D.getBeginLoc(), isOpenMPWorksharingDirective(EKind));1528 }1529 bool TeamsLoopCanBeParallel = false;1530 if (auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(&D))1531 TeamsLoopCanBeParallel = TTLD->canBeParallelFor();1532 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||1533 isOpenMPParallelDirective(EKind) ||1534 TeamsLoopCanBeParallel || ReductionKind == OMPD_simd;1535 bool SimpleReduction = ReductionKind == OMPD_simd;1536 // Emit nowait reduction if nowait clause is present or directive is a1537 // parallel directive (it always has implicit barrier).1538 CGM.getOpenMPRuntime().emitReduction(1539 *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,1540 {WithNowait, SimpleReduction, IsPrivateVarReduction, ReductionKind});1541 }1542}1543 1544static void emitPostUpdateForReductionClause(1545 CodeGenFunction &CGF, const OMPExecutableDirective &D,1546 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {1547 if (!CGF.HaveInsertPoint())1548 return;1549 llvm::BasicBlock *DoneBB = nullptr;1550 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {1551 if (const Expr *PostUpdate = C->getPostUpdateExpr()) {1552 if (!DoneBB) {1553 if (llvm::Value *Cond = CondGen(CGF)) {1554 // If the first post-update expression is found, emit conditional1555 // block if it was requested.1556 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");1557 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");1558 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);1559 CGF.EmitBlock(ThenBB);1560 }1561 }1562 CGF.EmitIgnoredExpr(PostUpdate);1563 }1564 }1565 if (DoneBB)1566 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);1567}1568 1569namespace {1570/// Codegen lambda for appending distribute lower and upper bounds to outlined1571/// parallel function. This is necessary for combined constructs such as1572/// 'distribute parallel for'1573typedef llvm::function_ref<void(CodeGenFunction &,1574 const OMPExecutableDirective &,1575 llvm::SmallVectorImpl<llvm::Value *> &)>1576 CodeGenBoundParametersTy;1577} // anonymous namespace1578 1579static void1580checkForLastprivateConditionalUpdate(CodeGenFunction &CGF,1581 const OMPExecutableDirective &S) {1582 if (CGF.getLangOpts().OpenMP < 50)1583 return;1584 llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> PrivateDecls;1585 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {1586 for (const Expr *Ref : C->varlist()) {1587 if (!Ref->getType()->isScalarType())1588 continue;1589 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());1590 if (!DRE)1591 continue;1592 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));1593 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref);1594 }1595 }1596 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {1597 for (const Expr *Ref : C->varlist()) {1598 if (!Ref->getType()->isScalarType())1599 continue;1600 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());1601 if (!DRE)1602 continue;1603 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));1604 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref);1605 }1606 }1607 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {1608 for (const Expr *Ref : C->varlist()) {1609 if (!Ref->getType()->isScalarType())1610 continue;1611 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());1612 if (!DRE)1613 continue;1614 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));1615 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref);1616 }1617 }1618 // Privates should ne analyzed since they are not captured at all.1619 // Task reductions may be skipped - tasks are ignored.1620 // Firstprivates do not return value but may be passed by reference - no need1621 // to check for updated lastprivate conditional.1622 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {1623 for (const Expr *Ref : C->varlist()) {1624 if (!Ref->getType()->isScalarType())1625 continue;1626 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());1627 if (!DRE)1628 continue;1629 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));1630 }1631 }1632 CGF.CGM.getOpenMPRuntime().checkAndEmitSharedLastprivateConditional(1633 CGF, S, PrivateDecls);1634}1635 1636static void emitCommonOMPParallelDirective(1637 CodeGenFunction &CGF, const OMPExecutableDirective &S,1638 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,1639 const CodeGenBoundParametersTy &CodeGenBoundParameters) {1640 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);1641 llvm::Value *NumThreads = nullptr;1642 OpenMPNumThreadsClauseModifier Modifier = OMPC_NUMTHREADS_unknown;1643 // OpenMP 6.0, 10.4: "If no severity clause is specified then the effect is as1644 // if sev-level is fatal."1645 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal;1646 clang::Expr *Message = nullptr;1647 SourceLocation SeverityLoc = SourceLocation();1648 SourceLocation MessageLoc = SourceLocation();1649 1650 llvm::Function *OutlinedFn =1651 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(1652 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind,1653 CodeGen);1654 1655 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {1656 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);1657 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),1658 /*IgnoreResultAssign=*/true);1659 Modifier = NumThreadsClause->getModifier();1660 if (const auto *MessageClause = S.getSingleClause<OMPMessageClause>()) {1661 Message = MessageClause->getMessageString();1662 MessageLoc = MessageClause->getBeginLoc();1663 }1664 if (const auto *SeverityClause = S.getSingleClause<OMPSeverityClause>()) {1665 Severity = SeverityClause->getSeverityKind();1666 SeverityLoc = SeverityClause->getBeginLoc();1667 }1668 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(1669 CGF, NumThreads, NumThreadsClause->getBeginLoc(), Modifier, Severity,1670 SeverityLoc, Message, MessageLoc);1671 }1672 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {1673 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);1674 CGF.CGM.getOpenMPRuntime().emitProcBindClause(1675 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());1676 }1677 const Expr *IfCond = nullptr;1678 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {1679 if (C->getNameModifier() == OMPD_unknown ||1680 C->getNameModifier() == OMPD_parallel) {1681 IfCond = C->getCondition();1682 break;1683 }1684 }1685 1686 OMPParallelScope Scope(CGF, S);1687 llvm::SmallVector<llvm::Value *, 16> CapturedVars;1688 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk1689 // lower and upper bounds with the pragma 'for' chunking mechanism.1690 // The following lambda takes care of appending the lower and upper bound1691 // parameters when necessary1692 CodeGenBoundParameters(CGF, S, CapturedVars);1693 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);1694 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,1695 CapturedVars, IfCond, NumThreads,1696 Modifier, Severity, Message);1697}1698 1699static bool isAllocatableDecl(const VarDecl *VD) {1700 const VarDecl *CVD = VD->getCanonicalDecl();1701 if (!CVD->hasAttr<OMPAllocateDeclAttr>())1702 return false;1703 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();1704 // Use the default allocation.1705 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||1706 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&1707 !AA->getAllocator());1708}1709 1710static void emitEmptyBoundParameters(CodeGenFunction &,1711 const OMPExecutableDirective &,1712 llvm::SmallVectorImpl<llvm::Value *> &) {}1713 1714static void emitOMPCopyinClause(CodeGenFunction &CGF,1715 const OMPExecutableDirective &S) {1716 bool Copyins = CGF.EmitOMPCopyinClause(S);1717 if (Copyins) {1718 // Emit implicit barrier to synchronize threads and avoid data races on1719 // propagation master's thread values of threadprivate variables to local1720 // instances of that variables of all other implicit threads.1721 CGF.CGM.getOpenMPRuntime().emitBarrierCall(1722 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,1723 /*ForceSimpleCall=*/true);1724 }1725}1726 1727Address CodeGenFunction::OMPBuilderCBHelpers::getAddressOfLocalVariable(1728 CodeGenFunction &CGF, const VarDecl *VD) {1729 CodeGenModule &CGM = CGF.CGM;1730 auto &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();1731 1732 if (!VD)1733 return Address::invalid();1734 const VarDecl *CVD = VD->getCanonicalDecl();1735 if (!isAllocatableDecl(CVD))1736 return Address::invalid();1737 llvm::Value *Size;1738 CharUnits Align = CGM.getContext().getDeclAlign(CVD);1739 if (CVD->getType()->isVariablyModifiedType()) {1740 Size = CGF.getTypeSize(CVD->getType());1741 // Align the size: ((size + align - 1) / align) * align1742 Size = CGF.Builder.CreateNUWAdd(1743 Size, CGM.getSize(Align - CharUnits::fromQuantity(1)));1744 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));1745 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));1746 } else {1747 CharUnits Sz = CGM.getContext().getTypeSizeInChars(CVD->getType());1748 Size = CGM.getSize(Sz.alignTo(Align));1749 }1750 1751 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();1752 assert(AA->getAllocator() &&1753 "Expected allocator expression for non-default allocator.");1754 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator());1755 // According to the standard, the original allocator type is a enum (integer).1756 // Convert to pointer type, if required.1757 if (Allocator->getType()->isIntegerTy())1758 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy);1759 else if (Allocator->getType()->isPointerTy())1760 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator,1761 CGM.VoidPtrTy);1762 1763 llvm::Value *Addr = OMPBuilder.createOMPAlloc(1764 CGF.Builder, Size, Allocator,1765 getNameWithSeparators({CVD->getName(), ".void.addr"}, ".", "."));1766 llvm::CallInst *FreeCI =1767 OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator);1768 1769 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI);1770 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(1771 Addr,1772 CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())),1773 getNameWithSeparators({CVD->getName(), ".addr"}, ".", "."));1774 return Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align);1775}1776 1777Address CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(1778 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr,1779 SourceLocation Loc) {1780 CodeGenModule &CGM = CGF.CGM;1781 if (CGM.getLangOpts().OpenMPUseTLS &&1782 CGM.getContext().getTargetInfo().isTLSSupported())1783 return VDAddr;1784 1785 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();1786 1787 llvm::Type *VarTy = VDAddr.getElementType();1788 llvm::Value *Data =1789 CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy);1790 llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy));1791 std::string Suffix = getNameWithSeparators({"cache", ""});1792 llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix);1793 1794 llvm::CallInst *ThreadPrivateCacheCall =1795 OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName);1796 1797 return Address(ThreadPrivateCacheCall, CGM.Int8Ty, VDAddr.getAlignment());1798}1799 1800std::string CodeGenFunction::OMPBuilderCBHelpers::getNameWithSeparators(1801 ArrayRef<StringRef> Parts, StringRef FirstSeparator, StringRef Separator) {1802 SmallString<128> Buffer;1803 llvm::raw_svector_ostream OS(Buffer);1804 StringRef Sep = FirstSeparator;1805 for (StringRef Part : Parts) {1806 OS << Sep << Part;1807 Sep = Separator;1808 }1809 return OS.str().str();1810}1811 1812void CodeGenFunction::OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(1813 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,1814 InsertPointTy CodeGenIP, Twine RegionName) {1815 CGBuilderTy &Builder = CGF.Builder;1816 Builder.restoreIP(CodeGenIP);1817 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,1818 "." + RegionName + ".after");1819 1820 {1821 OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);1822 CGF.EmitStmt(RegionBodyStmt);1823 }1824 1825 if (Builder.saveIP().isSet())1826 Builder.CreateBr(FiniBB);1827}1828 1829void CodeGenFunction::OMPBuilderCBHelpers::EmitOMPOutlinedRegionBody(1830 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,1831 InsertPointTy CodeGenIP, Twine RegionName) {1832 CGBuilderTy &Builder = CGF.Builder;1833 Builder.restoreIP(CodeGenIP);1834 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,1835 "." + RegionName + ".after");1836 1837 {1838 OMPBuilderCBHelpers::OutlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);1839 CGF.EmitStmt(RegionBodyStmt);1840 }1841 1842 if (Builder.saveIP().isSet())1843 Builder.CreateBr(FiniBB);1844}1845 1846void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {1847 if (CGM.getLangOpts().OpenMPIRBuilder) {1848 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();1849 // Check if we have any if clause associated with the directive.1850 llvm::Value *IfCond = nullptr;1851 if (const auto *C = S.getSingleClause<OMPIfClause>())1852 IfCond = EmitScalarExpr(C->getCondition(),1853 /*IgnoreResultAssign=*/true);1854 1855 llvm::Value *NumThreads = nullptr;1856 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>())1857 NumThreads = EmitScalarExpr(NumThreadsClause->getNumThreads(),1858 /*IgnoreResultAssign=*/true);1859 1860 ProcBindKind ProcBind = OMP_PROC_BIND_default;1861 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>())1862 ProcBind = ProcBindClause->getProcBindKind();1863 1864 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;1865 1866 // The cleanup callback that finalizes all variables at the given location,1867 // thus calls destructors etc.1868 auto FiniCB = [this](InsertPointTy IP) {1869 OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);1870 return llvm::Error::success();1871 };1872 1873 // Privatization callback that performs appropriate action for1874 // shared/private/firstprivate/lastprivate/copyin/... variables.1875 //1876 // TODO: This defaults to shared right now.1877 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,1878 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {1879 // The next line is appropriate only for variables (Val) with the1880 // data-sharing attribute "shared".1881 ReplVal = &Val;1882 1883 return CodeGenIP;1884 };1885 1886 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);1887 const Stmt *ParallelRegionBodyStmt = CS->getCapturedStmt();1888 1889 auto BodyGenCB = [&, this](InsertPointTy AllocaIP,1890 InsertPointTy CodeGenIP) {1891 OMPBuilderCBHelpers::EmitOMPOutlinedRegionBody(1892 *this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP, "parallel");1893 return llvm::Error::success();1894 };1895 1896 CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);1897 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);1898 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(1899 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());1900 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(1901 OMPBuilder.createParallel(Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,1902 IfCond, NumThreads, ProcBind, S.hasCancel()));1903 Builder.restoreIP(AfterIP);1904 return;1905 }1906 1907 // Emit parallel region as a standalone region.1908 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {1909 Action.Enter(CGF);1910 OMPPrivateScope PrivateScope(CGF);1911 emitOMPCopyinClause(CGF, S);1912 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);1913 CGF.EmitOMPPrivateClause(S, PrivateScope);1914 CGF.EmitOMPReductionClauseInit(S, PrivateScope);1915 (void)PrivateScope.Privatize();1916 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());1917 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);1918 };1919 {1920 auto LPCRegion =1921 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);1922 emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,1923 emitEmptyBoundParameters);1924 emitPostUpdateForReductionClause(*this, S,1925 [](CodeGenFunction &) { return nullptr; });1926 }1927 // Check for outer lastprivate conditional update.1928 checkForLastprivateConditionalUpdate(*this, S);1929}1930 1931void CodeGenFunction::EmitOMPMetaDirective(const OMPMetaDirective &S) {1932 EmitStmt(S.getIfStmt());1933}1934 1935namespace {1936/// RAII to handle scopes for loop transformation directives.1937class OMPTransformDirectiveScopeRAII {1938 OMPLoopScope *Scope = nullptr;1939 CodeGenFunction::CGCapturedStmtInfo *CGSI = nullptr;1940 CodeGenFunction::CGCapturedStmtRAII *CapInfoRAII = nullptr;1941 1942 OMPTransformDirectiveScopeRAII(const OMPTransformDirectiveScopeRAII &) =1943 delete;1944 OMPTransformDirectiveScopeRAII &1945 operator=(const OMPTransformDirectiveScopeRAII &) = delete;1946 1947public:1948 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) {1949 if (const auto *Dir = dyn_cast<OMPLoopBasedDirective>(S)) {1950 Scope = new OMPLoopScope(CGF, *Dir);1951 CGSI = new CodeGenFunction::CGCapturedStmtInfo(CR_OpenMP);1952 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);1953 } else if (const auto *Dir =1954 dyn_cast<OMPCanonicalLoopSequenceTransformationDirective>(1955 S)) {1956 // For simplicity we reuse the loop scope similarly to what we do with1957 // OMPCanonicalLoopNestTransformationDirective do by being a subclass1958 // of OMPLoopBasedDirective.1959 Scope = new OMPLoopScope(CGF, *Dir);1960 CGSI = new CodeGenFunction::CGCapturedStmtInfo(CR_OpenMP);1961 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);1962 }1963 }1964 ~OMPTransformDirectiveScopeRAII() {1965 if (!Scope)1966 return;1967 delete CapInfoRAII;1968 delete CGSI;1969 delete Scope;1970 }1971};1972} // namespace1973 1974static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,1975 int MaxLevel, int Level = 0) {1976 assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");1977 const Stmt *SimplifiedS = S->IgnoreContainers();1978 if (const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {1979 PrettyStackTraceLoc CrashInfo(1980 CGF.getContext().getSourceManager(), CS->getLBracLoc(),1981 "LLVM IR generation of compound statement ('{}')");1982 1983 // Keep track of the current cleanup stack depth, including debug scopes.1984 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange());1985 for (const Stmt *CurStmt : CS->body())1986 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);1987 return;1988 }1989 if (SimplifiedS == NextLoop) {1990 if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))1991 SimplifiedS = Dir->getTransformedStmt();1992 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))1993 SimplifiedS = CanonLoop->getLoopStmt();1994 if (const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {1995 S = For->getBody();1996 } else {1997 assert(isa<CXXForRangeStmt>(SimplifiedS) &&1998 "Expected canonical for loop or range-based for loop.");1999 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);2000 CGF.EmitStmt(CXXFor->getLoopVarStmt());2001 S = CXXFor->getBody();2002 }2003 if (Level + 1 < MaxLevel) {2004 NextLoop = OMPLoopDirective::tryToFindNextInnerLoop(2005 S, /*TryImperfectlyNestedLoops=*/true);2006 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);2007 return;2008 }2009 }2010 CGF.EmitStmt(S);2011}2012 2013void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D,2014 JumpDest LoopExit) {2015 RunCleanupsScope BodyScope(*this);2016 // Update counters values on current iteration.2017 for (const Expr *UE : D.updates())2018 EmitIgnoredExpr(UE);2019 // Update the linear variables.2020 // In distribute directives only loop counters may be marked as linear, no2021 // need to generate the code for them.2022 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(D);2023 if (!isOpenMPDistributeDirective(EKind)) {2024 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {2025 for (const Expr *UE : C->updates())2026 EmitIgnoredExpr(UE);2027 }2028 }2029 2030 // On a continue in the body, jump to the end.2031 JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");2032 BreakContinueStack.push_back(BreakContinue(D, LoopExit, Continue));2033 for (const Expr *E : D.finals_conditions()) {2034 if (!E)2035 continue;2036 // Check that loop counter in non-rectangular nest fits into the iteration2037 // space.2038 llvm::BasicBlock *NextBB = createBasicBlock("omp.body.next");2039 EmitBranchOnBoolExpr(E, NextBB, Continue.getBlock(),2040 getProfileCount(D.getBody()));2041 EmitBlock(NextBB);2042 }2043 2044 OMPPrivateScope InscanScope(*this);2045 EmitOMPReductionClauseInit(D, InscanScope, /*ForInscan=*/true);2046 bool IsInscanRegion = InscanScope.Privatize();2047 if (IsInscanRegion) {2048 // Need to remember the block before and after scan directive2049 // to dispatch them correctly depending on the clause used in2050 // this directive, inclusive or exclusive. For inclusive scan the natural2051 // order of the blocks is used, for exclusive clause the blocks must be2052 // executed in reverse order.2053 OMPBeforeScanBlock = createBasicBlock("omp.before.scan.bb");2054 OMPAfterScanBlock = createBasicBlock("omp.after.scan.bb");2055 // No need to allocate inscan exit block, in simd mode it is selected in the2056 // codegen for the scan directive.2057 if (EKind != OMPD_simd && !getLangOpts().OpenMPSimd)2058 OMPScanExitBlock = createBasicBlock("omp.exit.inscan.bb");2059 OMPScanDispatch = createBasicBlock("omp.inscan.dispatch");2060 EmitBranch(OMPScanDispatch);2061 EmitBlock(OMPBeforeScanBlock);2062 }2063 2064 // Emit loop variables for C++ range loops.2065 const Stmt *Body =2066 D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();2067 // Emit loop body.2068 emitBody(*this, Body,2069 OMPLoopBasedDirective::tryToFindNextInnerLoop(2070 Body, /*TryImperfectlyNestedLoops=*/true),2071 D.getLoopsNumber());2072 2073 // Jump to the dispatcher at the end of the loop body.2074 if (IsInscanRegion)2075 EmitBranch(OMPScanExitBlock);2076 2077 // The end (updates/cleanups).2078 EmitBlock(Continue.getBlock());2079 BreakContinueStack.pop_back();2080}2081 2082using EmittedClosureTy = std::pair<llvm::Function *, llvm::Value *>;2083 2084/// Emit a captured statement and return the function as well as its captured2085/// closure context.2086static EmittedClosureTy emitCapturedStmtFunc(CodeGenFunction &ParentCGF,2087 const CapturedStmt *S) {2088 LValue CapStruct = ParentCGF.InitCapturedStruct(*S);2089 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true);2090 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =2091 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);2092 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());2093 llvm::Function *F = CGF.GenerateCapturedStmtFunction(*S);2094 2095 return {F, CapStruct.getPointer(ParentCGF)};2096}2097 2098/// Emit a call to a previously captured closure.2099static llvm::CallInst *2100emitCapturedStmtCall(CodeGenFunction &ParentCGF, EmittedClosureTy Cap,2101 llvm::ArrayRef<llvm::Value *> Args) {2102 // Append the closure context to the argument.2103 SmallVector<llvm::Value *> EffectiveArgs;2104 EffectiveArgs.reserve(Args.size() + 1);2105 llvm::append_range(EffectiveArgs, Args);2106 EffectiveArgs.push_back(Cap.second);2107 2108 return ParentCGF.Builder.CreateCall(Cap.first, EffectiveArgs);2109}2110 2111llvm::CanonicalLoopInfo *2112CodeGenFunction::EmitOMPCollapsedCanonicalLoopNest(const Stmt *S, int Depth) {2113 assert(Depth == 1 && "Nested loops with OpenMPIRBuilder not yet implemented");2114 2115 // The caller is processing the loop-associated directive processing the \p2116 // Depth loops nested in \p S. Put the previous pending loop-associated2117 // directive to the stack. If the current loop-associated directive is a loop2118 // transformation directive, it will push its generated loops onto the stack2119 // such that together with the loops left here they form the combined loop2120 // nest for the parent loop-associated directive.2121 int ParentExpectedOMPLoopDepth = ExpectedOMPLoopDepth;2122 ExpectedOMPLoopDepth = Depth;2123 2124 EmitStmt(S);2125 assert(OMPLoopNestStack.size() >= (size_t)Depth && "Found too few loops");2126 2127 // The last added loop is the outermost one.2128 llvm::CanonicalLoopInfo *Result = OMPLoopNestStack.back();2129 2130 // Pop the \p Depth loops requested by the call from that stack and restore2131 // the previous context.2132 OMPLoopNestStack.pop_back_n(Depth);2133 ExpectedOMPLoopDepth = ParentExpectedOMPLoopDepth;2134 2135 return Result;2136}2137 2138void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) {2139 const Stmt *SyntacticalLoop = S->getLoopStmt();2140 if (!getLangOpts().OpenMPIRBuilder) {2141 // Ignore if OpenMPIRBuilder is not enabled.2142 EmitStmt(SyntacticalLoop);2143 return;2144 }2145 2146 LexicalScope ForScope(*this, S->getSourceRange());2147 2148 // Emit init statements. The Distance/LoopVar funcs may reference variable2149 // declarations they contain.2150 const Stmt *BodyStmt;2151 if (const auto *For = dyn_cast<ForStmt>(SyntacticalLoop)) {2152 if (const Stmt *InitStmt = For->getInit())2153 EmitStmt(InitStmt);2154 BodyStmt = For->getBody();2155 } else if (const auto *RangeFor =2156 dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {2157 if (const DeclStmt *RangeStmt = RangeFor->getRangeStmt())2158 EmitStmt(RangeStmt);2159 if (const DeclStmt *BeginStmt = RangeFor->getBeginStmt())2160 EmitStmt(BeginStmt);2161 if (const DeclStmt *EndStmt = RangeFor->getEndStmt())2162 EmitStmt(EndStmt);2163 if (const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())2164 EmitStmt(LoopVarStmt);2165 BodyStmt = RangeFor->getBody();2166 } else2167 llvm_unreachable("Expected for-stmt or range-based for-stmt");2168 2169 // Emit closure for later use. By-value captures will be captured here.2170 const CapturedStmt *DistanceFunc = S->getDistanceFunc();2171 EmittedClosureTy DistanceClosure = emitCapturedStmtFunc(*this, DistanceFunc);2172 const CapturedStmt *LoopVarFunc = S->getLoopVarFunc();2173 EmittedClosureTy LoopVarClosure = emitCapturedStmtFunc(*this, LoopVarFunc);2174 2175 // Call the distance function to get the number of iterations of the loop to2176 // come.2177 QualType LogicalTy = DistanceFunc->getCapturedDecl()2178 ->getParam(0)2179 ->getType()2180 .getNonReferenceType();2181 RawAddress CountAddr = CreateMemTemp(LogicalTy, ".count.addr");2182 emitCapturedStmtCall(*this, DistanceClosure, {CountAddr.getPointer()});2183 llvm::Value *DistVal = Builder.CreateLoad(CountAddr, ".count");2184 2185 // Emit the loop structure.2186 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();2187 auto BodyGen = [&, this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,2188 llvm::Value *IndVar) {2189 Builder.restoreIP(CodeGenIP);2190 2191 // Emit the loop body: Convert the logical iteration number to the loop2192 // variable and emit the body.2193 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();2194 LValue LCVal = EmitLValue(LoopVarRef);2195 Address LoopVarAddress = LCVal.getAddress();2196 emitCapturedStmtCall(*this, LoopVarClosure,2197 {LoopVarAddress.emitRawPointer(*this), IndVar});2198 2199 RunCleanupsScope BodyScope(*this);2200 EmitStmt(BodyStmt);2201 return llvm::Error::success();2202 };2203 2204 llvm::CanonicalLoopInfo *CL =2205 cantFail(OMPBuilder.createCanonicalLoop(Builder, BodyGen, DistVal));2206 2207 // Finish up the loop.2208 Builder.restoreIP(CL->getAfterIP());2209 ForScope.ForceCleanup();2210 2211 // Remember the CanonicalLoopInfo for parent AST nodes consuming it.2212 OMPLoopNestStack.push_back(CL);2213}2214 2215void CodeGenFunction::EmitOMPInnerLoop(2216 const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond,2217 const Expr *IncExpr,2218 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,2219 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {2220 auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");2221 2222 // Start the loop with a block that tests the condition.2223 auto CondBlock = createBasicBlock("omp.inner.for.cond");2224 EmitBlock(CondBlock);2225 const SourceRange R = S.getSourceRange();2226 2227 // If attributes are attached, push to the basic block with them.2228 const auto &OMPED = cast<OMPExecutableDirective>(S);2229 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();2230 const Stmt *SS = ICS->getCapturedStmt();2231 const AttributedStmt *AS = dyn_cast_or_null<AttributedStmt>(SS);2232 OMPLoopNestStack.clear();2233 if (AS)2234 LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(),2235 AS->getAttrs(), SourceLocToDebugLoc(R.getBegin()),2236 SourceLocToDebugLoc(R.getEnd()));2237 else2238 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),2239 SourceLocToDebugLoc(R.getEnd()));2240 2241 // If there are any cleanups between here and the loop-exit scope,2242 // create a block to stage a loop exit along.2243 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();2244 if (RequiresCleanup)2245 ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");2246 2247 llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");2248 2249 // Emit condition.2250 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));2251 if (ExitBlock != LoopExit.getBlock()) {2252 EmitBlock(ExitBlock);2253 EmitBranchThroughCleanup(LoopExit);2254 }2255 2256 EmitBlock(LoopBody);2257 incrementProfileCounter(&S);2258 2259 // Create a block for the increment.2260 JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");2261 BreakContinueStack.push_back(BreakContinue(S, LoopExit, Continue));2262 2263 BodyGen(*this);2264 2265 // Emit "IV = IV + 1" and a back-edge to the condition block.2266 EmitBlock(Continue.getBlock());2267 EmitIgnoredExpr(IncExpr);2268 PostIncGen(*this);2269 BreakContinueStack.pop_back();2270 EmitBranch(CondBlock);2271 LoopStack.pop();2272 // Emit the fall-through block.2273 EmitBlock(LoopExit.getBlock());2274}2275 2276bool CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) {2277 if (!HaveInsertPoint())2278 return false;2279 // Emit inits for the linear variables.2280 bool HasLinears = false;2281 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {2282 for (const Expr *Init : C->inits()) {2283 HasLinears = true;2284 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());2285 if (const auto *Ref =2286 dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {2287 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);2288 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());2289 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),2290 CapturedStmtInfo->lookup(OrigVD) != nullptr,2291 VD->getInit()->getType(), VK_LValue,2292 VD->getInit()->getExprLoc());2293 EmitExprAsInit(2294 &DRE, VD,2295 MakeAddrLValue(Emission.getAllocatedAddress(), VD->getType()),2296 /*capturedByInit=*/false);2297 EmitAutoVarCleanups(Emission);2298 } else {2299 EmitVarDecl(*VD);2300 }2301 }2302 // Emit the linear steps for the linear clauses.2303 // If a step is not constant, it is pre-calculated before the loop.2304 if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))2305 if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {2306 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));2307 // Emit calculation of the linear step.2308 EmitIgnoredExpr(CS);2309 }2310 }2311 return HasLinears;2312}2313 2314void CodeGenFunction::EmitOMPLinearClauseFinal(2315 const OMPLoopDirective &D,2316 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {2317 if (!HaveInsertPoint())2318 return;2319 llvm::BasicBlock *DoneBB = nullptr;2320 // Emit the final values of the linear variables.2321 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {2322 auto IC = C->varlist_begin();2323 for (const Expr *F : C->finals()) {2324 if (!DoneBB) {2325 if (llvm::Value *Cond = CondGen(*this)) {2326 // If the first post-update expression is found, emit conditional2327 // block if it was requested.2328 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");2329 DoneBB = createBasicBlock(".omp.linear.pu.done");2330 Builder.CreateCondBr(Cond, ThenBB, DoneBB);2331 EmitBlock(ThenBB);2332 }2333 }2334 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());2335 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),2336 CapturedStmtInfo->lookup(OrigVD) != nullptr,2337 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());2338 Address OrigAddr = EmitLValue(&DRE).getAddress();2339 CodeGenFunction::OMPPrivateScope VarScope(*this);2340 VarScope.addPrivate(OrigVD, OrigAddr);2341 (void)VarScope.Privatize();2342 EmitIgnoredExpr(F);2343 ++IC;2344 }2345 if (const Expr *PostUpdate = C->getPostUpdateExpr())2346 EmitIgnoredExpr(PostUpdate);2347 }2348 if (DoneBB)2349 EmitBlock(DoneBB, /*IsFinished=*/true);2350}2351 2352static void emitAlignedClause(CodeGenFunction &CGF,2353 const OMPExecutableDirective &D) {2354 if (!CGF.HaveInsertPoint())2355 return;2356 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {2357 llvm::APInt ClauseAlignment(64, 0);2358 if (const Expr *AlignmentExpr = Clause->getAlignment()) {2359 auto *AlignmentCI =2360 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));2361 ClauseAlignment = AlignmentCI->getValue();2362 }2363 for (const Expr *E : Clause->varlist()) {2364 llvm::APInt Alignment(ClauseAlignment);2365 if (Alignment == 0) {2366 // OpenMP [2.8.1, Description]2367 // If no optional parameter is specified, implementation-defined default2368 // alignments for SIMD instructions on the target platforms are assumed.2369 Alignment =2370 CGF.getContext()2371 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(2372 E->getType()->getPointeeType()))2373 .getQuantity();2374 }2375 assert((Alignment == 0 || Alignment.isPowerOf2()) &&2376 "alignment is not power of 2");2377 if (Alignment != 0) {2378 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);2379 CGF.emitAlignmentAssumption(2380 PtrValue, E, /*No second loc needed*/ SourceLocation(),2381 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment));2382 }2383 }2384 }2385}2386 2387void CodeGenFunction::EmitOMPPrivateLoopCounters(2388 const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {2389 if (!HaveInsertPoint())2390 return;2391 auto I = S.private_counters().begin();2392 for (const Expr *E : S.counters()) {2393 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());2394 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());2395 // Emit var without initialization.2396 AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);2397 EmitAutoVarCleanups(VarEmission);2398 LocalDeclMap.erase(PrivateVD);2399 (void)LoopScope.addPrivate(VD, VarEmission.getAllocatedAddress());2400 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||2401 VD->hasGlobalStorage()) {2402 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),2403 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),2404 E->getType(), VK_LValue, E->getExprLoc());2405 (void)LoopScope.addPrivate(PrivateVD, EmitLValue(&DRE).getAddress());2406 } else {2407 (void)LoopScope.addPrivate(PrivateVD, VarEmission.getAllocatedAddress());2408 }2409 ++I;2410 }2411 // Privatize extra loop counters used in loops for ordered(n) clauses.2412 for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {2413 if (!C->getNumForLoops())2414 continue;2415 for (unsigned I = S.getLoopsNumber(), E = C->getLoopNumIterations().size();2416 I < E; ++I) {2417 const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));2418 const auto *VD = cast<VarDecl>(DRE->getDecl());2419 // Override only those variables that can be captured to avoid re-emission2420 // of the variables declared within the loops.2421 if (DRE->refersToEnclosingVariableOrCapture()) {2422 (void)LoopScope.addPrivate(2423 VD, CreateMemTemp(DRE->getType(), VD->getName()));2424 }2425 }2426 }2427}2428 2429static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,2430 const Expr *Cond, llvm::BasicBlock *TrueBlock,2431 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {2432 if (!CGF.HaveInsertPoint())2433 return;2434 {2435 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);2436 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);2437 (void)PreCondScope.Privatize();2438 // Get initial values of real counters.2439 for (const Expr *I : S.inits()) {2440 CGF.EmitIgnoredExpr(I);2441 }2442 }2443 // Create temp loop control variables with their init values to support2444 // non-rectangular loops.2445 CodeGenFunction::OMPMapVars PreCondVars;2446 for (const Expr *E : S.dependent_counters()) {2447 if (!E)2448 continue;2449 assert(!E->getType().getNonReferenceType()->isRecordType() &&2450 "dependent counter must not be an iterator.");2451 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());2452 Address CounterAddr =2453 CGF.CreateMemTemp(VD->getType().getNonReferenceType());2454 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);2455 }2456 (void)PreCondVars.apply(CGF);2457 for (const Expr *E : S.dependent_inits()) {2458 if (!E)2459 continue;2460 CGF.EmitIgnoredExpr(E);2461 }2462 // Check that loop is executed at least one time.2463 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);2464 PreCondVars.restore(CGF);2465}2466 2467void CodeGenFunction::EmitOMPLinearClause(2468 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {2469 if (!HaveInsertPoint())2470 return;2471 llvm::DenseSet<const VarDecl *> SIMDLCVs;2472 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(D);2473 if (isOpenMPSimdDirective(EKind)) {2474 const auto *LoopDirective = cast<OMPLoopDirective>(&D);2475 for (const Expr *C : LoopDirective->counters()) {2476 SIMDLCVs.insert(2477 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());2478 }2479 }2480 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {2481 auto CurPrivate = C->privates().begin();2482 for (const Expr *E : C->varlist()) {2483 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());2484 const auto *PrivateVD =2485 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());2486 if (!SIMDLCVs.count(VD->getCanonicalDecl())) {2487 // Emit private VarDecl with copy init.2488 EmitVarDecl(*PrivateVD);2489 bool IsRegistered =2490 PrivateScope.addPrivate(VD, GetAddrOfLocalVar(PrivateVD));2491 assert(IsRegistered && "linear var already registered as private");2492 // Silence the warning about unused variable.2493 (void)IsRegistered;2494 } else {2495 EmitVarDecl(*PrivateVD);2496 }2497 ++CurPrivate;2498 }2499 }2500}2501 2502static void emitSimdlenSafelenClause(CodeGenFunction &CGF,2503 const OMPExecutableDirective &D) {2504 if (!CGF.HaveInsertPoint())2505 return;2506 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {2507 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),2508 /*ignoreResult=*/true);2509 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());2510 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());2511 // In presence of finite 'safelen', it may be unsafe to mark all2512 // the memory instructions parallel, because loop-carried2513 // dependences of 'safelen' iterations are possible.2514 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());2515 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {2516 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),2517 /*ignoreResult=*/true);2518 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());2519 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());2520 // In presence of finite 'safelen', it may be unsafe to mark all2521 // the memory instructions parallel, because loop-carried2522 // dependences of 'safelen' iterations are possible.2523 CGF.LoopStack.setParallel(/*Enable=*/false);2524 }2525}2526 2527// Check for the presence of an `OMPOrderedDirective`,2528// i.e., `ordered` in `#pragma omp ordered simd`.2529//2530// Consider the following source code:2531// ```2532// __attribute__((noinline)) void omp_simd_loop(float X[ARRAY_SIZE][ARRAY_SIZE])2533// {2534// for (int r = 1; r < ARRAY_SIZE; ++r) {2535// for (int c = 1; c < ARRAY_SIZE; ++c) {2536// #pragma omp simd2537// for (int k = 2; k < ARRAY_SIZE; ++k) {2538// #pragma omp ordered simd2539// X[r][k] = X[r][k - 2] + sinf((float)(r / c));2540// }2541// }2542// }2543// }2544// ```2545//2546// Suppose we are in `CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective2547// &D)`. By examining `D.dump()` we have the following AST containing2548// `OMPOrderedDirective`:2549//2550// ```2551// OMPSimdDirective 0x1c329502552// `-CapturedStmt 0x1c320282553// |-CapturedDecl 0x1c310e82554// | |-ForStmt 0x1c31e302555// | | |-DeclStmt 0x1c312982556// | | | `-VarDecl 0x1c31208 used k 'int' cinit2557// | | | `-IntegerLiteral 0x1c31278 'int' 22558// | | |-<<<NULL>>>2559// | | |-BinaryOperator 0x1c31308 'int' '<'2560// | | | |-ImplicitCastExpr 0x1c312f0 'int' <LValueToRValue>2561// | | | | `-DeclRefExpr 0x1c312b0 'int' lvalue Var 0x1c31208 'k' 'int'2562// | | | `-IntegerLiteral 0x1c312d0 'int' 2562563// | | |-UnaryOperator 0x1c31348 'int' prefix '++'2564// | | | `-DeclRefExpr 0x1c31328 'int' lvalue Var 0x1c31208 'k' 'int'2565// | | `-CompoundStmt 0x1c31e182566// | | `-OMPOrderedDirective 0x1c31dd82567// | | |-OMPSimdClause 0x1c313802568// | | `-CapturedStmt 0x1c31cd02569// ```2570//2571// Note the presence of `OMPOrderedDirective` above:2572// It's (transitively) nested in a `CapturedStmt` representing the pragma2573// annotated compound statement. Thus, we need to consider this nesting and2574// include checking the `getCapturedStmt` in this case.2575static bool hasOrderedDirective(const Stmt *S) {2576 if (isa<OMPOrderedDirective>(S))2577 return true;2578 2579 if (const auto *CS = dyn_cast<CapturedStmt>(S))2580 return hasOrderedDirective(CS->getCapturedStmt());2581 2582 for (const Stmt *Child : S->children()) {2583 if (Child && hasOrderedDirective(Child))2584 return true;2585 }2586 2587 return false;2588}2589 2590static void applyConservativeSimdOrderedDirective(const Stmt &AssociatedStmt,2591 LoopInfoStack &LoopStack) {2592 // Check for the presence of an `OMPOrderedDirective`2593 // i.e., `ordered` in `#pragma omp ordered simd`2594 bool HasOrderedDirective = hasOrderedDirective(&AssociatedStmt);2595 // If present then conservatively disable loop vectorization2596 // analogously to how `emitSimdlenSafelenClause` does.2597 if (HasOrderedDirective)2598 LoopStack.setParallel(/*Enable=*/false);2599}2600 2601void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D) {2602 // Walk clauses and process safelen/lastprivate.2603 LoopStack.setParallel(/*Enable=*/true);2604 LoopStack.setVectorizeEnable();2605 const Stmt *AssociatedStmt = D.getAssociatedStmt();2606 applyConservativeSimdOrderedDirective(*AssociatedStmt, LoopStack);2607 emitSimdlenSafelenClause(*this, D);2608 if (const auto *C = D.getSingleClause<OMPOrderClause>())2609 if (C->getKind() == OMPC_ORDER_concurrent)2610 LoopStack.setParallel(/*Enable=*/true);2611 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(D);2612 if ((EKind == OMPD_simd ||2613 (getLangOpts().OpenMPSimd && isOpenMPSimdDirective(EKind))) &&2614 llvm::any_of(D.getClausesOfKind<OMPReductionClause>(),2615 [](const OMPReductionClause *C) {2616 return C->getModifier() == OMPC_REDUCTION_inscan;2617 }))2618 // Disable parallel access in case of prefix sum.2619 LoopStack.setParallel(/*Enable=*/false);2620}2621 2622void CodeGenFunction::EmitOMPSimdFinal(2623 const OMPLoopDirective &D,2624 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {2625 if (!HaveInsertPoint())2626 return;2627 llvm::BasicBlock *DoneBB = nullptr;2628 auto IC = D.counters().begin();2629 auto IPC = D.private_counters().begin();2630 for (const Expr *F : D.finals()) {2631 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());2632 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());2633 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);2634 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||2635 OrigVD->hasGlobalStorage() || CED) {2636 if (!DoneBB) {2637 if (llvm::Value *Cond = CondGen(*this)) {2638 // If the first post-update expression is found, emit conditional2639 // block if it was requested.2640 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");2641 DoneBB = createBasicBlock(".omp.final.done");2642 Builder.CreateCondBr(Cond, ThenBB, DoneBB);2643 EmitBlock(ThenBB);2644 }2645 }2646 Address OrigAddr = Address::invalid();2647 if (CED) {2648 OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();2649 } else {2650 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),2651 /*RefersToEnclosingVariableOrCapture=*/false,2652 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());2653 OrigAddr = EmitLValue(&DRE).getAddress();2654 }2655 OMPPrivateScope VarScope(*this);2656 VarScope.addPrivate(OrigVD, OrigAddr);2657 (void)VarScope.Privatize();2658 EmitIgnoredExpr(F);2659 }2660 ++IC;2661 ++IPC;2662 }2663 if (DoneBB)2664 EmitBlock(DoneBB, /*IsFinished=*/true);2665}2666 2667static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF,2668 const OMPLoopDirective &S,2669 CodeGenFunction::JumpDest LoopExit) {2670 CGF.EmitOMPLoopBody(S, LoopExit);2671 CGF.EmitStopPoint(&S);2672}2673 2674/// Emit a helper variable and return corresponding lvalue.2675static LValue EmitOMPHelperVar(CodeGenFunction &CGF,2676 const DeclRefExpr *Helper) {2677 auto VDecl = cast<VarDecl>(Helper->getDecl());2678 CGF.EmitVarDecl(*VDecl);2679 return CGF.EmitLValue(Helper);2680}2681 2682static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S,2683 const RegionCodeGenTy &SimdInitGen,2684 const RegionCodeGenTy &BodyCodeGen) {2685 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF,2686 PrePostActionTy &) {2687 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S);2688 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);2689 SimdInitGen(CGF);2690 2691 BodyCodeGen(CGF);2692 };2693 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) {2694 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);2695 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false);2696 2697 BodyCodeGen(CGF);2698 };2699 const Expr *IfCond = nullptr;2700 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);2701 if (isOpenMPSimdDirective(EKind)) {2702 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {2703 if (CGF.getLangOpts().OpenMP >= 50 &&2704 (C->getNameModifier() == OMPD_unknown ||2705 C->getNameModifier() == OMPD_simd)) {2706 IfCond = C->getCondition();2707 break;2708 }2709 }2710 }2711 if (IfCond) {2712 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen);2713 } else {2714 RegionCodeGenTy ThenRCG(ThenGen);2715 ThenRCG(CGF);2716 }2717}2718 2719static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S,2720 PrePostActionTy &Action) {2721 Action.Enter(CGF);2722 OMPLoopScope PreInitScope(CGF, S);2723 // if (PreCond) {2724 // for (IV in 0..LastIteration) BODY;2725 // <Final counter/linear vars updates>;2726 // }2727 2728 // The presence of lower/upper bound variable depends on the actual directive2729 // kind in the AST node. The variables must be emitted because some of the2730 // expressions associated with the loop will use them.2731 OpenMPDirectiveKind DKind = S.getDirectiveKind();2732 if (isOpenMPDistributeDirective(DKind) ||2733 isOpenMPWorksharingDirective(DKind) || isOpenMPTaskLoopDirective(DKind) ||2734 isOpenMPGenericLoopDirective(DKind)) {2735 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));2736 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));2737 }2738 2739 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);2740 // Emit: if (PreCond) - begin.2741 // If the condition constant folds and can be elided, avoid emitting the2742 // whole loop.2743 bool CondConstant;2744 llvm::BasicBlock *ContBlock = nullptr;2745 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {2746 if (!CondConstant)2747 return;2748 } else {2749 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");2750 ContBlock = CGF.createBasicBlock("simd.if.end");2751 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,2752 CGF.getProfileCount(&S));2753 CGF.EmitBlock(ThenBlock);2754 CGF.incrementProfileCounter(&S);2755 }2756 2757 // Emit the loop iteration variable.2758 const Expr *IVExpr = S.getIterationVariable();2759 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());2760 CGF.EmitVarDecl(*IVDecl);2761 CGF.EmitIgnoredExpr(S.getInit());2762 2763 // Emit the iterations count variable.2764 // If it is not a variable, Sema decided to calculate iterations count on2765 // each iteration (e.g., it is foldable into a constant).2766 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {2767 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));2768 // Emit calculation of the iterations count.2769 CGF.EmitIgnoredExpr(S.getCalcLastIteration());2770 }2771 2772 emitAlignedClause(CGF, S);2773 (void)CGF.EmitOMPLinearClauseInit(S);2774 {2775 CodeGenFunction::OMPPrivateScope LoopScope(CGF);2776 CGF.EmitOMPPrivateClause(S, LoopScope);2777 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);2778 CGF.EmitOMPLinearClause(S, LoopScope);2779 CGF.EmitOMPReductionClauseInit(S, LoopScope);2780 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(2781 CGF, S, CGF.EmitLValue(S.getIterationVariable()));2782 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);2783 (void)LoopScope.Privatize();2784 if (isOpenMPTargetExecutionDirective(EKind))2785 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);2786 2787 emitCommonSimdLoop(2788 CGF, S,2789 [&S](CodeGenFunction &CGF, PrePostActionTy &) {2790 CGF.EmitOMPSimdInit(S);2791 },2792 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {2793 CGF.EmitOMPInnerLoop(2794 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),2795 [&S](CodeGenFunction &CGF) {2796 emitOMPLoopBodyWithStopPoint(CGF, S,2797 CodeGenFunction::JumpDest());2798 },2799 [](CodeGenFunction &) {});2800 });2801 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });2802 // Emit final copy of the lastprivate variables at the end of loops.2803 if (HasLastprivateClause)2804 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);2805 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);2806 emitPostUpdateForReductionClause(CGF, S,2807 [](CodeGenFunction &) { return nullptr; });2808 LoopScope.restoreMap();2809 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });2810 }2811 // Emit: if (PreCond) - end.2812 if (ContBlock) {2813 CGF.EmitBranch(ContBlock);2814 CGF.EmitBlock(ContBlock, true);2815 }2816}2817 2818// Pass OMPLoopDirective (instead of OMPSimdDirective) to make this function2819// available for "loop bind(thread)", which maps to "simd".2820static bool isSimdSupportedByOpenMPIRBuilder(const OMPLoopDirective &S) {2821 // Check for unsupported clauses2822 for (OMPClause *C : S.clauses()) {2823 // Currently only order, simdlen and safelen clauses are supported2824 if (!(isa<OMPSimdlenClause>(C) || isa<OMPSafelenClause>(C) ||2825 isa<OMPOrderClause>(C) || isa<OMPAlignedClause>(C)))2826 return false;2827 }2828 2829 // Check if we have a statement with the ordered directive.2830 // Visit the statement hierarchy to find a compound statement2831 // with a ordered directive in it.2832 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {2833 if (const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {2834 for (const Stmt *SubStmt : SyntacticalLoop->children()) {2835 if (!SubStmt)2836 continue;2837 if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(SubStmt)) {2838 for (const Stmt *CSSubStmt : CS->children()) {2839 if (!CSSubStmt)2840 continue;2841 if (isa<OMPOrderedDirective>(CSSubStmt)) {2842 return false;2843 }2844 }2845 }2846 }2847 }2848 }2849 return true;2850}2851 2852static llvm::MapVector<llvm::Value *, llvm::Value *>2853GetAlignedMapping(const OMPLoopDirective &S, CodeGenFunction &CGF) {2854 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;2855 for (const auto *Clause : S.getClausesOfKind<OMPAlignedClause>()) {2856 llvm::APInt ClauseAlignment(64, 0);2857 if (const Expr *AlignmentExpr = Clause->getAlignment()) {2858 auto *AlignmentCI =2859 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));2860 ClauseAlignment = AlignmentCI->getValue();2861 }2862 for (const Expr *E : Clause->varlist()) {2863 llvm::APInt Alignment(ClauseAlignment);2864 if (Alignment == 0) {2865 // OpenMP [2.8.1, Description]2866 // If no optional parameter is specified, implementation-defined default2867 // alignments for SIMD instructions on the target platforms are assumed.2868 Alignment =2869 CGF.getContext()2870 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(2871 E->getType()->getPointeeType()))2872 .getQuantity();2873 }2874 assert((Alignment == 0 || Alignment.isPowerOf2()) &&2875 "alignment is not power of 2");2876 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);2877 AlignedVars[PtrValue] = CGF.Builder.getInt64(Alignment.getSExtValue());2878 }2879 }2880 return AlignedVars;2881}2882 2883// Pass OMPLoopDirective (instead of OMPSimdDirective) to make this function2884// available for "loop bind(thread)", which maps to "simd".2885static void emitOMPSimdDirective(const OMPLoopDirective &S,2886 CodeGenFunction &CGF, CodeGenModule &CGM) {2887 bool UseOMPIRBuilder =2888 CGM.getLangOpts().OpenMPIRBuilder && isSimdSupportedByOpenMPIRBuilder(S);2889 if (UseOMPIRBuilder) {2890 auto &&CodeGenIRBuilder = [&S, &CGM, UseOMPIRBuilder](CodeGenFunction &CGF,2891 PrePostActionTy &) {2892 // Use the OpenMPIRBuilder if enabled.2893 if (UseOMPIRBuilder) {2894 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =2895 GetAlignedMapping(S, CGF);2896 // Emit the associated statement and get its loop representation.2897 const Stmt *Inner = S.getRawStmt();2898 llvm::CanonicalLoopInfo *CLI =2899 CGF.EmitOMPCollapsedCanonicalLoopNest(Inner, 1);2900 2901 llvm::OpenMPIRBuilder &OMPBuilder =2902 CGM.getOpenMPRuntime().getOMPBuilder();2903 // Add SIMD specific metadata2904 llvm::ConstantInt *Simdlen = nullptr;2905 if (const auto *C = S.getSingleClause<OMPSimdlenClause>()) {2906 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),2907 /*ignoreResult=*/true);2908 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());2909 Simdlen = Val;2910 }2911 llvm::ConstantInt *Safelen = nullptr;2912 if (const auto *C = S.getSingleClause<OMPSafelenClause>()) {2913 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),2914 /*ignoreResult=*/true);2915 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());2916 Safelen = Val;2917 }2918 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;2919 if (const auto *C = S.getSingleClause<OMPOrderClause>()) {2920 if (C->getKind() == OpenMPOrderClauseKind::OMPC_ORDER_concurrent) {2921 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;2922 }2923 }2924 // Add simd metadata to the collapsed loop. Do not generate2925 // another loop for if clause. Support for if clause is done earlier.2926 OMPBuilder.applySimd(CLI, AlignedVars,2927 /*IfCond*/ nullptr, Order, Simdlen, Safelen);2928 return;2929 }2930 };2931 {2932 auto LPCRegion =2933 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);2934 OMPLexicalScope Scope(CGF, S, OMPD_unknown);2935 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,2936 CodeGenIRBuilder);2937 }2938 return;2939 }2940 2941 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);2942 CGF.OMPFirstScanLoop = true;2943 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {2944 emitOMPSimdRegion(CGF, S, Action);2945 };2946 {2947 auto LPCRegion =2948 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);2949 OMPLexicalScope Scope(CGF, S, OMPD_unknown);2950 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd, CodeGen);2951 }2952 // Check for outer lastprivate conditional update.2953 checkForLastprivateConditionalUpdate(CGF, S);2954}2955 2956void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {2957 emitOMPSimdDirective(S, *this, CGM);2958}2959 2960void CodeGenFunction::EmitOMPTileDirective(const OMPTileDirective &S) {2961 // Emit the de-sugared statement.2962 OMPTransformDirectiveScopeRAII TileScope(*this, &S);2963 EmitStmt(S.getTransformedStmt());2964}2965 2966void CodeGenFunction::EmitOMPStripeDirective(const OMPStripeDirective &S) {2967 // Emit the de-sugared statement.2968 OMPTransformDirectiveScopeRAII StripeScope(*this, &S);2969 EmitStmt(S.getTransformedStmt());2970}2971 2972void CodeGenFunction::EmitOMPReverseDirective(const OMPReverseDirective &S) {2973 // Emit the de-sugared statement.2974 OMPTransformDirectiveScopeRAII ReverseScope(*this, &S);2975 EmitStmt(S.getTransformedStmt());2976}2977 2978void CodeGenFunction::EmitOMPInterchangeDirective(2979 const OMPInterchangeDirective &S) {2980 // Emit the de-sugared statement.2981 OMPTransformDirectiveScopeRAII InterchangeScope(*this, &S);2982 EmitStmt(S.getTransformedStmt());2983}2984 2985void CodeGenFunction::EmitOMPFuseDirective(const OMPFuseDirective &S) {2986 // Emit the de-sugared statement2987 OMPTransformDirectiveScopeRAII FuseScope(*this, &S);2988 EmitStmt(S.getTransformedStmt());2989}2990 2991void CodeGenFunction::EmitOMPUnrollDirective(const OMPUnrollDirective &S) {2992 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder;2993 2994 if (UseOMPIRBuilder) {2995 auto DL = SourceLocToDebugLoc(S.getBeginLoc());2996 const Stmt *Inner = S.getRawStmt();2997 2998 // Consume nested loop. Clear the entire remaining loop stack because a2999 // fully unrolled loop is non-transformable. For partial unrolling the3000 // generated outer loop is pushed back to the stack.3001 llvm::CanonicalLoopInfo *CLI = EmitOMPCollapsedCanonicalLoopNest(Inner, 1);3002 OMPLoopNestStack.clear();3003 3004 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();3005 3006 bool NeedsUnrolledCLI = ExpectedOMPLoopDepth >= 1;3007 llvm::CanonicalLoopInfo *UnrolledCLI = nullptr;3008 3009 if (S.hasClausesOfKind<OMPFullClause>()) {3010 assert(ExpectedOMPLoopDepth == 0);3011 OMPBuilder.unrollLoopFull(DL, CLI);3012 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {3013 uint64_t Factor = 0;3014 if (Expr *FactorExpr = PartialClause->getFactor()) {3015 Factor = FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();3016 assert(Factor >= 1 && "Only positive factors are valid");3017 }3018 OMPBuilder.unrollLoopPartial(DL, CLI, Factor,3019 NeedsUnrolledCLI ? &UnrolledCLI : nullptr);3020 } else {3021 OMPBuilder.unrollLoopHeuristic(DL, CLI);3022 }3023 3024 assert((!NeedsUnrolledCLI || UnrolledCLI) &&3025 "NeedsUnrolledCLI implies UnrolledCLI to be set");3026 if (UnrolledCLI)3027 OMPLoopNestStack.push_back(UnrolledCLI);3028 3029 return;3030 }3031 3032 // This function is only called if the unrolled loop is not consumed by any3033 // other loop-associated construct. Such a loop-associated construct will have3034 // used the transformed AST.3035 3036 // Set the unroll metadata for the next emitted loop.3037 LoopStack.setUnrollState(LoopAttributes::Enable);3038 3039 if (S.hasClausesOfKind<OMPFullClause>()) {3040 LoopStack.setUnrollState(LoopAttributes::Full);3041 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {3042 if (Expr *FactorExpr = PartialClause->getFactor()) {3043 uint64_t Factor =3044 FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();3045 assert(Factor >= 1 && "Only positive factors are valid");3046 LoopStack.setUnrollCount(Factor);3047 }3048 }3049 3050 EmitStmt(S.getAssociatedStmt());3051}3052 3053void CodeGenFunction::EmitOMPOuterLoop(3054 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,3055 CodeGenFunction::OMPPrivateScope &LoopScope,3056 const CodeGenFunction::OMPLoopArguments &LoopArgs,3057 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,3058 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {3059 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();3060 3061 const Expr *IVExpr = S.getIterationVariable();3062 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());3063 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();3064 3065 JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");3066 3067 // Start the loop with a block that tests the condition.3068 llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");3069 EmitBlock(CondBlock);3070 const SourceRange R = S.getSourceRange();3071 OMPLoopNestStack.clear();3072 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),3073 SourceLocToDebugLoc(R.getEnd()));3074 3075 llvm::Value *BoolCondVal = nullptr;3076 if (!DynamicOrOrdered) {3077 // UB = min(UB, GlobalUB) or3078 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.3079 // 'distribute parallel for')3080 EmitIgnoredExpr(LoopArgs.EUB);3081 // IV = LB3082 EmitIgnoredExpr(LoopArgs.Init);3083 // IV < UB3084 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);3085 } else {3086 BoolCondVal =3087 RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,3088 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);3089 }3090 3091 // If there are any cleanups between here and the loop-exit scope,3092 // create a block to stage a loop exit along.3093 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();3094 if (LoopScope.requiresCleanups())3095 ExitBlock = createBasicBlock("omp.dispatch.cleanup");3096 3097 llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");3098 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);3099 if (ExitBlock != LoopExit.getBlock()) {3100 EmitBlock(ExitBlock);3101 EmitBranchThroughCleanup(LoopExit);3102 }3103 EmitBlock(LoopBody);3104 3105 // Emit "IV = LB" (in case of static schedule, we have already calculated new3106 // LB for loop condition and emitted it above).3107 if (DynamicOrOrdered)3108 EmitIgnoredExpr(LoopArgs.Init);3109 3110 // Create a block for the increment.3111 JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");3112 BreakContinueStack.push_back(BreakContinue(S, LoopExit, Continue));3113 3114 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);3115 emitCommonSimdLoop(3116 *this, S,3117 [&S, IsMonotonic, EKind](CodeGenFunction &CGF, PrePostActionTy &) {3118 // Generate !llvm.loop.parallel metadata for loads and stores for loops3119 // with dynamic/guided scheduling and without ordered clause.3120 if (!isOpenMPSimdDirective(EKind)) {3121 CGF.LoopStack.setParallel(!IsMonotonic);3122 if (const auto *C = S.getSingleClause<OMPOrderClause>())3123 if (C->getKind() == OMPC_ORDER_concurrent)3124 CGF.LoopStack.setParallel(/*Enable=*/true);3125 } else {3126 CGF.EmitOMPSimdInit(S);3127 }3128 },3129 [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,3130 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {3131 SourceLocation Loc = S.getBeginLoc();3132 // when 'distribute' is not combined with a 'for':3133 // while (idx <= UB) { BODY; ++idx; }3134 // when 'distribute' is combined with a 'for'3135 // (e.g. 'distribute parallel for')3136 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }3137 CGF.EmitOMPInnerLoop(3138 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,3139 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {3140 CodeGenLoop(CGF, S, LoopExit);3141 },3142 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {3143 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);3144 });3145 });3146 3147 EmitBlock(Continue.getBlock());3148 BreakContinueStack.pop_back();3149 if (!DynamicOrOrdered) {3150 // Emit "LB = LB + Stride", "UB = UB + Stride".3151 EmitIgnoredExpr(LoopArgs.NextLB);3152 EmitIgnoredExpr(LoopArgs.NextUB);3153 }3154 3155 EmitBranch(CondBlock);3156 OMPLoopNestStack.clear();3157 LoopStack.pop();3158 // Emit the fall-through block.3159 EmitBlock(LoopExit.getBlock());3160 3161 // Tell the runtime we are done.3162 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](CodeGenFunction &CGF) {3163 if (!DynamicOrOrdered)3164 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),3165 LoopArgs.DKind);3166 };3167 OMPCancelStack.emitExit(*this, EKind, CodeGen);3168}3169 3170void CodeGenFunction::EmitOMPForOuterLoop(3171 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,3172 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,3173 const OMPLoopArguments &LoopArgs,3174 const CodeGenDispatchBoundsTy &CGDispatchBounds) {3175 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();3176 3177 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).3178 const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind.Schedule);3179 3180 assert((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule,3181 LoopArgs.Chunk != nullptr)) &&3182 "static non-chunked schedule does not need outer loop");3183 3184 // Emit outer loop.3185 //3186 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]3187 // When schedule(dynamic,chunk_size) is specified, the iterations are3188 // distributed to threads in the team in chunks as the threads request them.3189 // Each thread executes a chunk of iterations, then requests another chunk,3190 // until no chunks remain to be distributed. Each chunk contains chunk_size3191 // iterations, except for the last chunk to be distributed, which may have3192 // fewer iterations. When no chunk_size is specified, it defaults to 1.3193 //3194 // When schedule(guided,chunk_size) is specified, the iterations are assigned3195 // to threads in the team in chunks as the executing threads request them.3196 // Each thread executes a chunk of iterations, then requests another chunk,3197 // until no chunks remain to be assigned. For a chunk_size of 1, the size of3198 // each chunk is proportional to the number of unassigned iterations divided3199 // by the number of threads in the team, decreasing to 1. For a chunk_size3200 // with value k (greater than 1), the size of each chunk is determined in the3201 // same way, with the restriction that the chunks do not contain fewer than k3202 // iterations (except for the last chunk to be assigned, which may have fewer3203 // than k iterations).3204 //3205 // When schedule(auto) is specified, the decision regarding scheduling is3206 // delegated to the compiler and/or runtime system. The programmer gives the3207 // implementation the freedom to choose any possible mapping of iterations to3208 // threads in the team.3209 //3210 // When schedule(runtime) is specified, the decision regarding scheduling is3211 // deferred until run time, and the schedule and chunk size are taken from the3212 // run-sched-var ICV. If the ICV is set to auto, the schedule is3213 // implementation defined3214 //3215 // __kmpc_dispatch_init();3216 // while(__kmpc_dispatch_next(&LB, &UB)) {3217 // idx = LB;3218 // while (idx <= UB) { BODY; ++idx;3219 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.3220 // } // inner loop3221 // }3222 // __kmpc_dispatch_deinit();3223 //3224 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]3225 // When schedule(static, chunk_size) is specified, iterations are divided into3226 // chunks of size chunk_size, and the chunks are assigned to the threads in3227 // the team in a round-robin fashion in the order of the thread number.3228 //3229 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {3230 // while (idx <= UB) { BODY; ++idx; } // inner loop3231 // LB = LB + ST;3232 // UB = UB + ST;3233 // }3234 //3235 3236 const Expr *IVExpr = S.getIterationVariable();3237 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());3238 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();3239 3240 if (DynamicOrOrdered) {3241 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =3242 CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);3243 llvm::Value *LBVal = DispatchBounds.first;3244 llvm::Value *UBVal = DispatchBounds.second;3245 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,3246 LoopArgs.Chunk};3247 RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,3248 IVSigned, Ordered, DipatchRTInputValues);3249 } else {3250 CGOpenMPRuntime::StaticRTInput StaticInit(3251 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,3252 LoopArgs.ST, LoopArgs.Chunk);3253 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);3254 RT.emitForStaticInit(*this, S.getBeginLoc(), EKind, ScheduleKind,3255 StaticInit);3256 }3257 3258 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,3259 const unsigned IVSize,3260 const bool IVSigned) {3261 if (Ordered) {3262 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,3263 IVSigned);3264 }3265 };3266 3267 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,3268 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);3269 OuterLoopArgs.IncExpr = S.getInc();3270 OuterLoopArgs.Init = S.getInit();3271 OuterLoopArgs.Cond = S.getCond();3272 OuterLoopArgs.NextLB = S.getNextLowerBound();3273 OuterLoopArgs.NextUB = S.getNextUpperBound();3274 OuterLoopArgs.DKind = LoopArgs.DKind;3275 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,3276 emitOMPLoopBodyWithStopPoint, CodeGenOrdered);3277 if (DynamicOrOrdered) {3278 RT.emitForDispatchDeinit(*this, S.getBeginLoc());3279 }3280}3281 3282static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc,3283 const unsigned IVSize, const bool IVSigned) {}3284 3285void CodeGenFunction::EmitOMPDistributeOuterLoop(3286 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,3287 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,3288 const CodeGenLoopTy &CodeGenLoopContent) {3289 3290 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();3291 3292 // Emit outer loop.3293 // Same behavior as a OMPForOuterLoop, except that schedule cannot be3294 // dynamic3295 //3296 3297 const Expr *IVExpr = S.getIterationVariable();3298 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());3299 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();3300 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);3301 3302 CGOpenMPRuntime::StaticRTInput StaticInit(3303 IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,3304 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);3305 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);3306 3307 // for combined 'distribute' and 'for' the increment expression of distribute3308 // is stored in DistInc. For 'distribute' alone, it is in Inc.3309 Expr *IncExpr;3310 if (isOpenMPLoopBoundSharingDirective(EKind))3311 IncExpr = S.getDistInc();3312 else3313 IncExpr = S.getInc();3314 3315 // this routine is shared by 'omp distribute parallel for' and3316 // 'omp distribute': select the right EUB expression depending on the3317 // directive3318 OMPLoopArguments OuterLoopArgs;3319 OuterLoopArgs.LB = LoopArgs.LB;3320 OuterLoopArgs.UB = LoopArgs.UB;3321 OuterLoopArgs.ST = LoopArgs.ST;3322 OuterLoopArgs.IL = LoopArgs.IL;3323 OuterLoopArgs.Chunk = LoopArgs.Chunk;3324 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(EKind)3325 ? S.getCombinedEnsureUpperBound()3326 : S.getEnsureUpperBound();3327 OuterLoopArgs.IncExpr = IncExpr;3328 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(EKind)3329 ? S.getCombinedInit()3330 : S.getInit();3331 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(EKind)3332 ? S.getCombinedCond()3333 : S.getCond();3334 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(EKind)3335 ? S.getCombinedNextLowerBound()3336 : S.getNextLowerBound();3337 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(EKind)3338 ? S.getCombinedNextUpperBound()3339 : S.getNextUpperBound();3340 OuterLoopArgs.DKind = OMPD_distribute;3341 3342 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,3343 LoopScope, OuterLoopArgs, CodeGenLoopContent,3344 emitEmptyOrdered);3345}3346 3347static std::pair<LValue, LValue>3348emitDistributeParallelForInnerBounds(CodeGenFunction &CGF,3349 const OMPExecutableDirective &S) {3350 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);3351 LValue LB =3352 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));3353 LValue UB =3354 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));3355 3356 // When composing 'distribute' with 'for' (e.g. as in 'distribute3357 // parallel for') we need to use the 'distribute'3358 // chunk lower and upper bounds rather than the whole loop iteration3359 // space. These are parameters to the outlined function for 'parallel'3360 // and we copy the bounds of the previous schedule into the3361 // the current ones.3362 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());3363 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());3364 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(3365 PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());3366 PrevLBVal = CGF.EmitScalarConversion(3367 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),3368 LS.getIterationVariable()->getType(),3369 LS.getPrevLowerBoundVariable()->getExprLoc());3370 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(3371 PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());3372 PrevUBVal = CGF.EmitScalarConversion(3373 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),3374 LS.getIterationVariable()->getType(),3375 LS.getPrevUpperBoundVariable()->getExprLoc());3376 3377 CGF.EmitStoreOfScalar(PrevLBVal, LB);3378 CGF.EmitStoreOfScalar(PrevUBVal, UB);3379 3380 return {LB, UB};3381}3382 3383/// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then3384/// we need to use the LB and UB expressions generated by the worksharing3385/// code generation support, whereas in non combined situations we would3386/// just emit 0 and the LastIteration expression3387/// This function is necessary due to the difference of the LB and UB3388/// types for the RT emission routines for 'for_static_init' and3389/// 'for_dispatch_init'3390static std::pair<llvm::Value *, llvm::Value *>3391emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF,3392 const OMPExecutableDirective &S,3393 Address LB, Address UB) {3394 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);3395 const Expr *IVExpr = LS.getIterationVariable();3396 // when implementing a dynamic schedule for a 'for' combined with a3397 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop3398 // is not normalized as each team only executes its own assigned3399 // distribute chunk3400 QualType IteratorTy = IVExpr->getType();3401 llvm::Value *LBVal =3402 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());3403 llvm::Value *UBVal =3404 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());3405 return {LBVal, UBVal};3406}3407 3408static void emitDistributeParallelForDistributeInnerBoundParams(3409 CodeGenFunction &CGF, const OMPExecutableDirective &S,3410 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {3411 const auto &Dir = cast<OMPLoopDirective>(S);3412 LValue LB =3413 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));3414 llvm::Value *LBCast = CGF.Builder.CreateIntCast(3415 CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);3416 CapturedVars.push_back(LBCast);3417 LValue UB =3418 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));3419 3420 llvm::Value *UBCast = CGF.Builder.CreateIntCast(3421 CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);3422 CapturedVars.push_back(UBCast);3423}3424 3425static void3426emitInnerParallelForWhenCombined(CodeGenFunction &CGF,3427 const OMPLoopDirective &S,3428 CodeGenFunction::JumpDest LoopExit) {3429 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);3430 auto &&CGInlinedWorksharingLoop = [&S, EKind](CodeGenFunction &CGF,3431 PrePostActionTy &Action) {3432 Action.Enter(CGF);3433 bool HasCancel = false;3434 if (!isOpenMPSimdDirective(EKind)) {3435 if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))3436 HasCancel = D->hasCancel();3437 else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))3438 HasCancel = D->hasCancel();3439 else if (const auto *D =3440 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))3441 HasCancel = D->hasCancel();3442 }3443 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);3444 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),3445 emitDistributeParallelForInnerBounds,3446 emitDistributeParallelForDispatchBounds);3447 };3448 3449 emitCommonOMPParallelDirective(3450 CGF, S, isOpenMPSimdDirective(EKind) ? OMPD_for_simd : OMPD_for,3451 CGInlinedWorksharingLoop,3452 emitDistributeParallelForDistributeInnerBoundParams);3453}3454 3455void CodeGenFunction::EmitOMPDistributeParallelForDirective(3456 const OMPDistributeParallelForDirective &S) {3457 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {3458 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,3459 S.getDistInc());3460 };3461 OMPLexicalScope Scope(*this, S, OMPD_parallel);3462 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);3463}3464 3465void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective(3466 const OMPDistributeParallelForSimdDirective &S) {3467 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {3468 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,3469 S.getDistInc());3470 };3471 OMPLexicalScope Scope(*this, S, OMPD_parallel);3472 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);3473}3474 3475void CodeGenFunction::EmitOMPDistributeSimdDirective(3476 const OMPDistributeSimdDirective &S) {3477 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {3478 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());3479 };3480 OMPLexicalScope Scope(*this, S, OMPD_unknown);3481 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);3482}3483 3484void CodeGenFunction::EmitOMPTargetSimdDeviceFunction(3485 CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {3486 // Emit SPMD target parallel for region as a standalone region.3487 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {3488 emitOMPSimdRegion(CGF, S, Action);3489 };3490 llvm::Function *Fn;3491 llvm::Constant *Addr;3492 // Emit target region as a standalone region.3493 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(3494 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);3495 assert(Fn && Addr && "Target device function emission failed.");3496}3497 3498void CodeGenFunction::EmitOMPTargetSimdDirective(3499 const OMPTargetSimdDirective &S) {3500 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {3501 emitOMPSimdRegion(CGF, S, Action);3502 };3503 emitCommonOMPTargetDirective(*this, S, CodeGen);3504}3505 3506namespace {3507struct ScheduleKindModifiersTy {3508 OpenMPScheduleClauseKind Kind;3509 OpenMPScheduleClauseModifier M1;3510 OpenMPScheduleClauseModifier M2;3511 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,3512 OpenMPScheduleClauseModifier M1,3513 OpenMPScheduleClauseModifier M2)3514 : Kind(Kind), M1(M1), M2(M2) {}3515};3516} // namespace3517 3518bool CodeGenFunction::EmitOMPWorksharingLoop(3519 const OMPLoopDirective &S, Expr *EUB,3520 const CodeGenLoopBoundsTy &CodeGenLoopBounds,3521 const CodeGenDispatchBoundsTy &CGDispatchBounds) {3522 // Emit the loop iteration variable.3523 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());3524 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());3525 EmitVarDecl(*IVDecl);3526 3527 // Emit the iterations count variable.3528 // If it is not a variable, Sema decided to calculate iterations count on each3529 // iteration (e.g., it is foldable into a constant).3530 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {3531 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));3532 // Emit calculation of the iterations count.3533 EmitIgnoredExpr(S.getCalcLastIteration());3534 }3535 3536 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();3537 3538 bool HasLastprivateClause;3539 // Check pre-condition.3540 {3541 OMPLoopScope PreInitScope(*this, S);3542 // Skip the entire loop if we don't meet the precondition.3543 // If the condition constant folds and can be elided, avoid emitting the3544 // whole loop.3545 bool CondConstant;3546 llvm::BasicBlock *ContBlock = nullptr;3547 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {3548 if (!CondConstant)3549 return false;3550 } else {3551 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");3552 ContBlock = createBasicBlock("omp.precond.end");3553 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,3554 getProfileCount(&S));3555 EmitBlock(ThenBlock);3556 incrementProfileCounter(&S);3557 }3558 3559 RunCleanupsScope DoacrossCleanupScope(*this);3560 bool Ordered = false;3561 if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {3562 if (OrderedClause->getNumForLoops())3563 RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());3564 else3565 Ordered = true;3566 }3567 3568 emitAlignedClause(*this, S);3569 bool HasLinears = EmitOMPLinearClauseInit(S);3570 // Emit helper vars inits.3571 3572 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);3573 LValue LB = Bounds.first;3574 LValue UB = Bounds.second;3575 LValue ST =3576 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));3577 LValue IL =3578 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));3579 3580 // Emit 'then' code.3581 {3582 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);3583 OMPPrivateScope LoopScope(*this);3584 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {3585 // Emit implicit barrier to synchronize threads and avoid data races on3586 // initialization of firstprivate variables and post-update of3587 // lastprivate variables.3588 CGM.getOpenMPRuntime().emitBarrierCall(3589 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,3590 /*ForceSimpleCall=*/true);3591 }3592 EmitOMPPrivateClause(S, LoopScope);3593 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(3594 *this, S, EmitLValue(S.getIterationVariable()));3595 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);3596 EmitOMPReductionClauseInit(S, LoopScope);3597 EmitOMPPrivateLoopCounters(S, LoopScope);3598 EmitOMPLinearClause(S, LoopScope);3599 (void)LoopScope.Privatize();3600 if (isOpenMPTargetExecutionDirective(EKind))3601 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);3602 3603 // Detect the loop schedule kind and chunk.3604 const Expr *ChunkExpr = nullptr;3605 OpenMPScheduleTy ScheduleKind;3606 if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {3607 ScheduleKind.Schedule = C->getScheduleKind();3608 ScheduleKind.M1 = C->getFirstScheduleModifier();3609 ScheduleKind.M2 = C->getSecondScheduleModifier();3610 ChunkExpr = C->getChunkSize();3611 } else {3612 // Default behaviour for schedule clause.3613 CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(3614 *this, S, ScheduleKind.Schedule, ChunkExpr);3615 }3616 bool HasChunkSizeOne = false;3617 llvm::Value *Chunk = nullptr;3618 if (ChunkExpr) {3619 Chunk = EmitScalarExpr(ChunkExpr);3620 Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),3621 S.getIterationVariable()->getType(),3622 S.getBeginLoc());3623 Expr::EvalResult Result;3624 if (ChunkExpr->EvaluateAsInt(Result, getContext())) {3625 llvm::APSInt EvaluatedChunk = Result.Val.getInt();3626 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);3627 }3628 }3629 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());3630 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();3631 // OpenMP 4.5, 2.7.1 Loop Construct, Description.3632 // If the static schedule kind is specified or if the ordered clause is3633 // specified, and if no monotonic modifier is specified, the effect will3634 // be as if the monotonic modifier was specified.3635 bool StaticChunkedOne =3636 RT.isStaticChunked(ScheduleKind.Schedule,3637 /* Chunked */ Chunk != nullptr) &&3638 HasChunkSizeOne && isOpenMPLoopBoundSharingDirective(EKind);3639 bool IsMonotonic =3640 Ordered ||3641 (ScheduleKind.Schedule == OMPC_SCHEDULE_static &&3642 !(ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||3643 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||3644 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||3645 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;3646 if ((RT.isStaticNonchunked(ScheduleKind.Schedule,3647 /* Chunked */ Chunk != nullptr) ||3648 StaticChunkedOne) &&3649 !Ordered) {3650 JumpDest LoopExit =3651 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));3652 emitCommonSimdLoop(3653 *this, S,3654 [&S, EKind](CodeGenFunction &CGF, PrePostActionTy &) {3655 if (isOpenMPSimdDirective(EKind)) {3656 CGF.EmitOMPSimdInit(S);3657 } else if (const auto *C = S.getSingleClause<OMPOrderClause>()) {3658 if (C->getKind() == OMPC_ORDER_concurrent)3659 CGF.LoopStack.setParallel(/*Enable=*/true);3660 }3661 },3662 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,3663 &S, ScheduleKind, LoopExit, EKind,3664 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {3665 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]3666 // When no chunk_size is specified, the iteration space is divided3667 // into chunks that are approximately equal in size, and at most3668 // one chunk is distributed to each thread. Note that the size of3669 // the chunks is unspecified in this case.3670 CGOpenMPRuntime::StaticRTInput StaticInit(3671 IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),3672 UB.getAddress(), ST.getAddress(),3673 StaticChunkedOne ? Chunk : nullptr);3674 CGF.CGM.getOpenMPRuntime().emitForStaticInit(3675 CGF, S.getBeginLoc(), EKind, ScheduleKind, StaticInit);3676 // UB = min(UB, GlobalUB);3677 if (!StaticChunkedOne)3678 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());3679 // IV = LB;3680 CGF.EmitIgnoredExpr(S.getInit());3681 // For unchunked static schedule generate:3682 //3683 // while (idx <= UB) {3684 // BODY;3685 // ++idx;3686 // }3687 //3688 // For static schedule with chunk one:3689 //3690 // while (IV <= PrevUB) {3691 // BODY;3692 // IV += ST;3693 // }3694 CGF.EmitOMPInnerLoop(3695 S, LoopScope.requiresCleanups(),3696 StaticChunkedOne ? S.getCombinedParForInDistCond()3697 : S.getCond(),3698 StaticChunkedOne ? S.getDistInc() : S.getInc(),3699 [&S, LoopExit](CodeGenFunction &CGF) {3700 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);3701 },3702 [](CodeGenFunction &) {});3703 });3704 EmitBlock(LoopExit.getBlock());3705 // Tell the runtime we are done.3706 auto &&CodeGen = [&S](CodeGenFunction &CGF) {3707 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),3708 OMPD_for);3709 };3710 OMPCancelStack.emitExit(*this, EKind, CodeGen);3711 } else {3712 // Emit the outer loop, which requests its work chunk [LB..UB] from3713 // runtime and runs the inner loop to process it.3714 OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),3715 ST.getAddress(), IL.getAddress(), Chunk,3716 EUB);3717 LoopArguments.DKind = OMPD_for;3718 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,3719 LoopArguments, CGDispatchBounds);3720 }3721 if (isOpenMPSimdDirective(EKind)) {3722 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {3723 return CGF.Builder.CreateIsNotNull(3724 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));3725 });3726 }3727 EmitOMPReductionClauseFinal(3728 S, /*ReductionKind=*/isOpenMPSimdDirective(EKind)3729 ? /*Parallel and Simd*/ OMPD_parallel_for_simd3730 : /*Parallel only*/ OMPD_parallel);3731 // Emit post-update of the reduction variables if IsLastIter != 0.3732 emitPostUpdateForReductionClause(3733 *this, S, [IL, &S](CodeGenFunction &CGF) {3734 return CGF.Builder.CreateIsNotNull(3735 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));3736 });3737 // Emit final copy of the lastprivate variables if IsLastIter != 0.3738 if (HasLastprivateClause)3739 EmitOMPLastprivateClauseFinal(3740 S, isOpenMPSimdDirective(EKind),3741 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));3742 LoopScope.restoreMap();3743 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {3744 return CGF.Builder.CreateIsNotNull(3745 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));3746 });3747 }3748 DoacrossCleanupScope.ForceCleanup();3749 // We're now done with the loop, so jump to the continuation block.3750 if (ContBlock) {3751 EmitBranch(ContBlock);3752 EmitBlock(ContBlock, /*IsFinished=*/true);3753 }3754 }3755 return HasLastprivateClause;3756}3757 3758/// The following two functions generate expressions for the loop lower3759/// and upper bounds in case of static and dynamic (dispatch) schedule3760/// of the associated 'for' or 'distribute' loop.3761static std::pair<LValue, LValue>3762emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) {3763 const auto &LS = cast<OMPLoopDirective>(S);3764 LValue LB =3765 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));3766 LValue UB =3767 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));3768 return {LB, UB};3769}3770 3771/// When dealing with dispatch schedules (e.g. dynamic, guided) we do not3772/// consider the lower and upper bound expressions generated by the3773/// worksharing loop support, but we use 0 and the iteration space size as3774/// constants3775static std::pair<llvm::Value *, llvm::Value *>3776emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S,3777 Address LB, Address UB) {3778 const auto &LS = cast<OMPLoopDirective>(S);3779 const Expr *IVExpr = LS.getIterationVariable();3780 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());3781 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);3782 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());3783 return {LBVal, UBVal};3784}3785 3786/// Emits internal temp array declarations for the directive with inscan3787/// reductions.3788/// The code is the following:3789/// \code3790/// size num_iters = <num_iters>;3791/// <type> buffer[num_iters];3792/// \endcode3793static void emitScanBasedDirectiveDecls(3794 CodeGenFunction &CGF, const OMPLoopDirective &S,3795 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {3796 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(3797 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);3798 SmallVector<const Expr *, 4> Shareds;3799 SmallVector<const Expr *, 4> Privates;3800 SmallVector<const Expr *, 4> ReductionOps;3801 SmallVector<const Expr *, 4> CopyArrayTemps;3802 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {3803 assert(C->getModifier() == OMPC_REDUCTION_inscan &&3804 "Only inscan reductions are expected.");3805 Shareds.append(C->varlist_begin(), C->varlist_end());3806 Privates.append(C->privates().begin(), C->privates().end());3807 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());3808 CopyArrayTemps.append(C->copy_array_temps().begin(),3809 C->copy_array_temps().end());3810 }3811 {3812 // Emit buffers for each reduction variables.3813 // ReductionCodeGen is required to emit correctly the code for array3814 // reductions.3815 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);3816 unsigned Count = 0;3817 auto *ITA = CopyArrayTemps.begin();3818 for (const Expr *IRef : Privates) {3819 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());3820 // Emit variably modified arrays, used for arrays/array sections3821 // reductions.3822 if (PrivateVD->getType()->isVariablyModifiedType()) {3823 RedCG.emitSharedOrigLValue(CGF, Count);3824 RedCG.emitAggregateType(CGF, Count);3825 }3826 CodeGenFunction::OpaqueValueMapping DimMapping(3827 CGF,3828 cast<OpaqueValueExpr>(3829 cast<VariableArrayType>((*ITA)->getType()->getAsArrayTypeUnsafe())3830 ->getSizeExpr()),3831 RValue::get(OMPScanNumIterations));3832 // Emit temp buffer.3833 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));3834 ++ITA;3835 ++Count;3836 }3837 }3838}3839 3840/// Copies final inscan reductions values to the original variables.3841/// The code is the following:3842/// \code3843/// <orig_var> = buffer[num_iters-1];3844/// \endcode3845static void emitScanBasedDirectiveFinals(3846 CodeGenFunction &CGF, const OMPLoopDirective &S,3847 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {3848 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(3849 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);3850 SmallVector<const Expr *, 4> Shareds;3851 SmallVector<const Expr *, 4> LHSs;3852 SmallVector<const Expr *, 4> RHSs;3853 SmallVector<const Expr *, 4> Privates;3854 SmallVector<const Expr *, 4> CopyOps;3855 SmallVector<const Expr *, 4> CopyArrayElems;3856 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {3857 assert(C->getModifier() == OMPC_REDUCTION_inscan &&3858 "Only inscan reductions are expected.");3859 Shareds.append(C->varlist_begin(), C->varlist_end());3860 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());3861 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());3862 Privates.append(C->privates().begin(), C->privates().end());3863 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());3864 CopyArrayElems.append(C->copy_array_elems().begin(),3865 C->copy_array_elems().end());3866 }3867 // Create temp var and copy LHS value to this temp value.3868 // LHS = TMP[LastIter];3869 llvm::Value *OMPLast = CGF.Builder.CreateNSWSub(3870 OMPScanNumIterations,3871 llvm::ConstantInt::get(CGF.SizeTy, 1, /*isSigned=*/false));3872 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {3873 const Expr *PrivateExpr = Privates[I];3874 const Expr *OrigExpr = Shareds[I];3875 const Expr *CopyArrayElem = CopyArrayElems[I];3876 CodeGenFunction::OpaqueValueMapping IdxMapping(3877 CGF,3878 cast<OpaqueValueExpr>(3879 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),3880 RValue::get(OMPLast));3881 LValue DestLVal = CGF.EmitLValue(OrigExpr);3882 LValue SrcLVal = CGF.EmitLValue(CopyArrayElem);3883 CGF.EmitOMPCopy(3884 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),3885 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),3886 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);3887 }3888}3889 3890/// Emits the code for the directive with inscan reductions.3891/// The code is the following:3892/// \code3893/// #pragma omp ...3894/// for (i: 0..<num_iters>) {3895/// <input phase>;3896/// buffer[i] = red;3897/// }3898/// #pragma omp master // in parallel region3899/// for (int k = 0; k != ceil(log2(num_iters)); ++k)3900/// for (size cnt = last_iter; cnt >= pow(2, k); --k)3901/// buffer[i] op= buffer[i-pow(2,k)];3902/// #pragma omp barrier // in parallel region3903/// #pragma omp ...3904/// for (0..<num_iters>) {3905/// red = InclusiveScan ? buffer[i] : buffer[i-1];3906/// <scan phase>;3907/// }3908/// \endcode3909static void emitScanBasedDirective(3910 CodeGenFunction &CGF, const OMPLoopDirective &S,3911 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen,3912 llvm::function_ref<void(CodeGenFunction &)> FirstGen,3913 llvm::function_ref<void(CodeGenFunction &)> SecondGen) {3914 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(3915 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);3916 SmallVector<const Expr *, 4> Privates;3917 SmallVector<const Expr *, 4> ReductionOps;3918 SmallVector<const Expr *, 4> LHSs;3919 SmallVector<const Expr *, 4> RHSs;3920 SmallVector<const Expr *, 4> CopyArrayElems;3921 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {3922 assert(C->getModifier() == OMPC_REDUCTION_inscan &&3923 "Only inscan reductions are expected.");3924 Privates.append(C->privates().begin(), C->privates().end());3925 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());3926 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());3927 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());3928 CopyArrayElems.append(C->copy_array_elems().begin(),3929 C->copy_array_elems().end());3930 }3931 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);3932 {3933 // Emit loop with input phase:3934 // #pragma omp ...3935 // for (i: 0..<num_iters>) {3936 // <input phase>;3937 // buffer[i] = red;3938 // }3939 CGF.OMPFirstScanLoop = true;3940 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);3941 FirstGen(CGF);3942 }3943 // #pragma omp barrier // in parallel region3944 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,3945 &ReductionOps,3946 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) {3947 Action.Enter(CGF);3948 // Emit prefix reduction:3949 // #pragma omp master // in parallel region3950 // for (int k = 0; k <= ceil(log2(n)); ++k)3951 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();3952 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body");3953 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit");3954 llvm::Function *F =3955 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);3956 llvm::Value *Arg =3957 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);3958 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);3959 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);3960 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);3961 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);3962 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(3963 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));3964 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getBeginLoc());3965 CGF.EmitBlock(LoopBB);3966 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);3967 // size pow2k = 1;3968 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);3969 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);3970 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);3971 // for (size i = n - 1; i >= 2 ^ k; --i)3972 // tmp[i] op= tmp[i-pow2k];3973 llvm::BasicBlock *InnerLoopBB =3974 CGF.createBasicBlock("omp.inner.log.scan.body");3975 llvm::BasicBlock *InnerExitBB =3976 CGF.createBasicBlock("omp.inner.log.scan.exit");3977 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);3978 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);3979 CGF.EmitBlock(InnerLoopBB);3980 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);3981 IVal->addIncoming(NMin1, LoopBB);3982 {3983 CodeGenFunction::OMPPrivateScope PrivScope(CGF);3984 auto *ILHS = LHSs.begin();3985 auto *IRHS = RHSs.begin();3986 for (const Expr *CopyArrayElem : CopyArrayElems) {3987 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());3988 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());3989 Address LHSAddr = Address::invalid();3990 {3991 CodeGenFunction::OpaqueValueMapping IdxMapping(3992 CGF,3993 cast<OpaqueValueExpr>(3994 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),3995 RValue::get(IVal));3996 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();3997 }3998 PrivScope.addPrivate(LHSVD, LHSAddr);3999 Address RHSAddr = Address::invalid();4000 {4001 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);4002 CodeGenFunction::OpaqueValueMapping IdxMapping(4003 CGF,4004 cast<OpaqueValueExpr>(4005 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),4006 RValue::get(OffsetIVal));4007 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();4008 }4009 PrivScope.addPrivate(RHSVD, RHSAddr);4010 ++ILHS;4011 ++IRHS;4012 }4013 PrivScope.Privatize();4014 CGF.CGM.getOpenMPRuntime().emitReduction(4015 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,4016 {/*WithNowait=*/true, /*SimpleReduction=*/true,4017 /*IsPrivateVarReduction*/ {}, OMPD_unknown});4018 }4019 llvm::Value *NextIVal =4020 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));4021 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());4022 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);4023 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);4024 CGF.EmitBlock(InnerExitBB);4025 llvm::Value *Next =4026 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));4027 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());4028 // pow2k <<= 1;4029 llvm::Value *NextPow2K =4030 CGF.Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true);4031 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());4032 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);4033 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);4034 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getEndLoc());4035 CGF.EmitBlock(ExitBB);4036 };4037 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);4038 if (isOpenMPParallelDirective(EKind)) {4039 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());4040 CGF.CGM.getOpenMPRuntime().emitBarrierCall(4041 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,4042 /*ForceSimpleCall=*/true);4043 } else {4044 RegionCodeGenTy RCG(CodeGen);4045 RCG(CGF);4046 }4047 4048 CGF.OMPFirstScanLoop = false;4049 SecondGen(CGF);4050}4051 4052static bool emitWorksharingDirective(CodeGenFunction &CGF,4053 const OMPLoopDirective &S,4054 bool HasCancel) {4055 bool HasLastprivates;4056 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);4057 if (llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),4058 [](const OMPReductionClause *C) {4059 return C->getModifier() == OMPC_REDUCTION_inscan;4060 })) {4061 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {4062 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);4063 OMPLoopScope LoopScope(CGF, S);4064 return CGF.EmitScalarExpr(S.getNumIterations());4065 };4066 const auto &&FirstGen = [&S, HasCancel, EKind](CodeGenFunction &CGF) {4067 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);4068 (void)CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),4069 emitForLoopBounds,4070 emitDispatchForLoopBounds);4071 // Emit an implicit barrier at the end.4072 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(),4073 OMPD_for);4074 };4075 const auto &&SecondGen = [&S, HasCancel, EKind,4076 &HasLastprivates](CodeGenFunction &CGF) {4077 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);4078 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),4079 emitForLoopBounds,4080 emitDispatchForLoopBounds);4081 };4082 if (!isOpenMPParallelDirective(EKind))4083 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen);4084 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen);4085 if (!isOpenMPParallelDirective(EKind))4086 emitScanBasedDirectiveFinals(CGF, S, NumIteratorsGen);4087 } else {4088 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);4089 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),4090 emitForLoopBounds,4091 emitDispatchForLoopBounds);4092 }4093 return HasLastprivates;4094}4095 4096// Pass OMPLoopDirective (instead of OMPForDirective) to make this check4097// available for "loop bind(parallel)", which maps to "for".4098static bool isForSupportedByOpenMPIRBuilder(const OMPLoopDirective &S,4099 bool HasCancel) {4100 if (HasCancel)4101 return false;4102 for (OMPClause *C : S.clauses()) {4103 if (isa<OMPNowaitClause, OMPBindClause>(C))4104 continue;4105 4106 if (auto *SC = dyn_cast<OMPScheduleClause>(C)) {4107 if (SC->getFirstScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)4108 return false;4109 if (SC->getSecondScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)4110 return false;4111 switch (SC->getScheduleKind()) {4112 case OMPC_SCHEDULE_auto:4113 case OMPC_SCHEDULE_dynamic:4114 case OMPC_SCHEDULE_runtime:4115 case OMPC_SCHEDULE_guided:4116 case OMPC_SCHEDULE_static:4117 continue;4118 case OMPC_SCHEDULE_unknown:4119 return false;4120 }4121 }4122 4123 return false;4124 }4125 4126 return true;4127}4128 4129static llvm::omp::ScheduleKind4130convertClauseKindToSchedKind(OpenMPScheduleClauseKind ScheduleClauseKind) {4131 switch (ScheduleClauseKind) {4132 case OMPC_SCHEDULE_unknown:4133 return llvm::omp::OMP_SCHEDULE_Default;4134 case OMPC_SCHEDULE_auto:4135 return llvm::omp::OMP_SCHEDULE_Auto;4136 case OMPC_SCHEDULE_dynamic:4137 return llvm::omp::OMP_SCHEDULE_Dynamic;4138 case OMPC_SCHEDULE_guided:4139 return llvm::omp::OMP_SCHEDULE_Guided;4140 case OMPC_SCHEDULE_runtime:4141 return llvm::omp::OMP_SCHEDULE_Runtime;4142 case OMPC_SCHEDULE_static:4143 return llvm::omp::OMP_SCHEDULE_Static;4144 }4145 llvm_unreachable("Unhandled schedule kind");4146}4147 4148// Pass OMPLoopDirective (instead of OMPForDirective) to make this function4149// available for "loop bind(parallel)", which maps to "for".4150static void emitOMPForDirective(const OMPLoopDirective &S, CodeGenFunction &CGF,4151 CodeGenModule &CGM, bool HasCancel) {4152 bool HasLastprivates = false;4153 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder &&4154 isForSupportedByOpenMPIRBuilder(S, HasCancel);4155 auto &&CodeGen = [&S, &CGM, HasCancel, &HasLastprivates,4156 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) {4157 // Use the OpenMPIRBuilder if enabled.4158 if (UseOMPIRBuilder) {4159 bool NeedsBarrier = !S.getSingleClause<OMPNowaitClause>();4160 4161 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;4162 llvm::Value *ChunkSize = nullptr;4163 if (auto *SchedClause = S.getSingleClause<OMPScheduleClause>()) {4164 SchedKind =4165 convertClauseKindToSchedKind(SchedClause->getScheduleKind());4166 if (const Expr *ChunkSizeExpr = SchedClause->getChunkSize())4167 ChunkSize = CGF.EmitScalarExpr(ChunkSizeExpr);4168 }4169 4170 // Emit the associated statement and get its loop representation.4171 const Stmt *Inner = S.getRawStmt();4172 llvm::CanonicalLoopInfo *CLI =4173 CGF.EmitOMPCollapsedCanonicalLoopNest(Inner, 1);4174 4175 llvm::OpenMPIRBuilder &OMPBuilder =4176 CGM.getOpenMPRuntime().getOMPBuilder();4177 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(4178 CGF.AllocaInsertPt->getParent(), CGF.AllocaInsertPt->getIterator());4179 cantFail(OMPBuilder.applyWorkshareLoop(4180 CGF.Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,4181 SchedKind, ChunkSize, /*HasSimdModifier=*/false,4182 /*HasMonotonicModifier=*/false, /*HasNonmonotonicModifier=*/false,4183 /*HasOrderedClause=*/false));4184 return;4185 }4186 4187 HasLastprivates = emitWorksharingDirective(CGF, S, HasCancel);4188 };4189 {4190 auto LPCRegion =4191 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);4192 OMPLexicalScope Scope(CGF, S, OMPD_unknown);4193 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_for, CodeGen,4194 HasCancel);4195 }4196 4197 if (!UseOMPIRBuilder) {4198 // Emit an implicit barrier at the end.4199 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)4200 CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(), OMPD_for);4201 }4202 // Check for outer lastprivate conditional update.4203 checkForLastprivateConditionalUpdate(CGF, S);4204}4205 4206void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {4207 return emitOMPForDirective(S, *this, CGM, S.hasCancel());4208}4209 4210void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) {4211 bool HasLastprivates = false;4212 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,4213 PrePostActionTy &) {4214 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false);4215 };4216 {4217 auto LPCRegion =4218 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);4219 OMPLexicalScope Scope(*this, S, OMPD_unknown);4220 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);4221 }4222 4223 // Emit an implicit barrier at the end.4224 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)4225 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);4226 // Check for outer lastprivate conditional update.4227 checkForLastprivateConditionalUpdate(*this, S);4228}4229 4230static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,4231 const Twine &Name,4232 llvm::Value *Init = nullptr) {4233 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);4234 if (Init)4235 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);4236 return LVal;4237}4238 4239void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {4240 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();4241 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);4242 bool HasLastprivates = false;4243 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);4244 auto &&CodeGen = [&S, CapturedStmt, CS, EKind,4245 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {4246 const ASTContext &C = CGF.getContext();4247 QualType KmpInt32Ty =4248 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);4249 // Emit helper vars inits.4250 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",4251 CGF.Builder.getInt32(0));4252 llvm::ConstantInt *GlobalUBVal = CS != nullptr4253 ? CGF.Builder.getInt32(CS->size() - 1)4254 : CGF.Builder.getInt32(0);4255 LValue UB =4256 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);4257 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",4258 CGF.Builder.getInt32(1));4259 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",4260 CGF.Builder.getInt32(0));4261 // Loop counter.4262 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");4263 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);4264 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);4265 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);4266 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);4267 // Generate condition for loop.4268 BinaryOperator *Cond = BinaryOperator::Create(4269 C, &IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_PRValue, OK_Ordinary,4270 S.getBeginLoc(), FPOptionsOverride());4271 // Increment for loop counter.4272 UnaryOperator *Inc = UnaryOperator::Create(4273 C, &IVRefExpr, UO_PreInc, KmpInt32Ty, VK_PRValue, OK_Ordinary,4274 S.getBeginLoc(), true, FPOptionsOverride());4275 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {4276 // Iterate through all sections and emit a switch construct:4277 // switch (IV) {4278 // case 0:4279 // <SectionStmt[0]>;4280 // break;4281 // ...4282 // case <NumSection> - 1:4283 // <SectionStmt[<NumSection> - 1]>;4284 // break;4285 // }4286 // .omp.sections.exit:4287 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");4288 llvm::SwitchInst *SwitchStmt =4289 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),4290 ExitBB, CS == nullptr ? 1 : CS->size());4291 if (CS) {4292 unsigned CaseNumber = 0;4293 for (const Stmt *SubStmt : CS->children()) {4294 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");4295 CGF.EmitBlock(CaseBB);4296 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);4297 CGF.EmitStmt(SubStmt);4298 CGF.EmitBranch(ExitBB);4299 ++CaseNumber;4300 }4301 } else {4302 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");4303 CGF.EmitBlock(CaseBB);4304 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);4305 CGF.EmitStmt(CapturedStmt);4306 CGF.EmitBranch(ExitBB);4307 }4308 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);4309 };4310 4311 CodeGenFunction::OMPPrivateScope LoopScope(CGF);4312 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {4313 // Emit implicit barrier to synchronize threads and avoid data races on4314 // initialization of firstprivate variables and post-update of lastprivate4315 // variables.4316 CGF.CGM.getOpenMPRuntime().emitBarrierCall(4317 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,4318 /*ForceSimpleCall=*/true);4319 }4320 CGF.EmitOMPPrivateClause(S, LoopScope);4321 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);4322 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);4323 CGF.EmitOMPReductionClauseInit(S, LoopScope);4324 (void)LoopScope.Privatize();4325 if (isOpenMPTargetExecutionDirective(EKind))4326 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);4327 4328 // Emit static non-chunked loop.4329 OpenMPScheduleTy ScheduleKind;4330 ScheduleKind.Schedule = OMPC_SCHEDULE_static;4331 CGOpenMPRuntime::StaticRTInput StaticInit(4332 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),4333 LB.getAddress(), UB.getAddress(), ST.getAddress());4334 CGF.CGM.getOpenMPRuntime().emitForStaticInit(CGF, S.getBeginLoc(), EKind,4335 ScheduleKind, StaticInit);4336 // UB = min(UB, GlobalUB);4337 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());4338 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(4339 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);4340 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);4341 // IV = LB;4342 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);4343 // while (idx <= UB) { BODY; ++idx; }4344 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, Cond, Inc, BodyGen,4345 [](CodeGenFunction &) {});4346 // Tell the runtime we are done.4347 auto &&CodeGen = [&S](CodeGenFunction &CGF) {4348 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),4349 OMPD_sections);4350 };4351 CGF.OMPCancelStack.emitExit(CGF, EKind, CodeGen);4352 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);4353 // Emit post-update of the reduction variables if IsLastIter != 0.4354 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {4355 return CGF.Builder.CreateIsNotNull(4356 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));4357 });4358 4359 // Emit final copy of the lastprivate variables if IsLastIter != 0.4360 if (HasLastprivates)4361 CGF.EmitOMPLastprivateClauseFinal(4362 S, /*NoFinals=*/false,4363 CGF.Builder.CreateIsNotNull(4364 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));4365 };4366 4367 bool HasCancel = false;4368 if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))4369 HasCancel = OSD->hasCancel();4370 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))4371 HasCancel = OPSD->hasCancel();4372 OMPCancelStackRAII CancelRegion(*this, EKind, HasCancel);4373 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,4374 HasCancel);4375 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'4376 // clause. Otherwise the barrier will be generated by the codegen for the4377 // directive.4378 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {4379 // Emit implicit barrier to synchronize threads and avoid data races on4380 // initialization of firstprivate variables.4381 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),4382 OMPD_unknown);4383 }4384}4385 4386void CodeGenFunction::EmitOMPScopeDirective(const OMPScopeDirective &S) {4387 {4388 // Emit code for 'scope' region4389 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {4390 Action.Enter(CGF);4391 OMPPrivateScope PrivateScope(CGF);4392 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);4393 CGF.EmitOMPPrivateClause(S, PrivateScope);4394 CGF.EmitOMPReductionClauseInit(S, PrivateScope);4395 (void)PrivateScope.Privatize();4396 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());4397 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);4398 };4399 auto LPCRegion =4400 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);4401 OMPLexicalScope Scope(*this, S, OMPD_unknown);4402 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_scope, CodeGen);4403 }4404 // Emit an implicit barrier at the end.4405 if (!S.getSingleClause<OMPNowaitClause>()) {4406 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_scope);4407 }4408 // Check for outer lastprivate conditional update.4409 checkForLastprivateConditionalUpdate(*this, S);4410}4411 4412void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {4413 if (CGM.getLangOpts().OpenMPIRBuilder) {4414 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();4415 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;4416 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;4417 4418 auto FiniCB = [](InsertPointTy IP) {4419 // Don't FinalizeOMPRegion because this is done inside of OMPIRBuilder for4420 // sections.4421 return llvm::Error::success();4422 };4423 4424 const CapturedStmt *ICS = S.getInnermostCapturedStmt();4425 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();4426 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);4427 llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;4428 if (CS) {4429 for (const Stmt *SubStmt : CS->children()) {4430 auto SectionCB = [this, SubStmt](InsertPointTy AllocaIP,4431 InsertPointTy CodeGenIP) {4432 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(4433 *this, SubStmt, AllocaIP, CodeGenIP, "section");4434 return llvm::Error::success();4435 };4436 SectionCBVector.push_back(SectionCB);4437 }4438 } else {4439 auto SectionCB = [this, CapturedStmt](InsertPointTy AllocaIP,4440 InsertPointTy CodeGenIP) {4441 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(4442 *this, CapturedStmt, AllocaIP, CodeGenIP, "section");4443 return llvm::Error::success();4444 };4445 SectionCBVector.push_back(SectionCB);4446 }4447 4448 // Privatization callback that performs appropriate action for4449 // shared/private/firstprivate/lastprivate/copyin/... variables.4450 //4451 // TODO: This defaults to shared right now.4452 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,4453 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {4454 // The next line is appropriate only for variables (Val) with the4455 // data-sharing attribute "shared".4456 ReplVal = &Val;4457 4458 return CodeGenIP;4459 };4460 4461 CGCapturedStmtInfo CGSI(*ICS, CR_OpenMP);4462 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);4463 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(4464 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());4465 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =4466 cantFail(OMPBuilder.createSections(4467 Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),4468 S.getSingleClause<OMPNowaitClause>()));4469 Builder.restoreIP(AfterIP);4470 return;4471 }4472 {4473 auto LPCRegion =4474 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);4475 OMPLexicalScope Scope(*this, S, OMPD_unknown);4476 EmitSections(S);4477 }4478 // Emit an implicit barrier at the end.4479 if (!S.getSingleClause<OMPNowaitClause>()) {4480 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),4481 OMPD_sections);4482 }4483 // Check for outer lastprivate conditional update.4484 checkForLastprivateConditionalUpdate(*this, S);4485}4486 4487void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {4488 if (CGM.getLangOpts().OpenMPIRBuilder) {4489 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();4490 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;4491 4492 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();4493 auto FiniCB = [this](InsertPointTy IP) {4494 OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);4495 return llvm::Error::success();4496 };4497 4498 auto BodyGenCB = [SectionRegionBodyStmt, this](InsertPointTy AllocaIP,4499 InsertPointTy CodeGenIP) {4500 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(4501 *this, SectionRegionBodyStmt, AllocaIP, CodeGenIP, "section");4502 return llvm::Error::success();4503 };4504 4505 LexicalScope Scope(*this, S.getSourceRange());4506 EmitStopPoint(&S);4507 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =4508 cantFail(OMPBuilder.createSection(Builder, BodyGenCB, FiniCB));4509 Builder.restoreIP(AfterIP);4510 4511 return;4512 }4513 LexicalScope Scope(*this, S.getSourceRange());4514 EmitStopPoint(&S);4515 EmitStmt(S.getAssociatedStmt());4516}4517 4518void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {4519 llvm::SmallVector<const Expr *, 8> CopyprivateVars;4520 llvm::SmallVector<const Expr *, 8> DestExprs;4521 llvm::SmallVector<const Expr *, 8> SrcExprs;4522 llvm::SmallVector<const Expr *, 8> AssignmentOps;4523 // Check if there are any 'copyprivate' clauses associated with this4524 // 'single' construct.4525 // Build a list of copyprivate variables along with helper expressions4526 // (<source>, <destination>, <destination>=<source> expressions)4527 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {4528 CopyprivateVars.append(C->varlist_begin(), C->varlist_end());4529 DestExprs.append(C->destination_exprs().begin(),4530 C->destination_exprs().end());4531 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());4532 AssignmentOps.append(C->assignment_ops().begin(),4533 C->assignment_ops().end());4534 }4535 // Emit code for 'single' region along with 'copyprivate' clauses4536 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {4537 Action.Enter(CGF);4538 OMPPrivateScope SingleScope(CGF);4539 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);4540 CGF.EmitOMPPrivateClause(S, SingleScope);4541 (void)SingleScope.Privatize();4542 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());4543 };4544 {4545 auto LPCRegion =4546 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);4547 OMPLexicalScope Scope(*this, S, OMPD_unknown);4548 CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),4549 CopyprivateVars, DestExprs,4550 SrcExprs, AssignmentOps);4551 }4552 // Emit an implicit barrier at the end (to avoid data race on firstprivate4553 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).4554 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {4555 CGM.getOpenMPRuntime().emitBarrierCall(4556 *this, S.getBeginLoc(),4557 S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);4558 }4559 // Check for outer lastprivate conditional update.4560 checkForLastprivateConditionalUpdate(*this, S);4561}4562 4563static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) {4564 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {4565 Action.Enter(CGF);4566 CGF.EmitStmt(S.getRawStmt());4567 };4568 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());4569}4570 4571void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {4572 if (CGM.getLangOpts().OpenMPIRBuilder) {4573 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();4574 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;4575 4576 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();4577 4578 auto FiniCB = [this](InsertPointTy IP) {4579 OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);4580 return llvm::Error::success();4581 };4582 4583 auto BodyGenCB = [MasterRegionBodyStmt, this](InsertPointTy AllocaIP,4584 InsertPointTy CodeGenIP) {4585 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(4586 *this, MasterRegionBodyStmt, AllocaIP, CodeGenIP, "master");4587 return llvm::Error::success();4588 };4589 4590 LexicalScope Scope(*this, S.getSourceRange());4591 EmitStopPoint(&S);4592 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =4593 cantFail(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB));4594 Builder.restoreIP(AfterIP);4595 4596 return;4597 }4598 LexicalScope Scope(*this, S.getSourceRange());4599 EmitStopPoint(&S);4600 emitMaster(*this, S);4601}4602 4603static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S) {4604 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {4605 Action.Enter(CGF);4606 CGF.EmitStmt(S.getRawStmt());4607 };4608 Expr *Filter = nullptr;4609 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())4610 Filter = FilterClause->getThreadID();4611 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, CodeGen, S.getBeginLoc(),4612 Filter);4613}4614 4615void CodeGenFunction::EmitOMPMaskedDirective(const OMPMaskedDirective &S) {4616 if (CGM.getLangOpts().OpenMPIRBuilder) {4617 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();4618 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;4619 4620 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();4621 const Expr *Filter = nullptr;4622 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())4623 Filter = FilterClause->getThreadID();4624 llvm::Value *FilterVal = Filter4625 ? EmitScalarExpr(Filter, CGM.Int32Ty)4626 : llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/0);4627 4628 auto FiniCB = [this](InsertPointTy IP) {4629 OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);4630 return llvm::Error::success();4631 };4632 4633 auto BodyGenCB = [MaskedRegionBodyStmt, this](InsertPointTy AllocaIP,4634 InsertPointTy CodeGenIP) {4635 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(4636 *this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP, "masked");4637 return llvm::Error::success();4638 };4639 4640 LexicalScope Scope(*this, S.getSourceRange());4641 EmitStopPoint(&S);4642 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(4643 OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, FilterVal));4644 Builder.restoreIP(AfterIP);4645 4646 return;4647 }4648 LexicalScope Scope(*this, S.getSourceRange());4649 EmitStopPoint(&S);4650 emitMasked(*this, S);4651}4652 4653void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {4654 if (CGM.getLangOpts().OpenMPIRBuilder) {4655 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();4656 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;4657 4658 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();4659 const Expr *Hint = nullptr;4660 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())4661 Hint = HintClause->getHint();4662 4663 // TODO: This is slightly different from what's currently being done in4664 // clang. Fix the Int32Ty to IntPtrTy (pointer width size) when everything4665 // about typing is final.4666 llvm::Value *HintInst = nullptr;4667 if (Hint)4668 HintInst =4669 Builder.CreateIntCast(EmitScalarExpr(Hint), CGM.Int32Ty, false);4670 4671 auto FiniCB = [this](InsertPointTy IP) {4672 OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);4673 return llvm::Error::success();4674 };4675 4676 auto BodyGenCB = [CriticalRegionBodyStmt, this](InsertPointTy AllocaIP,4677 InsertPointTy CodeGenIP) {4678 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(4679 *this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP, "critical");4680 return llvm::Error::success();4681 };4682 4683 LexicalScope Scope(*this, S.getSourceRange());4684 EmitStopPoint(&S);4685 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =4686 cantFail(OMPBuilder.createCritical(Builder, BodyGenCB, FiniCB,4687 S.getDirectiveName().getAsString(),4688 HintInst));4689 Builder.restoreIP(AfterIP);4690 4691 return;4692 }4693 4694 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {4695 Action.Enter(CGF);4696 CGF.EmitStmt(S.getAssociatedStmt());4697 };4698 const Expr *Hint = nullptr;4699 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())4700 Hint = HintClause->getHint();4701 LexicalScope Scope(*this, S.getSourceRange());4702 EmitStopPoint(&S);4703 CGM.getOpenMPRuntime().emitCriticalRegion(*this,4704 S.getDirectiveName().getAsString(),4705 CodeGen, S.getBeginLoc(), Hint);4706}4707 4708void CodeGenFunction::EmitOMPParallelForDirective(4709 const OMPParallelForDirective &S) {4710 // Emit directive as a combined directive that consists of two implicit4711 // directives: 'parallel' with 'for' directive.4712 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {4713 Action.Enter(CGF);4714 emitOMPCopyinClause(CGF, S);4715 (void)emitWorksharingDirective(CGF, S, S.hasCancel());4716 };4717 {4718 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {4719 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);4720 CGCapturedStmtInfo CGSI(CR_OpenMP);4721 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);4722 OMPLoopScope LoopScope(CGF, S);4723 return CGF.EmitScalarExpr(S.getNumIterations());4724 };4725 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),4726 [](const OMPReductionClause *C) {4727 return C->getModifier() == OMPC_REDUCTION_inscan;4728 });4729 if (IsInscan)4730 emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);4731 auto LPCRegion =4732 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);4733 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,4734 emitEmptyBoundParameters);4735 if (IsInscan)4736 emitScanBasedDirectiveFinals(*this, S, NumIteratorsGen);4737 }4738 // Check for outer lastprivate conditional update.4739 checkForLastprivateConditionalUpdate(*this, S);4740}4741 4742void CodeGenFunction::EmitOMPParallelForSimdDirective(4743 const OMPParallelForSimdDirective &S) {4744 // Emit directive as a combined directive that consists of two implicit4745 // directives: 'parallel' with 'for' directive.4746 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {4747 Action.Enter(CGF);4748 emitOMPCopyinClause(CGF, S);4749 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);4750 };4751 {4752 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {4753 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);4754 CGCapturedStmtInfo CGSI(CR_OpenMP);4755 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);4756 OMPLoopScope LoopScope(CGF, S);4757 return CGF.EmitScalarExpr(S.getNumIterations());4758 };4759 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),4760 [](const OMPReductionClause *C) {4761 return C->getModifier() == OMPC_REDUCTION_inscan;4762 });4763 if (IsInscan)4764 emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);4765 auto LPCRegion =4766 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);4767 emitCommonOMPParallelDirective(*this, S, OMPD_for_simd, CodeGen,4768 emitEmptyBoundParameters);4769 if (IsInscan)4770 emitScanBasedDirectiveFinals(*this, S, NumIteratorsGen);4771 }4772 // Check for outer lastprivate conditional update.4773 checkForLastprivateConditionalUpdate(*this, S);4774}4775 4776void CodeGenFunction::EmitOMPParallelMasterDirective(4777 const OMPParallelMasterDirective &S) {4778 // Emit directive as a combined directive that consists of two implicit4779 // directives: 'parallel' with 'master' directive.4780 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {4781 Action.Enter(CGF);4782 OMPPrivateScope PrivateScope(CGF);4783 emitOMPCopyinClause(CGF, S);4784 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);4785 CGF.EmitOMPPrivateClause(S, PrivateScope);4786 CGF.EmitOMPReductionClauseInit(S, PrivateScope);4787 (void)PrivateScope.Privatize();4788 emitMaster(CGF, S);4789 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);4790 };4791 {4792 auto LPCRegion =4793 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);4794 emitCommonOMPParallelDirective(*this, S, OMPD_master, CodeGen,4795 emitEmptyBoundParameters);4796 emitPostUpdateForReductionClause(*this, S,4797 [](CodeGenFunction &) { return nullptr; });4798 }4799 // Check for outer lastprivate conditional update.4800 checkForLastprivateConditionalUpdate(*this, S);4801}4802 4803void CodeGenFunction::EmitOMPParallelMaskedDirective(4804 const OMPParallelMaskedDirective &S) {4805 // Emit directive as a combined directive that consists of two implicit4806 // directives: 'parallel' with 'masked' directive.4807 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {4808 Action.Enter(CGF);4809 OMPPrivateScope PrivateScope(CGF);4810 emitOMPCopyinClause(CGF, S);4811 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);4812 CGF.EmitOMPPrivateClause(S, PrivateScope);4813 CGF.EmitOMPReductionClauseInit(S, PrivateScope);4814 (void)PrivateScope.Privatize();4815 emitMasked(CGF, S);4816 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);4817 };4818 {4819 auto LPCRegion =4820 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);4821 emitCommonOMPParallelDirective(*this, S, OMPD_masked, CodeGen,4822 emitEmptyBoundParameters);4823 emitPostUpdateForReductionClause(*this, S,4824 [](CodeGenFunction &) { return nullptr; });4825 }4826 // Check for outer lastprivate conditional update.4827 checkForLastprivateConditionalUpdate(*this, S);4828}4829 4830void CodeGenFunction::EmitOMPParallelSectionsDirective(4831 const OMPParallelSectionsDirective &S) {4832 // Emit directive as a combined directive that consists of two implicit4833 // directives: 'parallel' with 'sections' directive.4834 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {4835 Action.Enter(CGF);4836 emitOMPCopyinClause(CGF, S);4837 CGF.EmitSections(S);4838 };4839 {4840 auto LPCRegion =4841 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);4842 emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,4843 emitEmptyBoundParameters);4844 }4845 // Check for outer lastprivate conditional update.4846 checkForLastprivateConditionalUpdate(*this, S);4847}4848 4849namespace {4850/// Get the list of variables declared in the context of the untied tasks.4851class CheckVarsEscapingUntiedTaskDeclContext final4852 : public ConstStmtVisitor<CheckVarsEscapingUntiedTaskDeclContext> {4853 llvm::SmallVector<const VarDecl *, 4> PrivateDecls;4854 4855public:4856 explicit CheckVarsEscapingUntiedTaskDeclContext() = default;4857 ~CheckVarsEscapingUntiedTaskDeclContext() = default;4858 void VisitDeclStmt(const DeclStmt *S) {4859 if (!S)4860 return;4861 // Need to privatize only local vars, static locals can be processed as is.4862 for (const Decl *D : S->decls()) {4863 if (const auto *VD = dyn_cast_or_null<VarDecl>(D))4864 if (VD->hasLocalStorage())4865 PrivateDecls.push_back(VD);4866 }4867 }4868 void VisitOMPExecutableDirective(const OMPExecutableDirective *) {}4869 void VisitCapturedStmt(const CapturedStmt *) {}4870 void VisitLambdaExpr(const LambdaExpr *) {}4871 void VisitBlockExpr(const BlockExpr *) {}4872 void VisitStmt(const Stmt *S) {4873 if (!S)4874 return;4875 for (const Stmt *Child : S->children())4876 if (Child)4877 Visit(Child);4878 }4879 4880 /// Swaps list of vars with the provided one.4881 ArrayRef<const VarDecl *> getPrivateDecls() const { return PrivateDecls; }4882};4883} // anonymous namespace4884 4885static void buildDependences(const OMPExecutableDirective &S,4886 OMPTaskDataTy &Data) {4887 4888 // First look for 'omp_all_memory' and add this first.4889 bool OmpAllMemory = false;4890 if (llvm::any_of(4891 S.getClausesOfKind<OMPDependClause>(), [](const OMPDependClause *C) {4892 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||4893 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;4894 })) {4895 OmpAllMemory = true;4896 // Since both OMPC_DEPEND_outallmemory and OMPC_DEPEND_inoutallmemory are4897 // equivalent to the runtime, always use OMPC_DEPEND_outallmemory to4898 // simplify.4899 OMPTaskDataTy::DependData &DD =4900 Data.Dependences.emplace_back(OMPC_DEPEND_outallmemory,4901 /*IteratorExpr=*/nullptr);4902 // Add a nullptr Expr to simplify the codegen in emitDependData.4903 DD.DepExprs.push_back(nullptr);4904 }4905 // Add remaining dependences skipping any 'out' or 'inout' if they are4906 // overridden by 'omp_all_memory'.4907 for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {4908 OpenMPDependClauseKind Kind = C->getDependencyKind();4909 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)4910 continue;4911 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))4912 continue;4913 OMPTaskDataTy::DependData &DD =4914 Data.Dependences.emplace_back(C->getDependencyKind(), C->getModifier());4915 DD.DepExprs.append(C->varlist_begin(), C->varlist_end());4916 }4917}4918 4919void CodeGenFunction::EmitOMPTaskBasedDirective(4920 const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,4921 const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,4922 OMPTaskDataTy &Data) {4923 // Emit outlined function for task construct.4924 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);4925 auto I = CS->getCapturedDecl()->param_begin();4926 auto PartId = std::next(I);4927 auto TaskT = std::next(I, 4);4928 // Check if the task is final4929 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {4930 // If the condition constant folds and can be elided, try to avoid emitting4931 // the condition and the dead arm of the if/else.4932 const Expr *Cond = Clause->getCondition();4933 bool CondConstant;4934 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))4935 Data.Final.setInt(CondConstant);4936 else4937 Data.Final.setPointer(EvaluateExprAsBool(Cond));4938 } else {4939 // By default the task is not final.4940 Data.Final.setInt(/*IntVal=*/false);4941 }4942 // Check if the task has 'priority' clause.4943 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {4944 const Expr *Prio = Clause->getPriority();4945 Data.Priority.setInt(/*IntVal=*/true);4946 Data.Priority.setPointer(EmitScalarConversion(4947 EmitScalarExpr(Prio), Prio->getType(),4948 getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),4949 Prio->getExprLoc()));4950 }4951 // The first function argument for tasks is a thread id, the second one is a4952 // part id (0 for tied tasks, >=0 for untied task).4953 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;4954 // Get list of private variables.4955 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {4956 auto IRef = C->varlist_begin();4957 for (const Expr *IInit : C->private_copies()) {4958 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());4959 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {4960 Data.PrivateVars.push_back(*IRef);4961 Data.PrivateCopies.push_back(IInit);4962 }4963 ++IRef;4964 }4965 }4966 EmittedAsPrivate.clear();4967 // Get list of firstprivate variables.4968 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {4969 auto IRef = C->varlist_begin();4970 auto IElemInitRef = C->inits().begin();4971 for (const Expr *IInit : C->private_copies()) {4972 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());4973 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {4974 Data.FirstprivateVars.push_back(*IRef);4975 Data.FirstprivateCopies.push_back(IInit);4976 Data.FirstprivateInits.push_back(*IElemInitRef);4977 }4978 ++IRef;4979 ++IElemInitRef;4980 }4981 }4982 // Get list of lastprivate variables (for taskloops).4983 llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;4984 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {4985 auto IRef = C->varlist_begin();4986 auto ID = C->destination_exprs().begin();4987 for (const Expr *IInit : C->private_copies()) {4988 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());4989 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {4990 Data.LastprivateVars.push_back(*IRef);4991 Data.LastprivateCopies.push_back(IInit);4992 }4993 LastprivateDstsOrigs.insert(4994 std::make_pair(cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),4995 cast<DeclRefExpr>(*IRef)));4996 ++IRef;4997 ++ID;4998 }4999 }5000 SmallVector<const Expr *, 4> LHSs;5001 SmallVector<const Expr *, 4> RHSs;5002 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {5003 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());5004 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());5005 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());5006 Data.ReductionOps.append(C->reduction_ops().begin(),5007 C->reduction_ops().end());5008 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());5009 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());5010 }5011 Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(5012 *this, S.getBeginLoc(), LHSs, RHSs, Data);5013 // Build list of dependences.5014 buildDependences(S, Data);5015 // Get list of local vars for untied tasks.5016 if (!Data.Tied) {5017 CheckVarsEscapingUntiedTaskDeclContext Checker;5018 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());5019 Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),5020 Checker.getPrivateDecls().end());5021 }5022 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,5023 CapturedRegion](CodeGenFunction &CGF,5024 PrePostActionTy &Action) {5025 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,5026 std::pair<Address, Address>>5027 UntiedLocalVars;5028 // Set proper addresses for generated private copies.5029 OMPPrivateScope Scope(CGF);5030 // Generate debug info for variables present in shared clause.5031 if (auto *DI = CGF.getDebugInfo()) {5032 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =5033 CGF.CapturedStmtInfo->getCaptureFields();5034 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();5035 if (CaptureFields.size() && ContextValue) {5036 unsigned CharWidth = CGF.getContext().getCharWidth();5037 // The shared variables are packed together as members of structure.5038 // So the address of each shared variable can be computed by adding5039 // offset of it (within record) to the base address of record. For each5040 // shared variable, debug intrinsic llvm.dbg.declare is generated with5041 // appropriate expressions (DIExpression).5042 // Ex:5043 // %12 = load %struct.anon*, %struct.anon** %__context.addr.i5044 // call void @llvm.dbg.declare(metadata %struct.anon* %12,5045 // metadata !svar1,5046 // metadata !DIExpression(DW_OP_deref))5047 // call void @llvm.dbg.declare(metadata %struct.anon* %12,5048 // metadata !svar2,5049 // metadata !DIExpression(DW_OP_plus_uconst, 8, DW_OP_deref))5050 for (auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {5051 const VarDecl *SharedVar = It->first;5052 RecordDecl *CaptureRecord = It->second->getParent();5053 const ASTRecordLayout &Layout =5054 CGF.getContext().getASTRecordLayout(CaptureRecord);5055 unsigned Offset =5056 Layout.getFieldOffset(It->second->getFieldIndex()) / CharWidth;5057 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())5058 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,5059 CGF.Builder, false);5060 // Get the call dbg.declare instruction we just created and update5061 // its DIExpression to add offset to base address.5062 auto UpdateExpr = [](llvm::LLVMContext &Ctx, auto *Declare,5063 unsigned Offset) {5064 SmallVector<uint64_t, 8> Ops;5065 // Add offset to the base address if non zero.5066 if (Offset) {5067 Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);5068 Ops.push_back(Offset);5069 }5070 Ops.push_back(llvm::dwarf::DW_OP_deref);5071 Declare->setExpression(llvm::DIExpression::get(Ctx, Ops));5072 };5073 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back();5074 if (auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&Last))5075 UpdateExpr(DDI->getContext(), DDI, Offset);5076 // If we're emitting using the new debug info format into a block5077 // without a terminator, the record will be "trailing".5078 assert(!Last.isTerminator() && "unexpected terminator");5079 if (auto *Marker =5080 CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) {5081 for (llvm::DbgVariableRecord &DVR : llvm::reverse(5082 llvm::filterDbgVars(Marker->getDbgRecordRange()))) {5083 UpdateExpr(Last.getContext(), &DVR, Offset);5084 break;5085 }5086 }5087 }5088 }5089 }5090 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> FirstprivatePtrs;5091 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||5092 !Data.LastprivateVars.empty() || !Data.PrivateLocals.empty()) {5093 enum { PrivatesParam = 2, CopyFnParam = 3 };5094 llvm::Value *CopyFn = CGF.Builder.CreateLoad(5095 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));5096 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(5097 CS->getCapturedDecl()->getParam(PrivatesParam)));5098 // Map privates.5099 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;5100 llvm::SmallVector<llvm::Value *, 16> CallArgs;5101 llvm::SmallVector<llvm::Type *, 4> ParamTypes;5102 CallArgs.push_back(PrivatesPtr);5103 ParamTypes.push_back(PrivatesPtr->getType());5104 for (const Expr *E : Data.PrivateVars) {5105 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());5106 RawAddress PrivatePtr = CGF.CreateMemTemp(5107 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");5108 PrivatePtrs.emplace_back(VD, PrivatePtr);5109 CallArgs.push_back(PrivatePtr.getPointer());5110 ParamTypes.push_back(PrivatePtr.getType());5111 }5112 for (const Expr *E : Data.FirstprivateVars) {5113 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());5114 RawAddress PrivatePtr =5115 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),5116 ".firstpriv.ptr.addr");5117 PrivatePtrs.emplace_back(VD, PrivatePtr);5118 FirstprivatePtrs.emplace_back(VD, PrivatePtr);5119 CallArgs.push_back(PrivatePtr.getPointer());5120 ParamTypes.push_back(PrivatePtr.getType());5121 }5122 for (const Expr *E : Data.LastprivateVars) {5123 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());5124 RawAddress PrivatePtr =5125 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),5126 ".lastpriv.ptr.addr");5127 PrivatePtrs.emplace_back(VD, PrivatePtr);5128 CallArgs.push_back(PrivatePtr.getPointer());5129 ParamTypes.push_back(PrivatePtr.getType());5130 }5131 for (const VarDecl *VD : Data.PrivateLocals) {5132 QualType Ty = VD->getType().getNonReferenceType();5133 if (VD->getType()->isLValueReferenceType())5134 Ty = CGF.getContext().getPointerType(Ty);5135 if (isAllocatableDecl(VD))5136 Ty = CGF.getContext().getPointerType(Ty);5137 RawAddress PrivatePtr = CGF.CreateMemTemp(5138 CGF.getContext().getPointerType(Ty), ".local.ptr.addr");5139 auto Result = UntiedLocalVars.insert(5140 std::make_pair(VD, std::make_pair(PrivatePtr, Address::invalid())));5141 // If key exists update in place.5142 if (Result.second == false)5143 *Result.first = std::make_pair(5144 VD, std::make_pair(PrivatePtr, Address::invalid()));5145 CallArgs.push_back(PrivatePtr.getPointer());5146 ParamTypes.push_back(PrivatePtr.getType());5147 }5148 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),5149 ParamTypes, /*isVarArg=*/false);5150 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(5151 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);5152 for (const auto &Pair : LastprivateDstsOrigs) {5153 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());5154 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),5155 /*RefersToEnclosingVariableOrCapture=*/5156 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,5157 Pair.second->getType(), VK_LValue,5158 Pair.second->getExprLoc());5159 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress());5160 }5161 for (const auto &Pair : PrivatePtrs) {5162 Address Replacement = Address(5163 CGF.Builder.CreateLoad(Pair.second),5164 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),5165 CGF.getContext().getDeclAlign(Pair.first));5166 Scope.addPrivate(Pair.first, Replacement);5167 if (auto *DI = CGF.getDebugInfo())5168 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())5169 (void)DI->EmitDeclareOfAutoVariable(5170 Pair.first, Pair.second.getBasePointer(), CGF.Builder,5171 /*UsePointerValue*/ true);5172 }5173 // Adjust mapping for internal locals by mapping actual memory instead of5174 // a pointer to this memory.5175 for (auto &Pair : UntiedLocalVars) {5176 QualType VDType = Pair.first->getType().getNonReferenceType();5177 if (Pair.first->getType()->isLValueReferenceType())5178 VDType = CGF.getContext().getPointerType(VDType);5179 if (isAllocatableDecl(Pair.first)) {5180 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);5181 Address Replacement(5182 Ptr,5183 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),5184 CGF.getPointerAlign());5185 Pair.second.first = Replacement;5186 Ptr = CGF.Builder.CreateLoad(Replacement);5187 Replacement = Address(Ptr, CGF.ConvertTypeForMem(VDType),5188 CGF.getContext().getDeclAlign(Pair.first));5189 Pair.second.second = Replacement;5190 } else {5191 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);5192 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType),5193 CGF.getContext().getDeclAlign(Pair.first));5194 Pair.second.first = Replacement;5195 }5196 }5197 }5198 if (Data.Reductions) {5199 OMPPrivateScope FirstprivateScope(CGF);5200 for (const auto &Pair : FirstprivatePtrs) {5201 Address Replacement(5202 CGF.Builder.CreateLoad(Pair.second),5203 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),5204 CGF.getContext().getDeclAlign(Pair.first));5205 FirstprivateScope.addPrivate(Pair.first, Replacement);5206 }5207 (void)FirstprivateScope.Privatize();5208 OMPLexicalScope LexScope(CGF, S, CapturedRegion);5209 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,5210 Data.ReductionCopies, Data.ReductionOps);5211 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(5212 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));5213 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {5214 RedCG.emitSharedOrigLValue(CGF, Cnt);5215 RedCG.emitAggregateType(CGF, Cnt);5216 // FIXME: This must removed once the runtime library is fixed.5217 // Emit required threadprivate variables for5218 // initializer/combiner/finalizer.5219 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),5220 RedCG, Cnt);5221 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(5222 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));5223 Replacement = Address(5224 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),5225 CGF.getContext().VoidPtrTy,5226 CGF.getContext().getPointerType(5227 Data.ReductionCopies[Cnt]->getType()),5228 Data.ReductionCopies[Cnt]->getExprLoc()),5229 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),5230 Replacement.getAlignment());5231 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);5232 Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);5233 }5234 }5235 // Privatize all private variables except for in_reduction items.5236 (void)Scope.Privatize();5237 SmallVector<const Expr *, 4> InRedVars;5238 SmallVector<const Expr *, 4> InRedPrivs;5239 SmallVector<const Expr *, 4> InRedOps;5240 SmallVector<const Expr *, 4> TaskgroupDescriptors;5241 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {5242 auto IPriv = C->privates().begin();5243 auto IRed = C->reduction_ops().begin();5244 auto ITD = C->taskgroup_descriptors().begin();5245 for (const Expr *Ref : C->varlist()) {5246 InRedVars.emplace_back(Ref);5247 InRedPrivs.emplace_back(*IPriv);5248 InRedOps.emplace_back(*IRed);5249 TaskgroupDescriptors.emplace_back(*ITD);5250 std::advance(IPriv, 1);5251 std::advance(IRed, 1);5252 std::advance(ITD, 1);5253 }5254 }5255 // Privatize in_reduction items here, because taskgroup descriptors must be5256 // privatized earlier.5257 OMPPrivateScope InRedScope(CGF);5258 if (!InRedVars.empty()) {5259 ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);5260 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {5261 RedCG.emitSharedOrigLValue(CGF, Cnt);5262 RedCG.emitAggregateType(CGF, Cnt);5263 // The taskgroup descriptor variable is always implicit firstprivate and5264 // privatized already during processing of the firstprivates.5265 // FIXME: This must removed once the runtime library is fixed.5266 // Emit required threadprivate variables for5267 // initializer/combiner/finalizer.5268 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),5269 RedCG, Cnt);5270 llvm::Value *ReductionsPtr;5271 if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {5272 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr),5273 TRExpr->getExprLoc());5274 } else {5275 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);5276 }5277 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(5278 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));5279 Replacement = Address(5280 CGF.EmitScalarConversion(5281 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,5282 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),5283 InRedPrivs[Cnt]->getExprLoc()),5284 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),5285 Replacement.getAlignment());5286 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);5287 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);5288 }5289 }5290 (void)InRedScope.Privatize();5291 5292 CGOpenMPRuntime::UntiedTaskLocalDeclsRAII LocalVarsScope(CGF,5293 UntiedLocalVars);5294 Action.Enter(CGF);5295 BodyGen(CGF);5296 };5297 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);5298 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(5299 S, *I, *PartId, *TaskT, EKind, CodeGen, Data.Tied, Data.NumberOfParts);5300 OMPLexicalScope Scope(*this, S, std::nullopt,5301 !isOpenMPParallelDirective(EKind) &&5302 !isOpenMPSimdDirective(EKind));5303 TaskGen(*this, OutlinedFn, Data);5304}5305 5306static ImplicitParamDecl *5307createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data,5308 QualType Ty, CapturedDecl *CD,5309 SourceLocation Loc) {5310 auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,5311 ImplicitParamKind::Other);5312 auto *OrigRef = DeclRefExpr::Create(5313 C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,5314 /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);5315 auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,5316 ImplicitParamKind::Other);5317 auto *PrivateRef = DeclRefExpr::Create(5318 C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,5319 /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);5320 QualType ElemType = C.getBaseElementType(Ty);5321 auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,5322 ImplicitParamKind::Other);5323 auto *InitRef = DeclRefExpr::Create(5324 C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,5325 /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);5326 PrivateVD->setInitStyle(VarDecl::CInit);5327 PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,5328 InitRef, /*BasePath=*/nullptr,5329 VK_PRValue, FPOptionsOverride()));5330 Data.FirstprivateVars.emplace_back(OrigRef);5331 Data.FirstprivateCopies.emplace_back(PrivateRef);5332 Data.FirstprivateInits.emplace_back(InitRef);5333 return OrigVD;5334}5335 5336void CodeGenFunction::EmitOMPTargetTaskBasedDirective(5337 const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,5338 OMPTargetDataInfo &InputInfo) {5339 // Emit outlined function for task construct.5340 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);5341 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);5342 CanQualType SharedsTy =5343 getContext().getCanonicalTagType(CS->getCapturedRecordDecl());5344 auto I = CS->getCapturedDecl()->param_begin();5345 auto PartId = std::next(I);5346 auto TaskT = std::next(I, 4);5347 OMPTaskDataTy Data;5348 // The task is not final.5349 Data.Final.setInt(/*IntVal=*/false);5350 // Get list of firstprivate variables.5351 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {5352 auto IRef = C->varlist_begin();5353 auto IElemInitRef = C->inits().begin();5354 for (auto *IInit : C->private_copies()) {5355 Data.FirstprivateVars.push_back(*IRef);5356 Data.FirstprivateCopies.push_back(IInit);5357 Data.FirstprivateInits.push_back(*IElemInitRef);5358 ++IRef;5359 ++IElemInitRef;5360 }5361 }5362 SmallVector<const Expr *, 4> LHSs;5363 SmallVector<const Expr *, 4> RHSs;5364 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {5365 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());5366 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());5367 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());5368 Data.ReductionOps.append(C->reduction_ops().begin(),5369 C->reduction_ops().end());5370 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());5371 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());5372 }5373 OMPPrivateScope TargetScope(*this);5374 VarDecl *BPVD = nullptr;5375 VarDecl *PVD = nullptr;5376 VarDecl *SVD = nullptr;5377 VarDecl *MVD = nullptr;5378 if (InputInfo.NumberOfTargetItems > 0) {5379 auto *CD = CapturedDecl::Create(5380 getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);5381 llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);5382 QualType BaseAndPointerAndMapperType = getContext().getConstantArrayType(5383 getContext().VoidPtrTy, ArrSize, nullptr, ArraySizeModifier::Normal,5384 /*IndexTypeQuals=*/0);5385 BPVD = createImplicitFirstprivateForType(5386 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());5387 PVD = createImplicitFirstprivateForType(5388 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());5389 QualType SizesType = getContext().getConstantArrayType(5390 getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1),5391 ArrSize, nullptr, ArraySizeModifier::Normal,5392 /*IndexTypeQuals=*/0);5393 SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,5394 S.getBeginLoc());5395 TargetScope.addPrivate(BPVD, InputInfo.BasePointersArray);5396 TargetScope.addPrivate(PVD, InputInfo.PointersArray);5397 TargetScope.addPrivate(SVD, InputInfo.SizesArray);5398 // If there is no user-defined mapper, the mapper array will be nullptr. In5399 // this case, we don't need to privatize it.5400 if (!isa_and_nonnull<llvm::ConstantPointerNull>(5401 InputInfo.MappersArray.emitRawPointer(*this))) {5402 MVD = createImplicitFirstprivateForType(5403 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());5404 TargetScope.addPrivate(MVD, InputInfo.MappersArray);5405 }5406 }5407 (void)TargetScope.Privatize();5408 buildDependences(S, Data);5409 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);5410 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD, EKind,5411 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {5412 // Set proper addresses for generated private copies.5413 OMPPrivateScope Scope(CGF);5414 if (!Data.FirstprivateVars.empty()) {5415 enum { PrivatesParam = 2, CopyFnParam = 3 };5416 llvm::Value *CopyFn = CGF.Builder.CreateLoad(5417 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));5418 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(5419 CS->getCapturedDecl()->getParam(PrivatesParam)));5420 // Map privates.5421 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;5422 llvm::SmallVector<llvm::Value *, 16> CallArgs;5423 llvm::SmallVector<llvm::Type *, 4> ParamTypes;5424 CallArgs.push_back(PrivatesPtr);5425 ParamTypes.push_back(PrivatesPtr->getType());5426 for (const Expr *E : Data.FirstprivateVars) {5427 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());5428 RawAddress PrivatePtr =5429 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),5430 ".firstpriv.ptr.addr");5431 PrivatePtrs.emplace_back(VD, PrivatePtr);5432 CallArgs.push_back(PrivatePtr.getPointer());5433 ParamTypes.push_back(PrivatePtr.getType());5434 }5435 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),5436 ParamTypes, /*isVarArg=*/false);5437 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(5438 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);5439 for (const auto &Pair : PrivatePtrs) {5440 Address Replacement(5441 CGF.Builder.CreateLoad(Pair.second),5442 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),5443 CGF.getContext().getDeclAlign(Pair.first));5444 Scope.addPrivate(Pair.first, Replacement);5445 }5446 }5447 CGF.processInReduction(S, Data, CGF, CS, Scope);5448 if (InputInfo.NumberOfTargetItems > 0) {5449 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(5450 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0);5451 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(5452 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0);5453 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(5454 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0);5455 // If MVD is nullptr, the mapper array is not privatized5456 if (MVD)5457 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP(5458 CGF.GetAddrOfLocalVar(MVD), /*Index=*/0);5459 }5460 5461 Action.Enter(CGF);5462 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);5463 auto *TL = S.getSingleClause<OMPThreadLimitClause>();5464 if (CGF.CGM.getLangOpts().OpenMP >= 51 &&5465 needsTaskBasedThreadLimit(EKind) && TL) {5466 // Emit __kmpc_set_thread_limit() to set the thread_limit for the task5467 // enclosing this target region. This will indirectly set the thread_limit5468 // for every applicable construct within target region.5469 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause(5470 CGF, TL->getThreadLimit().front(), S.getBeginLoc());5471 }5472 BodyGen(CGF);5473 };5474 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(5475 S, *I, *PartId, *TaskT, EKind, CodeGen, /*Tied=*/true,5476 Data.NumberOfParts);5477 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);5478 IntegerLiteral IfCond(getContext(), TrueOrFalse,5479 getContext().getIntTypeForBitwidth(32, /*Signed=*/0),5480 SourceLocation());5481 CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,5482 SharedsTy, CapturedStruct, &IfCond, Data);5483}5484 5485void CodeGenFunction::processInReduction(const OMPExecutableDirective &S,5486 OMPTaskDataTy &Data,5487 CodeGenFunction &CGF,5488 const CapturedStmt *CS,5489 OMPPrivateScope &Scope) {5490 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);5491 if (Data.Reductions) {5492 OpenMPDirectiveKind CapturedRegion = EKind;5493 OMPLexicalScope LexScope(CGF, S, CapturedRegion);5494 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,5495 Data.ReductionCopies, Data.ReductionOps);5496 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(5497 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(4)));5498 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {5499 RedCG.emitSharedOrigLValue(CGF, Cnt);5500 RedCG.emitAggregateType(CGF, Cnt);5501 // FIXME: This must removed once the runtime library is fixed.5502 // Emit required threadprivate variables for5503 // initializer/combiner/finalizer.5504 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),5505 RedCG, Cnt);5506 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(5507 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));5508 Replacement = Address(5509 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),5510 CGF.getContext().VoidPtrTy,5511 CGF.getContext().getPointerType(5512 Data.ReductionCopies[Cnt]->getType()),5513 Data.ReductionCopies[Cnt]->getExprLoc()),5514 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),5515 Replacement.getAlignment());5516 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);5517 Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);5518 }5519 }5520 (void)Scope.Privatize();5521 SmallVector<const Expr *, 4> InRedVars;5522 SmallVector<const Expr *, 4> InRedPrivs;5523 SmallVector<const Expr *, 4> InRedOps;5524 SmallVector<const Expr *, 4> TaskgroupDescriptors;5525 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {5526 auto IPriv = C->privates().begin();5527 auto IRed = C->reduction_ops().begin();5528 auto ITD = C->taskgroup_descriptors().begin();5529 for (const Expr *Ref : C->varlist()) {5530 InRedVars.emplace_back(Ref);5531 InRedPrivs.emplace_back(*IPriv);5532 InRedOps.emplace_back(*IRed);5533 TaskgroupDescriptors.emplace_back(*ITD);5534 std::advance(IPriv, 1);5535 std::advance(IRed, 1);5536 std::advance(ITD, 1);5537 }5538 }5539 OMPPrivateScope InRedScope(CGF);5540 if (!InRedVars.empty()) {5541 ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);5542 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {5543 RedCG.emitSharedOrigLValue(CGF, Cnt);5544 RedCG.emitAggregateType(CGF, Cnt);5545 // FIXME: This must removed once the runtime library is fixed.5546 // Emit required threadprivate variables for5547 // initializer/combiner/finalizer.5548 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),5549 RedCG, Cnt);5550 llvm::Value *ReductionsPtr;5551 if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {5552 ReductionsPtr =5553 CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), TRExpr->getExprLoc());5554 } else {5555 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);5556 }5557 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(5558 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));5559 Replacement = Address(5560 CGF.EmitScalarConversion(5561 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,5562 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),5563 InRedPrivs[Cnt]->getExprLoc()),5564 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),5565 Replacement.getAlignment());5566 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);5567 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);5568 }5569 }5570 (void)InRedScope.Privatize();5571}5572 5573void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {5574 // Emit outlined function for task construct.5575 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);5576 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);5577 CanQualType SharedsTy =5578 getContext().getCanonicalTagType(CS->getCapturedRecordDecl());5579 const Expr *IfCond = nullptr;5580 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {5581 if (C->getNameModifier() == OMPD_unknown ||5582 C->getNameModifier() == OMPD_task) {5583 IfCond = C->getCondition();5584 break;5585 }5586 }5587 5588 OMPTaskDataTy Data;5589 // Check if we should emit tied or untied task.5590 Data.Tied = !S.getSingleClause<OMPUntiedClause>();5591 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {5592 CGF.EmitStmt(CS->getCapturedStmt());5593 };5594 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,5595 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,5596 const OMPTaskDataTy &Data) {5597 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,5598 SharedsTy, CapturedStruct, IfCond,5599 Data);5600 };5601 auto LPCRegion =5602 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);5603 EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);5604}5605 5606void CodeGenFunction::EmitOMPTaskyieldDirective(5607 const OMPTaskyieldDirective &S) {5608 CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());5609}5610 5611void CodeGenFunction::EmitOMPErrorDirective(const OMPErrorDirective &S) {5612 const OMPMessageClause *MC = S.getSingleClause<OMPMessageClause>();5613 Expr *ME = MC ? MC->getMessageString() : nullptr;5614 const OMPSeverityClause *SC = S.getSingleClause<OMPSeverityClause>();5615 bool IsFatal = false;5616 if (!SC || SC->getSeverityKind() == OMPC_SEVERITY_fatal)5617 IsFatal = true;5618 CGM.getOpenMPRuntime().emitErrorCall(*this, S.getBeginLoc(), ME, IsFatal);5619}5620 5621void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {5622 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);5623}5624 5625void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {5626 OMPTaskDataTy Data;5627 // Build list of dependences5628 buildDependences(S, Data);5629 Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();5630 CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc(), Data);5631}5632 5633static bool isSupportedByOpenMPIRBuilder(const OMPTaskgroupDirective &T) {5634 return T.clauses().empty();5635}5636 5637void CodeGenFunction::EmitOMPTaskgroupDirective(5638 const OMPTaskgroupDirective &S) {5639 OMPLexicalScope Scope(*this, S, OMPD_unknown);5640 if (CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(S)) {5641 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();5642 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;5643 InsertPointTy AllocaIP(AllocaInsertPt->getParent(),5644 AllocaInsertPt->getIterator());5645 5646 auto BodyGenCB = [&, this](InsertPointTy AllocaIP,5647 InsertPointTy CodeGenIP) {5648 Builder.restoreIP(CodeGenIP);5649 EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());5650 return llvm::Error::success();5651 };5652 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;5653 if (!CapturedStmtInfo)5654 CapturedStmtInfo = &CapStmtInfo;5655 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =5656 cantFail(OMPBuilder.createTaskgroup(Builder, AllocaIP, BodyGenCB));5657 Builder.restoreIP(AfterIP);5658 return;5659 }5660 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {5661 Action.Enter(CGF);5662 if (const Expr *E = S.getReductionRef()) {5663 SmallVector<const Expr *, 4> LHSs;5664 SmallVector<const Expr *, 4> RHSs;5665 OMPTaskDataTy Data;5666 for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {5667 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());5668 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());5669 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());5670 Data.ReductionOps.append(C->reduction_ops().begin(),5671 C->reduction_ops().end());5672 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());5673 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());5674 }5675 llvm::Value *ReductionDesc =5676 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),5677 LHSs, RHSs, Data);5678 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());5679 CGF.EmitVarDecl(*VD);5680 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),5681 /*Volatile=*/false, E->getType());5682 }5683 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());5684 };5685 CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());5686}5687 5688void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {5689 llvm::AtomicOrdering AO = S.getSingleClause<OMPFlushClause>()5690 ? llvm::AtomicOrdering::NotAtomic5691 : llvm::AtomicOrdering::AcquireRelease;5692 CGM.getOpenMPRuntime().emitFlush(5693 *this,5694 [&S]() -> ArrayRef<const Expr *> {5695 if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())5696 return llvm::ArrayRef(FlushClause->varlist_begin(),5697 FlushClause->varlist_end());5698 return {};5699 }(),5700 S.getBeginLoc(), AO);5701}5702 5703void CodeGenFunction::EmitOMPDepobjDirective(const OMPDepobjDirective &S) {5704 const auto *DO = S.getSingleClause<OMPDepobjClause>();5705 LValue DOLVal = EmitLValue(DO->getDepobj());5706 if (const auto *DC = S.getSingleClause<OMPDependClause>()) {5707 // Build list and emit dependences5708 OMPTaskDataTy Data;5709 buildDependences(S, Data);5710 for (auto &Dep : Data.Dependences) {5711 Address DepAddr = CGM.getOpenMPRuntime().emitDepobjDependClause(5712 *this, Dep, DC->getBeginLoc());5713 EmitStoreOfScalar(DepAddr.emitRawPointer(*this), DOLVal);5714 }5715 return;5716 }5717 if (const auto *DC = S.getSingleClause<OMPDestroyClause>()) {5718 CGM.getOpenMPRuntime().emitDestroyClause(*this, DOLVal, DC->getBeginLoc());5719 return;5720 }5721 if (const auto *UC = S.getSingleClause<OMPUpdateClause>()) {5722 CGM.getOpenMPRuntime().emitUpdateClause(5723 *this, DOLVal, UC->getDependencyKind(), UC->getBeginLoc());5724 return;5725 }5726}5727 5728void CodeGenFunction::EmitOMPScanDirective(const OMPScanDirective &S) {5729 if (!OMPParentLoopDirectiveForScan)5730 return;5731 const OMPExecutableDirective &ParentDir = *OMPParentLoopDirectiveForScan;5732 bool IsInclusive = S.hasClausesOfKind<OMPInclusiveClause>();5733 SmallVector<const Expr *, 4> Shareds;5734 SmallVector<const Expr *, 4> Privates;5735 SmallVector<const Expr *, 4> LHSs;5736 SmallVector<const Expr *, 4> RHSs;5737 SmallVector<const Expr *, 4> ReductionOps;5738 SmallVector<const Expr *, 4> CopyOps;5739 SmallVector<const Expr *, 4> CopyArrayTemps;5740 SmallVector<const Expr *, 4> CopyArrayElems;5741 for (const auto *C : ParentDir.getClausesOfKind<OMPReductionClause>()) {5742 if (C->getModifier() != OMPC_REDUCTION_inscan)5743 continue;5744 Shareds.append(C->varlist_begin(), C->varlist_end());5745 Privates.append(C->privates().begin(), C->privates().end());5746 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());5747 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());5748 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());5749 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());5750 CopyArrayTemps.append(C->copy_array_temps().begin(),5751 C->copy_array_temps().end());5752 CopyArrayElems.append(C->copy_array_elems().begin(),5753 C->copy_array_elems().end());5754 }5755 if (ParentDir.getDirectiveKind() == OMPD_simd ||5756 (getLangOpts().OpenMPSimd &&5757 isOpenMPSimdDirective(ParentDir.getDirectiveKind()))) {5758 // For simd directive and simd-based directives in simd only mode, use the5759 // following codegen:5760 // int x = 0;5761 // #pragma omp simd reduction(inscan, +: x)5762 // for (..) {5763 // <first part>5764 // #pragma omp scan inclusive(x)5765 // <second part>5766 // }5767 // is transformed to:5768 // int x = 0;5769 // for (..) {5770 // int x_priv = 0;5771 // <first part>5772 // x = x_priv + x;5773 // x_priv = x;5774 // <second part>5775 // }5776 // and5777 // int x = 0;5778 // #pragma omp simd reduction(inscan, +: x)5779 // for (..) {5780 // <first part>5781 // #pragma omp scan exclusive(x)5782 // <second part>5783 // }5784 // to5785 // int x = 0;5786 // for (..) {5787 // int x_priv = 0;5788 // <second part>5789 // int temp = x;5790 // x = x_priv + x;5791 // x_priv = temp;5792 // <first part>5793 // }5794 llvm::BasicBlock *OMPScanReduce = createBasicBlock("omp.inscan.reduce");5795 EmitBranch(IsInclusive5796 ? OMPScanReduce5797 : BreakContinueStack.back().ContinueBlock.getBlock());5798 EmitBlock(OMPScanDispatch);5799 {5800 // New scope for correct construction/destruction of temp variables for5801 // exclusive scan.5802 LexicalScope Scope(*this, S.getSourceRange());5803 EmitBranch(IsInclusive ? OMPBeforeScanBlock : OMPAfterScanBlock);5804 EmitBlock(OMPScanReduce);5805 if (!IsInclusive) {5806 // Create temp var and copy LHS value to this temp value.5807 // TMP = LHS;5808 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {5809 const Expr *PrivateExpr = Privates[I];5810 const Expr *TempExpr = CopyArrayTemps[I];5811 EmitAutoVarDecl(5812 *cast<VarDecl>(cast<DeclRefExpr>(TempExpr)->getDecl()));5813 LValue DestLVal = EmitLValue(TempExpr);5814 LValue SrcLVal = EmitLValue(LHSs[I]);5815 EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(),5816 SrcLVal.getAddress(),5817 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),5818 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),5819 CopyOps[I]);5820 }5821 }5822 CGM.getOpenMPRuntime().emitReduction(5823 *this, ParentDir.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,5824 {/*WithNowait=*/true, /*SimpleReduction=*/true,5825 /*IsPrivateVarReduction*/ {}, OMPD_simd});5826 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {5827 const Expr *PrivateExpr = Privates[I];5828 LValue DestLVal;5829 LValue SrcLVal;5830 if (IsInclusive) {5831 DestLVal = EmitLValue(RHSs[I]);5832 SrcLVal = EmitLValue(LHSs[I]);5833 } else {5834 const Expr *TempExpr = CopyArrayTemps[I];5835 DestLVal = EmitLValue(RHSs[I]);5836 SrcLVal = EmitLValue(TempExpr);5837 }5838 EmitOMPCopy(5839 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),5840 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),5841 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);5842 }5843 }5844 EmitBranch(IsInclusive ? OMPAfterScanBlock : OMPBeforeScanBlock);5845 OMPScanExitBlock = IsInclusive5846 ? BreakContinueStack.back().ContinueBlock.getBlock()5847 : OMPScanReduce;5848 EmitBlock(OMPAfterScanBlock);5849 return;5850 }5851 if (!IsInclusive) {5852 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());5853 EmitBlock(OMPScanExitBlock);5854 }5855 if (OMPFirstScanLoop) {5856 // Emit buffer[i] = red; at the end of the input phase.5857 const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)5858 .getIterationVariable()5859 ->IgnoreParenImpCasts();5860 LValue IdxLVal = EmitLValue(IVExpr);5861 llvm::Value *IdxVal = EmitLoadOfScalar(IdxLVal, IVExpr->getExprLoc());5862 IdxVal = Builder.CreateIntCast(IdxVal, SizeTy, /*isSigned=*/false);5863 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {5864 const Expr *PrivateExpr = Privates[I];5865 const Expr *OrigExpr = Shareds[I];5866 const Expr *CopyArrayElem = CopyArrayElems[I];5867 OpaqueValueMapping IdxMapping(5868 *this,5869 cast<OpaqueValueExpr>(5870 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),5871 RValue::get(IdxVal));5872 LValue DestLVal = EmitLValue(CopyArrayElem);5873 LValue SrcLVal = EmitLValue(OrigExpr);5874 EmitOMPCopy(5875 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),5876 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),5877 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);5878 }5879 }5880 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());5881 if (IsInclusive) {5882 EmitBlock(OMPScanExitBlock);5883 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());5884 }5885 EmitBlock(OMPScanDispatch);5886 if (!OMPFirstScanLoop) {5887 // Emit red = buffer[i]; at the entrance to the scan phase.5888 const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)5889 .getIterationVariable()5890 ->IgnoreParenImpCasts();5891 LValue IdxLVal = EmitLValue(IVExpr);5892 llvm::Value *IdxVal = EmitLoadOfScalar(IdxLVal, IVExpr->getExprLoc());5893 IdxVal = Builder.CreateIntCast(IdxVal, SizeTy, /*isSigned=*/false);5894 llvm::BasicBlock *ExclusiveExitBB = nullptr;5895 if (!IsInclusive) {5896 llvm::BasicBlock *ContBB = createBasicBlock("omp.exclusive.dec");5897 ExclusiveExitBB = createBasicBlock("omp.exclusive.copy.exit");5898 llvm::Value *Cmp = Builder.CreateIsNull(IdxVal);5899 Builder.CreateCondBr(Cmp, ExclusiveExitBB, ContBB);5900 EmitBlock(ContBB);5901 // Use idx - 1 iteration for exclusive scan.5902 IdxVal = Builder.CreateNUWSub(IdxVal, llvm::ConstantInt::get(SizeTy, 1));5903 }5904 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {5905 const Expr *PrivateExpr = Privates[I];5906 const Expr *OrigExpr = Shareds[I];5907 const Expr *CopyArrayElem = CopyArrayElems[I];5908 OpaqueValueMapping IdxMapping(5909 *this,5910 cast<OpaqueValueExpr>(5911 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),5912 RValue::get(IdxVal));5913 LValue SrcLVal = EmitLValue(CopyArrayElem);5914 LValue DestLVal = EmitLValue(OrigExpr);5915 EmitOMPCopy(5916 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),5917 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),5918 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);5919 }5920 if (!IsInclusive) {5921 EmitBlock(ExclusiveExitBB);5922 }5923 }5924 EmitBranch((OMPFirstScanLoop == IsInclusive) ? OMPBeforeScanBlock5925 : OMPAfterScanBlock);5926 EmitBlock(OMPAfterScanBlock);5927}5928 5929void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S,5930 const CodeGenLoopTy &CodeGenLoop,5931 Expr *IncExpr) {5932 // Emit the loop iteration variable.5933 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());5934 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());5935 EmitVarDecl(*IVDecl);5936 5937 // Emit the iterations count variable.5938 // If it is not a variable, Sema decided to calculate iterations count on each5939 // iteration (e.g., it is foldable into a constant).5940 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {5941 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));5942 // Emit calculation of the iterations count.5943 EmitIgnoredExpr(S.getCalcLastIteration());5944 }5945 5946 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();5947 5948 bool HasLastprivateClause = false;5949 // Check pre-condition.5950 {5951 OMPLoopScope PreInitScope(*this, S);5952 // Skip the entire loop if we don't meet the precondition.5953 // If the condition constant folds and can be elided, avoid emitting the5954 // whole loop.5955 bool CondConstant;5956 llvm::BasicBlock *ContBlock = nullptr;5957 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {5958 if (!CondConstant)5959 return;5960 } else {5961 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");5962 ContBlock = createBasicBlock("omp.precond.end");5963 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,5964 getProfileCount(&S));5965 EmitBlock(ThenBlock);5966 incrementProfileCounter(&S);5967 }5968 5969 emitAlignedClause(*this, S);5970 // Emit 'then' code.5971 {5972 // Emit helper vars inits.5973 5974 LValue LB = EmitOMPHelperVar(5975 *this, cast<DeclRefExpr>(5976 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())5977 ? S.getCombinedLowerBoundVariable()5978 : S.getLowerBoundVariable())));5979 LValue UB = EmitOMPHelperVar(5980 *this, cast<DeclRefExpr>(5981 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())5982 ? S.getCombinedUpperBoundVariable()5983 : S.getUpperBoundVariable())));5984 LValue ST =5985 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));5986 LValue IL =5987 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));5988 5989 OMPPrivateScope LoopScope(*this);5990 if (EmitOMPFirstprivateClause(S, LoopScope)) {5991 // Emit implicit barrier to synchronize threads and avoid data races5992 // on initialization of firstprivate variables and post-update of5993 // lastprivate variables.5994 CGM.getOpenMPRuntime().emitBarrierCall(5995 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,5996 /*ForceSimpleCall=*/true);5997 }5998 EmitOMPPrivateClause(S, LoopScope);5999 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&6000 !isOpenMPParallelDirective(S.getDirectiveKind()) &&6001 !isOpenMPTeamsDirective(S.getDirectiveKind()))6002 EmitOMPReductionClauseInit(S, LoopScope);6003 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);6004 EmitOMPPrivateLoopCounters(S, LoopScope);6005 (void)LoopScope.Privatize();6006 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))6007 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);6008 6009 // Detect the distribute schedule kind and chunk.6010 llvm::Value *Chunk = nullptr;6011 OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown;6012 if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {6013 ScheduleKind = C->getDistScheduleKind();6014 if (const Expr *Ch = C->getChunkSize()) {6015 Chunk = EmitScalarExpr(Ch);6016 Chunk = EmitScalarConversion(Chunk, Ch->getType(),6017 S.getIterationVariable()->getType(),6018 S.getBeginLoc());6019 }6020 } else {6021 // Default behaviour for dist_schedule clause.6022 CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(6023 *this, S, ScheduleKind, Chunk);6024 }6025 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());6026 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();6027 6028 // OpenMP [2.10.8, distribute Construct, Description]6029 // If dist_schedule is specified, kind must be static. If specified,6030 // iterations are divided into chunks of size chunk_size, chunks are6031 // assigned to the teams of the league in a round-robin fashion in the6032 // order of the team number. When no chunk_size is specified, the6033 // iteration space is divided into chunks that are approximately equal6034 // in size, and at most one chunk is distributed to each team of the6035 // league. The size of the chunks is unspecified in this case.6036 bool StaticChunked =6037 RT.isStaticChunked(ScheduleKind, /* Chunked */ Chunk != nullptr) &&6038 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());6039 if (RT.isStaticNonchunked(ScheduleKind,6040 /* Chunked */ Chunk != nullptr) ||6041 StaticChunked) {6042 CGOpenMPRuntime::StaticRTInput StaticInit(6043 IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),6044 LB.getAddress(), UB.getAddress(), ST.getAddress(),6045 StaticChunked ? Chunk : nullptr);6046 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,6047 StaticInit);6048 JumpDest LoopExit =6049 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));6050 // UB = min(UB, GlobalUB);6051 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())6052 ? S.getCombinedEnsureUpperBound()6053 : S.getEnsureUpperBound());6054 // IV = LB;6055 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())6056 ? S.getCombinedInit()6057 : S.getInit());6058 6059 const Expr *Cond =6060 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())6061 ? S.getCombinedCond()6062 : S.getCond();6063 6064 if (StaticChunked)6065 Cond = S.getCombinedDistCond();6066 6067 // For static unchunked schedules generate:6068 //6069 // 1. For distribute alone, codegen6070 // while (idx <= UB) {6071 // BODY;6072 // ++idx;6073 // }6074 //6075 // 2. When combined with 'for' (e.g. as in 'distribute parallel for')6076 // while (idx <= UB) {6077 // <CodeGen rest of pragma>(LB, UB);6078 // idx += ST;6079 // }6080 //6081 // For static chunk one schedule generate:6082 //6083 // while (IV <= GlobalUB) {6084 // <CodeGen rest of pragma>(LB, UB);6085 // LB += ST;6086 // UB += ST;6087 // UB = min(UB, GlobalUB);6088 // IV = LB;6089 // }6090 //6091 emitCommonSimdLoop(6092 *this, S,6093 [&S](CodeGenFunction &CGF, PrePostActionTy &) {6094 if (isOpenMPSimdDirective(S.getDirectiveKind()))6095 CGF.EmitOMPSimdInit(S);6096 },6097 [&S, &LoopScope, Cond, IncExpr, LoopExit, &CodeGenLoop,6098 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) {6099 CGF.EmitOMPInnerLoop(6100 S, LoopScope.requiresCleanups(), Cond, IncExpr,6101 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {6102 CodeGenLoop(CGF, S, LoopExit);6103 },6104 [&S, StaticChunked](CodeGenFunction &CGF) {6105 if (StaticChunked) {6106 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());6107 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());6108 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());6109 CGF.EmitIgnoredExpr(S.getCombinedInit());6110 }6111 });6112 });6113 EmitBlock(LoopExit.getBlock());6114 // Tell the runtime we are done.6115 RT.emitForStaticFinish(*this, S.getEndLoc(), OMPD_distribute);6116 } else {6117 // Emit the outer loop, which requests its work chunk [LB..UB] from6118 // runtime and runs the inner loop to process it.6119 const OMPLoopArguments LoopArguments = {6120 LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),6121 Chunk};6122 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,6123 CodeGenLoop);6124 }6125 if (isOpenMPSimdDirective(S.getDirectiveKind())) {6126 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {6127 return CGF.Builder.CreateIsNotNull(6128 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));6129 });6130 }6131 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&6132 !isOpenMPParallelDirective(S.getDirectiveKind()) &&6133 !isOpenMPTeamsDirective(S.getDirectiveKind())) {6134 EmitOMPReductionClauseFinal(S, OMPD_simd);6135 // Emit post-update of the reduction variables if IsLastIter != 0.6136 emitPostUpdateForReductionClause(6137 *this, S, [IL, &S](CodeGenFunction &CGF) {6138 return CGF.Builder.CreateIsNotNull(6139 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));6140 });6141 }6142 // Emit final copy of the lastprivate variables if IsLastIter != 0.6143 if (HasLastprivateClause) {6144 EmitOMPLastprivateClauseFinal(6145 S, /*NoFinals=*/false,6146 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));6147 }6148 }6149 6150 // We're now done with the loop, so jump to the continuation block.6151 if (ContBlock) {6152 EmitBranch(ContBlock);6153 EmitBlock(ContBlock, true);6154 }6155 }6156}6157 6158// Pass OMPLoopDirective (instead of OMPDistributeDirective) to make this6159// function available for "loop bind(teams)", which maps to "distribute".6160static void emitOMPDistributeDirective(const OMPLoopDirective &S,6161 CodeGenFunction &CGF,6162 CodeGenModule &CGM) {6163 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {6164 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());6165 };6166 OMPLexicalScope Scope(CGF, S, OMPD_unknown);6167 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, CodeGen);6168}6169 6170void CodeGenFunction::EmitOMPDistributeDirective(6171 const OMPDistributeDirective &S) {6172 emitOMPDistributeDirective(S, *this, CGM);6173}6174 6175static llvm::Function *6176emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S,6177 const OMPExecutableDirective &D) {6178 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);6179 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;6180 CGF.CapturedStmtInfo = &CapStmtInfo;6181 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S, D);6182 Fn->setDoesNotRecurse();6183 return Fn;6184}6185 6186template <typename T>6187static void emitRestoreIP(CodeGenFunction &CGF, const T *C,6188 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,6189 llvm::OpenMPIRBuilder &OMPBuilder) {6190 6191 unsigned NumLoops = C->getNumLoops();6192 QualType Int64Ty = CGF.CGM.getContext().getIntTypeForBitwidth(6193 /*DestWidth=*/64, /*Signed=*/1);6194 llvm::SmallVector<llvm::Value *> StoreValues;6195 for (unsigned I = 0; I < NumLoops; I++) {6196 const Expr *CounterVal = C->getLoopData(I);6197 assert(CounterVal);6198 llvm::Value *StoreValue = CGF.EmitScalarConversion(6199 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty,6200 CounterVal->getExprLoc());6201 StoreValues.emplace_back(StoreValue);6202 }6203 OMPDoacrossKind<T> ODK;6204 bool IsDependSource = ODK.isSource(C);6205 CGF.Builder.restoreIP(6206 OMPBuilder.createOrderedDepend(CGF.Builder, AllocaIP, NumLoops,6207 StoreValues, ".cnt.addr", IsDependSource));6208}6209 6210void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {6211 if (CGM.getLangOpts().OpenMPIRBuilder) {6212 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();6213 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;6214 6215 if (S.hasClausesOfKind<OMPDependClause>() ||6216 S.hasClausesOfKind<OMPDoacrossClause>()) {6217 // The ordered directive with depend clause.6218 assert(!S.hasAssociatedStmt() && "No associated statement must be in "6219 "ordered depend|doacross construct.");6220 InsertPointTy AllocaIP(AllocaInsertPt->getParent(),6221 AllocaInsertPt->getIterator());6222 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())6223 emitRestoreIP(*this, DC, AllocaIP, OMPBuilder);6224 for (const auto *DC : S.getClausesOfKind<OMPDoacrossClause>())6225 emitRestoreIP(*this, DC, AllocaIP, OMPBuilder);6226 } else {6227 // The ordered directive with threads or simd clause, or without clause.6228 // Without clause, it behaves as if the threads clause is specified.6229 const auto *C = S.getSingleClause<OMPSIMDClause>();6230 6231 auto FiniCB = [this](InsertPointTy IP) {6232 OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);6233 return llvm::Error::success();6234 };6235 6236 auto BodyGenCB = [&S, C, this](InsertPointTy AllocaIP,6237 InsertPointTy CodeGenIP) {6238 Builder.restoreIP(CodeGenIP);6239 6240 const CapturedStmt *CS = S.getInnermostCapturedStmt();6241 if (C) {6242 llvm::BasicBlock *FiniBB = splitBBWithSuffix(6243 Builder, /*CreateBranch=*/false, ".ordered.after");6244 llvm::SmallVector<llvm::Value *, 16> CapturedVars;6245 GenerateOpenMPCapturedVars(*CS, CapturedVars);6246 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS, S);6247 assert(S.getBeginLoc().isValid() &&6248 "Outlined function call location must be valid.");6249 ApplyDebugLocation::CreateDefaultArtificial(*this, S.getBeginLoc());6250 OMPBuilderCBHelpers::EmitCaptureStmt(*this, CodeGenIP, *FiniBB,6251 OutlinedFn, CapturedVars);6252 } else {6253 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(6254 *this, CS->getCapturedStmt(), AllocaIP, CodeGenIP, "ordered");6255 }6256 return llvm::Error::success();6257 };6258 6259 OMPLexicalScope Scope(*this, S, OMPD_unknown);6260 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(6261 OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, !C));6262 Builder.restoreIP(AfterIP);6263 }6264 return;6265 }6266 6267 if (S.hasClausesOfKind<OMPDependClause>()) {6268 assert(!S.hasAssociatedStmt() &&6269 "No associated statement must be in ordered depend construct.");6270 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())6271 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);6272 return;6273 }6274 if (S.hasClausesOfKind<OMPDoacrossClause>()) {6275 assert(!S.hasAssociatedStmt() &&6276 "No associated statement must be in ordered doacross construct.");6277 for (const auto *DC : S.getClausesOfKind<OMPDoacrossClause>())6278 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);6279 return;6280 }6281 const auto *C = S.getSingleClause<OMPSIMDClause>();6282 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,6283 PrePostActionTy &Action) {6284 const CapturedStmt *CS = S.getInnermostCapturedStmt();6285 if (C) {6286 llvm::SmallVector<llvm::Value *, 16> CapturedVars;6287 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);6288 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS, S);6289 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),6290 OutlinedFn, CapturedVars);6291 } else {6292 Action.Enter(CGF);6293 CGF.EmitStmt(CS->getCapturedStmt());6294 }6295 };6296 OMPLexicalScope Scope(*this, S, OMPD_unknown);6297 CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);6298}6299 6300static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,6301 QualType SrcType, QualType DestType,6302 SourceLocation Loc) {6303 assert(CGF.hasScalarEvaluationKind(DestType) &&6304 "DestType must have scalar evaluation kind.");6305 assert(!Val.isAggregate() && "Must be a scalar or complex.");6306 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,6307 DestType, Loc)6308 : CGF.EmitComplexToScalarConversion(6309 Val.getComplexVal(), SrcType, DestType, Loc);6310}6311 6312static CodeGenFunction::ComplexPairTy6313convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,6314 QualType DestType, SourceLocation Loc) {6315 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&6316 "DestType must have complex evaluation kind.");6317 CodeGenFunction::ComplexPairTy ComplexVal;6318 if (Val.isScalar()) {6319 // Convert the input element to the element type of the complex.6320 QualType DestElementType =6321 DestType->castAs<ComplexType>()->getElementType();6322 llvm::Value *ScalarVal = CGF.EmitScalarConversion(6323 Val.getScalarVal(), SrcType, DestElementType, Loc);6324 ComplexVal = CodeGenFunction::ComplexPairTy(6325 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));6326 } else {6327 assert(Val.isComplex() && "Must be a scalar or complex.");6328 QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();6329 QualType DestElementType =6330 DestType->castAs<ComplexType>()->getElementType();6331 ComplexVal.first = CGF.EmitScalarConversion(6332 Val.getComplexVal().first, SrcElementType, DestElementType, Loc);6333 ComplexVal.second = CGF.EmitScalarConversion(6334 Val.getComplexVal().second, SrcElementType, DestElementType, Loc);6335 }6336 return ComplexVal;6337}6338 6339static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO,6340 LValue LVal, RValue RVal) {6341 if (LVal.isGlobalReg())6342 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);6343 else6344 CGF.EmitAtomicStore(RVal, LVal, AO, LVal.isVolatile(), /*isInit=*/false);6345}6346 6347static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF,6348 llvm::AtomicOrdering AO, LValue LVal,6349 SourceLocation Loc) {6350 if (LVal.isGlobalReg())6351 return CGF.EmitLoadOfLValue(LVal, Loc);6352 return CGF.EmitAtomicLoad(6353 LVal, Loc, llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO),6354 LVal.isVolatile());6355}6356 6357void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal,6358 QualType RValTy, SourceLocation Loc) {6359 switch (getEvaluationKind(LVal.getType())) {6360 case TEK_Scalar:6361 EmitStoreThroughLValue(RValue::get(convertToScalarValue(6362 *this, RVal, RValTy, LVal.getType(), Loc)),6363 LVal);6364 break;6365 case TEK_Complex:6366 EmitStoreOfComplex(6367 convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,6368 /*isInit=*/false);6369 break;6370 case TEK_Aggregate:6371 llvm_unreachable("Must be a scalar or complex.");6372 }6373}6374 6375static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO,6376 const Expr *X, const Expr *V,6377 SourceLocation Loc) {6378 // v = x;6379 assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");6380 assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");6381 LValue XLValue = CGF.EmitLValue(X);6382 LValue VLValue = CGF.EmitLValue(V);6383 RValue Res = emitSimpleAtomicLoad(CGF, AO, XLValue, Loc);6384 // OpenMP, 2.17.7, atomic Construct6385 // If the read or capture clause is specified and the acquire, acq_rel, or6386 // seq_cst clause is specified then the strong flush on exit from the atomic6387 // operation is also an acquire flush.6388 switch (AO) {6389 case llvm::AtomicOrdering::Acquire:6390 case llvm::AtomicOrdering::AcquireRelease:6391 case llvm::AtomicOrdering::SequentiallyConsistent:6392 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,6393 llvm::AtomicOrdering::Acquire);6394 break;6395 case llvm::AtomicOrdering::Monotonic:6396 case llvm::AtomicOrdering::Release:6397 break;6398 case llvm::AtomicOrdering::NotAtomic:6399 case llvm::AtomicOrdering::Unordered:6400 llvm_unreachable("Unexpected ordering.");6401 }6402 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);6403 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V);6404}6405 6406static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF,6407 llvm::AtomicOrdering AO, const Expr *X,6408 const Expr *E, SourceLocation Loc) {6409 // x = expr;6410 assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");6411 emitSimpleAtomicStore(CGF, AO, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));6412 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X);6413 // OpenMP, 2.17.7, atomic Construct6414 // If the write, update, or capture clause is specified and the release,6415 // acq_rel, or seq_cst clause is specified then the strong flush on entry to6416 // the atomic operation is also a release flush.6417 switch (AO) {6418 case llvm::AtomicOrdering::Release:6419 case llvm::AtomicOrdering::AcquireRelease:6420 case llvm::AtomicOrdering::SequentiallyConsistent:6421 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,6422 llvm::AtomicOrdering::Release);6423 break;6424 case llvm::AtomicOrdering::Acquire:6425 case llvm::AtomicOrdering::Monotonic:6426 break;6427 case llvm::AtomicOrdering::NotAtomic:6428 case llvm::AtomicOrdering::Unordered:6429 llvm_unreachable("Unexpected ordering.");6430 }6431}6432 6433static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,6434 RValue Update,6435 BinaryOperatorKind BO,6436 llvm::AtomicOrdering AO,6437 bool IsXLHSInRHSPart) {6438 ASTContext &Context = CGF.getContext();6439 // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'6440 // expression is simple and atomic is allowed for the given type for the6441 // target platform.6442 if (BO == BO_Comma || !Update.isScalar() || !X.isSimple() ||6443 (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&6444 (Update.getScalarVal()->getType() != X.getAddress().getElementType())) ||6445 !Context.getTargetInfo().hasBuiltinAtomic(6446 Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))6447 return std::make_pair(false, RValue::get(nullptr));6448 6449 auto &&CheckAtomicSupport = [&CGF](llvm::Type *T, BinaryOperatorKind BO) {6450 if (T->isIntegerTy())6451 return true;6452 6453 if (T->isFloatingPointTy() && (BO == BO_Add || BO == BO_Sub))6454 return llvm::isPowerOf2_64(CGF.CGM.getDataLayout().getTypeStoreSize(T));6455 6456 return false;6457 };6458 6459 if (!CheckAtomicSupport(Update.getScalarVal()->getType(), BO) ||6460 !CheckAtomicSupport(X.getAddress().getElementType(), BO))6461 return std::make_pair(false, RValue::get(nullptr));6462 6463 bool IsInteger = X.getAddress().getElementType()->isIntegerTy();6464 llvm::AtomicRMWInst::BinOp RMWOp;6465 switch (BO) {6466 case BO_Add:6467 RMWOp = IsInteger ? llvm::AtomicRMWInst::Add : llvm::AtomicRMWInst::FAdd;6468 break;6469 case BO_Sub:6470 if (!IsXLHSInRHSPart)6471 return std::make_pair(false, RValue::get(nullptr));6472 RMWOp = IsInteger ? llvm::AtomicRMWInst::Sub : llvm::AtomicRMWInst::FSub;6473 break;6474 case BO_And:6475 RMWOp = llvm::AtomicRMWInst::And;6476 break;6477 case BO_Or:6478 RMWOp = llvm::AtomicRMWInst::Or;6479 break;6480 case BO_Xor:6481 RMWOp = llvm::AtomicRMWInst::Xor;6482 break;6483 case BO_LT:6484 if (IsInteger)6485 RMWOp = X.getType()->hasSignedIntegerRepresentation()6486 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min6487 : llvm::AtomicRMWInst::Max)6488 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin6489 : llvm::AtomicRMWInst::UMax);6490 else6491 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMin6492 : llvm::AtomicRMWInst::FMax;6493 break;6494 case BO_GT:6495 if (IsInteger)6496 RMWOp = X.getType()->hasSignedIntegerRepresentation()6497 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max6498 : llvm::AtomicRMWInst::Min)6499 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax6500 : llvm::AtomicRMWInst::UMin);6501 else6502 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMax6503 : llvm::AtomicRMWInst::FMin;6504 break;6505 case BO_Assign:6506 RMWOp = llvm::AtomicRMWInst::Xchg;6507 break;6508 case BO_Mul:6509 case BO_Div:6510 case BO_Rem:6511 case BO_Shl:6512 case BO_Shr:6513 case BO_LAnd:6514 case BO_LOr:6515 return std::make_pair(false, RValue::get(nullptr));6516 case BO_PtrMemD:6517 case BO_PtrMemI:6518 case BO_LE:6519 case BO_GE:6520 case BO_EQ:6521 case BO_NE:6522 case BO_Cmp:6523 case BO_AddAssign:6524 case BO_SubAssign:6525 case BO_AndAssign:6526 case BO_OrAssign:6527 case BO_XorAssign:6528 case BO_MulAssign:6529 case BO_DivAssign:6530 case BO_RemAssign:6531 case BO_ShlAssign:6532 case BO_ShrAssign:6533 case BO_Comma:6534 llvm_unreachable("Unsupported atomic update operation");6535 }6536 llvm::Value *UpdateVal = Update.getScalarVal();6537 if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {6538 if (IsInteger)6539 UpdateVal = CGF.Builder.CreateIntCast(6540 IC, X.getAddress().getElementType(),6541 X.getType()->hasSignedIntegerRepresentation());6542 else6543 UpdateVal = CGF.Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC,6544 X.getAddress().getElementType());6545 }6546 llvm::AtomicRMWInst *Res =6547 CGF.emitAtomicRMWInst(RMWOp, X.getAddress(), UpdateVal, AO);6548 return std::make_pair(true, RValue::get(Res));6549}6550 6551std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr(6552 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,6553 llvm::AtomicOrdering AO, SourceLocation Loc,6554 const llvm::function_ref<RValue(RValue)> CommonGen) {6555 // Update expressions are allowed to have the following forms:6556 // x binop= expr; -> xrval + expr;6557 // x++, ++x -> xrval + 1;6558 // x--, --x -> xrval - 1;6559 // x = x binop expr; -> xrval binop expr6560 // x = expr Op x; - > expr binop xrval;6561 auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);6562 if (!Res.first) {6563 if (X.isGlobalReg()) {6564 // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop6565 // 'xrval'.6566 EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);6567 } else {6568 // Perform compare-and-swap procedure.6569 EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());6570 }6571 }6572 return Res;6573}6574 6575static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF,6576 llvm::AtomicOrdering AO, const Expr *X,6577 const Expr *E, const Expr *UE,6578 bool IsXLHSInRHSPart, SourceLocation Loc) {6579 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&6580 "Update expr in 'atomic update' must be a binary operator.");6581 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());6582 // Update expressions are allowed to have the following forms:6583 // x binop= expr; -> xrval + expr;6584 // x++, ++x -> xrval + 1;6585 // x--, --x -> xrval - 1;6586 // x = x binop expr; -> xrval binop expr6587 // x = expr Op x; - > expr binop xrval;6588 assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");6589 LValue XLValue = CGF.EmitLValue(X);6590 RValue ExprRValue = CGF.EmitAnyExpr(E);6591 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());6592 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());6593 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;6594 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;6595 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {6596 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);6597 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);6598 return CGF.EmitAnyExpr(UE);6599 };6600 (void)CGF.EmitOMPAtomicSimpleUpdateExpr(6601 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);6602 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X);6603 // OpenMP, 2.17.7, atomic Construct6604 // If the write, update, or capture clause is specified and the release,6605 // acq_rel, or seq_cst clause is specified then the strong flush on entry to6606 // the atomic operation is also a release flush.6607 switch (AO) {6608 case llvm::AtomicOrdering::Release:6609 case llvm::AtomicOrdering::AcquireRelease:6610 case llvm::AtomicOrdering::SequentiallyConsistent:6611 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,6612 llvm::AtomicOrdering::Release);6613 break;6614 case llvm::AtomicOrdering::Acquire:6615 case llvm::AtomicOrdering::Monotonic:6616 break;6617 case llvm::AtomicOrdering::NotAtomic:6618 case llvm::AtomicOrdering::Unordered:6619 llvm_unreachable("Unexpected ordering.");6620 }6621}6622 6623static RValue convertToType(CodeGenFunction &CGF, RValue Value,6624 QualType SourceType, QualType ResType,6625 SourceLocation Loc) {6626 switch (CGF.getEvaluationKind(ResType)) {6627 case TEK_Scalar:6628 return RValue::get(6629 convertToScalarValue(CGF, Value, SourceType, ResType, Loc));6630 case TEK_Complex: {6631 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);6632 return RValue::getComplex(Res.first, Res.second);6633 }6634 case TEK_Aggregate:6635 break;6636 }6637 llvm_unreachable("Must be a scalar or complex.");6638}6639 6640static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF,6641 llvm::AtomicOrdering AO,6642 bool IsPostfixUpdate, const Expr *V,6643 const Expr *X, const Expr *E,6644 const Expr *UE, bool IsXLHSInRHSPart,6645 SourceLocation Loc) {6646 assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");6647 assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");6648 RValue NewVVal;6649 LValue VLValue = CGF.EmitLValue(V);6650 LValue XLValue = CGF.EmitLValue(X);6651 RValue ExprRValue = CGF.EmitAnyExpr(E);6652 QualType NewVValType;6653 if (UE) {6654 // 'x' is updated with some additional value.6655 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&6656 "Update expr in 'atomic capture' must be a binary operator.");6657 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());6658 // Update expressions are allowed to have the following forms:6659 // x binop= expr; -> xrval + expr;6660 // x++, ++x -> xrval + 1;6661 // x--, --x -> xrval - 1;6662 // x = x binop expr; -> xrval binop expr6663 // x = expr Op x; - > expr binop xrval;6664 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());6665 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());6666 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;6667 NewVValType = XRValExpr->getType();6668 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;6669 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,6670 IsPostfixUpdate](RValue XRValue) {6671 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);6672 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);6673 RValue Res = CGF.EmitAnyExpr(UE);6674 NewVVal = IsPostfixUpdate ? XRValue : Res;6675 return Res;6676 };6677 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(6678 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);6679 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X);6680 if (Res.first) {6681 // 'atomicrmw' instruction was generated.6682 if (IsPostfixUpdate) {6683 // Use old value from 'atomicrmw'.6684 NewVVal = Res.second;6685 } else {6686 // 'atomicrmw' does not provide new value, so evaluate it using old6687 // value of 'x'.6688 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);6689 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);6690 NewVVal = CGF.EmitAnyExpr(UE);6691 }6692 }6693 } else {6694 // 'x' is simply rewritten with some 'expr'.6695 NewVValType = X->getType().getNonReferenceType();6696 ExprRValue = convertToType(CGF, ExprRValue, E->getType(),6697 X->getType().getNonReferenceType(), Loc);6698 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {6699 NewVVal = XRValue;6700 return ExprRValue;6701 };6702 // Try to perform atomicrmw xchg, otherwise simple exchange.6703 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(6704 XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,6705 Loc, Gen);6706 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X);6707 if (Res.first) {6708 // 'atomicrmw' instruction was generated.6709 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;6710 }6711 }6712 // Emit post-update store to 'v' of old/new 'x' value.6713 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);6714 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V);6715 // OpenMP 5.1 removes the required flush for capture clause.6716 if (CGF.CGM.getLangOpts().OpenMP < 51) {6717 // OpenMP, 2.17.7, atomic Construct6718 // If the write, update, or capture clause is specified and the release,6719 // acq_rel, or seq_cst clause is specified then the strong flush on entry to6720 // the atomic operation is also a release flush.6721 // If the read or capture clause is specified and the acquire, acq_rel, or6722 // seq_cst clause is specified then the strong flush on exit from the atomic6723 // operation is also an acquire flush.6724 switch (AO) {6725 case llvm::AtomicOrdering::Release:6726 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,6727 llvm::AtomicOrdering::Release);6728 break;6729 case llvm::AtomicOrdering::Acquire:6730 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,6731 llvm::AtomicOrdering::Acquire);6732 break;6733 case llvm::AtomicOrdering::AcquireRelease:6734 case llvm::AtomicOrdering::SequentiallyConsistent:6735 CGF.CGM.getOpenMPRuntime().emitFlush(6736 CGF, {}, Loc, llvm::AtomicOrdering::AcquireRelease);6737 break;6738 case llvm::AtomicOrdering::Monotonic:6739 break;6740 case llvm::AtomicOrdering::NotAtomic:6741 case llvm::AtomicOrdering::Unordered:6742 llvm_unreachable("Unexpected ordering.");6743 }6744 }6745}6746 6747static void emitOMPAtomicCompareExpr(6748 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO,6749 const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D,6750 const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly,6751 SourceLocation Loc) {6752 llvm::OpenMPIRBuilder &OMPBuilder =6753 CGF.CGM.getOpenMPRuntime().getOMPBuilder();6754 6755 OMPAtomicCompareOp Op;6756 assert(isa<BinaryOperator>(CE) && "CE is not a BinaryOperator");6757 switch (cast<BinaryOperator>(CE)->getOpcode()) {6758 case BO_EQ:6759 Op = OMPAtomicCompareOp::EQ;6760 break;6761 case BO_LT:6762 Op = OMPAtomicCompareOp::MIN;6763 break;6764 case BO_GT:6765 Op = OMPAtomicCompareOp::MAX;6766 break;6767 default:6768 llvm_unreachable("unsupported atomic compare binary operator");6769 }6770 6771 LValue XLVal = CGF.EmitLValue(X);6772 Address XAddr = XLVal.getAddress();6773 6774 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](const Expr *X, const Expr *E) {6775 if (X->getType() == E->getType())6776 return CGF.EmitScalarExpr(E);6777 const Expr *NewE = E->IgnoreImplicitAsWritten();6778 llvm::Value *V = CGF.EmitScalarExpr(NewE);6779 if (NewE->getType() == X->getType())6780 return V;6781 return CGF.EmitScalarConversion(V, NewE->getType(), X->getType(), Loc);6782 };6783 6784 llvm::Value *EVal = EmitRValueWithCastIfNeeded(X, E);6785 llvm::Value *DVal = D ? EmitRValueWithCastIfNeeded(X, D) : nullptr;6786 if (auto *CI = dyn_cast<llvm::ConstantInt>(EVal))6787 EVal = CGF.Builder.CreateIntCast(6788 CI, XLVal.getAddress().getElementType(),6789 E->getType()->hasSignedIntegerRepresentation());6790 if (DVal)6791 if (auto *CI = dyn_cast<llvm::ConstantInt>(DVal))6792 DVal = CGF.Builder.CreateIntCast(6793 CI, XLVal.getAddress().getElementType(),6794 D->getType()->hasSignedIntegerRepresentation());6795 6796 llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{6797 XAddr.emitRawPointer(CGF), XAddr.getElementType(),6798 X->getType()->hasSignedIntegerRepresentation(),6799 X->getType().isVolatileQualified()};6800 llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal;6801 if (V) {6802 LValue LV = CGF.EmitLValue(V);6803 Address Addr = LV.getAddress();6804 VOpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(),6805 V->getType()->hasSignedIntegerRepresentation(),6806 V->getType().isVolatileQualified()};6807 }6808 if (R) {6809 LValue LV = CGF.EmitLValue(R);6810 Address Addr = LV.getAddress();6811 ROpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(),6812 R->getType()->hasSignedIntegerRepresentation(),6813 R->getType().isVolatileQualified()};6814 }6815 6816 if (FailAO == llvm::AtomicOrdering::NotAtomic) {6817 // fail clause was not mentioned on the6818 // "#pragma omp atomic compare" construct.6819 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(6820 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,6821 IsPostfixUpdate, IsFailOnly));6822 } else6823 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(6824 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,6825 IsPostfixUpdate, IsFailOnly, FailAO));6826}6827 6828static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,6829 llvm::AtomicOrdering AO,6830 llvm::AtomicOrdering FailAO, bool IsPostfixUpdate,6831 const Expr *X, const Expr *V, const Expr *R,6832 const Expr *E, const Expr *UE, const Expr *D,6833 const Expr *CE, bool IsXLHSInRHSPart,6834 bool IsFailOnly, SourceLocation Loc) {6835 switch (Kind) {6836 case OMPC_read:6837 emitOMPAtomicReadExpr(CGF, AO, X, V, Loc);6838 break;6839 case OMPC_write:6840 emitOMPAtomicWriteExpr(CGF, AO, X, E, Loc);6841 break;6842 case OMPC_unknown:6843 case OMPC_update:6844 emitOMPAtomicUpdateExpr(CGF, AO, X, E, UE, IsXLHSInRHSPart, Loc);6845 break;6846 case OMPC_capture:6847 emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE,6848 IsXLHSInRHSPart, Loc);6849 break;6850 case OMPC_compare: {6851 emitOMPAtomicCompareExpr(CGF, AO, FailAO, X, V, R, E, D, CE,6852 IsXLHSInRHSPart, IsPostfixUpdate, IsFailOnly, Loc);6853 break;6854 }6855 default:6856 llvm_unreachable("Clause is not allowed in 'omp atomic'.");6857 }6858}6859 6860void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {6861 llvm::AtomicOrdering AO = CGM.getOpenMPRuntime().getDefaultMemoryOrdering();6862 // Fail Memory Clause Ordering.6863 llvm::AtomicOrdering FailAO = llvm::AtomicOrdering::NotAtomic;6864 bool MemOrderingSpecified = false;6865 if (S.getSingleClause<OMPSeqCstClause>()) {6866 AO = llvm::AtomicOrdering::SequentiallyConsistent;6867 MemOrderingSpecified = true;6868 } else if (S.getSingleClause<OMPAcqRelClause>()) {6869 AO = llvm::AtomicOrdering::AcquireRelease;6870 MemOrderingSpecified = true;6871 } else if (S.getSingleClause<OMPAcquireClause>()) {6872 AO = llvm::AtomicOrdering::Acquire;6873 MemOrderingSpecified = true;6874 } else if (S.getSingleClause<OMPReleaseClause>()) {6875 AO = llvm::AtomicOrdering::Release;6876 MemOrderingSpecified = true;6877 } else if (S.getSingleClause<OMPRelaxedClause>()) {6878 AO = llvm::AtomicOrdering::Monotonic;6879 MemOrderingSpecified = true;6880 }6881 llvm::SmallSet<OpenMPClauseKind, 2> KindsEncountered;6882 OpenMPClauseKind Kind = OMPC_unknown;6883 for (const OMPClause *C : S.clauses()) {6884 // Find first clause (skip seq_cst|acq_rel|aqcuire|release|relaxed clause,6885 // if it is first).6886 OpenMPClauseKind K = C->getClauseKind();6887 // TBD6888 if (K == OMPC_weak)6889 return;6890 if (K == OMPC_seq_cst || K == OMPC_acq_rel || K == OMPC_acquire ||6891 K == OMPC_release || K == OMPC_relaxed || K == OMPC_hint)6892 continue;6893 Kind = K;6894 KindsEncountered.insert(K);6895 }6896 // We just need to correct Kind here. No need to set a bool saying it is6897 // actually compare capture because we can tell from whether V and R are6898 // nullptr.6899 if (KindsEncountered.contains(OMPC_compare) &&6900 KindsEncountered.contains(OMPC_capture))6901 Kind = OMPC_compare;6902 if (!MemOrderingSpecified) {6903 llvm::AtomicOrdering DefaultOrder =6904 CGM.getOpenMPRuntime().getDefaultMemoryOrdering();6905 if (DefaultOrder == llvm::AtomicOrdering::Monotonic ||6906 DefaultOrder == llvm::AtomicOrdering::SequentiallyConsistent ||6907 (DefaultOrder == llvm::AtomicOrdering::AcquireRelease &&6908 Kind == OMPC_capture)) {6909 AO = DefaultOrder;6910 } else if (DefaultOrder == llvm::AtomicOrdering::AcquireRelease) {6911 if (Kind == OMPC_unknown || Kind == OMPC_update || Kind == OMPC_write) {6912 AO = llvm::AtomicOrdering::Release;6913 } else if (Kind == OMPC_read) {6914 assert(Kind == OMPC_read && "Unexpected atomic kind.");6915 AO = llvm::AtomicOrdering::Acquire;6916 }6917 }6918 }6919 6920 if (KindsEncountered.contains(OMPC_compare) &&6921 KindsEncountered.contains(OMPC_fail)) {6922 Kind = OMPC_compare;6923 const auto *FailClause = S.getSingleClause<OMPFailClause>();6924 if (FailClause) {6925 OpenMPClauseKind FailParameter = FailClause->getFailParameter();6926 if (FailParameter == llvm::omp::OMPC_relaxed)6927 FailAO = llvm::AtomicOrdering::Monotonic;6928 else if (FailParameter == llvm::omp::OMPC_acquire)6929 FailAO = llvm::AtomicOrdering::Acquire;6930 else if (FailParameter == llvm::omp::OMPC_seq_cst)6931 FailAO = llvm::AtomicOrdering::SequentiallyConsistent;6932 }6933 }6934 6935 LexicalScope Scope(*this, S.getSourceRange());6936 EmitStopPoint(S.getAssociatedStmt());6937 emitOMPAtomicExpr(*this, Kind, AO, FailAO, S.isPostfixUpdate(), S.getX(),6938 S.getV(), S.getR(), S.getExpr(), S.getUpdateExpr(),6939 S.getD(), S.getCondExpr(), S.isXLHSInRHSPart(),6940 S.isFailOnly(), S.getBeginLoc());6941}6942 6943static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,6944 const OMPExecutableDirective &S,6945 const RegionCodeGenTy &CodeGen) {6946 assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()));6947 CodeGenModule &CGM = CGF.CGM;6948 6949 // On device emit this construct as inlined code.6950 if (CGM.getLangOpts().OpenMPIsTargetDevice) {6951 OMPLexicalScope Scope(CGF, S, OMPD_target);6952 CGM.getOpenMPRuntime().emitInlinedDirective(6953 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {6954 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());6955 });6956 return;6957 }6958 6959 auto LPCRegion = CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);6960 llvm::Function *Fn = nullptr;6961 llvm::Constant *FnID = nullptr;6962 6963 const Expr *IfCond = nullptr;6964 // Check for the at most one if clause associated with the target region.6965 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {6966 if (C->getNameModifier() == OMPD_unknown ||6967 C->getNameModifier() == OMPD_target) {6968 IfCond = C->getCondition();6969 break;6970 }6971 }6972 6973 // Check if we have any device clause associated with the directive.6974 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device(6975 nullptr, OMPC_DEVICE_unknown);6976 if (auto *C = S.getSingleClause<OMPDeviceClause>())6977 Device.setPointerAndInt(C->getDevice(), C->getModifier());6978 6979 // Check if we have an if clause whose conditional always evaluates to false6980 // or if we do not have any targets specified. If so the target region is not6981 // an offload entry point.6982 bool IsOffloadEntry = true;6983 if (IfCond) {6984 bool Val;6985 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)6986 IsOffloadEntry = false;6987 }6988 if (CGM.getLangOpts().OMPTargetTriples.empty())6989 IsOffloadEntry = false;6990 6991 if (CGM.getLangOpts().OpenMPOffloadMandatory && !IsOffloadEntry) {6992 unsigned DiagID = CGM.getDiags().getCustomDiagID(6993 DiagnosticsEngine::Error,6994 "No offloading entry generated while offloading is mandatory.");6995 CGM.getDiags().Report(DiagID);6996 }6997 6998 assert(CGF.CurFuncDecl && "No parent declaration for target region!");6999 StringRef ParentName;7000 // In case we have Ctors/Dtors we use the complete type variant to produce7001 // the mangling of the device outlined kernel.7002 if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))7003 ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));7004 else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))7005 ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));7006 else7007 ParentName =7008 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));7009 7010 // Emit target region as a standalone region.7011 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,7012 IsOffloadEntry, CodeGen);7013 OMPLexicalScope Scope(CGF, S, OMPD_task);7014 auto &&SizeEmitter =7015 [IsOffloadEntry](CodeGenFunction &CGF,7016 const OMPLoopDirective &D) -> llvm::Value * {7017 if (IsOffloadEntry) {7018 OMPLoopScope(CGF, D);7019 // Emit calculation of the iterations count.7020 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());7021 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,7022 /*isSigned=*/false);7023 return NumIterations;7024 }7025 return nullptr;7026 };7027 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,7028 SizeEmitter);7029}7030 7031static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,7032 PrePostActionTy &Action) {7033 Action.Enter(CGF);7034 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);7035 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);7036 CGF.EmitOMPPrivateClause(S, PrivateScope);7037 (void)PrivateScope.Privatize();7038 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))7039 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);7040 7041 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());7042 CGF.EnsureInsertPoint();7043}7044 7045void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM,7046 StringRef ParentName,7047 const OMPTargetDirective &S) {7048 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7049 emitTargetRegion(CGF, S, Action);7050 };7051 llvm::Function *Fn;7052 llvm::Constant *Addr;7053 // Emit target region as a standalone region.7054 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(7055 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);7056 assert(Fn && Addr && "Target device function emission failed.");7057}7058 7059void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) {7060 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7061 emitTargetRegion(CGF, S, Action);7062 };7063 emitCommonOMPTargetDirective(*this, S, CodeGen);7064}7065 7066static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,7067 const OMPExecutableDirective &S,7068 OpenMPDirectiveKind InnermostKind,7069 const RegionCodeGenTy &CodeGen) {7070 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);7071 llvm::Function *OutlinedFn =7072 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(7073 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind,7074 CodeGen);7075 7076 const auto *NT = S.getSingleClause<OMPNumTeamsClause>();7077 const auto *TL = S.getSingleClause<OMPThreadLimitClause>();7078 if (NT || TL) {7079 const Expr *NumTeams = NT ? NT->getNumTeams().front() : nullptr;7080 const Expr *ThreadLimit = TL ? TL->getThreadLimit().front() : nullptr;7081 7082 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,7083 S.getBeginLoc());7084 }7085 7086 OMPTeamsScope Scope(CGF, S);7087 llvm::SmallVector<llvm::Value *, 16> CapturedVars;7088 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);7089 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,7090 CapturedVars);7091}7092 7093void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) {7094 // Emit teams region as a standalone region.7095 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7096 Action.Enter(CGF);7097 OMPPrivateScope PrivateScope(CGF);7098 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);7099 CGF.EmitOMPPrivateClause(S, PrivateScope);7100 CGF.EmitOMPReductionClauseInit(S, PrivateScope);7101 (void)PrivateScope.Privatize();7102 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());7103 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);7104 };7105 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);7106 emitPostUpdateForReductionClause(*this, S,7107 [](CodeGenFunction &) { return nullptr; });7108}7109 7110static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action,7111 const OMPTargetTeamsDirective &S) {7112 auto *CS = S.getCapturedStmt(OMPD_teams);7113 Action.Enter(CGF);7114 // Emit teams region as a standalone region.7115 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {7116 Action.Enter(CGF);7117 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);7118 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);7119 CGF.EmitOMPPrivateClause(S, PrivateScope);7120 CGF.EmitOMPReductionClauseInit(S, PrivateScope);7121 (void)PrivateScope.Privatize();7122 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))7123 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);7124 CGF.EmitStmt(CS->getCapturedStmt());7125 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);7126 };7127 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);7128 emitPostUpdateForReductionClause(CGF, S,7129 [](CodeGenFunction &) { return nullptr; });7130}7131 7132void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(7133 CodeGenModule &CGM, StringRef ParentName,7134 const OMPTargetTeamsDirective &S) {7135 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7136 emitTargetTeamsRegion(CGF, Action, S);7137 };7138 llvm::Function *Fn;7139 llvm::Constant *Addr;7140 // Emit target region as a standalone region.7141 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(7142 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);7143 assert(Fn && Addr && "Target device function emission failed.");7144}7145 7146void CodeGenFunction::EmitOMPTargetTeamsDirective(7147 const OMPTargetTeamsDirective &S) {7148 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7149 emitTargetTeamsRegion(CGF, Action, S);7150 };7151 emitCommonOMPTargetDirective(*this, S, CodeGen);7152}7153 7154static void7155emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action,7156 const OMPTargetTeamsDistributeDirective &S) {7157 Action.Enter(CGF);7158 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {7159 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());7160 };7161 7162 // Emit teams region as a standalone region.7163 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,7164 PrePostActionTy &Action) {7165 Action.Enter(CGF);7166 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);7167 CGF.EmitOMPReductionClauseInit(S, PrivateScope);7168 (void)PrivateScope.Privatize();7169 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,7170 CodeGenDistribute);7171 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);7172 };7173 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);7174 emitPostUpdateForReductionClause(CGF, S,7175 [](CodeGenFunction &) { return nullptr; });7176}7177 7178void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(7179 CodeGenModule &CGM, StringRef ParentName,7180 const OMPTargetTeamsDistributeDirective &S) {7181 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7182 emitTargetTeamsDistributeRegion(CGF, Action, S);7183 };7184 llvm::Function *Fn;7185 llvm::Constant *Addr;7186 // Emit target region as a standalone region.7187 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(7188 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);7189 assert(Fn && Addr && "Target device function emission failed.");7190}7191 7192void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective(7193 const OMPTargetTeamsDistributeDirective &S) {7194 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7195 emitTargetTeamsDistributeRegion(CGF, Action, S);7196 };7197 emitCommonOMPTargetDirective(*this, S, CodeGen);7198}7199 7200static void emitTargetTeamsDistributeSimdRegion(7201 CodeGenFunction &CGF, PrePostActionTy &Action,7202 const OMPTargetTeamsDistributeSimdDirective &S) {7203 Action.Enter(CGF);7204 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {7205 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());7206 };7207 7208 // Emit teams region as a standalone region.7209 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,7210 PrePostActionTy &Action) {7211 Action.Enter(CGF);7212 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);7213 CGF.EmitOMPReductionClauseInit(S, PrivateScope);7214 (void)PrivateScope.Privatize();7215 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,7216 CodeGenDistribute);7217 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);7218 };7219 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);7220 emitPostUpdateForReductionClause(CGF, S,7221 [](CodeGenFunction &) { return nullptr; });7222}7223 7224void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(7225 CodeGenModule &CGM, StringRef ParentName,7226 const OMPTargetTeamsDistributeSimdDirective &S) {7227 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7228 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);7229 };7230 llvm::Function *Fn;7231 llvm::Constant *Addr;7232 // Emit target region as a standalone region.7233 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(7234 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);7235 assert(Fn && Addr && "Target device function emission failed.");7236}7237 7238void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective(7239 const OMPTargetTeamsDistributeSimdDirective &S) {7240 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7241 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);7242 };7243 emitCommonOMPTargetDirective(*this, S, CodeGen);7244}7245 7246void CodeGenFunction::EmitOMPTeamsDistributeDirective(7247 const OMPTeamsDistributeDirective &S) {7248 7249 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {7250 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());7251 };7252 7253 // Emit teams region as a standalone region.7254 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,7255 PrePostActionTy &Action) {7256 Action.Enter(CGF);7257 OMPPrivateScope PrivateScope(CGF);7258 CGF.EmitOMPReductionClauseInit(S, PrivateScope);7259 (void)PrivateScope.Privatize();7260 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,7261 CodeGenDistribute);7262 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);7263 };7264 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);7265 emitPostUpdateForReductionClause(*this, S,7266 [](CodeGenFunction &) { return nullptr; });7267}7268 7269void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective(7270 const OMPTeamsDistributeSimdDirective &S) {7271 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {7272 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());7273 };7274 7275 // Emit teams region as a standalone region.7276 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,7277 PrePostActionTy &Action) {7278 Action.Enter(CGF);7279 OMPPrivateScope PrivateScope(CGF);7280 CGF.EmitOMPReductionClauseInit(S, PrivateScope);7281 (void)PrivateScope.Privatize();7282 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,7283 CodeGenDistribute);7284 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);7285 };7286 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);7287 emitPostUpdateForReductionClause(*this, S,7288 [](CodeGenFunction &) { return nullptr; });7289}7290 7291void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective(7292 const OMPTeamsDistributeParallelForDirective &S) {7293 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {7294 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,7295 S.getDistInc());7296 };7297 7298 // Emit teams region as a standalone region.7299 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,7300 PrePostActionTy &Action) {7301 Action.Enter(CGF);7302 OMPPrivateScope PrivateScope(CGF);7303 CGF.EmitOMPReductionClauseInit(S, PrivateScope);7304 (void)PrivateScope.Privatize();7305 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,7306 CodeGenDistribute);7307 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);7308 };7309 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);7310 emitPostUpdateForReductionClause(*this, S,7311 [](CodeGenFunction &) { return nullptr; });7312}7313 7314void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective(7315 const OMPTeamsDistributeParallelForSimdDirective &S) {7316 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {7317 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,7318 S.getDistInc());7319 };7320 7321 // Emit teams region as a standalone region.7322 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,7323 PrePostActionTy &Action) {7324 Action.Enter(CGF);7325 OMPPrivateScope PrivateScope(CGF);7326 CGF.EmitOMPReductionClauseInit(S, PrivateScope);7327 (void)PrivateScope.Privatize();7328 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(7329 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);7330 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);7331 };7332 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for_simd,7333 CodeGen);7334 emitPostUpdateForReductionClause(*this, S,7335 [](CodeGenFunction &) { return nullptr; });7336}7337 7338void CodeGenFunction::EmitOMPInteropDirective(const OMPInteropDirective &S) {7339 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();7340 llvm::Value *Device = nullptr;7341 llvm::Value *NumDependences = nullptr;7342 llvm::Value *DependenceList = nullptr;7343 7344 if (const auto *C = S.getSingleClause<OMPDeviceClause>())7345 Device = EmitScalarExpr(C->getDevice());7346 7347 // Build list and emit dependences7348 OMPTaskDataTy Data;7349 buildDependences(S, Data);7350 if (!Data.Dependences.empty()) {7351 Address DependenciesArray = Address::invalid();7352 std::tie(NumDependences, DependenciesArray) =7353 CGM.getOpenMPRuntime().emitDependClause(*this, Data.Dependences,7354 S.getBeginLoc());7355 DependenceList = DependenciesArray.emitRawPointer(*this);7356 }7357 Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();7358 7359 assert(!(Data.HasNowaitClause && !(S.getSingleClause<OMPInitClause>() ||7360 S.getSingleClause<OMPDestroyClause>() ||7361 S.getSingleClause<OMPUseClause>())) &&7362 "OMPNowaitClause clause is used separately in OMPInteropDirective.");7363 7364 auto ItOMPInitClause = S.getClausesOfKind<OMPInitClause>();7365 if (!ItOMPInitClause.empty()) {7366 // Look at the multiple init clauses7367 for (const OMPInitClause *C : ItOMPInitClause) {7368 llvm::Value *InteropvarPtr =7369 EmitLValue(C->getInteropVar()).getPointer(*this);7370 llvm::omp::OMPInteropType InteropType =7371 llvm::omp::OMPInteropType::Unknown;7372 if (C->getIsTarget()) {7373 InteropType = llvm::omp::OMPInteropType::Target;7374 } else {7375 assert(C->getIsTargetSync() &&7376 "Expected interop-type target/targetsync");7377 InteropType = llvm::omp::OMPInteropType::TargetSync;7378 }7379 OMPBuilder.createOMPInteropInit(Builder, InteropvarPtr, InteropType,7380 Device, NumDependences, DependenceList,7381 Data.HasNowaitClause);7382 }7383 }7384 auto ItOMPDestroyClause = S.getClausesOfKind<OMPDestroyClause>();7385 if (!ItOMPDestroyClause.empty()) {7386 // Look at the multiple destroy clauses7387 for (const OMPDestroyClause *C : ItOMPDestroyClause) {7388 llvm::Value *InteropvarPtr =7389 EmitLValue(C->getInteropVar()).getPointer(*this);7390 OMPBuilder.createOMPInteropDestroy(Builder, InteropvarPtr, Device,7391 NumDependences, DependenceList,7392 Data.HasNowaitClause);7393 }7394 }7395 auto ItOMPUseClause = S.getClausesOfKind<OMPUseClause>();7396 if (!ItOMPUseClause.empty()) {7397 // Look at the multiple use clauses7398 for (const OMPUseClause *C : ItOMPUseClause) {7399 llvm::Value *InteropvarPtr =7400 EmitLValue(C->getInteropVar()).getPointer(*this);7401 OMPBuilder.createOMPInteropUse(Builder, InteropvarPtr, Device,7402 NumDependences, DependenceList,7403 Data.HasNowaitClause);7404 }7405 }7406}7407 7408static void emitTargetTeamsDistributeParallelForRegion(7409 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S,7410 PrePostActionTy &Action) {7411 Action.Enter(CGF);7412 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {7413 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,7414 S.getDistInc());7415 };7416 7417 // Emit teams region as a standalone region.7418 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,7419 PrePostActionTy &Action) {7420 Action.Enter(CGF);7421 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);7422 CGF.EmitOMPReductionClauseInit(S, PrivateScope);7423 (void)PrivateScope.Privatize();7424 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(7425 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);7426 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);7427 };7428 7429 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,7430 CodeGenTeams);7431 emitPostUpdateForReductionClause(CGF, S,7432 [](CodeGenFunction &) { return nullptr; });7433}7434 7435void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(7436 CodeGenModule &CGM, StringRef ParentName,7437 const OMPTargetTeamsDistributeParallelForDirective &S) {7438 // Emit SPMD target teams distribute parallel for region as a standalone7439 // region.7440 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7441 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);7442 };7443 llvm::Function *Fn;7444 llvm::Constant *Addr;7445 // Emit target region as a standalone region.7446 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(7447 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);7448 assert(Fn && Addr && "Target device function emission failed.");7449}7450 7451void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective(7452 const OMPTargetTeamsDistributeParallelForDirective &S) {7453 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7454 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);7455 };7456 emitCommonOMPTargetDirective(*this, S, CodeGen);7457}7458 7459static void emitTargetTeamsDistributeParallelForSimdRegion(7460 CodeGenFunction &CGF,7461 const OMPTargetTeamsDistributeParallelForSimdDirective &S,7462 PrePostActionTy &Action) {7463 Action.Enter(CGF);7464 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {7465 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,7466 S.getDistInc());7467 };7468 7469 // Emit teams region as a standalone region.7470 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,7471 PrePostActionTy &Action) {7472 Action.Enter(CGF);7473 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);7474 CGF.EmitOMPReductionClauseInit(S, PrivateScope);7475 (void)PrivateScope.Privatize();7476 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(7477 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);7478 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);7479 };7480 7481 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,7482 CodeGenTeams);7483 emitPostUpdateForReductionClause(CGF, S,7484 [](CodeGenFunction &) { return nullptr; });7485}7486 7487void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(7488 CodeGenModule &CGM, StringRef ParentName,7489 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {7490 // Emit SPMD target teams distribute parallel for simd region as a standalone7491 // region.7492 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7493 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);7494 };7495 llvm::Function *Fn;7496 llvm::Constant *Addr;7497 // Emit target region as a standalone region.7498 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(7499 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);7500 assert(Fn && Addr && "Target device function emission failed.");7501}7502 7503void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective(7504 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {7505 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7506 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);7507 };7508 emitCommonOMPTargetDirective(*this, S, CodeGen);7509}7510 7511void CodeGenFunction::EmitOMPCancellationPointDirective(7512 const OMPCancellationPointDirective &S) {7513 CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),7514 S.getCancelRegion());7515}7516 7517void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {7518 const Expr *IfCond = nullptr;7519 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {7520 if (C->getNameModifier() == OMPD_unknown ||7521 C->getNameModifier() == OMPD_cancel) {7522 IfCond = C->getCondition();7523 break;7524 }7525 }7526 if (CGM.getLangOpts().OpenMPIRBuilder) {7527 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();7528 // TODO: This check is necessary as we only generate `omp parallel` through7529 // the OpenMPIRBuilder for now.7530 if (S.getCancelRegion() == OMPD_parallel ||7531 S.getCancelRegion() == OMPD_sections ||7532 S.getCancelRegion() == OMPD_section) {7533 llvm::Value *IfCondition = nullptr;7534 if (IfCond)7535 IfCondition = EmitScalarExpr(IfCond,7536 /*IgnoreResultAssign=*/true);7537 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(7538 OMPBuilder.createCancel(Builder, IfCondition, S.getCancelRegion()));7539 return Builder.restoreIP(AfterIP);7540 }7541 }7542 7543 CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,7544 S.getCancelRegion());7545}7546 7547CodeGenFunction::JumpDest7548CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) {7549 if (Kind == OMPD_parallel || Kind == OMPD_task ||7550 Kind == OMPD_target_parallel || Kind == OMPD_taskloop ||7551 Kind == OMPD_master_taskloop || Kind == OMPD_parallel_master_taskloop)7552 return ReturnBlock;7553 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||7554 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||7555 Kind == OMPD_distribute_parallel_for ||7556 Kind == OMPD_target_parallel_for ||7557 Kind == OMPD_teams_distribute_parallel_for ||7558 Kind == OMPD_target_teams_distribute_parallel_for);7559 return OMPCancelStack.getExitBlock();7560}7561 7562void CodeGenFunction::EmitOMPUseDevicePtrClause(7563 const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,7564 const llvm::DenseMap<const ValueDecl *, llvm::Value *>7565 CaptureDeviceAddrMap) {7566 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;7567 for (const Expr *OrigVarIt : C.varlist()) {7568 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(OrigVarIt)->getDecl());7569 if (!Processed.insert(OrigVD).second)7570 continue;7571 7572 // In order to identify the right initializer we need to match the7573 // declaration used by the mapping logic. In some cases we may get7574 // OMPCapturedExprDecl that refers to the original declaration.7575 const ValueDecl *MatchingVD = OrigVD;7576 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {7577 // OMPCapturedExprDecl are used to privative fields of the current7578 // structure.7579 const auto *ME = cast<MemberExpr>(OED->getInit());7580 assert(isa<CXXThisExpr>(ME->getBase()->IgnoreImpCasts()) &&7581 "Base should be the current struct!");7582 MatchingVD = ME->getMemberDecl();7583 }7584 7585 // If we don't have information about the current list item, move on to7586 // the next one.7587 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);7588 if (InitAddrIt == CaptureDeviceAddrMap.end())7589 continue;7590 7591 llvm::Type *Ty = ConvertTypeForMem(OrigVD->getType().getNonReferenceType());7592 7593 // Return the address of the private variable.7594 bool IsRegistered = PrivateScope.addPrivate(7595 OrigVD,7596 Address(InitAddrIt->second, Ty,7597 getContext().getTypeAlignInChars(getContext().VoidPtrTy)));7598 assert(IsRegistered && "firstprivate var already registered as private");7599 // Silence the warning about unused variable.7600 (void)IsRegistered;7601 }7602}7603 7604static const VarDecl *getBaseDecl(const Expr *Ref) {7605 const Expr *Base = Ref->IgnoreParenImpCasts();7606 while (const auto *OASE = dyn_cast<ArraySectionExpr>(Base))7607 Base = OASE->getBase()->IgnoreParenImpCasts();7608 while (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Base))7609 Base = ASE->getBase()->IgnoreParenImpCasts();7610 return cast<VarDecl>(cast<DeclRefExpr>(Base)->getDecl());7611}7612 7613void CodeGenFunction::EmitOMPUseDeviceAddrClause(7614 const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,7615 const llvm::DenseMap<const ValueDecl *, llvm::Value *>7616 CaptureDeviceAddrMap) {7617 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;7618 for (const Expr *Ref : C.varlist()) {7619 const VarDecl *OrigVD = getBaseDecl(Ref);7620 if (!Processed.insert(OrigVD).second)7621 continue;7622 // In order to identify the right initializer we need to match the7623 // declaration used by the mapping logic. In some cases we may get7624 // OMPCapturedExprDecl that refers to the original declaration.7625 const ValueDecl *MatchingVD = OrigVD;7626 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {7627 // OMPCapturedExprDecl are used to privative fields of the current7628 // structure.7629 const auto *ME = cast<MemberExpr>(OED->getInit());7630 assert(isa<CXXThisExpr>(ME->getBase()) &&7631 "Base should be the current struct!");7632 MatchingVD = ME->getMemberDecl();7633 }7634 7635 // If we don't have information about the current list item, move on to7636 // the next one.7637 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);7638 if (InitAddrIt == CaptureDeviceAddrMap.end())7639 continue;7640 7641 llvm::Type *Ty = ConvertTypeForMem(OrigVD->getType().getNonReferenceType());7642 7643 Address PrivAddr =7644 Address(InitAddrIt->second, Ty,7645 getContext().getTypeAlignInChars(getContext().VoidPtrTy));7646 // For declrefs and variable length array need to load the pointer for7647 // correct mapping, since the pointer to the data was passed to the runtime.7648 if (isa<DeclRefExpr>(Ref->IgnoreParenImpCasts()) ||7649 MatchingVD->getType()->isArrayType()) {7650 QualType PtrTy = getContext().getPointerType(7651 OrigVD->getType().getNonReferenceType());7652 PrivAddr =7653 EmitLoadOfPointer(PrivAddr.withElementType(ConvertTypeForMem(PtrTy)),7654 PtrTy->castAs<PointerType>());7655 }7656 7657 (void)PrivateScope.addPrivate(OrigVD, PrivAddr);7658 }7659}7660 7661// Generate the instructions for '#pragma omp target data' directive.7662void CodeGenFunction::EmitOMPTargetDataDirective(7663 const OMPTargetDataDirective &S) {7664 CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true,7665 /*SeparateBeginEndCalls=*/true);7666 7667 // Create a pre/post action to signal the privatization of the device pointer.7668 // This action can be replaced by the OpenMP runtime code generation to7669 // deactivate privatization.7670 bool PrivatizeDevicePointers = false;7671 class DevicePointerPrivActionTy : public PrePostActionTy {7672 bool &PrivatizeDevicePointers;7673 7674 public:7675 explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)7676 : PrivatizeDevicePointers(PrivatizeDevicePointers) {}7677 void Enter(CodeGenFunction &CGF) override {7678 PrivatizeDevicePointers = true;7679 }7680 };7681 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);7682 7683 auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {7684 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {7685 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());7686 };7687 7688 // Codegen that selects whether to generate the privatization code or not.7689 auto &&PrivCodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {7690 RegionCodeGenTy RCG(InnermostCodeGen);7691 PrivatizeDevicePointers = false;7692 7693 // Call the pre-action to change the status of PrivatizeDevicePointers if7694 // needed.7695 Action.Enter(CGF);7696 7697 if (PrivatizeDevicePointers) {7698 OMPPrivateScope PrivateScope(CGF);7699 // Emit all instances of the use_device_ptr clause.7700 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())7701 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,7702 Info.CaptureDeviceAddrMap);7703 for (const auto *C : S.getClausesOfKind<OMPUseDeviceAddrClause>())7704 CGF.EmitOMPUseDeviceAddrClause(*C, PrivateScope,7705 Info.CaptureDeviceAddrMap);7706 (void)PrivateScope.Privatize();7707 RCG(CGF);7708 } else {7709 // If we don't have target devices, don't bother emitting the data7710 // mapping code.7711 std::optional<OpenMPDirectiveKind> CaptureRegion;7712 if (CGM.getLangOpts().OMPTargetTriples.empty()) {7713 // Emit helper decls of the use_device_ptr/use_device_addr clauses.7714 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())7715 for (const Expr *E : C->varlist()) {7716 const Decl *D = cast<DeclRefExpr>(E)->getDecl();7717 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))7718 CGF.EmitVarDecl(*OED);7719 }7720 for (const auto *C : S.getClausesOfKind<OMPUseDeviceAddrClause>())7721 for (const Expr *E : C->varlist()) {7722 const Decl *D = getBaseDecl(E);7723 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))7724 CGF.EmitVarDecl(*OED);7725 }7726 } else {7727 CaptureRegion = OMPD_unknown;7728 }7729 7730 OMPLexicalScope Scope(CGF, S, CaptureRegion);7731 RCG(CGF);7732 }7733 };7734 7735 // Forward the provided action to the privatization codegen.7736 RegionCodeGenTy PrivRCG(PrivCodeGen);7737 PrivRCG.setAction(Action);7738 7739 // Notwithstanding the body of the region is emitted as inlined directive,7740 // we don't use an inline scope as changes in the references inside the7741 // region are expected to be visible outside, so we do not privative them.7742 OMPLexicalScope Scope(CGF, S);7743 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,7744 PrivRCG);7745 };7746 7747 RegionCodeGenTy RCG(CodeGen);7748 7749 // If we don't have target devices, don't bother emitting the data mapping7750 // code.7751 if (CGM.getLangOpts().OMPTargetTriples.empty()) {7752 RCG(*this);7753 return;7754 }7755 7756 // Check if we have any if clause associated with the directive.7757 const Expr *IfCond = nullptr;7758 if (const auto *C = S.getSingleClause<OMPIfClause>())7759 IfCond = C->getCondition();7760 7761 // Check if we have any device clause associated with the directive.7762 const Expr *Device = nullptr;7763 if (const auto *C = S.getSingleClause<OMPDeviceClause>())7764 Device = C->getDevice();7765 7766 // Set the action to signal privatization of device pointers.7767 RCG.setAction(PrivAction);7768 7769 // Emit region code.7770 CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,7771 Info);7772}7773 7774void CodeGenFunction::EmitOMPTargetEnterDataDirective(7775 const OMPTargetEnterDataDirective &S) {7776 // If we don't have target devices, don't bother emitting the data mapping7777 // code.7778 if (CGM.getLangOpts().OMPTargetTriples.empty())7779 return;7780 7781 // Check if we have any if clause associated with the directive.7782 const Expr *IfCond = nullptr;7783 if (const auto *C = S.getSingleClause<OMPIfClause>())7784 IfCond = C->getCondition();7785 7786 // Check if we have any device clause associated with the directive.7787 const Expr *Device = nullptr;7788 if (const auto *C = S.getSingleClause<OMPDeviceClause>())7789 Device = C->getDevice();7790 7791 OMPLexicalScope Scope(*this, S, OMPD_task);7792 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);7793}7794 7795void CodeGenFunction::EmitOMPTargetExitDataDirective(7796 const OMPTargetExitDataDirective &S) {7797 // If we don't have target devices, don't bother emitting the data mapping7798 // code.7799 if (CGM.getLangOpts().OMPTargetTriples.empty())7800 return;7801 7802 // Check if we have any if clause associated with the directive.7803 const Expr *IfCond = nullptr;7804 if (const auto *C = S.getSingleClause<OMPIfClause>())7805 IfCond = C->getCondition();7806 7807 // Check if we have any device clause associated with the directive.7808 const Expr *Device = nullptr;7809 if (const auto *C = S.getSingleClause<OMPDeviceClause>())7810 Device = C->getDevice();7811 7812 OMPLexicalScope Scope(*this, S, OMPD_task);7813 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);7814}7815 7816static void emitTargetParallelRegion(CodeGenFunction &CGF,7817 const OMPTargetParallelDirective &S,7818 PrePostActionTy &Action) {7819 // Get the captured statement associated with the 'parallel' region.7820 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);7821 Action.Enter(CGF);7822 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {7823 Action.Enter(CGF);7824 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);7825 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);7826 CGF.EmitOMPPrivateClause(S, PrivateScope);7827 CGF.EmitOMPReductionClauseInit(S, PrivateScope);7828 (void)PrivateScope.Privatize();7829 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))7830 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);7831 // TODO: Add support for clauses.7832 CGF.EmitStmt(CS->getCapturedStmt());7833 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);7834 };7835 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,7836 emitEmptyBoundParameters);7837 emitPostUpdateForReductionClause(CGF, S,7838 [](CodeGenFunction &) { return nullptr; });7839}7840 7841void CodeGenFunction::EmitOMPTargetParallelDeviceFunction(7842 CodeGenModule &CGM, StringRef ParentName,7843 const OMPTargetParallelDirective &S) {7844 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7845 emitTargetParallelRegion(CGF, S, Action);7846 };7847 llvm::Function *Fn;7848 llvm::Constant *Addr;7849 // Emit target region as a standalone region.7850 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(7851 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);7852 assert(Fn && Addr && "Target device function emission failed.");7853}7854 7855void CodeGenFunction::EmitOMPTargetParallelDirective(7856 const OMPTargetParallelDirective &S) {7857 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7858 emitTargetParallelRegion(CGF, S, Action);7859 };7860 emitCommonOMPTargetDirective(*this, S, CodeGen);7861}7862 7863static void emitTargetParallelForRegion(CodeGenFunction &CGF,7864 const OMPTargetParallelForDirective &S,7865 PrePostActionTy &Action) {7866 Action.Enter(CGF);7867 // Emit directive as a combined directive that consists of two implicit7868 // directives: 'parallel' with 'for' directive.7869 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7870 Action.Enter(CGF);7871 CodeGenFunction::OMPCancelStackRAII CancelRegion(7872 CGF, OMPD_target_parallel_for, S.hasCancel());7873 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,7874 emitDispatchForLoopBounds);7875 };7876 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,7877 emitEmptyBoundParameters);7878}7879 7880void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(7881 CodeGenModule &CGM, StringRef ParentName,7882 const OMPTargetParallelForDirective &S) {7883 // Emit SPMD target parallel for region as a standalone region.7884 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7885 emitTargetParallelForRegion(CGF, S, Action);7886 };7887 llvm::Function *Fn;7888 llvm::Constant *Addr;7889 // Emit target region as a standalone region.7890 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(7891 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);7892 assert(Fn && Addr && "Target device function emission failed.");7893}7894 7895void CodeGenFunction::EmitOMPTargetParallelForDirective(7896 const OMPTargetParallelForDirective &S) {7897 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7898 emitTargetParallelForRegion(CGF, S, Action);7899 };7900 emitCommonOMPTargetDirective(*this, S, CodeGen);7901}7902 7903static void7904emitTargetParallelForSimdRegion(CodeGenFunction &CGF,7905 const OMPTargetParallelForSimdDirective &S,7906 PrePostActionTy &Action) {7907 Action.Enter(CGF);7908 // Emit directive as a combined directive that consists of two implicit7909 // directives: 'parallel' with 'for' directive.7910 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7911 Action.Enter(CGF);7912 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,7913 emitDispatchForLoopBounds);7914 };7915 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,7916 emitEmptyBoundParameters);7917}7918 7919void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(7920 CodeGenModule &CGM, StringRef ParentName,7921 const OMPTargetParallelForSimdDirective &S) {7922 // Emit SPMD target parallel for region as a standalone region.7923 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7924 emitTargetParallelForSimdRegion(CGF, S, Action);7925 };7926 llvm::Function *Fn;7927 llvm::Constant *Addr;7928 // Emit target region as a standalone region.7929 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(7930 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);7931 assert(Fn && Addr && "Target device function emission failed.");7932}7933 7934void CodeGenFunction::EmitOMPTargetParallelForSimdDirective(7935 const OMPTargetParallelForSimdDirective &S) {7936 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {7937 emitTargetParallelForSimdRegion(CGF, S, Action);7938 };7939 emitCommonOMPTargetDirective(*this, S, CodeGen);7940}7941 7942/// Emit a helper variable and return corresponding lvalue.7943static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,7944 const ImplicitParamDecl *PVD,7945 CodeGenFunction::OMPPrivateScope &Privates) {7946 const auto *VDecl = cast<VarDecl>(Helper->getDecl());7947 Privates.addPrivate(VDecl, CGF.GetAddrOfLocalVar(PVD));7948}7949 7950void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {7951 assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()));7952 // Emit outlined function for task construct.7953 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);7954 Address CapturedStruct = Address::invalid();7955 {7956 OMPLexicalScope Scope(*this, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);7957 CapturedStruct = GenerateCapturedStmtArgument(*CS);7958 }7959 CanQualType SharedsTy =7960 getContext().getCanonicalTagType(CS->getCapturedRecordDecl());7961 const Expr *IfCond = nullptr;7962 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {7963 if (C->getNameModifier() == OMPD_unknown ||7964 C->getNameModifier() == OMPD_taskloop) {7965 IfCond = C->getCondition();7966 break;7967 }7968 }7969 7970 OMPTaskDataTy Data;7971 // Check if taskloop must be emitted without taskgroup.7972 Data.Nogroup = S.getSingleClause<OMPNogroupClause>();7973 // TODO: Check if we should emit tied or untied task.7974 Data.Tied = true;7975 // Set scheduling for taskloop7976 if (const auto *Clause = S.getSingleClause<OMPGrainsizeClause>()) {7977 // grainsize clause7978 Data.Schedule.setInt(/*IntVal=*/false);7979 Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));7980 Data.HasModifier =7981 (Clause->getModifier() == OMPC_GRAINSIZE_strict) ? true : false;7982 } else if (const auto *Clause = S.getSingleClause<OMPNumTasksClause>()) {7983 // num_tasks clause7984 Data.Schedule.setInt(/*IntVal=*/true);7985 Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));7986 Data.HasModifier =7987 (Clause->getModifier() == OMPC_NUMTASKS_strict) ? true : false;7988 }7989 7990 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {7991 // if (PreCond) {7992 // for (IV in 0..LastIteration) BODY;7993 // <Final counter/linear vars updates>;7994 // }7995 //7996 7997 // Emit: if (PreCond) - begin.7998 // If the condition constant folds and can be elided, avoid emitting the7999 // whole loop.8000 bool CondConstant;8001 llvm::BasicBlock *ContBlock = nullptr;8002 OMPLoopScope PreInitScope(CGF, S);8003 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {8004 if (!CondConstant)8005 return;8006 } else {8007 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");8008 ContBlock = CGF.createBasicBlock("taskloop.if.end");8009 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,8010 CGF.getProfileCount(&S));8011 CGF.EmitBlock(ThenBlock);8012 CGF.incrementProfileCounter(&S);8013 }8014 8015 (void)CGF.EmitOMPLinearClauseInit(S);8016 8017 OMPPrivateScope LoopScope(CGF);8018 // Emit helper vars inits.8019 enum { LowerBound = 5, UpperBound, Stride, LastIter };8020 auto *I = CS->getCapturedDecl()->param_begin();8021 auto *LBP = std::next(I, LowerBound);8022 auto *UBP = std::next(I, UpperBound);8023 auto *STP = std::next(I, Stride);8024 auto *LIP = std::next(I, LastIter);8025 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,8026 LoopScope);8027 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,8028 LoopScope);8029 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);8030 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,8031 LoopScope);8032 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);8033 CGF.EmitOMPLinearClause(S, LoopScope);8034 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);8035 (void)LoopScope.Privatize();8036 // Emit the loop iteration variable.8037 const Expr *IVExpr = S.getIterationVariable();8038 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());8039 CGF.EmitVarDecl(*IVDecl);8040 CGF.EmitIgnoredExpr(S.getInit());8041 8042 // Emit the iterations count variable.8043 // If it is not a variable, Sema decided to calculate iterations count on8044 // each iteration (e.g., it is foldable into a constant).8045 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {8046 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));8047 // Emit calculation of the iterations count.8048 CGF.EmitIgnoredExpr(S.getCalcLastIteration());8049 }8050 8051 {8052 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);8053 emitCommonSimdLoop(8054 CGF, S,8055 [&S](CodeGenFunction &CGF, PrePostActionTy &) {8056 if (isOpenMPSimdDirective(S.getDirectiveKind()))8057 CGF.EmitOMPSimdInit(S);8058 },8059 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {8060 CGF.EmitOMPInnerLoop(8061 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),8062 [&S](CodeGenFunction &CGF) {8063 emitOMPLoopBodyWithStopPoint(CGF, S,8064 CodeGenFunction::JumpDest());8065 },8066 [](CodeGenFunction &) {});8067 });8068 }8069 // Emit: if (PreCond) - end.8070 if (ContBlock) {8071 CGF.EmitBranch(ContBlock);8072 CGF.EmitBlock(ContBlock, true);8073 }8074 // Emit final copy of the lastprivate variables if IsLastIter != 0.8075 if (HasLastprivateClause) {8076 CGF.EmitOMPLastprivateClauseFinal(8077 S, isOpenMPSimdDirective(S.getDirectiveKind()),8078 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(8079 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,8080 (*LIP)->getType(), S.getBeginLoc())));8081 }8082 LoopScope.restoreMap();8083 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) {8084 return CGF.Builder.CreateIsNotNull(8085 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,8086 (*LIP)->getType(), S.getBeginLoc()));8087 });8088 };8089 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,8090 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,8091 const OMPTaskDataTy &Data) {8092 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,8093 &Data](CodeGenFunction &CGF, PrePostActionTy &) {8094 OMPLoopScope PreInitScope(CGF, S);8095 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,8096 OutlinedFn, SharedsTy,8097 CapturedStruct, IfCond, Data);8098 };8099 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,8100 CodeGen);8101 };8102 if (Data.Nogroup) {8103 EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);8104 } else {8105 CGM.getOpenMPRuntime().emitTaskgroupRegion(8106 *this,8107 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,8108 PrePostActionTy &Action) {8109 Action.Enter(CGF);8110 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,8111 Data);8112 },8113 S.getBeginLoc());8114 }8115}8116 8117void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) {8118 auto LPCRegion =8119 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);8120 EmitOMPTaskLoopBasedDirective(S);8121}8122 8123void CodeGenFunction::EmitOMPTaskLoopSimdDirective(8124 const OMPTaskLoopSimdDirective &S) {8125 auto LPCRegion =8126 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);8127 OMPLexicalScope Scope(*this, S);8128 EmitOMPTaskLoopBasedDirective(S);8129}8130 8131void CodeGenFunction::EmitOMPMasterTaskLoopDirective(8132 const OMPMasterTaskLoopDirective &S) {8133 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {8134 Action.Enter(CGF);8135 EmitOMPTaskLoopBasedDirective(S);8136 };8137 auto LPCRegion =8138 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);8139 OMPLexicalScope Scope(*this, S, std::nullopt, /*EmitPreInitStmt=*/false);8140 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());8141}8142 8143void CodeGenFunction::EmitOMPMaskedTaskLoopDirective(8144 const OMPMaskedTaskLoopDirective &S) {8145 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {8146 Action.Enter(CGF);8147 EmitOMPTaskLoopBasedDirective(S);8148 };8149 auto LPCRegion =8150 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);8151 OMPLexicalScope Scope(*this, S, std::nullopt, /*EmitPreInitStmt=*/false);8152 CGM.getOpenMPRuntime().emitMaskedRegion(*this, CodeGen, S.getBeginLoc());8153}8154 8155void CodeGenFunction::EmitOMPMasterTaskLoopSimdDirective(8156 const OMPMasterTaskLoopSimdDirective &S) {8157 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {8158 Action.Enter(CGF);8159 EmitOMPTaskLoopBasedDirective(S);8160 };8161 auto LPCRegion =8162 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);8163 OMPLexicalScope Scope(*this, S);8164 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());8165}8166 8167void CodeGenFunction::EmitOMPMaskedTaskLoopSimdDirective(8168 const OMPMaskedTaskLoopSimdDirective &S) {8169 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {8170 Action.Enter(CGF);8171 EmitOMPTaskLoopBasedDirective(S);8172 };8173 auto LPCRegion =8174 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);8175 OMPLexicalScope Scope(*this, S);8176 CGM.getOpenMPRuntime().emitMaskedRegion(*this, CodeGen, S.getBeginLoc());8177}8178 8179void CodeGenFunction::EmitOMPParallelMasterTaskLoopDirective(8180 const OMPParallelMasterTaskLoopDirective &S) {8181 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {8182 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,8183 PrePostActionTy &Action) {8184 Action.Enter(CGF);8185 CGF.EmitOMPTaskLoopBasedDirective(S);8186 };8187 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);8188 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,8189 S.getBeginLoc());8190 };8191 auto LPCRegion =8192 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);8193 emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop, CodeGen,8194 emitEmptyBoundParameters);8195}8196 8197void CodeGenFunction::EmitOMPParallelMaskedTaskLoopDirective(8198 const OMPParallelMaskedTaskLoopDirective &S) {8199 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {8200 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,8201 PrePostActionTy &Action) {8202 Action.Enter(CGF);8203 CGF.EmitOMPTaskLoopBasedDirective(S);8204 };8205 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);8206 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, TaskLoopCodeGen,8207 S.getBeginLoc());8208 };8209 auto LPCRegion =8210 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);8211 emitCommonOMPParallelDirective(*this, S, OMPD_masked_taskloop, CodeGen,8212 emitEmptyBoundParameters);8213}8214 8215void CodeGenFunction::EmitOMPParallelMasterTaskLoopSimdDirective(8216 const OMPParallelMasterTaskLoopSimdDirective &S) {8217 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {8218 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,8219 PrePostActionTy &Action) {8220 Action.Enter(CGF);8221 CGF.EmitOMPTaskLoopBasedDirective(S);8222 };8223 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);8224 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,8225 S.getBeginLoc());8226 };8227 auto LPCRegion =8228 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);8229 emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop_simd, CodeGen,8230 emitEmptyBoundParameters);8231}8232 8233void CodeGenFunction::EmitOMPParallelMaskedTaskLoopSimdDirective(8234 const OMPParallelMaskedTaskLoopSimdDirective &S) {8235 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {8236 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,8237 PrePostActionTy &Action) {8238 Action.Enter(CGF);8239 CGF.EmitOMPTaskLoopBasedDirective(S);8240 };8241 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);8242 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, TaskLoopCodeGen,8243 S.getBeginLoc());8244 };8245 auto LPCRegion =8246 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);8247 emitCommonOMPParallelDirective(*this, S, OMPD_masked_taskloop_simd, CodeGen,8248 emitEmptyBoundParameters);8249}8250 8251// Generate the instructions for '#pragma omp target update' directive.8252void CodeGenFunction::EmitOMPTargetUpdateDirective(8253 const OMPTargetUpdateDirective &S) {8254 // If we don't have target devices, don't bother emitting the data mapping8255 // code.8256 if (CGM.getLangOpts().OMPTargetTriples.empty())8257 return;8258 8259 // Check if we have any if clause associated with the directive.8260 const Expr *IfCond = nullptr;8261 if (const auto *C = S.getSingleClause<OMPIfClause>())8262 IfCond = C->getCondition();8263 8264 // Check if we have any device clause associated with the directive.8265 const Expr *Device = nullptr;8266 if (const auto *C = S.getSingleClause<OMPDeviceClause>())8267 Device = C->getDevice();8268 8269 OMPLexicalScope Scope(*this, S, OMPD_task);8270 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);8271}8272 8273void CodeGenFunction::EmitOMPGenericLoopDirective(8274 const OMPGenericLoopDirective &S) {8275 // Always expect a bind clause on the loop directive. It it wasn't8276 // in the source, it should have been added in sema.8277 8278 OpenMPBindClauseKind BindKind = OMPC_BIND_unknown;8279 if (const auto *C = S.getSingleClause<OMPBindClause>())8280 BindKind = C->getBindKind();8281 8282 switch (BindKind) {8283 case OMPC_BIND_parallel: // for8284 return emitOMPForDirective(S, *this, CGM, /*HasCancel=*/false);8285 case OMPC_BIND_teams: // distribute8286 return emitOMPDistributeDirective(S, *this, CGM);8287 case OMPC_BIND_thread: // simd8288 return emitOMPSimdDirective(S, *this, CGM);8289 case OMPC_BIND_unknown:8290 break;8291 }8292 8293 // Unimplemented, just inline the underlying statement for now.8294 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {8295 // Emit the loop iteration variable.8296 const Stmt *CS =8297 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt();8298 const auto *ForS = dyn_cast<ForStmt>(CS);8299 if (ForS && !isa<DeclStmt>(ForS->getInit())) {8300 OMPPrivateScope LoopScope(CGF);8301 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);8302 (void)LoopScope.Privatize();8303 CGF.EmitStmt(CS);8304 LoopScope.restoreMap();8305 } else {8306 CGF.EmitStmt(CS);8307 }8308 };8309 OMPLexicalScope Scope(*this, S, OMPD_unknown);8310 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_loop, CodeGen);8311}8312 8313void CodeGenFunction::EmitOMPParallelGenericLoopDirective(8314 const OMPLoopDirective &S) {8315 // Emit combined directive as if its constituent constructs are 'parallel'8316 // and 'for'.8317 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {8318 Action.Enter(CGF);8319 emitOMPCopyinClause(CGF, S);8320 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);8321 };8322 {8323 auto LPCRegion =8324 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);8325 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,8326 emitEmptyBoundParameters);8327 }8328 // Check for outer lastprivate conditional update.8329 checkForLastprivateConditionalUpdate(*this, S);8330}8331 8332void CodeGenFunction::EmitOMPTeamsGenericLoopDirective(8333 const OMPTeamsGenericLoopDirective &S) {8334 // To be consistent with current behavior of 'target teams loop', emit8335 // 'teams loop' as if its constituent constructs are 'teams' and 'distribute'.8336 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {8337 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());8338 };8339 8340 // Emit teams region as a standalone region.8341 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,8342 PrePostActionTy &Action) {8343 Action.Enter(CGF);8344 OMPPrivateScope PrivateScope(CGF);8345 CGF.EmitOMPReductionClauseInit(S, PrivateScope);8346 (void)PrivateScope.Privatize();8347 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,8348 CodeGenDistribute);8349 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);8350 };8351 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);8352 emitPostUpdateForReductionClause(*this, S,8353 [](CodeGenFunction &) { return nullptr; });8354}8355 8356#ifndef NDEBUG8357static void emitTargetTeamsLoopCodegenStatus(CodeGenFunction &CGF,8358 std::string StatusMsg,8359 const OMPExecutableDirective &D) {8360 bool IsDevice = CGF.CGM.getLangOpts().OpenMPIsTargetDevice;8361 if (IsDevice)8362 StatusMsg += ": DEVICE";8363 else8364 StatusMsg += ": HOST";8365 SourceLocation L = D.getBeginLoc();8366 auto &SM = CGF.getContext().getSourceManager();8367 PresumedLoc PLoc = SM.getPresumedLoc(L);8368 const char *FileName = PLoc.isValid() ? PLoc.getFilename() : nullptr;8369 unsigned LineNo =8370 PLoc.isValid() ? PLoc.getLine() : SM.getExpansionLineNumber(L);8371 llvm::dbgs() << StatusMsg << ": " << FileName << ": " << LineNo << "\n";8372}8373#endif8374 8375static void emitTargetTeamsGenericLoopRegionAsParallel(8376 CodeGenFunction &CGF, PrePostActionTy &Action,8377 const OMPTargetTeamsGenericLoopDirective &S) {8378 Action.Enter(CGF);8379 // Emit 'teams loop' as if its constituent constructs are 'distribute,8380 // 'parallel, and 'for'.8381 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {8382 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,8383 S.getDistInc());8384 };8385 8386 // Emit teams region as a standalone region.8387 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,8388 PrePostActionTy &Action) {8389 Action.Enter(CGF);8390 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);8391 CGF.EmitOMPReductionClauseInit(S, PrivateScope);8392 (void)PrivateScope.Privatize();8393 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(8394 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);8395 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);8396 };8397 DEBUG_WITH_TYPE(TTL_CODEGEN_TYPE,8398 emitTargetTeamsLoopCodegenStatus(8399 CGF, TTL_CODEGEN_TYPE " as parallel for", S));8400 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,8401 CodeGenTeams);8402 emitPostUpdateForReductionClause(CGF, S,8403 [](CodeGenFunction &) { return nullptr; });8404}8405 8406static void emitTargetTeamsGenericLoopRegionAsDistribute(8407 CodeGenFunction &CGF, PrePostActionTy &Action,8408 const OMPTargetTeamsGenericLoopDirective &S) {8409 Action.Enter(CGF);8410 // Emit 'teams loop' as if its constituent construct is 'distribute'.8411 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {8412 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());8413 };8414 8415 // Emit teams region as a standalone region.8416 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,8417 PrePostActionTy &Action) {8418 Action.Enter(CGF);8419 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);8420 CGF.EmitOMPReductionClauseInit(S, PrivateScope);8421 (void)PrivateScope.Privatize();8422 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(8423 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);8424 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);8425 };8426 DEBUG_WITH_TYPE(TTL_CODEGEN_TYPE,8427 emitTargetTeamsLoopCodegenStatus(8428 CGF, TTL_CODEGEN_TYPE " as distribute", S));8429 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);8430 emitPostUpdateForReductionClause(CGF, S,8431 [](CodeGenFunction &) { return nullptr; });8432}8433 8434void CodeGenFunction::EmitOMPTargetTeamsGenericLoopDirective(8435 const OMPTargetTeamsGenericLoopDirective &S) {8436 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {8437 if (S.canBeParallelFor())8438 emitTargetTeamsGenericLoopRegionAsParallel(CGF, Action, S);8439 else8440 emitTargetTeamsGenericLoopRegionAsDistribute(CGF, Action, S);8441 };8442 emitCommonOMPTargetDirective(*this, S, CodeGen);8443}8444 8445void CodeGenFunction::EmitOMPTargetTeamsGenericLoopDeviceFunction(8446 CodeGenModule &CGM, StringRef ParentName,8447 const OMPTargetTeamsGenericLoopDirective &S) {8448 // Emit SPMD target parallel loop region as a standalone region.8449 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {8450 if (S.canBeParallelFor())8451 emitTargetTeamsGenericLoopRegionAsParallel(CGF, Action, S);8452 else8453 emitTargetTeamsGenericLoopRegionAsDistribute(CGF, Action, S);8454 };8455 llvm::Function *Fn;8456 llvm::Constant *Addr;8457 // Emit target region as a standalone region.8458 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(8459 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);8460 assert(Fn && Addr &&8461 "Target device function emission failed for 'target teams loop'.");8462}8463 8464static void emitTargetParallelGenericLoopRegion(8465 CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S,8466 PrePostActionTy &Action) {8467 Action.Enter(CGF);8468 // Emit as 'parallel for'.8469 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {8470 Action.Enter(CGF);8471 CodeGenFunction::OMPCancelStackRAII CancelRegion(8472 CGF, OMPD_target_parallel_loop, /*hasCancel=*/false);8473 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,8474 emitDispatchForLoopBounds);8475 };8476 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,8477 emitEmptyBoundParameters);8478}8479 8480void CodeGenFunction::EmitOMPTargetParallelGenericLoopDeviceFunction(8481 CodeGenModule &CGM, StringRef ParentName,8482 const OMPTargetParallelGenericLoopDirective &S) {8483 // Emit target parallel loop region as a standalone region.8484 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {8485 emitTargetParallelGenericLoopRegion(CGF, S, Action);8486 };8487 llvm::Function *Fn;8488 llvm::Constant *Addr;8489 // Emit target region as a standalone region.8490 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(8491 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);8492 assert(Fn && Addr && "Target device function emission failed.");8493}8494 8495/// Emit combined directive 'target parallel loop' as if its constituent8496/// constructs are 'target', 'parallel', and 'for'.8497void CodeGenFunction::EmitOMPTargetParallelGenericLoopDirective(8498 const OMPTargetParallelGenericLoopDirective &S) {8499 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {8500 emitTargetParallelGenericLoopRegion(CGF, S, Action);8501 };8502 emitCommonOMPTargetDirective(*this, S, CodeGen);8503}8504 8505void CodeGenFunction::EmitSimpleOMPExecutableDirective(8506 const OMPExecutableDirective &D) {8507 if (const auto *SD = dyn_cast<OMPScanDirective>(&D)) {8508 EmitOMPScanDirective(*SD);8509 return;8510 }8511 if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())8512 return;8513 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {8514 OMPPrivateScope GlobalsScope(CGF);8515 if (isOpenMPTaskingDirective(D.getDirectiveKind())) {8516 // Capture global firstprivates to avoid crash.8517 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {8518 for (const Expr *Ref : C->varlist()) {8519 const auto *DRE = cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());8520 if (!DRE)8521 continue;8522 const auto *VD = dyn_cast<VarDecl>(DRE->getDecl());8523 if (!VD || VD->hasLocalStorage())8524 continue;8525 if (!CGF.LocalDeclMap.count(VD)) {8526 LValue GlobLVal = CGF.EmitLValue(Ref);8527 GlobalsScope.addPrivate(VD, GlobLVal.getAddress());8528 }8529 }8530 }8531 }8532 if (isOpenMPSimdDirective(D.getDirectiveKind())) {8533 (void)GlobalsScope.Privatize();8534 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D);8535 emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);8536 } else {8537 if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {8538 for (const Expr *E : LD->counters()) {8539 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());8540 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {8541 LValue GlobLVal = CGF.EmitLValue(E);8542 GlobalsScope.addPrivate(VD, GlobLVal.getAddress());8543 }8544 if (isa<OMPCapturedExprDecl>(VD)) {8545 // Emit only those that were not explicitly referenced in clauses.8546 if (!CGF.LocalDeclMap.count(VD))8547 CGF.EmitVarDecl(*VD);8548 }8549 }8550 for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {8551 if (!C->getNumForLoops())8552 continue;8553 for (unsigned I = LD->getLoopsNumber(),8554 E = C->getLoopNumIterations().size();8555 I < E; ++I) {8556 if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(8557 cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {8558 // Emit only those that were not explicitly referenced in clauses.8559 if (!CGF.LocalDeclMap.count(VD))8560 CGF.EmitVarDecl(*VD);8561 }8562 }8563 }8564 }8565 (void)GlobalsScope.Privatize();8566 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt());8567 }8568 };8569 if (D.getDirectiveKind() == OMPD_atomic ||8570 D.getDirectiveKind() == OMPD_critical ||8571 D.getDirectiveKind() == OMPD_section ||8572 D.getDirectiveKind() == OMPD_master ||8573 D.getDirectiveKind() == OMPD_masked ||8574 D.getDirectiveKind() == OMPD_unroll ||8575 D.getDirectiveKind() == OMPD_assume) {8576 EmitStmt(D.getAssociatedStmt());8577 } else {8578 auto LPCRegion =8579 CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, D);8580 OMPSimdLexicalScope Scope(*this, D);8581 CGM.getOpenMPRuntime().emitInlinedDirective(8582 *this,8583 isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd8584 : D.getDirectiveKind(),8585 CodeGen);8586 }8587 // Check for outer lastprivate conditional update.8588 checkForLastprivateConditionalUpdate(*this, D);8589}8590 8591void CodeGenFunction::EmitOMPAssumeDirective(const OMPAssumeDirective &S) {8592 EmitStmt(S.getAssociatedStmt());8593}8594