12901 lines · cpp
1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This provides a class for OpenMP runtime code generation.10//11//===----------------------------------------------------------------------===//12 13#include "CGOpenMPRuntime.h"14#include "ABIInfoImpl.h"15#include "CGCXXABI.h"16#include "CGCleanup.h"17#include "CGDebugInfo.h"18#include "CGRecordLayout.h"19#include "CodeGenFunction.h"20#include "TargetInfo.h"21#include "clang/AST/APValue.h"22#include "clang/AST/Attr.h"23#include "clang/AST/Decl.h"24#include "clang/AST/OpenMPClause.h"25#include "clang/AST/StmtOpenMP.h"26#include "clang/AST/StmtVisitor.h"27#include "clang/Basic/OpenMPKinds.h"28#include "clang/Basic/SourceManager.h"29#include "clang/CodeGen/ConstantInitBuilder.h"30#include "llvm/ADT/ArrayRef.h"31#include "llvm/ADT/SmallVector.h"32#include "llvm/ADT/StringExtras.h"33#include "llvm/Bitcode/BitcodeReader.h"34#include "llvm/IR/Constants.h"35#include "llvm/IR/DerivedTypes.h"36#include "llvm/IR/GlobalValue.h"37#include "llvm/IR/InstrTypes.h"38#include "llvm/IR/Value.h"39#include "llvm/Support/AtomicOrdering.h"40#include "llvm/Support/raw_ostream.h"41#include <cassert>42#include <cstdint>43#include <numeric>44#include <optional>45 46using namespace clang;47using namespace CodeGen;48using namespace llvm::omp;49 50namespace {51/// Base class for handling code generation inside OpenMP regions.52class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {53public:54 /// Kinds of OpenMP regions used in codegen.55 enum CGOpenMPRegionKind {56 /// Region with outlined function for standalone 'parallel'57 /// directive.58 ParallelOutlinedRegion,59 /// Region with outlined function for standalone 'task' directive.60 TaskOutlinedRegion,61 /// Region for constructs that do not require function outlining,62 /// like 'for', 'sections', 'atomic' etc. directives.63 InlinedRegion,64 /// Region with outlined function for standalone 'target' directive.65 TargetRegion,66 };67 68 CGOpenMPRegionInfo(const CapturedStmt &CS,69 const CGOpenMPRegionKind RegionKind,70 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,71 bool HasCancel)72 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),73 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}74 75 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,76 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,77 bool HasCancel)78 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),79 Kind(Kind), HasCancel(HasCancel) {}80 81 /// Get a variable or parameter for storing global thread id82 /// inside OpenMP construct.83 virtual const VarDecl *getThreadIDVariable() const = 0;84 85 /// Emit the captured statement body.86 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;87 88 /// Get an LValue for the current ThreadID variable.89 /// \return LValue for thread id variable. This LValue always has type int32*.90 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);91 92 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}93 94 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }95 96 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }97 98 bool hasCancel() const { return HasCancel; }99 100 static bool classof(const CGCapturedStmtInfo *Info) {101 return Info->getKind() == CR_OpenMP;102 }103 104 ~CGOpenMPRegionInfo() override = default;105 106protected:107 CGOpenMPRegionKind RegionKind;108 RegionCodeGenTy CodeGen;109 OpenMPDirectiveKind Kind;110 bool HasCancel;111};112 113/// API for captured statement code generation in OpenMP constructs.114class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {115public:116 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,117 const RegionCodeGenTy &CodeGen,118 OpenMPDirectiveKind Kind, bool HasCancel,119 StringRef HelperName)120 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,121 HasCancel),122 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {123 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");124 }125 126 /// Get a variable or parameter for storing global thread id127 /// inside OpenMP construct.128 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }129 130 /// Get the name of the capture helper.131 StringRef getHelperName() const override { return HelperName; }132 133 static bool classof(const CGCapturedStmtInfo *Info) {134 return CGOpenMPRegionInfo::classof(Info) &&135 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==136 ParallelOutlinedRegion;137 }138 139private:140 /// A variable or parameter storing global thread id for OpenMP141 /// constructs.142 const VarDecl *ThreadIDVar;143 StringRef HelperName;144};145 146/// API for captured statement code generation in OpenMP constructs.147class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {148public:149 class UntiedTaskActionTy final : public PrePostActionTy {150 bool Untied;151 const VarDecl *PartIDVar;152 const RegionCodeGenTy UntiedCodeGen;153 llvm::SwitchInst *UntiedSwitch = nullptr;154 155 public:156 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,157 const RegionCodeGenTy &UntiedCodeGen)158 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}159 void Enter(CodeGenFunction &CGF) override {160 if (Untied) {161 // Emit task switching point.162 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(163 CGF.GetAddrOfLocalVar(PartIDVar),164 PartIDVar->getType()->castAs<PointerType>());165 llvm::Value *Res =166 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());167 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");168 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);169 CGF.EmitBlock(DoneBB);170 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);171 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));172 UntiedSwitch->addCase(CGF.Builder.getInt32(0),173 CGF.Builder.GetInsertBlock());174 emitUntiedSwitch(CGF);175 }176 }177 void emitUntiedSwitch(CodeGenFunction &CGF) const {178 if (Untied) {179 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(180 CGF.GetAddrOfLocalVar(PartIDVar),181 PartIDVar->getType()->castAs<PointerType>());182 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),183 PartIdLVal);184 UntiedCodeGen(CGF);185 CodeGenFunction::JumpDest CurPoint =186 CGF.getJumpDestInCurrentScope(".untied.next.");187 CGF.EmitBranch(CGF.ReturnBlock.getBlock());188 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));189 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),190 CGF.Builder.GetInsertBlock());191 CGF.EmitBranchThroughCleanup(CurPoint);192 CGF.EmitBlock(CurPoint.getBlock());193 }194 }195 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }196 };197 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,198 const VarDecl *ThreadIDVar,199 const RegionCodeGenTy &CodeGen,200 OpenMPDirectiveKind Kind, bool HasCancel,201 const UntiedTaskActionTy &Action)202 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),203 ThreadIDVar(ThreadIDVar), Action(Action) {204 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");205 }206 207 /// Get a variable or parameter for storing global thread id208 /// inside OpenMP construct.209 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }210 211 /// Get an LValue for the current ThreadID variable.212 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;213 214 /// Get the name of the capture helper.215 StringRef getHelperName() const override { return ".omp_outlined."; }216 217 void emitUntiedSwitch(CodeGenFunction &CGF) override {218 Action.emitUntiedSwitch(CGF);219 }220 221 static bool classof(const CGCapturedStmtInfo *Info) {222 return CGOpenMPRegionInfo::classof(Info) &&223 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==224 TaskOutlinedRegion;225 }226 227private:228 /// A variable or parameter storing global thread id for OpenMP229 /// constructs.230 const VarDecl *ThreadIDVar;231 /// Action for emitting code for untied tasks.232 const UntiedTaskActionTy &Action;233};234 235/// API for inlined captured statement code generation in OpenMP236/// constructs.237class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {238public:239 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,240 const RegionCodeGenTy &CodeGen,241 OpenMPDirectiveKind Kind, bool HasCancel)242 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),243 OldCSI(OldCSI),244 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}245 246 // Retrieve the value of the context parameter.247 llvm::Value *getContextValue() const override {248 if (OuterRegionInfo)249 return OuterRegionInfo->getContextValue();250 llvm_unreachable("No context value for inlined OpenMP region");251 }252 253 void setContextValue(llvm::Value *V) override {254 if (OuterRegionInfo) {255 OuterRegionInfo->setContextValue(V);256 return;257 }258 llvm_unreachable("No context value for inlined OpenMP region");259 }260 261 /// Lookup the captured field decl for a variable.262 const FieldDecl *lookup(const VarDecl *VD) const override {263 if (OuterRegionInfo)264 return OuterRegionInfo->lookup(VD);265 // If there is no outer outlined region,no need to lookup in a list of266 // captured variables, we can use the original one.267 return nullptr;268 }269 270 FieldDecl *getThisFieldDecl() const override {271 if (OuterRegionInfo)272 return OuterRegionInfo->getThisFieldDecl();273 return nullptr;274 }275 276 /// Get a variable or parameter for storing global thread id277 /// inside OpenMP construct.278 const VarDecl *getThreadIDVariable() const override {279 if (OuterRegionInfo)280 return OuterRegionInfo->getThreadIDVariable();281 return nullptr;282 }283 284 /// Get an LValue for the current ThreadID variable.285 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {286 if (OuterRegionInfo)287 return OuterRegionInfo->getThreadIDVariableLValue(CGF);288 llvm_unreachable("No LValue for inlined OpenMP construct");289 }290 291 /// Get the name of the capture helper.292 StringRef getHelperName() const override {293 if (auto *OuterRegionInfo = getOldCSI())294 return OuterRegionInfo->getHelperName();295 llvm_unreachable("No helper name for inlined OpenMP construct");296 }297 298 void emitUntiedSwitch(CodeGenFunction &CGF) override {299 if (OuterRegionInfo)300 OuterRegionInfo->emitUntiedSwitch(CGF);301 }302 303 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }304 305 static bool classof(const CGCapturedStmtInfo *Info) {306 return CGOpenMPRegionInfo::classof(Info) &&307 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;308 }309 310 ~CGOpenMPInlinedRegionInfo() override = default;311 312private:313 /// CodeGen info about outer OpenMP region.314 CodeGenFunction::CGCapturedStmtInfo *OldCSI;315 CGOpenMPRegionInfo *OuterRegionInfo;316};317 318/// API for captured statement code generation in OpenMP target319/// constructs. For this captures, implicit parameters are used instead of the320/// captured fields. The name of the target region has to be unique in a given321/// application so it is provided by the client, because only the client has322/// the information to generate that.323class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {324public:325 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,326 const RegionCodeGenTy &CodeGen, StringRef HelperName)327 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,328 /*HasCancel=*/false),329 HelperName(HelperName) {}330 331 /// This is unused for target regions because each starts executing332 /// with a single thread.333 const VarDecl *getThreadIDVariable() const override { return nullptr; }334 335 /// Get the name of the capture helper.336 StringRef getHelperName() const override { return HelperName; }337 338 static bool classof(const CGCapturedStmtInfo *Info) {339 return CGOpenMPRegionInfo::classof(Info) &&340 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;341 }342 343private:344 StringRef HelperName;345};346 347static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {348 llvm_unreachable("No codegen for expressions");349}350/// API for generation of expressions captured in a innermost OpenMP351/// region.352class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {353public:354 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)355 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,356 OMPD_unknown,357 /*HasCancel=*/false),358 PrivScope(CGF) {359 // Make sure the globals captured in the provided statement are local by360 // using the privatization logic. We assume the same variable is not361 // captured more than once.362 for (const auto &C : CS.captures()) {363 if (!C.capturesVariable() && !C.capturesVariableByCopy())364 continue;365 366 const VarDecl *VD = C.getCapturedVar();367 if (VD->isLocalVarDeclOrParm())368 continue;369 370 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),371 /*RefersToEnclosingVariableOrCapture=*/false,372 VD->getType().getNonReferenceType(), VK_LValue,373 C.getLocation());374 PrivScope.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());375 }376 (void)PrivScope.Privatize();377 }378 379 /// Lookup the captured field decl for a variable.380 const FieldDecl *lookup(const VarDecl *VD) const override {381 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))382 return FD;383 return nullptr;384 }385 386 /// Emit the captured statement body.387 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {388 llvm_unreachable("No body for expressions");389 }390 391 /// Get a variable or parameter for storing global thread id392 /// inside OpenMP construct.393 const VarDecl *getThreadIDVariable() const override {394 llvm_unreachable("No thread id for expressions");395 }396 397 /// Get the name of the capture helper.398 StringRef getHelperName() const override {399 llvm_unreachable("No helper name for expressions");400 }401 402 static bool classof(const CGCapturedStmtInfo *Info) { return false; }403 404private:405 /// Private scope to capture global variables.406 CodeGenFunction::OMPPrivateScope PrivScope;407};408 409/// RAII for emitting code of OpenMP constructs.410class InlinedOpenMPRegionRAII {411 CodeGenFunction &CGF;412 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;413 FieldDecl *LambdaThisCaptureField = nullptr;414 const CodeGen::CGBlockInfo *BlockInfo = nullptr;415 bool NoInheritance = false;416 417public:418 /// Constructs region for combined constructs.419 /// \param CodeGen Code generation sequence for combined directives. Includes420 /// a list of functions used for code generation of implicitly inlined421 /// regions.422 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,423 OpenMPDirectiveKind Kind, bool HasCancel,424 bool NoInheritance = true)425 : CGF(CGF), NoInheritance(NoInheritance) {426 // Start emission for the construct.427 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(428 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);429 if (NoInheritance) {430 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);431 LambdaThisCaptureField = CGF.LambdaThisCaptureField;432 CGF.LambdaThisCaptureField = nullptr;433 BlockInfo = CGF.BlockInfo;434 CGF.BlockInfo = nullptr;435 }436 }437 438 ~InlinedOpenMPRegionRAII() {439 // Restore original CapturedStmtInfo only if we're done with code emission.440 auto *OldCSI =441 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();442 delete CGF.CapturedStmtInfo;443 CGF.CapturedStmtInfo = OldCSI;444 if (NoInheritance) {445 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);446 CGF.LambdaThisCaptureField = LambdaThisCaptureField;447 CGF.BlockInfo = BlockInfo;448 }449 }450};451 452/// Values for bit flags used in the ident_t to describe the fields.453/// All enumeric elements are named and described in accordance with the code454/// from https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h455enum OpenMPLocationFlags : unsigned {456 /// Use trampoline for internal microtask.457 OMP_IDENT_IMD = 0x01,458 /// Use c-style ident structure.459 OMP_IDENT_KMPC = 0x02,460 /// Atomic reduction option for kmpc_reduce.461 OMP_ATOMIC_REDUCE = 0x10,462 /// Explicit 'barrier' directive.463 OMP_IDENT_BARRIER_EXPL = 0x20,464 /// Implicit barrier in code.465 OMP_IDENT_BARRIER_IMPL = 0x40,466 /// Implicit barrier in 'for' directive.467 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,468 /// Implicit barrier in 'sections' directive.469 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,470 /// Implicit barrier in 'single' directive.471 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,472 /// Call of __kmp_for_static_init for static loop.473 OMP_IDENT_WORK_LOOP = 0x200,474 /// Call of __kmp_for_static_init for sections.475 OMP_IDENT_WORK_SECTIONS = 0x400,476 /// Call of __kmp_for_static_init for distribute.477 OMP_IDENT_WORK_DISTRIBUTE = 0x800,478 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)479};480 481/// Describes ident structure that describes a source location.482/// All descriptions are taken from483/// https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h484/// Original structure:485/// typedef struct ident {486/// kmp_int32 reserved_1; /**< might be used in Fortran;487/// see above */488/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;489/// KMP_IDENT_KMPC identifies this union490/// member */491/// kmp_int32 reserved_2; /**< not really used in Fortran any more;492/// see above */493///#if USE_ITT_BUILD494/// /* but currently used for storing495/// region-specific ITT */496/// /* contextual information. */497///#endif /* USE_ITT_BUILD */498/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for499/// C++ */500/// char const *psource; /**< String describing the source location.501/// The string is composed of semi-colon separated502// fields which describe the source file,503/// the function and a pair of line numbers that504/// delimit the construct.505/// */506/// } ident_t;507enum IdentFieldIndex {508 /// might be used in Fortran509 IdentField_Reserved_1,510 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.511 IdentField_Flags,512 /// Not really used in Fortran any more513 IdentField_Reserved_2,514 /// Source[4] in Fortran, do not use for C++515 IdentField_Reserved_3,516 /// String describing the source location. The string is composed of517 /// semi-colon separated fields which describe the source file, the function518 /// and a pair of line numbers that delimit the construct.519 IdentField_PSource520};521 522/// Schedule types for 'omp for' loops (these enumerators are taken from523/// the enum sched_type in kmp.h).524enum OpenMPSchedType {525 /// Lower bound for default (unordered) versions.526 OMP_sch_lower = 32,527 OMP_sch_static_chunked = 33,528 OMP_sch_static = 34,529 OMP_sch_dynamic_chunked = 35,530 OMP_sch_guided_chunked = 36,531 OMP_sch_runtime = 37,532 OMP_sch_auto = 38,533 /// static with chunk adjustment (e.g., simd)534 OMP_sch_static_balanced_chunked = 45,535 /// Lower bound for 'ordered' versions.536 OMP_ord_lower = 64,537 OMP_ord_static_chunked = 65,538 OMP_ord_static = 66,539 OMP_ord_dynamic_chunked = 67,540 OMP_ord_guided_chunked = 68,541 OMP_ord_runtime = 69,542 OMP_ord_auto = 70,543 OMP_sch_default = OMP_sch_static,544 /// dist_schedule types545 OMP_dist_sch_static_chunked = 91,546 OMP_dist_sch_static = 92,547 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.548 /// Set if the monotonic schedule modifier was present.549 OMP_sch_modifier_monotonic = (1 << 29),550 /// Set if the nonmonotonic schedule modifier was present.551 OMP_sch_modifier_nonmonotonic = (1 << 30),552};553 554/// A basic class for pre|post-action for advanced codegen sequence for OpenMP555/// region.556class CleanupTy final : public EHScopeStack::Cleanup {557 PrePostActionTy *Action;558 559public:560 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}561 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {562 if (!CGF.HaveInsertPoint())563 return;564 Action->Exit(CGF);565 }566};567 568} // anonymous namespace569 570void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {571 CodeGenFunction::RunCleanupsScope Scope(CGF);572 if (PrePostAction) {573 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);574 Callback(CodeGen, CGF, *PrePostAction);575 } else {576 PrePostActionTy Action;577 Callback(CodeGen, CGF, Action);578 }579}580 581/// Check if the combiner is a call to UDR combiner and if it is so return the582/// UDR decl used for reduction.583static const OMPDeclareReductionDecl *584getReductionInit(const Expr *ReductionOp) {585 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))586 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))587 if (const auto *DRE =588 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))589 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))590 return DRD;591 return nullptr;592}593 594static void emitInitWithReductionInitializer(CodeGenFunction &CGF,595 const OMPDeclareReductionDecl *DRD,596 const Expr *InitOp,597 Address Private, Address Original,598 QualType Ty) {599 if (DRD->getInitializer()) {600 std::pair<llvm::Function *, llvm::Function *> Reduction =601 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);602 const auto *CE = cast<CallExpr>(InitOp);603 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());604 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();605 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();606 const auto *LHSDRE =607 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());608 const auto *RHSDRE =609 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());610 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);611 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()), Private);612 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()), Original);613 (void)PrivateScope.Privatize();614 RValue Func = RValue::get(Reduction.second);615 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);616 CGF.EmitIgnoredExpr(InitOp);617 } else {618 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);619 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});620 auto *GV = new llvm::GlobalVariable(621 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,622 llvm::GlobalValue::PrivateLinkage, Init, Name);623 LValue LV = CGF.MakeNaturalAlignRawAddrLValue(GV, Ty);624 RValue InitRVal;625 switch (CGF.getEvaluationKind(Ty)) {626 case TEK_Scalar:627 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());628 break;629 case TEK_Complex:630 InitRVal =631 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));632 break;633 case TEK_Aggregate: {634 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_LValue);635 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);636 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),637 /*IsInitializer=*/false);638 return;639 }640 }641 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_PRValue);642 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);643 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),644 /*IsInitializer=*/false);645 }646}647 648/// Emit initialization of arrays of complex types.649/// \param DestAddr Address of the array.650/// \param Type Type of array.651/// \param Init Initial expression of array.652/// \param SrcAddr Address of the original array.653static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,654 QualType Type, bool EmitDeclareReductionInit,655 const Expr *Init,656 const OMPDeclareReductionDecl *DRD,657 Address SrcAddr = Address::invalid()) {658 // Perform element-by-element initialization.659 QualType ElementTy;660 661 // Drill down to the base element type on both arrays.662 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();663 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);664 if (DRD)665 SrcAddr = SrcAddr.withElementType(DestAddr.getElementType());666 667 llvm::Value *SrcBegin = nullptr;668 if (DRD)669 SrcBegin = SrcAddr.emitRawPointer(CGF);670 llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF);671 // Cast from pointer to array type to pointer to single element.672 llvm::Value *DestEnd =673 CGF.Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements);674 // The basic structure here is a while-do loop.675 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");676 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");677 llvm::Value *IsEmpty =678 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");679 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);680 681 // Enter the loop body, making that address the current address.682 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();683 CGF.EmitBlock(BodyBB);684 685 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);686 687 llvm::PHINode *SrcElementPHI = nullptr;688 Address SrcElementCurrent = Address::invalid();689 if (DRD) {690 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,691 "omp.arraycpy.srcElementPast");692 SrcElementPHI->addIncoming(SrcBegin, EntryBB);693 SrcElementCurrent =694 Address(SrcElementPHI, SrcAddr.getElementType(),695 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));696 }697 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(698 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");699 DestElementPHI->addIncoming(DestBegin, EntryBB);700 Address DestElementCurrent =701 Address(DestElementPHI, DestAddr.getElementType(),702 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));703 704 // Emit copy.705 {706 CodeGenFunction::RunCleanupsScope InitScope(CGF);707 if (EmitDeclareReductionInit) {708 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,709 SrcElementCurrent, ElementTy);710 } else711 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),712 /*IsInitializer=*/false);713 }714 715 if (DRD) {716 // Shift the address forward by one element.717 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(718 SrcAddr.getElementType(), SrcElementPHI, /*Idx0=*/1,719 "omp.arraycpy.dest.element");720 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());721 }722 723 // Shift the address forward by one element.724 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(725 DestAddr.getElementType(), DestElementPHI, /*Idx0=*/1,726 "omp.arraycpy.dest.element");727 // Check whether we've reached the end.728 llvm::Value *Done =729 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");730 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);731 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());732 733 // Done.734 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);735}736 737LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {738 return CGF.EmitOMPSharedLValue(E);739}740 741LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,742 const Expr *E) {743 if (const auto *OASE = dyn_cast<ArraySectionExpr>(E))744 return CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false);745 return LValue();746}747 748void ReductionCodeGen::emitAggregateInitialization(749 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,750 const OMPDeclareReductionDecl *DRD) {751 // Emit VarDecl with copy init for arrays.752 // Get the address of the original variable captured in current753 // captured region.754 const auto *PrivateVD =755 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());756 bool EmitDeclareReductionInit =757 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());758 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),759 EmitDeclareReductionInit,760 EmitDeclareReductionInit ? ClausesData[N].ReductionOp761 : PrivateVD->getInit(),762 DRD, SharedAddr);763}764 765ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,766 ArrayRef<const Expr *> Origs,767 ArrayRef<const Expr *> Privates,768 ArrayRef<const Expr *> ReductionOps) {769 ClausesData.reserve(Shareds.size());770 SharedAddresses.reserve(Shareds.size());771 Sizes.reserve(Shareds.size());772 BaseDecls.reserve(Shareds.size());773 const auto *IOrig = Origs.begin();774 const auto *IPriv = Privates.begin();775 const auto *IRed = ReductionOps.begin();776 for (const Expr *Ref : Shareds) {777 ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);778 std::advance(IOrig, 1);779 std::advance(IPriv, 1);780 std::advance(IRed, 1);781 }782}783 784void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) {785 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&786 "Number of generated lvalues must be exactly N.");787 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared);788 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);789 SharedAddresses.emplace_back(First, Second);790 if (ClausesData[N].Shared == ClausesData[N].Ref) {791 OrigAddresses.emplace_back(First, Second);792 } else {793 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);794 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);795 OrigAddresses.emplace_back(First, Second);796 }797}798 799void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {800 QualType PrivateType = getPrivateType(N);801 bool AsArraySection = isa<ArraySectionExpr>(ClausesData[N].Ref);802 if (!PrivateType->isVariablyModifiedType()) {803 Sizes.emplace_back(804 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()),805 nullptr);806 return;807 }808 llvm::Value *Size;809 llvm::Value *SizeInChars;810 auto *ElemType = OrigAddresses[N].first.getAddress().getElementType();811 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);812 if (AsArraySection) {813 Size = CGF.Builder.CreatePtrDiff(ElemType,814 OrigAddresses[N].second.getPointer(CGF),815 OrigAddresses[N].first.getPointer(CGF));816 Size = CGF.Builder.CreateNUWAdd(817 Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));818 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);819 } else {820 SizeInChars =821 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType());822 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);823 }824 Sizes.emplace_back(SizeInChars, Size);825 CodeGenFunction::OpaqueValueMapping OpaqueMap(826 CGF,827 cast<OpaqueValueExpr>(828 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),829 RValue::get(Size));830 CGF.EmitVariablyModifiedType(PrivateType);831}832 833void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,834 llvm::Value *Size) {835 QualType PrivateType = getPrivateType(N);836 if (!PrivateType->isVariablyModifiedType()) {837 assert(!Size && !Sizes[N].second &&838 "Size should be nullptr for non-variably modified reduction "839 "items.");840 return;841 }842 CodeGenFunction::OpaqueValueMapping OpaqueMap(843 CGF,844 cast<OpaqueValueExpr>(845 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),846 RValue::get(Size));847 CGF.EmitVariablyModifiedType(PrivateType);848}849 850void ReductionCodeGen::emitInitialization(851 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,852 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {853 assert(SharedAddresses.size() > N && "No variable was generated");854 const auto *PrivateVD =855 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());856 const OMPDeclareReductionDecl *DRD =857 getReductionInit(ClausesData[N].ReductionOp);858 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {859 if (DRD && DRD->getInitializer())860 (void)DefaultInit(CGF);861 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);862 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {863 (void)DefaultInit(CGF);864 QualType SharedType = SharedAddresses[N].first.getType();865 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,866 PrivateAddr, SharedAddr, SharedType);867 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&868 !CGF.isTrivialInitializer(PrivateVD->getInit())) {869 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,870 PrivateVD->getType().getQualifiers(),871 /*IsInitializer=*/false);872 }873}874 875bool ReductionCodeGen::needCleanups(unsigned N) {876 QualType PrivateType = getPrivateType(N);877 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();878 return DTorKind != QualType::DK_none;879}880 881void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,882 Address PrivateAddr) {883 QualType PrivateType = getPrivateType(N);884 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();885 if (needCleanups(N)) {886 PrivateAddr =887 PrivateAddr.withElementType(CGF.ConvertTypeForMem(PrivateType));888 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);889 }890}891 892static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,893 LValue BaseLV) {894 BaseTy = BaseTy.getNonReferenceType();895 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&896 !CGF.getContext().hasSameType(BaseTy, ElTy)) {897 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {898 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy);899 } else {900 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(), BaseTy);901 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);902 }903 BaseTy = BaseTy->getPointeeType();904 }905 return CGF.MakeAddrLValue(906 BaseLV.getAddress().withElementType(CGF.ConvertTypeForMem(ElTy)),907 BaseLV.getType(), BaseLV.getBaseInfo(),908 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));909}910 911static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,912 Address OriginalBaseAddress, llvm::Value *Addr) {913 RawAddress Tmp = RawAddress::invalid();914 Address TopTmp = Address::invalid();915 Address MostTopTmp = Address::invalid();916 BaseTy = BaseTy.getNonReferenceType();917 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&918 !CGF.getContext().hasSameType(BaseTy, ElTy)) {919 Tmp = CGF.CreateMemTemp(BaseTy);920 if (TopTmp.isValid())921 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);922 else923 MostTopTmp = Tmp;924 TopTmp = Tmp;925 BaseTy = BaseTy->getPointeeType();926 }927 928 if (Tmp.isValid()) {929 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(930 Addr, Tmp.getElementType());931 CGF.Builder.CreateStore(Addr, Tmp);932 return MostTopTmp;933 }934 935 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(936 Addr, OriginalBaseAddress.getType());937 return OriginalBaseAddress.withPointer(Addr, NotKnownNonNull);938}939 940static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {941 const VarDecl *OrigVD = nullptr;942 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Ref)) {943 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();944 while (const auto *TempOASE = dyn_cast<ArraySectionExpr>(Base))945 Base = TempOASE->getBase()->IgnoreParenImpCasts();946 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))947 Base = TempASE->getBase()->IgnoreParenImpCasts();948 DE = cast<DeclRefExpr>(Base);949 OrigVD = cast<VarDecl>(DE->getDecl());950 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {951 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();952 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))953 Base = TempASE->getBase()->IgnoreParenImpCasts();954 DE = cast<DeclRefExpr>(Base);955 OrigVD = cast<VarDecl>(DE->getDecl());956 }957 return OrigVD;958}959 960Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,961 Address PrivateAddr) {962 const DeclRefExpr *DE;963 if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {964 BaseDecls.emplace_back(OrigVD);965 LValue OriginalBaseLValue = CGF.EmitLValue(DE);966 LValue BaseLValue =967 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),968 OriginalBaseLValue);969 Address SharedAddr = SharedAddresses[N].first.getAddress();970 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(971 SharedAddr.getElementType(), BaseLValue.getPointer(CGF),972 SharedAddr.emitRawPointer(CGF));973 llvm::Value *PrivatePointer =974 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(975 PrivateAddr.emitRawPointer(CGF), SharedAddr.getType());976 llvm::Value *Ptr = CGF.Builder.CreateGEP(977 SharedAddr.getElementType(), PrivatePointer, Adjustment);978 return castToBase(CGF, OrigVD->getType(),979 SharedAddresses[N].first.getType(),980 OriginalBaseLValue.getAddress(), Ptr);981 }982 BaseDecls.emplace_back(983 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));984 return PrivateAddr;985}986 987bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {988 const OMPDeclareReductionDecl *DRD =989 getReductionInit(ClausesData[N].ReductionOp);990 return DRD && DRD->getInitializer();991}992 993LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {994 return CGF.EmitLoadOfPointerLValue(995 CGF.GetAddrOfLocalVar(getThreadIDVariable()),996 getThreadIDVariable()->getType()->castAs<PointerType>());997}998 999void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) {1000 if (!CGF.HaveInsertPoint())1001 return;1002 // 1.2.2 OpenMP Language Terminology1003 // Structured block - An executable statement with a single entry at the1004 // top and a single exit at the bottom.1005 // The point of exit cannot be a branch out of the structured block.1006 // longjmp() and throw() must not violate the entry/exit criteria.1007 CGF.EHStack.pushTerminate();1008 if (S)1009 CGF.incrementProfileCounter(S);1010 CodeGen(CGF);1011 CGF.EHStack.popTerminate();1012}1013 1014LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(1015 CodeGenFunction &CGF) {1016 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),1017 getThreadIDVariable()->getType(),1018 AlignmentSource::Decl);1019}1020 1021static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,1022 QualType FieldTy) {1023 auto *Field = FieldDecl::Create(1024 C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,1025 C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),1026 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);1027 Field->setAccess(AS_public);1028 DC->addDecl(Field);1029 return Field;1030}1031 1032CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM)1033 : CGM(CGM), OMPBuilder(CGM.getModule()) {1034 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);1035 llvm::OpenMPIRBuilderConfig Config(1036 CGM.getLangOpts().OpenMPIsTargetDevice, isGPU(),1037 CGM.getLangOpts().OpenMPOffloadMandatory,1038 /*HasRequiresReverseOffload*/ false, /*HasRequiresUnifiedAddress*/ false,1039 hasRequiresUnifiedSharedMemory(), /*HasRequiresDynamicAllocators*/ false);1040 Config.setDefaultTargetAS(1041 CGM.getContext().getTargetInfo().getTargetAddressSpace(LangAS::Default));1042 Config.setRuntimeCC(CGM.getRuntimeCC());1043 1044 OMPBuilder.setConfig(Config);1045 OMPBuilder.initialize();1046 OMPBuilder.loadOffloadInfoMetadata(*CGM.getFileSystem(),1047 CGM.getLangOpts().OpenMPIsTargetDevice1048 ? CGM.getLangOpts().OMPHostIRFile1049 : StringRef{});1050 1051 // The user forces the compiler to behave as if omp requires1052 // unified_shared_memory was given.1053 if (CGM.getLangOpts().OpenMPForceUSM) {1054 HasRequiresUnifiedSharedMemory = true;1055 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true);1056 }1057}1058 1059void CGOpenMPRuntime::clear() {1060 InternalVars.clear();1061 // Clean non-target variable declarations possibly used only in debug info.1062 for (const auto &Data : EmittedNonTargetVariables) {1063 if (!Data.getValue().pointsToAliveValue())1064 continue;1065 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());1066 if (!GV)1067 continue;1068 if (!GV->isDeclaration() || GV->getNumUses() > 0)1069 continue;1070 GV->eraseFromParent();1071 }1072}1073 1074std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {1075 return OMPBuilder.createPlatformSpecificName(Parts);1076}1077 1078static llvm::Function *1079emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,1080 const Expr *CombinerInitializer, const VarDecl *In,1081 const VarDecl *Out, bool IsCombiner) {1082 // void .omp_combiner.(Ty *in, Ty *out);1083 ASTContext &C = CGM.getContext();1084 QualType PtrTy = C.getPointerType(Ty).withRestrict();1085 FunctionArgList Args;1086 ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),1087 /*Id=*/nullptr, PtrTy, ImplicitParamKind::Other);1088 ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),1089 /*Id=*/nullptr, PtrTy, ImplicitParamKind::Other);1090 Args.push_back(&OmpOutParm);1091 Args.push_back(&OmpInParm);1092 const CGFunctionInfo &FnInfo =1093 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);1094 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);1095 std::string Name = CGM.getOpenMPRuntime().getName(1096 {IsCombiner ? "omp_combiner" : "omp_initializer", ""});1097 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,1098 Name, &CGM.getModule());1099 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);1100 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {1101 Fn->removeFnAttr(llvm::Attribute::NoInline);1102 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);1103 Fn->addFnAttr(llvm::Attribute::AlwaysInline);1104 }1105 CodeGenFunction CGF(CGM);1106 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.1107 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.1108 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),1109 Out->getLocation());1110 CodeGenFunction::OMPPrivateScope Scope(CGF);1111 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);1112 Scope.addPrivate(1113 In, CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())1114 .getAddress());1115 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);1116 Scope.addPrivate(1117 Out, CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())1118 .getAddress());1119 (void)Scope.Privatize();1120 if (!IsCombiner && Out->hasInit() &&1121 !CGF.isTrivialInitializer(Out->getInit())) {1122 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),1123 Out->getType().getQualifiers(),1124 /*IsInitializer=*/true);1125 }1126 if (CombinerInitializer)1127 CGF.EmitIgnoredExpr(CombinerInitializer);1128 Scope.ForceCleanup();1129 CGF.FinishFunction();1130 return Fn;1131}1132 1133void CGOpenMPRuntime::emitUserDefinedReduction(1134 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {1135 if (UDRMap.count(D) > 0)1136 return;1137 llvm::Function *Combiner = emitCombinerOrInitializer(1138 CGM, D->getType(), D->getCombiner(),1139 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()),1140 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()),1141 /*IsCombiner=*/true);1142 llvm::Function *Initializer = nullptr;1143 if (const Expr *Init = D->getInitializer()) {1144 Initializer = emitCombinerOrInitializer(1145 CGM, D->getType(),1146 D->getInitializerKind() == OMPDeclareReductionInitKind::Call ? Init1147 : nullptr,1148 cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()),1149 cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()),1150 /*IsCombiner=*/false);1151 }1152 UDRMap.try_emplace(D, Combiner, Initializer);1153 if (CGF)1154 FunctionUDRMap[CGF->CurFn].push_back(D);1155}1156 1157std::pair<llvm::Function *, llvm::Function *>1158CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {1159 auto I = UDRMap.find(D);1160 if (I != UDRMap.end())1161 return I->second;1162 emitUserDefinedReduction(/*CGF=*/nullptr, D);1163 return UDRMap.lookup(D);1164}1165 1166namespace {1167// Temporary RAII solution to perform a push/pop stack event on the OpenMP IR1168// Builder if one is present.1169struct PushAndPopStackRAII {1170 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,1171 bool HasCancel, llvm::omp::Directive Kind)1172 : OMPBuilder(OMPBuilder) {1173 if (!OMPBuilder)1174 return;1175 1176 // The following callback is the crucial part of clangs cleanup process.1177 //1178 // NOTE:1179 // Once the OpenMPIRBuilder is used to create parallel regions (and1180 // similar), the cancellation destination (Dest below) is determined via1181 // IP. That means if we have variables to finalize we split the block at IP,1182 // use the new block (=BB) as destination to build a JumpDest (via1183 // getJumpDestInCurrentScope(BB)) which then is fed to1184 // EmitBranchThroughCleanup. Furthermore, there will not be the need1185 // to push & pop an FinalizationInfo object.1186 // The FiniCB will still be needed but at the point where the1187 // OpenMPIRBuilder is asked to construct a parallel (or similar) construct.1188 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {1189 assert(IP.getBlock()->end() == IP.getPoint() &&1190 "Clang CG should cause non-terminated block!");1191 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);1192 CGF.Builder.restoreIP(IP);1193 CodeGenFunction::JumpDest Dest =1194 CGF.getOMPCancelDestination(OMPD_parallel);1195 CGF.EmitBranchThroughCleanup(Dest);1196 return llvm::Error::success();1197 };1198 1199 // TODO: Remove this once we emit parallel regions through the1200 // OpenMPIRBuilder as it can do this setup internally.1201 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB, Kind, HasCancel});1202 OMPBuilder->pushFinalizationCB(std::move(FI));1203 }1204 ~PushAndPopStackRAII() {1205 if (OMPBuilder)1206 OMPBuilder->popFinalizationCB();1207 }1208 llvm::OpenMPIRBuilder *OMPBuilder;1209};1210} // namespace1211 1212static llvm::Function *emitParallelOrTeamsOutlinedFunction(1213 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,1214 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,1215 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {1216 assert(ThreadIDVar->getType()->isPointerType() &&1217 "thread id variable must be of type kmp_int32 *");1218 CodeGenFunction CGF(CGM, true);1219 bool HasCancel = false;1220 if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))1221 HasCancel = OPD->hasCancel();1222 else if (const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&D))1223 HasCancel = OPD->hasCancel();1224 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))1225 HasCancel = OPSD->hasCancel();1226 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))1227 HasCancel = OPFD->hasCancel();1228 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))1229 HasCancel = OPFD->hasCancel();1230 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))1231 HasCancel = OPFD->hasCancel();1232 else if (const auto *OPFD =1233 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))1234 HasCancel = OPFD->hasCancel();1235 else if (const auto *OPFD =1236 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))1237 HasCancel = OPFD->hasCancel();1238 1239 // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new1240 // parallel region to make cancellation barriers work properly.1241 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();1242 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);1243 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,1244 HasCancel, OutlinedHelperName);1245 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);1246 return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D);1247}1248 1249std::string CGOpenMPRuntime::getOutlinedHelperName(StringRef Name) const {1250 std::string Suffix = getName({"omp_outlined"});1251 return (Name + Suffix).str();1252}1253 1254std::string CGOpenMPRuntime::getOutlinedHelperName(CodeGenFunction &CGF) const {1255 return getOutlinedHelperName(CGF.CurFn->getName());1256}1257 1258std::string CGOpenMPRuntime::getReductionFuncName(StringRef Name) const {1259 std::string Suffix = getName({"omp", "reduction", "reduction_func"});1260 return (Name + Suffix).str();1261}1262 1263llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(1264 CodeGenFunction &CGF, const OMPExecutableDirective &D,1265 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,1266 const RegionCodeGenTy &CodeGen) {1267 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);1268 return emitParallelOrTeamsOutlinedFunction(1269 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(CGF),1270 CodeGen);1271}1272 1273llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(1274 CodeGenFunction &CGF, const OMPExecutableDirective &D,1275 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,1276 const RegionCodeGenTy &CodeGen) {1277 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);1278 return emitParallelOrTeamsOutlinedFunction(1279 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(CGF),1280 CodeGen);1281}1282 1283llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(1284 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,1285 const VarDecl *PartIDVar, const VarDecl *TaskTVar,1286 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,1287 bool Tied, unsigned &NumberOfParts) {1288 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,1289 PrePostActionTy &) {1290 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());1291 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());1292 llvm::Value *TaskArgs[] = {1293 UpLoc, ThreadID,1294 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),1295 TaskTVar->getType()->castAs<PointerType>())1296 .getPointer(CGF)};1297 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(1298 CGM.getModule(), OMPRTL___kmpc_omp_task),1299 TaskArgs);1300 };1301 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,1302 UntiedCodeGen);1303 CodeGen.setAction(Action);1304 assert(!ThreadIDVar->getType()->isPointerType() &&1305 "thread id variable must be of type kmp_int32 for tasks");1306 const OpenMPDirectiveKind Region =1307 isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop1308 : OMPD_task;1309 const CapturedStmt *CS = D.getCapturedStmt(Region);1310 bool HasCancel = false;1311 if (const auto *TD = dyn_cast<OMPTaskDirective>(&D))1312 HasCancel = TD->hasCancel();1313 else if (const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))1314 HasCancel = TD->hasCancel();1315 else if (const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))1316 HasCancel = TD->hasCancel();1317 else if (const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))1318 HasCancel = TD->hasCancel();1319 1320 CodeGenFunction CGF(CGM, true);1321 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,1322 InnermostKind, HasCancel, Action);1323 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);1324 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);1325 if (!Tied)1326 NumberOfParts = Action.getNumberOfParts();1327 return Res;1328}1329 1330void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,1331 bool AtCurrentPoint) {1332 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];1333 assert(!Elem.ServiceInsertPt && "Insert point is set already.");1334 1335 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);1336 if (AtCurrentPoint) {1337 Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt",1338 CGF.Builder.GetInsertBlock());1339 } else {1340 Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");1341 Elem.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt->getIterator());1342 }1343}1344 1345void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {1346 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];1347 if (Elem.ServiceInsertPt) {1348 llvm::Instruction *Ptr = Elem.ServiceInsertPt;1349 Elem.ServiceInsertPt = nullptr;1350 Ptr->eraseFromParent();1351 }1352}1353 1354static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF,1355 SourceLocation Loc,1356 SmallString<128> &Buffer) {1357 llvm::raw_svector_ostream OS(Buffer);1358 // Build debug location1359 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);1360 OS << ";";1361 if (auto *DbgInfo = CGF.getDebugInfo())1362 OS << DbgInfo->remapDIPath(PLoc.getFilename());1363 else1364 OS << PLoc.getFilename();1365 OS << ";";1366 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))1367 OS << FD->getQualifiedNameAsString();1368 OS << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";1369 return OS.str();1370}1371 1372llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,1373 SourceLocation Loc,1374 unsigned Flags, bool EmitLoc) {1375 uint32_t SrcLocStrSize;1376 llvm::Constant *SrcLocStr;1377 if ((!EmitLoc && CGM.getCodeGenOpts().getDebugInfo() ==1378 llvm::codegenoptions::NoDebugInfo) ||1379 Loc.isInvalid()) {1380 SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);1381 } else {1382 std::string FunctionName;1383 std::string FileName;1384 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))1385 FunctionName = FD->getQualifiedNameAsString();1386 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);1387 if (auto *DbgInfo = CGF.getDebugInfo())1388 FileName = DbgInfo->remapDIPath(PLoc.getFilename());1389 else1390 FileName = PLoc.getFilename();1391 unsigned Line = PLoc.getLine();1392 unsigned Column = PLoc.getColumn();1393 SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(FunctionName, FileName, Line,1394 Column, SrcLocStrSize);1395 }1396 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();1397 return OMPBuilder.getOrCreateIdent(1398 SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(Flags), Reserved2Flags);1399}1400 1401llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,1402 SourceLocation Loc) {1403 assert(CGF.CurFn && "No function in current CodeGenFunction.");1404 // If the OpenMPIRBuilder is used we need to use it for all thread id calls as1405 // the clang invariants used below might be broken.1406 if (CGM.getLangOpts().OpenMPIRBuilder) {1407 SmallString<128> Buffer;1408 OMPBuilder.updateToLocation(CGF.Builder.saveIP());1409 uint32_t SrcLocStrSize;1410 auto *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(1411 getIdentStringFromSourceLocation(CGF, Loc, Buffer), SrcLocStrSize);1412 return OMPBuilder.getOrCreateThreadID(1413 OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));1414 }1415 1416 llvm::Value *ThreadID = nullptr;1417 // Check whether we've already cached a load of the thread id in this1418 // function.1419 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);1420 if (I != OpenMPLocThreadIDMap.end()) {1421 ThreadID = I->second.ThreadID;1422 if (ThreadID != nullptr)1423 return ThreadID;1424 }1425 // If exceptions are enabled, do not use parameter to avoid possible crash.1426 if (auto *OMPRegionInfo =1427 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {1428 if (OMPRegionInfo->getThreadIDVariable()) {1429 // Check if this an outlined function with thread id passed as argument.1430 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);1431 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();1432 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||1433 !CGF.getLangOpts().CXXExceptions ||1434 CGF.Builder.GetInsertBlock() == TopBlock ||1435 !isa<llvm::Instruction>(LVal.getPointer(CGF)) ||1436 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==1437 TopBlock ||1438 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==1439 CGF.Builder.GetInsertBlock()) {1440 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);1441 // If value loaded in entry block, cache it and use it everywhere in1442 // function.1443 if (CGF.Builder.GetInsertBlock() == TopBlock)1444 OpenMPLocThreadIDMap[CGF.CurFn].ThreadID = ThreadID;1445 return ThreadID;1446 }1447 }1448 }1449 1450 // This is not an outlined function region - need to call __kmpc_int321451 // kmpc_global_thread_num(ident_t *loc).1452 // Generate thread id value and cache this value for use across the1453 // function.1454 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];1455 if (!Elem.ServiceInsertPt)1456 setLocThreadIdInsertPt(CGF);1457 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);1458 CGF.Builder.SetInsertPoint(Elem.ServiceInsertPt);1459 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);1460 llvm::CallInst *Call = CGF.Builder.CreateCall(1461 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),1462 OMPRTL___kmpc_global_thread_num),1463 emitUpdateLocation(CGF, Loc));1464 Call->setCallingConv(CGF.getRuntimeCC());1465 Elem.ThreadID = Call;1466 return Call;1467}1468 1469void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {1470 assert(CGF.CurFn && "No function in current CodeGenFunction.");1471 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {1472 clearLocThreadIdInsertPt(CGF);1473 OpenMPLocThreadIDMap.erase(CGF.CurFn);1474 }1475 if (auto I = FunctionUDRMap.find(CGF.CurFn); I != FunctionUDRMap.end()) {1476 for (const auto *D : I->second)1477 UDRMap.erase(D);1478 FunctionUDRMap.erase(I);1479 }1480 if (auto I = FunctionUDMMap.find(CGF.CurFn); I != FunctionUDMMap.end()) {1481 for (const auto *D : I->second)1482 UDMMap.erase(D);1483 FunctionUDMMap.erase(I);1484 }1485 LastprivateConditionalToTypes.erase(CGF.CurFn);1486 FunctionToUntiedTaskStackMap.erase(CGF.CurFn);1487}1488 1489llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {1490 return OMPBuilder.IdentPtr;1491}1492 1493static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind1494convertDeviceClause(const VarDecl *VD) {1495 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =1496 OMPDeclareTargetDeclAttr::getDeviceType(VD);1497 if (!DevTy)1498 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;1499 1500 switch ((int)*DevTy) { // Avoid -Wcovered-switch-default1501 case OMPDeclareTargetDeclAttr::DT_Host:1502 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;1503 break;1504 case OMPDeclareTargetDeclAttr::DT_NoHost:1505 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;1506 break;1507 case OMPDeclareTargetDeclAttr::DT_Any:1508 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;1509 break;1510 default:1511 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;1512 break;1513 }1514}1515 1516static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind1517convertCaptureClause(const VarDecl *VD) {1518 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =1519 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);1520 if (!MapType)1521 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;1522 switch ((int)*MapType) { // Avoid -Wcovered-switch-default1523 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:1524 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;1525 break;1526 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:1527 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;1528 break;1529 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:1530 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;1531 break;1532 default:1533 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;1534 break;1535 }1536}1537 1538static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(1539 CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder,1540 SourceLocation BeginLoc, llvm::StringRef ParentName = "") {1541 1542 auto FileInfoCallBack = [&]() {1543 SourceManager &SM = CGM.getContext().getSourceManager();1544 PresumedLoc PLoc = SM.getPresumedLoc(BeginLoc);1545 1546 if (!CGM.getFileSystem()->exists(PLoc.getFilename()))1547 PLoc = SM.getPresumedLoc(BeginLoc, /*UseLineDirectives=*/false);1548 1549 return std::pair<std::string, uint64_t>(PLoc.getFilename(), PLoc.getLine());1550 };1551 1552 return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack,1553 *CGM.getFileSystem(), ParentName);1554}1555 1556ConstantAddress CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {1557 auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(VD); };1558 1559 auto LinkageForVariable = [&VD, this]() {1560 return CGM.getLLVMLinkageVarDefinition(VD);1561 };1562 1563 std::vector<llvm::GlobalVariable *> GeneratedRefs;1564 1565 llvm::Type *LlvmPtrTy = CGM.getTypes().ConvertTypeForMem(1566 CGM.getContext().getPointerType(VD->getType()));1567 llvm::Constant *addr = OMPBuilder.getAddrOfDeclareTargetVar(1568 convertCaptureClause(VD), convertDeviceClause(VD),1569 VD->hasDefinition(CGM.getContext()) == VarDecl::DeclarationOnly,1570 VD->isExternallyVisible(),1571 getEntryInfoFromPresumedLoc(CGM, OMPBuilder,1572 VD->getCanonicalDecl()->getBeginLoc()),1573 CGM.getMangledName(VD), GeneratedRefs, CGM.getLangOpts().OpenMPSimd,1574 CGM.getLangOpts().OMPTargetTriples, LlvmPtrTy, AddrOfGlobal,1575 LinkageForVariable);1576 1577 if (!addr)1578 return ConstantAddress::invalid();1579 return ConstantAddress(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(VD));1580}1581 1582llvm::Constant *1583CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {1584 assert(!CGM.getLangOpts().OpenMPUseTLS ||1585 !CGM.getContext().getTargetInfo().isTLSSupported());1586 // Lookup the entry, lazily creating it if necessary.1587 std::string Suffix = getName({"cache", ""});1588 return OMPBuilder.getOrCreateInternalVariable(1589 CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix).str());1590}1591 1592Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,1593 const VarDecl *VD,1594 Address VDAddr,1595 SourceLocation Loc) {1596 if (CGM.getLangOpts().OpenMPUseTLS &&1597 CGM.getContext().getTargetInfo().isTLSSupported())1598 return VDAddr;1599 1600 llvm::Type *VarTy = VDAddr.getElementType();1601 llvm::Value *Args[] = {1602 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),1603 CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy),1604 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),1605 getOrCreateThreadPrivateCache(VD)};1606 return Address(1607 CGF.EmitRuntimeCall(1608 OMPBuilder.getOrCreateRuntimeFunction(1609 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),1610 Args),1611 CGF.Int8Ty, VDAddr.getAlignment());1612}1613 1614void CGOpenMPRuntime::emitThreadPrivateVarInit(1615 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,1616 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {1617 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime1618 // library.1619 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);1620 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(1621 CGM.getModule(), OMPRTL___kmpc_global_thread_num),1622 OMPLoc);1623 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)1624 // to register constructor/destructor for variable.1625 llvm::Value *Args[] = {1626 OMPLoc,1627 CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.VoidPtrTy),1628 Ctor, CopyCtor, Dtor};1629 CGF.EmitRuntimeCall(1630 OMPBuilder.getOrCreateRuntimeFunction(1631 CGM.getModule(), OMPRTL___kmpc_threadprivate_register),1632 Args);1633}1634 1635llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(1636 const VarDecl *VD, Address VDAddr, SourceLocation Loc,1637 bool PerformInit, CodeGenFunction *CGF) {1638 if (CGM.getLangOpts().OpenMPUseTLS &&1639 CGM.getContext().getTargetInfo().isTLSSupported())1640 return nullptr;1641 1642 VD = VD->getDefinition(CGM.getContext());1643 if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) {1644 QualType ASTTy = VD->getType();1645 1646 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;1647 const Expr *Init = VD->getAnyInitializer();1648 if (CGM.getLangOpts().CPlusPlus && PerformInit) {1649 // Generate function that re-emits the declaration's initializer into the1650 // threadprivate copy of the variable VD1651 CodeGenFunction CtorCGF(CGM);1652 FunctionArgList Args;1653 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,1654 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,1655 ImplicitParamKind::Other);1656 Args.push_back(&Dst);1657 1658 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(1659 CGM.getContext().VoidPtrTy, Args);1660 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);1661 std::string Name = getName({"__kmpc_global_ctor_", ""});1662 llvm::Function *Fn =1663 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);1664 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,1665 Args, Loc, Loc);1666 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(1667 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,1668 CGM.getContext().VoidPtrTy, Dst.getLocation());1669 Address Arg(ArgVal, CtorCGF.ConvertTypeForMem(ASTTy),1670 VDAddr.getAlignment());1671 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),1672 /*IsInitializer=*/true);1673 ArgVal = CtorCGF.EmitLoadOfScalar(1674 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,1675 CGM.getContext().VoidPtrTy, Dst.getLocation());1676 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);1677 CtorCGF.FinishFunction();1678 Ctor = Fn;1679 }1680 if (VD->getType().isDestructedType() != QualType::DK_none) {1681 // Generate function that emits destructor call for the threadprivate copy1682 // of the variable VD1683 CodeGenFunction DtorCGF(CGM);1684 FunctionArgList Args;1685 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,1686 /*Id=*/nullptr, CGM.getContext().VoidPtrTy,1687 ImplicitParamKind::Other);1688 Args.push_back(&Dst);1689 1690 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(1691 CGM.getContext().VoidTy, Args);1692 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);1693 std::string Name = getName({"__kmpc_global_dtor_", ""});1694 llvm::Function *Fn =1695 CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc);1696 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);1697 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,1698 Loc, Loc);1699 // Create a scope with an artificial location for the body of this function.1700 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);1701 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(1702 DtorCGF.GetAddrOfLocalVar(&Dst),1703 /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());1704 DtorCGF.emitDestroy(1705 Address(ArgVal, DtorCGF.Int8Ty, VDAddr.getAlignment()), ASTTy,1706 DtorCGF.getDestroyer(ASTTy.isDestructedType()),1707 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));1708 DtorCGF.FinishFunction();1709 Dtor = Fn;1710 }1711 // Do not emit init function if it is not required.1712 if (!Ctor && !Dtor)1713 return nullptr;1714 1715 // Copying constructor for the threadprivate variable.1716 // Must be NULL - reserved by runtime, but currently it requires that this1717 // parameter is always NULL. Otherwise it fires assertion.1718 CopyCtor = llvm::Constant::getNullValue(CGM.DefaultPtrTy);1719 if (Ctor == nullptr) {1720 Ctor = llvm::Constant::getNullValue(CGM.DefaultPtrTy);1721 }1722 if (Dtor == nullptr) {1723 Dtor = llvm::Constant::getNullValue(CGM.DefaultPtrTy);1724 }1725 if (!CGF) {1726 auto *InitFunctionTy =1727 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);1728 std::string Name = getName({"__omp_threadprivate_init_", ""});1729 llvm::Function *InitFunction = CGM.CreateGlobalInitOrCleanUpFunction(1730 InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());1731 CodeGenFunction InitCGF(CGM);1732 FunctionArgList ArgList;1733 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,1734 CGM.getTypes().arrangeNullaryFunction(), ArgList,1735 Loc, Loc);1736 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);1737 InitCGF.FinishFunction();1738 return InitFunction;1739 }1740 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);1741 }1742 return nullptr;1743}1744 1745void CGOpenMPRuntime::emitDeclareTargetFunction(const FunctionDecl *FD,1746 llvm::GlobalValue *GV) {1747 std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =1748 OMPDeclareTargetDeclAttr::getActiveAttr(FD);1749 1750 // We only need to handle active 'indirect' declare target functions.1751 if (!ActiveAttr || !(*ActiveAttr)->getIndirect())1752 return;1753 1754 // Get a mangled name to store the new device global in.1755 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(1756 CGM, OMPBuilder, FD->getCanonicalDecl()->getBeginLoc(), FD->getName());1757 SmallString<128> Name;1758 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);1759 1760 // We need to generate a new global to hold the address of the indirectly1761 // called device function. Doing this allows us to keep the visibility and1762 // linkage of the associated function unchanged while allowing the runtime to1763 // access its value.1764 llvm::GlobalValue *Addr = GV;1765 if (CGM.getLangOpts().OpenMPIsTargetDevice) {1766 llvm::PointerType *FnPtrTy = llvm::PointerType::get(1767 CGM.getLLVMContext(),1768 CGM.getModule().getDataLayout().getProgramAddressSpace());1769 Addr = new llvm::GlobalVariable(1770 CGM.getModule(), FnPtrTy,1771 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, GV, Name,1772 nullptr, llvm::GlobalValue::NotThreadLocal,1773 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());1774 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);1775 }1776 1777 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(1778 Name, Addr, CGM.GetTargetTypeStoreSize(CGM.VoidPtrTy).getQuantity(),1779 llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,1780 llvm::GlobalValue::WeakODRLinkage);1781}1782 1783Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,1784 QualType VarType,1785 StringRef Name) {1786 std::string Suffix = getName({"artificial", ""});1787 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);1788 llvm::GlobalVariable *GAddr = OMPBuilder.getOrCreateInternalVariable(1789 VarLVType, Twine(Name).concat(Suffix).str());1790 if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS &&1791 CGM.getTarget().isTLSSupported()) {1792 GAddr->setThreadLocal(/*Val=*/true);1793 return Address(GAddr, GAddr->getValueType(),1794 CGM.getContext().getTypeAlignInChars(VarType));1795 }1796 std::string CacheSuffix = getName({"cache", ""});1797 llvm::Value *Args[] = {1798 emitUpdateLocation(CGF, SourceLocation()),1799 getThreadID(CGF, SourceLocation()),1800 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),1801 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,1802 /*isSigned=*/false),1803 OMPBuilder.getOrCreateInternalVariable(1804 CGM.VoidPtrPtrTy,1805 Twine(Name).concat(Suffix).concat(CacheSuffix).str())};1806 return Address(1807 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(1808 CGF.EmitRuntimeCall(1809 OMPBuilder.getOrCreateRuntimeFunction(1810 CGM.getModule(), OMPRTL___kmpc_threadprivate_cached),1811 Args),1812 CGF.Builder.getPtrTy(0)),1813 VarLVType, CGM.getContext().getTypeAlignInChars(VarType));1814}1815 1816void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,1817 const RegionCodeGenTy &ThenGen,1818 const RegionCodeGenTy &ElseGen) {1819 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());1820 1821 // If the condition constant folds and can be elided, try to avoid emitting1822 // the condition and the dead arm of the if/else.1823 bool CondConstant;1824 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {1825 if (CondConstant)1826 ThenGen(CGF);1827 else1828 ElseGen(CGF);1829 return;1830 }1831 1832 // Otherwise, the condition did not fold, or we couldn't elide it. Just1833 // emit the conditional branch.1834 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");1835 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");1836 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");1837 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);1838 1839 // Emit the 'then' code.1840 CGF.EmitBlock(ThenBlock);1841 ThenGen(CGF);1842 CGF.EmitBranch(ContBlock);1843 // Emit the 'else' code if present.1844 // There is no need to emit line number for unconditional branch.1845 (void)ApplyDebugLocation::CreateEmpty(CGF);1846 CGF.EmitBlock(ElseBlock);1847 ElseGen(CGF);1848 // There is no need to emit line number for unconditional branch.1849 (void)ApplyDebugLocation::CreateEmpty(CGF);1850 CGF.EmitBranch(ContBlock);1851 // Emit the continuation block for code after the if.1852 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);1853}1854 1855void CGOpenMPRuntime::emitParallelCall(1856 CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn,1857 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,1858 llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier,1859 OpenMPSeverityClauseKind Severity, const Expr *Message) {1860 if (!CGF.HaveInsertPoint())1861 return;1862 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);1863 auto &M = CGM.getModule();1864 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,1865 this](CodeGenFunction &CGF, PrePostActionTy &) {1866 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);1867 llvm::Value *Args[] = {1868 RTLoc,1869 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars1870 OutlinedFn};1871 llvm::SmallVector<llvm::Value *, 16> RealArgs;1872 RealArgs.append(std::begin(Args), std::end(Args));1873 RealArgs.append(CapturedVars.begin(), CapturedVars.end());1874 1875 llvm::FunctionCallee RTLFn =1876 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);1877 CGF.EmitRuntimeCall(RTLFn, RealArgs);1878 };1879 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,1880 this](CodeGenFunction &CGF, PrePostActionTy &) {1881 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();1882 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);1883 // Build calls:1884 // __kmpc_serialized_parallel(&Loc, GTid);1885 llvm::Value *Args[] = {RTLoc, ThreadID};1886 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(1887 M, OMPRTL___kmpc_serialized_parallel),1888 Args);1889 1890 // OutlinedFn(>id, &zero_bound, CapturedStruct);1891 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);1892 RawAddress ZeroAddrBound =1893 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,1894 /*Name=*/".bound.zero.addr");1895 CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddrBound);1896 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;1897 // ThreadId for serialized parallels is 0.1898 OutlinedFnArgs.push_back(ThreadIDAddr.emitRawPointer(CGF));1899 OutlinedFnArgs.push_back(ZeroAddrBound.getPointer());1900 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());1901 1902 // Ensure we do not inline the function. This is trivially true for the ones1903 // passed to __kmpc_fork_call but the ones called in serialized regions1904 // could be inlined. This is not a perfect but it is closer to the invariant1905 // we want, namely, every data environment starts with a new function.1906 // TODO: We should pass the if condition to the runtime function and do the1907 // handling there. Much cleaner code.1908 OutlinedFn->removeFnAttr(llvm::Attribute::AlwaysInline);1909 OutlinedFn->addFnAttr(llvm::Attribute::NoInline);1910 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);1911 1912 // __kmpc_end_serialized_parallel(&Loc, GTid);1913 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};1914 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(1915 M, OMPRTL___kmpc_end_serialized_parallel),1916 EndArgs);1917 };1918 if (IfCond) {1919 emitIfClause(CGF, IfCond, ThenGen, ElseGen);1920 } else {1921 RegionCodeGenTy ThenRCG(ThenGen);1922 ThenRCG(CGF);1923 }1924}1925 1926// If we're inside an (outlined) parallel region, use the region info's1927// thread-ID variable (it is passed in a first argument of the outlined function1928// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in1929// regular serial code region, get thread ID by calling kmp_int321930// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and1931// return the address of that temp.1932Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,1933 SourceLocation Loc) {1934 if (auto *OMPRegionInfo =1935 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))1936 if (OMPRegionInfo->getThreadIDVariable())1937 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();1938 1939 llvm::Value *ThreadID = getThreadID(CGF, Loc);1940 QualType Int32Ty =1941 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);1942 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");1943 CGF.EmitStoreOfScalar(ThreadID,1944 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));1945 1946 return ThreadIDTemp;1947}1948 1949llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {1950 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();1951 std::string Name = getName({Prefix, "var"});1952 return OMPBuilder.getOrCreateInternalVariable(KmpCriticalNameTy, Name);1953}1954 1955namespace {1956/// Common pre(post)-action for different OpenMP constructs.1957class CommonActionTy final : public PrePostActionTy {1958 llvm::FunctionCallee EnterCallee;1959 ArrayRef<llvm::Value *> EnterArgs;1960 llvm::FunctionCallee ExitCallee;1961 ArrayRef<llvm::Value *> ExitArgs;1962 bool Conditional;1963 llvm::BasicBlock *ContBlock = nullptr;1964 1965public:1966 CommonActionTy(llvm::FunctionCallee EnterCallee,1967 ArrayRef<llvm::Value *> EnterArgs,1968 llvm::FunctionCallee ExitCallee,1969 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)1970 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),1971 ExitArgs(ExitArgs), Conditional(Conditional) {}1972 void Enter(CodeGenFunction &CGF) override {1973 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);1974 if (Conditional) {1975 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);1976 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");1977 ContBlock = CGF.createBasicBlock("omp_if.end");1978 // Generate the branch (If-stmt)1979 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);1980 CGF.EmitBlock(ThenBlock);1981 }1982 }1983 void Done(CodeGenFunction &CGF) {1984 // Emit the rest of blocks/branches1985 CGF.EmitBranch(ContBlock);1986 CGF.EmitBlock(ContBlock, true);1987 }1988 void Exit(CodeGenFunction &CGF) override {1989 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);1990 }1991};1992} // anonymous namespace1993 1994void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,1995 StringRef CriticalName,1996 const RegionCodeGenTy &CriticalOpGen,1997 SourceLocation Loc, const Expr *Hint) {1998 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);1999 // CriticalOpGen();2000 // __kmpc_end_critical(ident_t *, gtid, Lock);2001 // Prepare arguments and build a call to __kmpc_critical2002 if (!CGF.HaveInsertPoint())2003 return;2004 llvm::FunctionCallee RuntimeFcn = OMPBuilder.getOrCreateRuntimeFunction(2005 CGM.getModule(),2006 Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical);2007 llvm::Value *LockVar = getCriticalRegionLock(CriticalName);2008 unsigned LockVarArgIdx = 2;2009 if (cast<llvm::GlobalVariable>(LockVar)->getAddressSpace() !=2010 RuntimeFcn.getFunctionType()2011 ->getParamType(LockVarArgIdx)2012 ->getPointerAddressSpace())2013 LockVar = CGF.Builder.CreateAddrSpaceCast(2014 LockVar, RuntimeFcn.getFunctionType()->getParamType(LockVarArgIdx));2015 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),2016 LockVar};2017 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),2018 std::end(Args));2019 if (Hint) {2020 EnterArgs.push_back(CGF.Builder.CreateIntCast(2021 CGF.EmitScalarExpr(Hint), CGM.Int32Ty, /*isSigned=*/false));2022 }2023 CommonActionTy Action(RuntimeFcn, EnterArgs,2024 OMPBuilder.getOrCreateRuntimeFunction(2025 CGM.getModule(), OMPRTL___kmpc_end_critical),2026 Args);2027 CriticalOpGen.setAction(Action);2028 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);2029}2030 2031void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,2032 const RegionCodeGenTy &MasterOpGen,2033 SourceLocation Loc) {2034 if (!CGF.HaveInsertPoint())2035 return;2036 // if(__kmpc_master(ident_t *, gtid)) {2037 // MasterOpGen();2038 // __kmpc_end_master(ident_t *, gtid);2039 // }2040 // Prepare arguments and build a call to __kmpc_master2041 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};2042 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(2043 CGM.getModule(), OMPRTL___kmpc_master),2044 Args,2045 OMPBuilder.getOrCreateRuntimeFunction(2046 CGM.getModule(), OMPRTL___kmpc_end_master),2047 Args,2048 /*Conditional=*/true);2049 MasterOpGen.setAction(Action);2050 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);2051 Action.Done(CGF);2052}2053 2054void CGOpenMPRuntime::emitMaskedRegion(CodeGenFunction &CGF,2055 const RegionCodeGenTy &MaskedOpGen,2056 SourceLocation Loc, const Expr *Filter) {2057 if (!CGF.HaveInsertPoint())2058 return;2059 // if(__kmpc_masked(ident_t *, gtid, filter)) {2060 // MaskedOpGen();2061 // __kmpc_end_masked(iden_t *, gtid);2062 // }2063 // Prepare arguments and build a call to __kmpc_masked2064 llvm::Value *FilterVal = Filter2065 ? CGF.EmitScalarExpr(Filter, CGF.Int32Ty)2066 : llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/0);2067 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),2068 FilterVal};2069 llvm::Value *ArgsEnd[] = {emitUpdateLocation(CGF, Loc),2070 getThreadID(CGF, Loc)};2071 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(2072 CGM.getModule(), OMPRTL___kmpc_masked),2073 Args,2074 OMPBuilder.getOrCreateRuntimeFunction(2075 CGM.getModule(), OMPRTL___kmpc_end_masked),2076 ArgsEnd,2077 /*Conditional=*/true);2078 MaskedOpGen.setAction(Action);2079 emitInlinedDirective(CGF, OMPD_masked, MaskedOpGen);2080 Action.Done(CGF);2081}2082 2083void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,2084 SourceLocation Loc) {2085 if (!CGF.HaveInsertPoint())2086 return;2087 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {2088 OMPBuilder.createTaskyield(CGF.Builder);2089 } else {2090 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);2091 llvm::Value *Args[] = {2092 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),2093 llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};2094 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(2095 CGM.getModule(), OMPRTL___kmpc_omp_taskyield),2096 Args);2097 }2098 2099 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))2100 Region->emitUntiedSwitch(CGF);2101}2102 2103void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,2104 const RegionCodeGenTy &TaskgroupOpGen,2105 SourceLocation Loc) {2106 if (!CGF.HaveInsertPoint())2107 return;2108 // __kmpc_taskgroup(ident_t *, gtid);2109 // TaskgroupOpGen();2110 // __kmpc_end_taskgroup(ident_t *, gtid);2111 // Prepare arguments and build a call to __kmpc_taskgroup2112 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};2113 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(2114 CGM.getModule(), OMPRTL___kmpc_taskgroup),2115 Args,2116 OMPBuilder.getOrCreateRuntimeFunction(2117 CGM.getModule(), OMPRTL___kmpc_end_taskgroup),2118 Args);2119 TaskgroupOpGen.setAction(Action);2120 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);2121}2122 2123/// Given an array of pointers to variables, project the address of a2124/// given variable.2125static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,2126 unsigned Index, const VarDecl *Var) {2127 // Pull out the pointer to the variable.2128 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);2129 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);2130 2131 llvm::Type *ElemTy = CGF.ConvertTypeForMem(Var->getType());2132 return Address(Ptr, ElemTy, CGF.getContext().getDeclAlign(Var));2133}2134 2135static llvm::Value *emitCopyprivateCopyFunction(2136 CodeGenModule &CGM, llvm::Type *ArgsElemType,2137 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,2138 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,2139 SourceLocation Loc) {2140 ASTContext &C = CGM.getContext();2141 // void copy_func(void *LHSArg, void *RHSArg);2142 FunctionArgList Args;2143 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,2144 ImplicitParamKind::Other);2145 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,2146 ImplicitParamKind::Other);2147 Args.push_back(&LHSArg);2148 Args.push_back(&RHSArg);2149 const auto &CGFI =2150 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);2151 std::string Name =2152 CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});2153 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),2154 llvm::GlobalValue::InternalLinkage, Name,2155 &CGM.getModule());2156 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);2157 Fn->setDoesNotRecurse();2158 CodeGenFunction CGF(CGM);2159 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);2160 // Dest = (void*[n])(LHSArg);2161 // Src = (void*[n])(RHSArg);2162 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(2163 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),2164 CGF.Builder.getPtrTy(0)),2165 ArgsElemType, CGF.getPointerAlign());2166 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(2167 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),2168 CGF.Builder.getPtrTy(0)),2169 ArgsElemType, CGF.getPointerAlign());2170 // *(Type0*)Dst[0] = *(Type0*)Src[0];2171 // *(Type1*)Dst[1] = *(Type1*)Src[1];2172 // ...2173 // *(Typen*)Dst[n] = *(Typen*)Src[n];2174 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {2175 const auto *DestVar =2176 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());2177 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);2178 2179 const auto *SrcVar =2180 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());2181 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);2182 2183 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();2184 QualType Type = VD->getType();2185 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);2186 }2187 CGF.FinishFunction();2188 return Fn;2189}2190 2191void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,2192 const RegionCodeGenTy &SingleOpGen,2193 SourceLocation Loc,2194 ArrayRef<const Expr *> CopyprivateVars,2195 ArrayRef<const Expr *> SrcExprs,2196 ArrayRef<const Expr *> DstExprs,2197 ArrayRef<const Expr *> AssignmentOps) {2198 if (!CGF.HaveInsertPoint())2199 return;2200 assert(CopyprivateVars.size() == SrcExprs.size() &&2201 CopyprivateVars.size() == DstExprs.size() &&2202 CopyprivateVars.size() == AssignmentOps.size());2203 ASTContext &C = CGM.getContext();2204 // int32 did_it = 0;2205 // if(__kmpc_single(ident_t *, gtid)) {2206 // SingleOpGen();2207 // __kmpc_end_single(ident_t *, gtid);2208 // did_it = 1;2209 // }2210 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,2211 // <copy_func>, did_it);2212 2213 Address DidIt = Address::invalid();2214 if (!CopyprivateVars.empty()) {2215 // int32 did_it = 0;2216 QualType KmpInt32Ty =2217 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);2218 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");2219 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);2220 }2221 // Prepare arguments and build a call to __kmpc_single2222 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};2223 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(2224 CGM.getModule(), OMPRTL___kmpc_single),2225 Args,2226 OMPBuilder.getOrCreateRuntimeFunction(2227 CGM.getModule(), OMPRTL___kmpc_end_single),2228 Args,2229 /*Conditional=*/true);2230 SingleOpGen.setAction(Action);2231 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);2232 if (DidIt.isValid()) {2233 // did_it = 1;2234 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);2235 }2236 Action.Done(CGF);2237 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,2238 // <copy_func>, did_it);2239 if (DidIt.isValid()) {2240 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());2241 QualType CopyprivateArrayTy = C.getConstantArrayType(2242 C.VoidPtrTy, ArraySize, nullptr, ArraySizeModifier::Normal,2243 /*IndexTypeQuals=*/0);2244 // Create a list of all private variables for copyprivate.2245 Address CopyprivateList =2246 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");2247 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {2248 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);2249 CGF.Builder.CreateStore(2250 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(2251 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF),2252 CGF.VoidPtrTy),2253 Elem);2254 }2255 // Build function that copies private values from single region to all other2256 // threads in the corresponding parallel region.2257 llvm::Value *CpyFn = emitCopyprivateCopyFunction(2258 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy), CopyprivateVars,2259 SrcExprs, DstExprs, AssignmentOps, Loc);2260 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);2261 Address CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(2262 CopyprivateList, CGF.VoidPtrTy, CGF.Int8Ty);2263 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);2264 llvm::Value *Args[] = {2265 emitUpdateLocation(CGF, Loc), // ident_t *<loc>2266 getThreadID(CGF, Loc), // i32 <gtid>2267 BufSize, // size_t <buf_size>2268 CL.emitRawPointer(CGF), // void *<copyprivate list>2269 CpyFn, // void (*) (void *, void *) <copy_func>2270 DidItVal // i32 did_it2271 };2272 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(2273 CGM.getModule(), OMPRTL___kmpc_copyprivate),2274 Args);2275 }2276}2277 2278void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,2279 const RegionCodeGenTy &OrderedOpGen,2280 SourceLocation Loc, bool IsThreads) {2281 if (!CGF.HaveInsertPoint())2282 return;2283 // __kmpc_ordered(ident_t *, gtid);2284 // OrderedOpGen();2285 // __kmpc_end_ordered(ident_t *, gtid);2286 // Prepare arguments and build a call to __kmpc_ordered2287 if (IsThreads) {2288 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};2289 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(2290 CGM.getModule(), OMPRTL___kmpc_ordered),2291 Args,2292 OMPBuilder.getOrCreateRuntimeFunction(2293 CGM.getModule(), OMPRTL___kmpc_end_ordered),2294 Args);2295 OrderedOpGen.setAction(Action);2296 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);2297 return;2298 }2299 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);2300}2301 2302unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {2303 unsigned Flags;2304 if (Kind == OMPD_for)2305 Flags = OMP_IDENT_BARRIER_IMPL_FOR;2306 else if (Kind == OMPD_sections)2307 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;2308 else if (Kind == OMPD_single)2309 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;2310 else if (Kind == OMPD_barrier)2311 Flags = OMP_IDENT_BARRIER_EXPL;2312 else2313 Flags = OMP_IDENT_BARRIER_IMPL;2314 return Flags;2315}2316 2317void CGOpenMPRuntime::getDefaultScheduleAndChunk(2318 CodeGenFunction &CGF, const OMPLoopDirective &S,2319 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {2320 // Check if the loop directive is actually a doacross loop directive. In this2321 // case choose static, 1 schedule.2322 if (llvm::any_of(2323 S.getClausesOfKind<OMPOrderedClause>(),2324 [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {2325 ScheduleKind = OMPC_SCHEDULE_static;2326 // Chunk size is 1 in this case.2327 llvm::APInt ChunkSize(32, 1);2328 ChunkExpr = IntegerLiteral::Create(2329 CGF.getContext(), ChunkSize,2330 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),2331 SourceLocation());2332 }2333}2334 2335void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,2336 OpenMPDirectiveKind Kind, bool EmitChecks,2337 bool ForceSimpleCall) {2338 // Check if we should use the OMPBuilder2339 auto *OMPRegionInfo =2340 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo);2341 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {2342 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =2343 cantFail(OMPBuilder.createBarrier(CGF.Builder, Kind, ForceSimpleCall,2344 EmitChecks));2345 CGF.Builder.restoreIP(AfterIP);2346 return;2347 }2348 2349 if (!CGF.HaveInsertPoint())2350 return;2351 // Build call __kmpc_cancel_barrier(loc, thread_id);2352 // Build call __kmpc_barrier(loc, thread_id);2353 unsigned Flags = getDefaultFlagsForBarriers(Kind);2354 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,2355 // thread_id);2356 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),2357 getThreadID(CGF, Loc)};2358 if (OMPRegionInfo) {2359 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {2360 llvm::Value *Result = CGF.EmitRuntimeCall(2361 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),2362 OMPRTL___kmpc_cancel_barrier),2363 Args);2364 if (EmitChecks) {2365 // if (__kmpc_cancel_barrier()) {2366 // exit from construct;2367 // }2368 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");2369 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");2370 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);2371 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);2372 CGF.EmitBlock(ExitBB);2373 // exit from construct;2374 CodeGenFunction::JumpDest CancelDestination =2375 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());2376 CGF.EmitBranchThroughCleanup(CancelDestination);2377 CGF.EmitBlock(ContBB, /*IsFinished=*/true);2378 }2379 return;2380 }2381 }2382 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(2383 CGM.getModule(), OMPRTL___kmpc_barrier),2384 Args);2385}2386 2387void CGOpenMPRuntime::emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc,2388 Expr *ME, bool IsFatal) {2389 llvm::Value *MVL = ME ? CGF.EmitScalarExpr(ME)2390 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy);2391 // Build call void __kmpc_error(ident_t *loc, int severity, const char2392 // *message)2393 llvm::Value *Args[] = {2394 emitUpdateLocation(CGF, Loc, /*Flags=*/0, /*GenLoc=*/true),2395 llvm::ConstantInt::get(CGM.Int32Ty, IsFatal ? 2 : 1),2396 CGF.Builder.CreatePointerCast(MVL, CGM.Int8PtrTy)};2397 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(2398 CGM.getModule(), OMPRTL___kmpc_error),2399 Args);2400}2401 2402/// Map the OpenMP loop schedule to the runtime enumeration.2403static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,2404 bool Chunked, bool Ordered) {2405 switch (ScheduleKind) {2406 case OMPC_SCHEDULE_static:2407 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)2408 : (Ordered ? OMP_ord_static : OMP_sch_static);2409 case OMPC_SCHEDULE_dynamic:2410 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;2411 case OMPC_SCHEDULE_guided:2412 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;2413 case OMPC_SCHEDULE_runtime:2414 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;2415 case OMPC_SCHEDULE_auto:2416 return Ordered ? OMP_ord_auto : OMP_sch_auto;2417 case OMPC_SCHEDULE_unknown:2418 assert(!Chunked && "chunk was specified but schedule kind not known");2419 return Ordered ? OMP_ord_static : OMP_sch_static;2420 }2421 llvm_unreachable("Unexpected runtime schedule");2422}2423 2424/// Map the OpenMP distribute schedule to the runtime enumeration.2425static OpenMPSchedType2426getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {2427 // only static is allowed for dist_schedule2428 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;2429}2430 2431bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,2432 bool Chunked) const {2433 OpenMPSchedType Schedule =2434 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);2435 return Schedule == OMP_sch_static;2436}2437 2438bool CGOpenMPRuntime::isStaticNonchunked(2439 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {2440 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);2441 return Schedule == OMP_dist_sch_static;2442}2443 2444bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,2445 bool Chunked) const {2446 OpenMPSchedType Schedule =2447 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);2448 return Schedule == OMP_sch_static_chunked;2449}2450 2451bool CGOpenMPRuntime::isStaticChunked(2452 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {2453 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);2454 return Schedule == OMP_dist_sch_static_chunked;2455}2456 2457bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {2458 OpenMPSchedType Schedule =2459 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);2460 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here");2461 return Schedule != OMP_sch_static;2462}2463 2464static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,2465 OpenMPScheduleClauseModifier M1,2466 OpenMPScheduleClauseModifier M2) {2467 int Modifier = 0;2468 switch (M1) {2469 case OMPC_SCHEDULE_MODIFIER_monotonic:2470 Modifier = OMP_sch_modifier_monotonic;2471 break;2472 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:2473 Modifier = OMP_sch_modifier_nonmonotonic;2474 break;2475 case OMPC_SCHEDULE_MODIFIER_simd:2476 if (Schedule == OMP_sch_static_chunked)2477 Schedule = OMP_sch_static_balanced_chunked;2478 break;2479 case OMPC_SCHEDULE_MODIFIER_last:2480 case OMPC_SCHEDULE_MODIFIER_unknown:2481 break;2482 }2483 switch (M2) {2484 case OMPC_SCHEDULE_MODIFIER_monotonic:2485 Modifier = OMP_sch_modifier_monotonic;2486 break;2487 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:2488 Modifier = OMP_sch_modifier_nonmonotonic;2489 break;2490 case OMPC_SCHEDULE_MODIFIER_simd:2491 if (Schedule == OMP_sch_static_chunked)2492 Schedule = OMP_sch_static_balanced_chunked;2493 break;2494 case OMPC_SCHEDULE_MODIFIER_last:2495 case OMPC_SCHEDULE_MODIFIER_unknown:2496 break;2497 }2498 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.2499 // If the static schedule kind is specified or if the ordered clause is2500 // specified, and if the nonmonotonic modifier is not specified, the effect is2501 // as if the monotonic modifier is specified. Otherwise, unless the monotonic2502 // modifier is specified, the effect is as if the nonmonotonic modifier is2503 // specified.2504 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {2505 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||2506 Schedule == OMP_sch_static_balanced_chunked ||2507 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||2508 Schedule == OMP_dist_sch_static_chunked ||2509 Schedule == OMP_dist_sch_static))2510 Modifier = OMP_sch_modifier_nonmonotonic;2511 }2512 return Schedule | Modifier;2513}2514 2515void CGOpenMPRuntime::emitForDispatchInit(2516 CodeGenFunction &CGF, SourceLocation Loc,2517 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,2518 bool Ordered, const DispatchRTInput &DispatchValues) {2519 if (!CGF.HaveInsertPoint())2520 return;2521 OpenMPSchedType Schedule = getRuntimeSchedule(2522 ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);2523 assert(Ordered ||2524 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&2525 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&2526 Schedule != OMP_sch_static_balanced_chunked));2527 // Call __kmpc_dispatch_init(2528 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,2529 // kmp_int[32|64] lower, kmp_int[32|64] upper,2530 // kmp_int[32|64] stride, kmp_int[32|64] chunk);2531 2532 // If the Chunk was not specified in the clause - use default value 1.2533 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk2534 : CGF.Builder.getIntN(IVSize, 1);2535 llvm::Value *Args[] = {2536 emitUpdateLocation(CGF, Loc),2537 getThreadID(CGF, Loc),2538 CGF.Builder.getInt32(addMonoNonMonoModifier(2539 CGM, Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type2540 DispatchValues.LB, // Lower2541 DispatchValues.UB, // Upper2542 CGF.Builder.getIntN(IVSize, 1), // Stride2543 Chunk // Chunk2544 };2545 CGF.EmitRuntimeCall(OMPBuilder.createDispatchInitFunction(IVSize, IVSigned),2546 Args);2547}2548 2549void CGOpenMPRuntime::emitForDispatchDeinit(CodeGenFunction &CGF,2550 SourceLocation Loc) {2551 if (!CGF.HaveInsertPoint())2552 return;2553 // Call __kmpc_dispatch_deinit(ident_t *loc, kmp_int32 tid);2554 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};2555 CGF.EmitRuntimeCall(OMPBuilder.createDispatchDeinitFunction(), Args);2556}2557 2558static void emitForStaticInitCall(2559 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,2560 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,2561 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,2562 const CGOpenMPRuntime::StaticRTInput &Values) {2563 if (!CGF.HaveInsertPoint())2564 return;2565 2566 assert(!Values.Ordered);2567 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||2568 Schedule == OMP_sch_static_balanced_chunked ||2569 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||2570 Schedule == OMP_dist_sch_static ||2571 Schedule == OMP_dist_sch_static_chunked);2572 2573 // Call __kmpc_for_static_init(2574 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,2575 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,2576 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,2577 // kmp_int[32|64] incr, kmp_int[32|64] chunk);2578 llvm::Value *Chunk = Values.Chunk;2579 if (Chunk == nullptr) {2580 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||2581 Schedule == OMP_dist_sch_static) &&2582 "expected static non-chunked schedule");2583 // If the Chunk was not specified in the clause - use default value 1.2584 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);2585 } else {2586 assert((Schedule == OMP_sch_static_chunked ||2587 Schedule == OMP_sch_static_balanced_chunked ||2588 Schedule == OMP_ord_static_chunked ||2589 Schedule == OMP_dist_sch_static_chunked) &&2590 "expected static chunked schedule");2591 }2592 llvm::Value *Args[] = {2593 UpdateLocation,2594 ThreadId,2595 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,2596 M2)), // Schedule type2597 Values.IL.emitRawPointer(CGF), // &isLastIter2598 Values.LB.emitRawPointer(CGF), // &LB2599 Values.UB.emitRawPointer(CGF), // &UB2600 Values.ST.emitRawPointer(CGF), // &Stride2601 CGF.Builder.getIntN(Values.IVSize, 1), // Incr2602 Chunk // Chunk2603 };2604 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);2605}2606 2607void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,2608 SourceLocation Loc,2609 OpenMPDirectiveKind DKind,2610 const OpenMPScheduleTy &ScheduleKind,2611 const StaticRTInput &Values) {2612 OpenMPSchedType ScheduleNum = getRuntimeSchedule(2613 ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);2614 assert((isOpenMPWorksharingDirective(DKind) || (DKind == OMPD_loop)) &&2615 "Expected loop-based or sections-based directive.");2616 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,2617 isOpenMPLoopDirective(DKind)2618 ? OMP_IDENT_WORK_LOOP2619 : OMP_IDENT_WORK_SECTIONS);2620 llvm::Value *ThreadId = getThreadID(CGF, Loc);2621 llvm::FunctionCallee StaticInitFunction =2622 OMPBuilder.createForStaticInitFunction(Values.IVSize, Values.IVSigned,2623 false);2624 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);2625 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,2626 ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);2627}2628 2629void CGOpenMPRuntime::emitDistributeStaticInit(2630 CodeGenFunction &CGF, SourceLocation Loc,2631 OpenMPDistScheduleClauseKind SchedKind,2632 const CGOpenMPRuntime::StaticRTInput &Values) {2633 OpenMPSchedType ScheduleNum =2634 getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);2635 llvm::Value *UpdatedLocation =2636 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);2637 llvm::Value *ThreadId = getThreadID(CGF, Loc);2638 llvm::FunctionCallee StaticInitFunction;2639 bool isGPUDistribute =2640 CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU();2641 StaticInitFunction = OMPBuilder.createForStaticInitFunction(2642 Values.IVSize, Values.IVSigned, isGPUDistribute);2643 2644 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,2645 ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,2646 OMPC_SCHEDULE_MODIFIER_unknown, Values);2647}2648 2649void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,2650 SourceLocation Loc,2651 OpenMPDirectiveKind DKind) {2652 assert((DKind == OMPD_distribute || DKind == OMPD_for ||2653 DKind == OMPD_sections) &&2654 "Expected distribute, for, or sections directive kind");2655 if (!CGF.HaveInsertPoint())2656 return;2657 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);2658 llvm::Value *Args[] = {2659 emitUpdateLocation(CGF, Loc,2660 isOpenMPDistributeDirective(DKind) ||2661 (DKind == OMPD_target_teams_loop)2662 ? OMP_IDENT_WORK_DISTRIBUTE2663 : isOpenMPLoopDirective(DKind)2664 ? OMP_IDENT_WORK_LOOP2665 : OMP_IDENT_WORK_SECTIONS),2666 getThreadID(CGF, Loc)};2667 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);2668 if (isOpenMPDistributeDirective(DKind) &&2669 CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU())2670 CGF.EmitRuntimeCall(2671 OMPBuilder.getOrCreateRuntimeFunction(2672 CGM.getModule(), OMPRTL___kmpc_distribute_static_fini),2673 Args);2674 else2675 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(2676 CGM.getModule(), OMPRTL___kmpc_for_static_fini),2677 Args);2678}2679 2680void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,2681 SourceLocation Loc,2682 unsigned IVSize,2683 bool IVSigned) {2684 if (!CGF.HaveInsertPoint())2685 return;2686 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);2687 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};2688 CGF.EmitRuntimeCall(OMPBuilder.createDispatchFiniFunction(IVSize, IVSigned),2689 Args);2690}2691 2692llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,2693 SourceLocation Loc, unsigned IVSize,2694 bool IVSigned, Address IL,2695 Address LB, Address UB,2696 Address ST) {2697 // Call __kmpc_dispatch_next(2698 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,2699 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,2700 // kmp_int[32|64] *p_stride);2701 llvm::Value *Args[] = {2702 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),2703 IL.emitRawPointer(CGF), // &isLastIter2704 LB.emitRawPointer(CGF), // &Lower2705 UB.emitRawPointer(CGF), // &Upper2706 ST.emitRawPointer(CGF) // &Stride2707 };2708 llvm::Value *Call = CGF.EmitRuntimeCall(2709 OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args);2710 return CGF.EmitScalarConversion(2711 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),2712 CGF.getContext().BoolTy, Loc);2713}2714 2715llvm::Value *CGOpenMPRuntime::emitMessageClause(CodeGenFunction &CGF,2716 const Expr *Message,2717 SourceLocation Loc) {2718 if (!Message)2719 return llvm::ConstantPointerNull::get(CGF.VoidPtrTy);2720 return CGF.EmitScalarExpr(Message);2721}2722 2723llvm::Value *2724CGOpenMPRuntime::emitSeverityClause(OpenMPSeverityClauseKind Severity,2725 SourceLocation Loc) {2726 // OpenMP 6.0, 10.4: "If no severity clause is specified then the effect is2727 // as if sev-level is fatal."2728 return llvm::ConstantInt::get(CGM.Int32Ty,2729 Severity == OMPC_SEVERITY_warning ? 1 : 2);2730}2731 2732void CGOpenMPRuntime::emitNumThreadsClause(2733 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,2734 OpenMPNumThreadsClauseModifier Modifier, OpenMPSeverityClauseKind Severity,2735 SourceLocation SeverityLoc, const Expr *Message,2736 SourceLocation MessageLoc) {2737 if (!CGF.HaveInsertPoint())2738 return;2739 llvm::SmallVector<llvm::Value *, 4> Args(2740 {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),2741 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)});2742 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)2743 // or __kmpc_push_num_threads_strict(&loc, global_tid, num_threads, severity,2744 // messsage) if strict modifier is used.2745 RuntimeFunction FnID = OMPRTL___kmpc_push_num_threads;2746 if (Modifier == OMPC_NUMTHREADS_strict) {2747 FnID = OMPRTL___kmpc_push_num_threads_strict;2748 Args.push_back(emitSeverityClause(Severity, SeverityLoc));2749 Args.push_back(emitMessageClause(CGF, Message, MessageLoc));2750 }2751 CGF.EmitRuntimeCall(2752 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), FnID), Args);2753}2754 2755void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,2756 ProcBindKind ProcBind,2757 SourceLocation Loc) {2758 if (!CGF.HaveInsertPoint())2759 return;2760 assert(ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value.");2761 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)2762 llvm::Value *Args[] = {2763 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),2764 llvm::ConstantInt::get(CGM.IntTy, unsigned(ProcBind), /*isSigned=*/true)};2765 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(2766 CGM.getModule(), OMPRTL___kmpc_push_proc_bind),2767 Args);2768}2769 2770void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,2771 SourceLocation Loc, llvm::AtomicOrdering AO) {2772 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {2773 OMPBuilder.createFlush(CGF.Builder);2774 } else {2775 if (!CGF.HaveInsertPoint())2776 return;2777 // Build call void __kmpc_flush(ident_t *loc)2778 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(2779 CGM.getModule(), OMPRTL___kmpc_flush),2780 emitUpdateLocation(CGF, Loc));2781 }2782}2783 2784namespace {2785/// Indexes of fields for type kmp_task_t.2786enum KmpTaskTFields {2787 /// List of shared variables.2788 KmpTaskTShareds,2789 /// Task routine.2790 KmpTaskTRoutine,2791 /// Partition id for the untied tasks.2792 KmpTaskTPartId,2793 /// Function with call of destructors for private variables.2794 Data1,2795 /// Task priority.2796 Data2,2797 /// (Taskloops only) Lower bound.2798 KmpTaskTLowerBound,2799 /// (Taskloops only) Upper bound.2800 KmpTaskTUpperBound,2801 /// (Taskloops only) Stride.2802 KmpTaskTStride,2803 /// (Taskloops only) Is last iteration flag.2804 KmpTaskTLastIter,2805 /// (Taskloops only) Reduction data.2806 KmpTaskTReductions,2807};2808} // anonymous namespace2809 2810void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {2811 // If we are in simd mode or there are no entries, we don't need to do2812 // anything.2813 if (CGM.getLangOpts().OpenMPSimd || OMPBuilder.OffloadInfoManager.empty())2814 return;2815 2816 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =2817 [this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,2818 const llvm::TargetRegionEntryInfo &EntryInfo) -> void {2819 SourceLocation Loc;2820 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {2821 for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),2822 E = CGM.getContext().getSourceManager().fileinfo_end();2823 I != E; ++I) {2824 if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&2825 I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {2826 Loc = CGM.getContext().getSourceManager().translateFileLineCol(2827 I->getFirst(), EntryInfo.Line, 1);2828 break;2829 }2830 }2831 }2832 switch (Kind) {2833 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {2834 unsigned DiagID = CGM.getDiags().getCustomDiagID(2835 DiagnosticsEngine::Error, "Offloading entry for target region in "2836 "%0 is incorrect: either the "2837 "address or the ID is invalid.");2838 CGM.getDiags().Report(Loc, DiagID) << EntryInfo.ParentName;2839 } break;2840 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {2841 unsigned DiagID = CGM.getDiags().getCustomDiagID(2842 DiagnosticsEngine::Error, "Offloading entry for declare target "2843 "variable %0 is incorrect: the "2844 "address is invalid.");2845 CGM.getDiags().Report(Loc, DiagID) << EntryInfo.ParentName;2846 } break;2847 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {2848 unsigned DiagID = CGM.getDiags().getCustomDiagID(2849 DiagnosticsEngine::Error,2850 "Offloading entry for declare target variable is incorrect: the "2851 "address is invalid.");2852 CGM.getDiags().Report(DiagID);2853 } break;2854 }2855 };2856 2857 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFn);2858}2859 2860void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {2861 if (!KmpRoutineEntryPtrTy) {2862 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.2863 ASTContext &C = CGM.getContext();2864 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};2865 FunctionProtoType::ExtProtoInfo EPI;2866 KmpRoutineEntryPtrQTy = C.getPointerType(2867 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));2868 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);2869 }2870}2871 2872namespace {2873struct PrivateHelpersTy {2874 PrivateHelpersTy(const Expr *OriginalRef, const VarDecl *Original,2875 const VarDecl *PrivateCopy, const VarDecl *PrivateElemInit)2876 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),2877 PrivateElemInit(PrivateElemInit) {}2878 PrivateHelpersTy(const VarDecl *Original) : Original(Original) {}2879 const Expr *OriginalRef = nullptr;2880 const VarDecl *Original = nullptr;2881 const VarDecl *PrivateCopy = nullptr;2882 const VarDecl *PrivateElemInit = nullptr;2883 bool isLocalPrivate() const {2884 return !OriginalRef && !PrivateCopy && !PrivateElemInit;2885 }2886};2887typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;2888} // anonymous namespace2889 2890static bool isAllocatableDecl(const VarDecl *VD) {2891 const VarDecl *CVD = VD->getCanonicalDecl();2892 if (!CVD->hasAttr<OMPAllocateDeclAttr>())2893 return false;2894 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();2895 // Use the default allocation.2896 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&2897 !AA->getAllocator());2898}2899 2900static RecordDecl *2901createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {2902 if (!Privates.empty()) {2903 ASTContext &C = CGM.getContext();2904 // Build struct .kmp_privates_t. {2905 // /* private vars */2906 // };2907 RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");2908 RD->startDefinition();2909 for (const auto &Pair : Privates) {2910 const VarDecl *VD = Pair.second.Original;2911 QualType Type = VD->getType().getNonReferenceType();2912 // If the private variable is a local variable with lvalue ref type,2913 // allocate the pointer instead of the pointee type.2914 if (Pair.second.isLocalPrivate()) {2915 if (VD->getType()->isLValueReferenceType())2916 Type = C.getPointerType(Type);2917 if (isAllocatableDecl(VD))2918 Type = C.getPointerType(Type);2919 }2920 FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);2921 if (VD->hasAttrs()) {2922 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),2923 E(VD->getAttrs().end());2924 I != E; ++I)2925 FD->addAttr(*I);2926 }2927 }2928 RD->completeDefinition();2929 return RD;2930 }2931 return nullptr;2932}2933 2934static RecordDecl *2935createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,2936 QualType KmpInt32Ty,2937 QualType KmpRoutineEntryPointerQTy) {2938 ASTContext &C = CGM.getContext();2939 // Build struct kmp_task_t {2940 // void * shareds;2941 // kmp_routine_entry_t routine;2942 // kmp_int32 part_id;2943 // kmp_cmplrdata_t data1;2944 // kmp_cmplrdata_t data2;2945 // For taskloops additional fields:2946 // kmp_uint64 lb;2947 // kmp_uint64 ub;2948 // kmp_int64 st;2949 // kmp_int32 liter;2950 // void * reductions;2951 // };2952 RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TagTypeKind::Union);2953 UD->startDefinition();2954 addFieldToRecordDecl(C, UD, KmpInt32Ty);2955 addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);2956 UD->completeDefinition();2957 CanQualType KmpCmplrdataTy = C.getCanonicalTagType(UD);2958 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");2959 RD->startDefinition();2960 addFieldToRecordDecl(C, RD, C.VoidPtrTy);2961 addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);2962 addFieldToRecordDecl(C, RD, KmpInt32Ty);2963 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);2964 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);2965 if (isOpenMPTaskLoopDirective(Kind)) {2966 QualType KmpUInt64Ty =2967 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);2968 QualType KmpInt64Ty =2969 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);2970 addFieldToRecordDecl(C, RD, KmpUInt64Ty);2971 addFieldToRecordDecl(C, RD, KmpUInt64Ty);2972 addFieldToRecordDecl(C, RD, KmpInt64Ty);2973 addFieldToRecordDecl(C, RD, KmpInt32Ty);2974 addFieldToRecordDecl(C, RD, C.VoidPtrTy);2975 }2976 RD->completeDefinition();2977 return RD;2978}2979 2980static RecordDecl *2981createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,2982 ArrayRef<PrivateDataTy> Privates) {2983 ASTContext &C = CGM.getContext();2984 // Build struct kmp_task_t_with_privates {2985 // kmp_task_t task_data;2986 // .kmp_privates_t. privates;2987 // };2988 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");2989 RD->startDefinition();2990 addFieldToRecordDecl(C, RD, KmpTaskTQTy);2991 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))2992 addFieldToRecordDecl(C, RD, C.getCanonicalTagType(PrivateRD));2993 RD->completeDefinition();2994 return RD;2995}2996 2997/// Emit a proxy function which accepts kmp_task_t as the second2998/// argument.2999/// \code3000/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {3001/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,3002/// For taskloops:3003/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,3004/// tt->reductions, tt->shareds);3005/// return 0;3006/// }3007/// \endcode3008static llvm::Function *3009emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,3010 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,3011 QualType KmpTaskTWithPrivatesPtrQTy,3012 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,3013 QualType SharedsPtrTy, llvm::Function *TaskFunction,3014 llvm::Value *TaskPrivatesMap) {3015 ASTContext &C = CGM.getContext();3016 FunctionArgList Args;3017 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,3018 ImplicitParamKind::Other);3019 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,3020 KmpTaskTWithPrivatesPtrQTy.withRestrict(),3021 ImplicitParamKind::Other);3022 Args.push_back(&GtidArg);3023 Args.push_back(&TaskTypeArg);3024 const auto &TaskEntryFnInfo =3025 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);3026 llvm::FunctionType *TaskEntryTy =3027 CGM.getTypes().GetFunctionType(TaskEntryFnInfo);3028 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});3029 auto *TaskEntry = llvm::Function::Create(3030 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());3031 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);3032 TaskEntry->setDoesNotRecurse();3033 CodeGenFunction CGF(CGM);3034 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,3035 Loc, Loc);3036 3037 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,3038 // tt,3039 // For taskloops:3040 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,3041 // tt->task_data.shareds);3042 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(3043 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);3044 LValue TDBase = CGF.EmitLoadOfPointerLValue(3045 CGF.GetAddrOfLocalVar(&TaskTypeArg),3046 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());3047 const auto *KmpTaskTWithPrivatesQTyRD =3048 KmpTaskTWithPrivatesQTy->castAsRecordDecl();3049 LValue Base =3050 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());3051 const auto *KmpTaskTQTyRD = KmpTaskTQTy->castAsRecordDecl();3052 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);3053 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);3054 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);3055 3056 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);3057 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);3058 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(3059 CGF.EmitLoadOfScalar(SharedsLVal, Loc),3060 CGF.ConvertTypeForMem(SharedsPtrTy));3061 3062 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);3063 llvm::Value *PrivatesParam;3064 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {3065 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);3066 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(3067 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy);3068 } else {3069 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);3070 }3071 3072 llvm::Value *CommonArgs[] = {3073 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,3074 CGF.Builder3075 .CreatePointerBitCastOrAddrSpaceCast(TDBase.getAddress(),3076 CGF.VoidPtrTy, CGF.Int8Ty)3077 .emitRawPointer(CGF)};3078 SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),3079 std::end(CommonArgs));3080 if (isOpenMPTaskLoopDirective(Kind)) {3081 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);3082 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);3083 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);3084 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);3085 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);3086 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);3087 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);3088 LValue StLVal = CGF.EmitLValueForField(Base, *StFI);3089 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);3090 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);3091 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);3092 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);3093 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);3094 LValue RLVal = CGF.EmitLValueForField(Base, *RFI);3095 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);3096 CallArgs.push_back(LBParam);3097 CallArgs.push_back(UBParam);3098 CallArgs.push_back(StParam);3099 CallArgs.push_back(LIParam);3100 CallArgs.push_back(RParam);3101 }3102 CallArgs.push_back(SharedsParam);3103 3104 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,3105 CallArgs);3106 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),3107 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));3108 CGF.FinishFunction();3109 return TaskEntry;3110}3111 3112static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,3113 SourceLocation Loc,3114 QualType KmpInt32Ty,3115 QualType KmpTaskTWithPrivatesPtrQTy,3116 QualType KmpTaskTWithPrivatesQTy) {3117 ASTContext &C = CGM.getContext();3118 FunctionArgList Args;3119 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,3120 ImplicitParamKind::Other);3121 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,3122 KmpTaskTWithPrivatesPtrQTy.withRestrict(),3123 ImplicitParamKind::Other);3124 Args.push_back(&GtidArg);3125 Args.push_back(&TaskTypeArg);3126 const auto &DestructorFnInfo =3127 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);3128 llvm::FunctionType *DestructorFnTy =3129 CGM.getTypes().GetFunctionType(DestructorFnInfo);3130 std::string Name =3131 CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});3132 auto *DestructorFn =3133 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,3134 Name, &CGM.getModule());3135 CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,3136 DestructorFnInfo);3137 DestructorFn->setDoesNotRecurse();3138 CodeGenFunction CGF(CGM);3139 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,3140 Args, Loc, Loc);3141 3142 LValue Base = CGF.EmitLoadOfPointerLValue(3143 CGF.GetAddrOfLocalVar(&TaskTypeArg),3144 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());3145 const auto *KmpTaskTWithPrivatesQTyRD =3146 KmpTaskTWithPrivatesQTy->castAsRecordDecl();3147 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());3148 Base = CGF.EmitLValueForField(Base, *FI);3149 for (const auto *Field : FI->getType()->castAsRecordDecl()->fields()) {3150 if (QualType::DestructionKind DtorKind =3151 Field->getType().isDestructedType()) {3152 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);3153 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());3154 }3155 }3156 CGF.FinishFunction();3157 return DestructorFn;3158}3159 3160/// Emit a privates mapping function for correct handling of private and3161/// firstprivate variables.3162/// \code3163/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>3164/// **noalias priv1,..., <tyn> **noalias privn) {3165/// *priv1 = &.privates.priv1;3166/// ...;3167/// *privn = &.privates.privn;3168/// }3169/// \endcode3170static llvm::Value *3171emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,3172 const OMPTaskDataTy &Data, QualType PrivatesQTy,3173 ArrayRef<PrivateDataTy> Privates) {3174 ASTContext &C = CGM.getContext();3175 FunctionArgList Args;3176 ImplicitParamDecl TaskPrivatesArg(3177 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,3178 C.getPointerType(PrivatesQTy).withConst().withRestrict(),3179 ImplicitParamKind::Other);3180 Args.push_back(&TaskPrivatesArg);3181 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, unsigned> PrivateVarsPos;3182 unsigned Counter = 1;3183 for (const Expr *E : Data.PrivateVars) {3184 Args.push_back(ImplicitParamDecl::Create(3185 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,3186 C.getPointerType(C.getPointerType(E->getType()))3187 .withConst()3188 .withRestrict(),3189 ImplicitParamKind::Other));3190 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());3191 PrivateVarsPos[VD] = Counter;3192 ++Counter;3193 }3194 for (const Expr *E : Data.FirstprivateVars) {3195 Args.push_back(ImplicitParamDecl::Create(3196 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,3197 C.getPointerType(C.getPointerType(E->getType()))3198 .withConst()3199 .withRestrict(),3200 ImplicitParamKind::Other));3201 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());3202 PrivateVarsPos[VD] = Counter;3203 ++Counter;3204 }3205 for (const Expr *E : Data.LastprivateVars) {3206 Args.push_back(ImplicitParamDecl::Create(3207 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,3208 C.getPointerType(C.getPointerType(E->getType()))3209 .withConst()3210 .withRestrict(),3211 ImplicitParamKind::Other));3212 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());3213 PrivateVarsPos[VD] = Counter;3214 ++Counter;3215 }3216 for (const VarDecl *VD : Data.PrivateLocals) {3217 QualType Ty = VD->getType().getNonReferenceType();3218 if (VD->getType()->isLValueReferenceType())3219 Ty = C.getPointerType(Ty);3220 if (isAllocatableDecl(VD))3221 Ty = C.getPointerType(Ty);3222 Args.push_back(ImplicitParamDecl::Create(3223 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,3224 C.getPointerType(C.getPointerType(Ty)).withConst().withRestrict(),3225 ImplicitParamKind::Other));3226 PrivateVarsPos[VD] = Counter;3227 ++Counter;3228 }3229 const auto &TaskPrivatesMapFnInfo =3230 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);3231 llvm::FunctionType *TaskPrivatesMapTy =3232 CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);3233 std::string Name =3234 CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});3235 auto *TaskPrivatesMap = llvm::Function::Create(3236 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,3237 &CGM.getModule());3238 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,3239 TaskPrivatesMapFnInfo);3240 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {3241 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);3242 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);3243 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);3244 }3245 CodeGenFunction CGF(CGM);3246 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,3247 TaskPrivatesMapFnInfo, Args, Loc, Loc);3248 3249 // *privi = &.privates.privi;3250 LValue Base = CGF.EmitLoadOfPointerLValue(3251 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),3252 TaskPrivatesArg.getType()->castAs<PointerType>());3253 const auto *PrivatesQTyRD = PrivatesQTy->castAsRecordDecl();3254 Counter = 0;3255 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {3256 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);3257 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];3258 LValue RefLVal =3259 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());3260 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(3261 RefLVal.getAddress(), RefLVal.getType()->castAs<PointerType>());3262 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal);3263 ++Counter;3264 }3265 CGF.FinishFunction();3266 return TaskPrivatesMap;3267}3268 3269/// Emit initialization for private variables in task-based directives.3270static void emitPrivatesInit(CodeGenFunction &CGF,3271 const OMPExecutableDirective &D,3272 Address KmpTaskSharedsPtr, LValue TDBase,3273 const RecordDecl *KmpTaskTWithPrivatesQTyRD,3274 QualType SharedsTy, QualType SharedsPtrTy,3275 const OMPTaskDataTy &Data,3276 ArrayRef<PrivateDataTy> Privates, bool ForDup) {3277 ASTContext &C = CGF.getContext();3278 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());3279 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);3280 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())3281 ? OMPD_taskloop3282 : OMPD_task;3283 const CapturedStmt &CS = *D.getCapturedStmt(Kind);3284 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);3285 LValue SrcBase;3286 bool IsTargetTask =3287 isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||3288 isOpenMPTargetExecutionDirective(D.getDirectiveKind());3289 // For target-based directives skip 4 firstprivate arrays BasePointersArray,3290 // PointersArray, SizesArray, and MappersArray. The original variables for3291 // these arrays are not captured and we get their addresses explicitly.3292 if ((!IsTargetTask && !Data.FirstprivateVars.empty() && ForDup) ||3293 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {3294 SrcBase = CGF.MakeAddrLValue(3295 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(3296 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy),3297 CGF.ConvertTypeForMem(SharedsTy)),3298 SharedsTy);3299 }3300 FI = FI->getType()->castAsRecordDecl()->field_begin();3301 for (const PrivateDataTy &Pair : Privates) {3302 // Do not initialize private locals.3303 if (Pair.second.isLocalPrivate()) {3304 ++FI;3305 continue;3306 }3307 const VarDecl *VD = Pair.second.PrivateCopy;3308 const Expr *Init = VD->getAnyInitializer();3309 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&3310 !CGF.isTrivialInitializer(Init)))) {3311 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);3312 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {3313 const VarDecl *OriginalVD = Pair.second.Original;3314 // Check if the variable is the target-based BasePointersArray,3315 // PointersArray, SizesArray, or MappersArray.3316 LValue SharedRefLValue;3317 QualType Type = PrivateLValue.getType();3318 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);3319 if (IsTargetTask && !SharedField) {3320 assert(isa<ImplicitParamDecl>(OriginalVD) &&3321 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&3322 cast<CapturedDecl>(OriginalVD->getDeclContext())3323 ->getNumParams() == 0 &&3324 isa<TranslationUnitDecl>(3325 cast<CapturedDecl>(OriginalVD->getDeclContext())3326 ->getDeclContext()) &&3327 "Expected artificial target data variable.");3328 SharedRefLValue =3329 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);3330 } else if (ForDup) {3331 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);3332 SharedRefLValue = CGF.MakeAddrLValue(3333 SharedRefLValue.getAddress().withAlignment(3334 C.getDeclAlign(OriginalVD)),3335 SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),3336 SharedRefLValue.getTBAAInfo());3337 } else if (CGF.LambdaCaptureFields.count(3338 Pair.second.Original->getCanonicalDecl()) > 0 ||3339 isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl)) {3340 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);3341 } else {3342 // Processing for implicitly captured variables.3343 InlinedOpenMPRegionRAII Region(3344 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,3345 /*HasCancel=*/false, /*NoInheritance=*/true);3346 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);3347 }3348 if (Type->isArrayType()) {3349 // Initialize firstprivate array.3350 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {3351 // Perform simple memcpy.3352 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);3353 } else {3354 // Initialize firstprivate array using element-by-element3355 // initialization.3356 CGF.EmitOMPAggregateAssign(3357 PrivateLValue.getAddress(), SharedRefLValue.getAddress(), Type,3358 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,3359 Address SrcElement) {3360 // Clean up any temporaries needed by the initialization.3361 CodeGenFunction::OMPPrivateScope InitScope(CGF);3362 InitScope.addPrivate(Elem, SrcElement);3363 (void)InitScope.Privatize();3364 // Emit initialization for single element.3365 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(3366 CGF, &CapturesInfo);3367 CGF.EmitAnyExprToMem(Init, DestElement,3368 Init->getType().getQualifiers(),3369 /*IsInitializer=*/false);3370 });3371 }3372 } else {3373 CodeGenFunction::OMPPrivateScope InitScope(CGF);3374 InitScope.addPrivate(Elem, SharedRefLValue.getAddress());3375 (void)InitScope.Privatize();3376 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);3377 CGF.EmitExprAsInit(Init, VD, PrivateLValue,3378 /*capturedByInit=*/false);3379 }3380 } else {3381 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);3382 }3383 }3384 ++FI;3385 }3386}3387 3388/// Check if duplication function is required for taskloops.3389static bool checkInitIsRequired(CodeGenFunction &CGF,3390 ArrayRef<PrivateDataTy> Privates) {3391 bool InitRequired = false;3392 for (const PrivateDataTy &Pair : Privates) {3393 if (Pair.second.isLocalPrivate())3394 continue;3395 const VarDecl *VD = Pair.second.PrivateCopy;3396 const Expr *Init = VD->getAnyInitializer();3397 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(Init) &&3398 !CGF.isTrivialInitializer(Init));3399 if (InitRequired)3400 break;3401 }3402 return InitRequired;3403}3404 3405 3406/// Emit task_dup function (for initialization of3407/// private/firstprivate/lastprivate vars and last_iter flag)3408/// \code3409/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int3410/// lastpriv) {3411/// // setup lastprivate flag3412/// task_dst->last = lastpriv;3413/// // could be constructor calls here...3414/// }3415/// \endcode3416static llvm::Value *3417emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,3418 const OMPExecutableDirective &D,3419 QualType KmpTaskTWithPrivatesPtrQTy,3420 const RecordDecl *KmpTaskTWithPrivatesQTyRD,3421 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,3422 QualType SharedsPtrTy, const OMPTaskDataTy &Data,3423 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {3424 ASTContext &C = CGM.getContext();3425 FunctionArgList Args;3426 ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,3427 KmpTaskTWithPrivatesPtrQTy,3428 ImplicitParamKind::Other);3429 ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,3430 KmpTaskTWithPrivatesPtrQTy,3431 ImplicitParamKind::Other);3432 ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,3433 ImplicitParamKind::Other);3434 Args.push_back(&DstArg);3435 Args.push_back(&SrcArg);3436 Args.push_back(&LastprivArg);3437 const auto &TaskDupFnInfo =3438 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);3439 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);3440 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});3441 auto *TaskDup = llvm::Function::Create(3442 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());3443 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);3444 TaskDup->setDoesNotRecurse();3445 CodeGenFunction CGF(CGM);3446 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,3447 Loc);3448 3449 LValue TDBase = CGF.EmitLoadOfPointerLValue(3450 CGF.GetAddrOfLocalVar(&DstArg),3451 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());3452 // task_dst->liter = lastpriv;3453 if (WithLastIter) {3454 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);3455 LValue Base = CGF.EmitLValueForField(3456 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());3457 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);3458 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(3459 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);3460 CGF.EmitStoreOfScalar(Lastpriv, LILVal);3461 }3462 3463 // Emit initial values for private copies (if any).3464 assert(!Privates.empty());3465 Address KmpTaskSharedsPtr = Address::invalid();3466 if (!Data.FirstprivateVars.empty()) {3467 LValue TDBase = CGF.EmitLoadOfPointerLValue(3468 CGF.GetAddrOfLocalVar(&SrcArg),3469 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());3470 LValue Base = CGF.EmitLValueForField(3471 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());3472 KmpTaskSharedsPtr = Address(3473 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(3474 Base, *std::next(KmpTaskTQTyRD->field_begin(),3475 KmpTaskTShareds)),3476 Loc),3477 CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy));3478 }3479 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,3480 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);3481 CGF.FinishFunction();3482 return TaskDup;3483}3484 3485/// Checks if destructor function is required to be generated.3486/// \return true if cleanups are required, false otherwise.3487static bool3488checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD,3489 ArrayRef<PrivateDataTy> Privates) {3490 for (const PrivateDataTy &P : Privates) {3491 if (P.second.isLocalPrivate())3492 continue;3493 QualType Ty = P.second.Original->getType().getNonReferenceType();3494 if (Ty.isDestructedType())3495 return true;3496 }3497 return false;3498}3499 3500namespace {3501/// Loop generator for OpenMP iterator expression.3502class OMPIteratorGeneratorScope final3503 : public CodeGenFunction::OMPPrivateScope {3504 CodeGenFunction &CGF;3505 const OMPIteratorExpr *E = nullptr;3506 SmallVector<CodeGenFunction::JumpDest, 4> ContDests;3507 SmallVector<CodeGenFunction::JumpDest, 4> ExitDests;3508 OMPIteratorGeneratorScope() = delete;3509 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) = delete;3510 3511public:3512 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E)3513 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) {3514 if (!E)3515 return;3516 SmallVector<llvm::Value *, 4> Uppers;3517 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {3518 Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper));3519 const auto *VD = cast<VarDecl>(E->getIteratorDecl(I));3520 addPrivate(VD, CGF.CreateMemTemp(VD->getType(), VD->getName()));3521 const OMPIteratorHelperData &HelperData = E->getHelper(I);3522 addPrivate(3523 HelperData.CounterVD,3524 CGF.CreateMemTemp(HelperData.CounterVD->getType(), "counter.addr"));3525 }3526 Privatize();3527 3528 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {3529 const OMPIteratorHelperData &HelperData = E->getHelper(I);3530 LValue CLVal =3531 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD),3532 HelperData.CounterVD->getType());3533 // Counter = 0;3534 CGF.EmitStoreOfScalar(3535 llvm::ConstantInt::get(CLVal.getAddress().getElementType(), 0),3536 CLVal);3537 CodeGenFunction::JumpDest &ContDest =3538 ContDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.cont"));3539 CodeGenFunction::JumpDest &ExitDest =3540 ExitDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.exit"));3541 // N = <number-of_iterations>;3542 llvm::Value *N = Uppers[I];3543 // cont:3544 // if (Counter < N) goto body; else goto exit;3545 CGF.EmitBlock(ContDest.getBlock());3546 auto *CVal =3547 CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation());3548 llvm::Value *Cmp =3549 HelperData.CounterVD->getType()->isSignedIntegerOrEnumerationType()3550 ? CGF.Builder.CreateICmpSLT(CVal, N)3551 : CGF.Builder.CreateICmpULT(CVal, N);3552 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("iter.body");3553 CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());3554 // body:3555 CGF.EmitBlock(BodyBB);3556 // Iteri = Begini + Counter * Stepi;3557 CGF.EmitIgnoredExpr(HelperData.Update);3558 }3559 }3560 ~OMPIteratorGeneratorScope() {3561 if (!E)3562 return;3563 for (unsigned I = E->numOfIterators(); I > 0; --I) {3564 // Counter = Counter + 1;3565 const OMPIteratorHelperData &HelperData = E->getHelper(I - 1);3566 CGF.EmitIgnoredExpr(HelperData.CounterUpdate);3567 // goto cont;3568 CGF.EmitBranchThroughCleanup(ContDests[I - 1]);3569 // exit:3570 CGF.EmitBlock(ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1);3571 }3572 }3573};3574} // namespace3575 3576static std::pair<llvm::Value *, llvm::Value *>3577getPointerAndSize(CodeGenFunction &CGF, const Expr *E) {3578 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E);3579 llvm::Value *Addr;3580 if (OASE) {3581 const Expr *Base = OASE->getBase();3582 Addr = CGF.EmitScalarExpr(Base);3583 } else {3584 Addr = CGF.EmitLValue(E).getPointer(CGF);3585 }3586 llvm::Value *SizeVal;3587 QualType Ty = E->getType();3588 if (OASE) {3589 SizeVal = CGF.getTypeSize(OASE->getBase()->getType()->getPointeeType());3590 for (const Expr *SE : OASE->getDimensions()) {3591 llvm::Value *Sz = CGF.EmitScalarExpr(SE);3592 Sz = CGF.EmitScalarConversion(3593 Sz, SE->getType(), CGF.getContext().getSizeType(), SE->getExprLoc());3594 SizeVal = CGF.Builder.CreateNUWMul(SizeVal, Sz);3595 }3596 } else if (const auto *ASE =3597 dyn_cast<ArraySectionExpr>(E->IgnoreParenImpCasts())) {3598 LValue UpAddrLVal = CGF.EmitArraySectionExpr(ASE, /*IsLowerBound=*/false);3599 Address UpAddrAddress = UpAddrLVal.getAddress();3600 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32(3601 UpAddrAddress.getElementType(), UpAddrAddress.emitRawPointer(CGF),3602 /*Idx0=*/1);3603 llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy);3604 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy);3605 SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);3606 } else {3607 SizeVal = CGF.getTypeSize(Ty);3608 }3609 return std::make_pair(Addr, SizeVal);3610}3611 3612/// Builds kmp_depend_info, if it is not built yet, and builds flags type.3613static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy) {3614 QualType FlagsTy = C.getIntTypeForBitwidth(32, /*Signed=*/false);3615 if (KmpTaskAffinityInfoTy.isNull()) {3616 RecordDecl *KmpAffinityInfoRD =3617 C.buildImplicitRecord("kmp_task_affinity_info_t");3618 KmpAffinityInfoRD->startDefinition();3619 addFieldToRecordDecl(C, KmpAffinityInfoRD, C.getIntPtrType());3620 addFieldToRecordDecl(C, KmpAffinityInfoRD, C.getSizeType());3621 addFieldToRecordDecl(C, KmpAffinityInfoRD, FlagsTy);3622 KmpAffinityInfoRD->completeDefinition();3623 KmpTaskAffinityInfoTy = C.getCanonicalTagType(KmpAffinityInfoRD);3624 }3625}3626 3627CGOpenMPRuntime::TaskResultTy3628CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,3629 const OMPExecutableDirective &D,3630 llvm::Function *TaskFunction, QualType SharedsTy,3631 Address Shareds, const OMPTaskDataTy &Data) {3632 ASTContext &C = CGM.getContext();3633 llvm::SmallVector<PrivateDataTy, 4> Privates;3634 // Aggregate privates and sort them by the alignment.3635 const auto *I = Data.PrivateCopies.begin();3636 for (const Expr *E : Data.PrivateVars) {3637 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());3638 Privates.emplace_back(3639 C.getDeclAlign(VD),3640 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),3641 /*PrivateElemInit=*/nullptr));3642 ++I;3643 }3644 I = Data.FirstprivateCopies.begin();3645 const auto *IElemInitRef = Data.FirstprivateInits.begin();3646 for (const Expr *E : Data.FirstprivateVars) {3647 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());3648 Privates.emplace_back(3649 C.getDeclAlign(VD),3650 PrivateHelpersTy(3651 E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),3652 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));3653 ++I;3654 ++IElemInitRef;3655 }3656 I = Data.LastprivateCopies.begin();3657 for (const Expr *E : Data.LastprivateVars) {3658 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());3659 Privates.emplace_back(3660 C.getDeclAlign(VD),3661 PrivateHelpersTy(E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),3662 /*PrivateElemInit=*/nullptr));3663 ++I;3664 }3665 for (const VarDecl *VD : Data.PrivateLocals) {3666 if (isAllocatableDecl(VD))3667 Privates.emplace_back(CGM.getPointerAlign(), PrivateHelpersTy(VD));3668 else3669 Privates.emplace_back(C.getDeclAlign(VD), PrivateHelpersTy(VD));3670 }3671 llvm::stable_sort(Privates,3672 [](const PrivateDataTy &L, const PrivateDataTy &R) {3673 return L.first > R.first;3674 });3675 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);3676 // Build type kmp_routine_entry_t (if not built yet).3677 emitKmpRoutineEntryT(KmpInt32Ty);3678 // Build type kmp_task_t (if not built yet).3679 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {3680 if (SavedKmpTaskloopTQTy.isNull()) {3681 SavedKmpTaskloopTQTy = C.getCanonicalTagType(createKmpTaskTRecordDecl(3682 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));3683 }3684 KmpTaskTQTy = SavedKmpTaskloopTQTy;3685 } else {3686 assert((D.getDirectiveKind() == OMPD_task ||3687 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||3688 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&3689 "Expected taskloop, task or target directive");3690 if (SavedKmpTaskTQTy.isNull()) {3691 SavedKmpTaskTQTy = C.getCanonicalTagType(createKmpTaskTRecordDecl(3692 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));3693 }3694 KmpTaskTQTy = SavedKmpTaskTQTy;3695 }3696 const auto *KmpTaskTQTyRD = KmpTaskTQTy->castAsRecordDecl();3697 // Build particular struct kmp_task_t for the given task.3698 const RecordDecl *KmpTaskTWithPrivatesQTyRD =3699 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);3700 CanQualType KmpTaskTWithPrivatesQTy =3701 C.getCanonicalTagType(KmpTaskTWithPrivatesQTyRD);3702 QualType KmpTaskTWithPrivatesPtrQTy =3703 C.getPointerType(KmpTaskTWithPrivatesQTy);3704 llvm::Type *KmpTaskTWithPrivatesPtrTy = CGF.Builder.getPtrTy(0);3705 llvm::Value *KmpTaskTWithPrivatesTySize =3706 CGF.getTypeSize(KmpTaskTWithPrivatesQTy);3707 QualType SharedsPtrTy = C.getPointerType(SharedsTy);3708 3709 // Emit initial values for private copies (if any).3710 llvm::Value *TaskPrivatesMap = nullptr;3711 llvm::Type *TaskPrivatesMapTy =3712 std::next(TaskFunction->arg_begin(), 3)->getType();3713 if (!Privates.empty()) {3714 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());3715 TaskPrivatesMap =3716 emitTaskPrivateMappingFunction(CGM, Loc, Data, FI->getType(), Privates);3717 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(3718 TaskPrivatesMap, TaskPrivatesMapTy);3719 } else {3720 TaskPrivatesMap = llvm::ConstantPointerNull::get(3721 cast<llvm::PointerType>(TaskPrivatesMapTy));3722 }3723 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,3724 // kmp_task_t *tt);3725 llvm::Function *TaskEntry = emitProxyTaskFunction(3726 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,3727 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,3728 TaskPrivatesMap);3729 3730 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,3731 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,3732 // kmp_routine_entry_t *task_entry);3733 // Task flags. Format is taken from3734 // https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h,3735 // description of kmp_tasking_flags struct.3736 enum {3737 TiedFlag = 0x1,3738 FinalFlag = 0x2,3739 DestructorsFlag = 0x8,3740 PriorityFlag = 0x20,3741 DetachableFlag = 0x40,3742 FreeAgentFlag = 0x80,3743 };3744 unsigned Flags = Data.Tied ? TiedFlag : 0;3745 bool NeedsCleanup = false;3746 if (!Privates.empty()) {3747 NeedsCleanup =3748 checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD, Privates);3749 if (NeedsCleanup)3750 Flags = Flags | DestructorsFlag;3751 }3752 if (const auto *Clause = D.getSingleClause<OMPThreadsetClause>()) {3753 OpenMPThreadsetKind Kind = Clause->getThreadsetKind();3754 if (Kind == OMPC_THREADSET_omp_pool)3755 Flags = Flags | FreeAgentFlag;3756 }3757 if (Data.Priority.getInt())3758 Flags = Flags | PriorityFlag;3759 if (D.hasClausesOfKind<OMPDetachClause>())3760 Flags = Flags | DetachableFlag;3761 llvm::Value *TaskFlags =3762 Data.Final.getPointer()3763 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),3764 CGF.Builder.getInt32(FinalFlag),3765 CGF.Builder.getInt32(/*C=*/0))3766 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);3767 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));3768 llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));3769 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),3770 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,3771 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(3772 TaskEntry, KmpRoutineEntryPtrTy)};3773 llvm::Value *NewTask;3774 if (D.hasClausesOfKind<OMPNowaitClause>()) {3775 // Check if we have any device clause associated with the directive.3776 const Expr *Device = nullptr;3777 if (auto *C = D.getSingleClause<OMPDeviceClause>())3778 Device = C->getDevice();3779 // Emit device ID if any otherwise use default value.3780 llvm::Value *DeviceID;3781 if (Device)3782 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),3783 CGF.Int64Ty, /*isSigned=*/true);3784 else3785 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);3786 AllocArgs.push_back(DeviceID);3787 NewTask = CGF.EmitRuntimeCall(3788 OMPBuilder.getOrCreateRuntimeFunction(3789 CGM.getModule(), OMPRTL___kmpc_omp_target_task_alloc),3790 AllocArgs);3791 } else {3792 NewTask =3793 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(3794 CGM.getModule(), OMPRTL___kmpc_omp_task_alloc),3795 AllocArgs);3796 }3797 // Emit detach clause initialization.3798 // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,3799 // task_descriptor);3800 if (const auto *DC = D.getSingleClause<OMPDetachClause>()) {3801 const Expr *Evt = DC->getEventHandler()->IgnoreParenImpCasts();3802 LValue EvtLVal = CGF.EmitLValue(Evt);3803 3804 // Build kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref,3805 // int gtid, kmp_task_t *task);3806 llvm::Value *Loc = emitUpdateLocation(CGF, DC->getBeginLoc());3807 llvm::Value *Tid = getThreadID(CGF, DC->getBeginLoc());3808 Tid = CGF.Builder.CreateIntCast(Tid, CGF.IntTy, /*isSigned=*/false);3809 llvm::Value *EvtVal = CGF.EmitRuntimeCall(3810 OMPBuilder.getOrCreateRuntimeFunction(3811 CGM.getModule(), OMPRTL___kmpc_task_allow_completion_event),3812 {Loc, Tid, NewTask});3813 EvtVal = CGF.EmitScalarConversion(EvtVal, C.VoidPtrTy, Evt->getType(),3814 Evt->getExprLoc());3815 CGF.EmitStoreOfScalar(EvtVal, EvtLVal);3816 }3817 // Process affinity clauses.3818 if (D.hasClausesOfKind<OMPAffinityClause>()) {3819 // Process list of affinity data.3820 ASTContext &C = CGM.getContext();3821 Address AffinitiesArray = Address::invalid();3822 // Calculate number of elements to form the array of affinity data.3823 llvm::Value *NumOfElements = nullptr;3824 unsigned NumAffinities = 0;3825 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {3826 if (const Expr *Modifier = C->getModifier()) {3827 const auto *IE = cast<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts());3828 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {3829 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);3830 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false);3831 NumOfElements =3832 NumOfElements ? CGF.Builder.CreateNUWMul(NumOfElements, Sz) : Sz;3833 }3834 } else {3835 NumAffinities += C->varlist_size();3836 }3837 }3838 getKmpAffinityType(CGM.getContext(), KmpTaskAffinityInfoTy);3839 // Fields ids in kmp_task_affinity_info record.3840 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };3841 3842 QualType KmpTaskAffinityInfoArrayTy;3843 if (NumOfElements) {3844 NumOfElements = CGF.Builder.CreateNUWAdd(3845 llvm::ConstantInt::get(CGF.SizeTy, NumAffinities), NumOfElements);3846 auto *OVE = new (C) OpaqueValueExpr(3847 Loc,3848 C.getIntTypeForBitwidth(C.getTypeSize(C.getSizeType()), /*Signed=*/0),3849 VK_PRValue);3850 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,3851 RValue::get(NumOfElements));3852 KmpTaskAffinityInfoArrayTy = C.getVariableArrayType(3853 KmpTaskAffinityInfoTy, OVE, ArraySizeModifier::Normal,3854 /*IndexTypeQuals=*/0);3855 // Properly emit variable-sized array.3856 auto *PD = ImplicitParamDecl::Create(C, KmpTaskAffinityInfoArrayTy,3857 ImplicitParamKind::Other);3858 CGF.EmitVarDecl(*PD);3859 AffinitiesArray = CGF.GetAddrOfLocalVar(PD);3860 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty,3861 /*isSigned=*/false);3862 } else {3863 KmpTaskAffinityInfoArrayTy = C.getConstantArrayType(3864 KmpTaskAffinityInfoTy,3865 llvm::APInt(C.getTypeSize(C.getSizeType()), NumAffinities), nullptr,3866 ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);3867 AffinitiesArray =3868 CGF.CreateMemTemp(KmpTaskAffinityInfoArrayTy, ".affs.arr.addr");3869 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(AffinitiesArray, 0);3870 NumOfElements = llvm::ConstantInt::get(CGM.Int32Ty, NumAffinities,3871 /*isSigned=*/false);3872 }3873 3874 const auto *KmpAffinityInfoRD = KmpTaskAffinityInfoTy->getAsRecordDecl();3875 // Fill array by elements without iterators.3876 unsigned Pos = 0;3877 bool HasIterator = false;3878 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {3879 if (C->getModifier()) {3880 HasIterator = true;3881 continue;3882 }3883 for (const Expr *E : C->varlist()) {3884 llvm::Value *Addr;3885 llvm::Value *Size;3886 std::tie(Addr, Size) = getPointerAndSize(CGF, E);3887 LValue Base =3888 CGF.MakeAddrLValue(CGF.Builder.CreateConstGEP(AffinitiesArray, Pos),3889 KmpTaskAffinityInfoTy);3890 // affs[i].base_addr = &<Affinities[i].second>;3891 LValue BaseAddrLVal = CGF.EmitLValueForField(3892 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));3893 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),3894 BaseAddrLVal);3895 // affs[i].len = sizeof(<Affinities[i].second>);3896 LValue LenLVal = CGF.EmitLValueForField(3897 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));3898 CGF.EmitStoreOfScalar(Size, LenLVal);3899 ++Pos;3900 }3901 }3902 LValue PosLVal;3903 if (HasIterator) {3904 PosLVal = CGF.MakeAddrLValue(3905 CGF.CreateMemTemp(C.getSizeType(), "affs.counter.addr"),3906 C.getSizeType());3907 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal);3908 }3909 // Process elements with iterators.3910 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {3911 const Expr *Modifier = C->getModifier();3912 if (!Modifier)3913 continue;3914 OMPIteratorGeneratorScope IteratorScope(3915 CGF, cast_or_null<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts()));3916 for (const Expr *E : C->varlist()) {3917 llvm::Value *Addr;3918 llvm::Value *Size;3919 std::tie(Addr, Size) = getPointerAndSize(CGF, E);3920 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());3921 LValue Base =3922 CGF.MakeAddrLValue(CGF.Builder.CreateGEP(CGF, AffinitiesArray, Idx),3923 KmpTaskAffinityInfoTy);3924 // affs[i].base_addr = &<Affinities[i].second>;3925 LValue BaseAddrLVal = CGF.EmitLValueForField(3926 Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));3927 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),3928 BaseAddrLVal);3929 // affs[i].len = sizeof(<Affinities[i].second>);3930 LValue LenLVal = CGF.EmitLValueForField(3931 Base, *std::next(KmpAffinityInfoRD->field_begin(), Len));3932 CGF.EmitStoreOfScalar(Size, LenLVal);3933 Idx = CGF.Builder.CreateNUWAdd(3934 Idx, llvm::ConstantInt::get(Idx->getType(), 1));3935 CGF.EmitStoreOfScalar(Idx, PosLVal);3936 }3937 }3938 // Call to kmp_int32 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref,3939 // kmp_int32 gtid, kmp_task_t *new_task, kmp_int323940 // naffins, kmp_task_affinity_info_t *affin_list);3941 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc);3942 llvm::Value *GTid = getThreadID(CGF, Loc);3943 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(3944 AffinitiesArray.emitRawPointer(CGF), CGM.VoidPtrTy);3945 // FIXME: Emit the function and ignore its result for now unless the3946 // runtime function is properly implemented.3947 (void)CGF.EmitRuntimeCall(3948 OMPBuilder.getOrCreateRuntimeFunction(3949 CGM.getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity),3950 {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});3951 }3952 llvm::Value *NewTaskNewTaskTTy =3953 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(3954 NewTask, KmpTaskTWithPrivatesPtrTy);3955 LValue Base = CGF.MakeNaturalAlignRawAddrLValue(NewTaskNewTaskTTy,3956 KmpTaskTWithPrivatesQTy);3957 LValue TDBase =3958 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());3959 // Fill the data in the resulting kmp_task_t record.3960 // Copy shareds if there are any.3961 Address KmpTaskSharedsPtr = Address::invalid();3962 if (!SharedsTy->castAsRecordDecl()->field_empty()) {3963 KmpTaskSharedsPtr = Address(3964 CGF.EmitLoadOfScalar(3965 CGF.EmitLValueForField(3966 TDBase,3967 *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),3968 Loc),3969 CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy));3970 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);3971 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);3972 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);3973 }3974 // Emit initial values for private copies (if any).3975 TaskResultTy Result;3976 if (!Privates.empty()) {3977 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,3978 SharedsTy, SharedsPtrTy, Data, Privates,3979 /*ForDup=*/false);3980 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&3981 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {3982 Result.TaskDupFn = emitTaskDupFunction(3983 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,3984 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,3985 /*WithLastIter=*/!Data.LastprivateVars.empty());3986 }3987 }3988 // Fields of union "kmp_cmplrdata_t" for destructors and priority.3989 enum { Priority = 0, Destructors = 1 };3990 // Provide pointer to function with destructors for privates.3991 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);3992 const auto *KmpCmplrdataUD = (*FI)->getType()->castAsRecordDecl();3993 assert(KmpCmplrdataUD->isUnion());3994 if (NeedsCleanup) {3995 llvm::Value *DestructorFn = emitDestructorsFunction(3996 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,3997 KmpTaskTWithPrivatesQTy);3998 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);3999 LValue DestructorsLV = CGF.EmitLValueForField(4000 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));4001 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(4002 DestructorFn, KmpRoutineEntryPtrTy),4003 DestructorsLV);4004 }4005 // Set priority.4006 if (Data.Priority.getInt()) {4007 LValue Data2LV = CGF.EmitLValueForField(4008 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));4009 LValue PriorityLV = CGF.EmitLValueForField(4010 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));4011 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);4012 }4013 Result.NewTask = NewTask;4014 Result.TaskEntry = TaskEntry;4015 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;4016 Result.TDBase = TDBase;4017 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;4018 return Result;4019}4020 4021/// Translates internal dependency kind into the runtime kind.4022static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K) {4023 RTLDependenceKindTy DepKind;4024 switch (K) {4025 case OMPC_DEPEND_in:4026 DepKind = RTLDependenceKindTy::DepIn;4027 break;4028 // Out and InOut dependencies must use the same code.4029 case OMPC_DEPEND_out:4030 case OMPC_DEPEND_inout:4031 DepKind = RTLDependenceKindTy::DepInOut;4032 break;4033 case OMPC_DEPEND_mutexinoutset:4034 DepKind = RTLDependenceKindTy::DepMutexInOutSet;4035 break;4036 case OMPC_DEPEND_inoutset:4037 DepKind = RTLDependenceKindTy::DepInOutSet;4038 break;4039 case OMPC_DEPEND_outallmemory:4040 DepKind = RTLDependenceKindTy::DepOmpAllMem;4041 break;4042 case OMPC_DEPEND_source:4043 case OMPC_DEPEND_sink:4044 case OMPC_DEPEND_depobj:4045 case OMPC_DEPEND_inoutallmemory:4046 case OMPC_DEPEND_unknown:4047 llvm_unreachable("Unknown task dependence type");4048 }4049 return DepKind;4050}4051 4052/// Builds kmp_depend_info, if it is not built yet, and builds flags type.4053static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy,4054 QualType &FlagsTy) {4055 FlagsTy = C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);4056 if (KmpDependInfoTy.isNull()) {4057 RecordDecl *KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");4058 KmpDependInfoRD->startDefinition();4059 addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());4060 addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());4061 addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);4062 KmpDependInfoRD->completeDefinition();4063 KmpDependInfoTy = C.getCanonicalTagType(KmpDependInfoRD);4064 }4065}4066 4067std::pair<llvm::Value *, LValue>4068CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal,4069 SourceLocation Loc) {4070 ASTContext &C = CGM.getContext();4071 QualType FlagsTy;4072 getDependTypes(C, KmpDependInfoTy, FlagsTy);4073 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();4074 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);4075 LValue Base = CGF.EmitLoadOfPointerLValue(4076 DepobjLVal.getAddress().withElementType(4077 CGF.ConvertTypeForMem(KmpDependInfoPtrTy)),4078 KmpDependInfoPtrTy->castAs<PointerType>());4079 Address DepObjAddr = CGF.Builder.CreateGEP(4080 CGF, Base.getAddress(),4081 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));4082 LValue NumDepsBase = CGF.MakeAddrLValue(4083 DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo());4084 // NumDeps = deps[i].base_addr;4085 LValue BaseAddrLVal = CGF.EmitLValueForField(4086 NumDepsBase,4087 *std::next(KmpDependInfoRD->field_begin(),4088 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));4089 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(BaseAddrLVal, Loc);4090 return std::make_pair(NumDeps, Base);4091}4092 4093static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,4094 llvm::PointerUnion<unsigned *, LValue *> Pos,4095 const OMPTaskDataTy::DependData &Data,4096 Address DependenciesArray) {4097 CodeGenModule &CGM = CGF.CGM;4098 ASTContext &C = CGM.getContext();4099 QualType FlagsTy;4100 getDependTypes(C, KmpDependInfoTy, FlagsTy);4101 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();4102 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);4103 4104 OMPIteratorGeneratorScope IteratorScope(4105 CGF, cast_or_null<OMPIteratorExpr>(4106 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()4107 : nullptr));4108 for (const Expr *E : Data.DepExprs) {4109 llvm::Value *Addr;4110 llvm::Value *Size;4111 4112 // The expression will be a nullptr in the 'omp_all_memory' case.4113 if (E) {4114 std::tie(Addr, Size) = getPointerAndSize(CGF, E);4115 Addr = CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy);4116 } else {4117 Addr = llvm::ConstantInt::get(CGF.IntPtrTy, 0);4118 Size = llvm::ConstantInt::get(CGF.SizeTy, 0);4119 }4120 LValue Base;4121 if (unsigned *P = dyn_cast<unsigned *>(Pos)) {4122 Base = CGF.MakeAddrLValue(4123 CGF.Builder.CreateConstGEP(DependenciesArray, *P), KmpDependInfoTy);4124 } else {4125 assert(E && "Expected a non-null expression");4126 LValue &PosLVal = *cast<LValue *>(Pos);4127 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());4128 Base = CGF.MakeAddrLValue(4129 CGF.Builder.CreateGEP(CGF, DependenciesArray, Idx), KmpDependInfoTy);4130 }4131 // deps[i].base_addr = &<Dependencies[i].second>;4132 LValue BaseAddrLVal = CGF.EmitLValueForField(4133 Base,4134 *std::next(KmpDependInfoRD->field_begin(),4135 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));4136 CGF.EmitStoreOfScalar(Addr, BaseAddrLVal);4137 // deps[i].len = sizeof(<Dependencies[i].second>);4138 LValue LenLVal = CGF.EmitLValueForField(4139 Base, *std::next(KmpDependInfoRD->field_begin(),4140 static_cast<unsigned int>(RTLDependInfoFields::Len)));4141 CGF.EmitStoreOfScalar(Size, LenLVal);4142 // deps[i].flags = <Dependencies[i].first>;4143 RTLDependenceKindTy DepKind = translateDependencyKind(Data.DepKind);4144 LValue FlagsLVal = CGF.EmitLValueForField(4145 Base,4146 *std::next(KmpDependInfoRD->field_begin(),4147 static_cast<unsigned int>(RTLDependInfoFields::Flags)));4148 CGF.EmitStoreOfScalar(4149 llvm::ConstantInt::get(LLVMFlagsTy, static_cast<unsigned int>(DepKind)),4150 FlagsLVal);4151 if (unsigned *P = dyn_cast<unsigned *>(Pos)) {4152 ++(*P);4153 } else {4154 LValue &PosLVal = *cast<LValue *>(Pos);4155 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());4156 Idx = CGF.Builder.CreateNUWAdd(Idx,4157 llvm::ConstantInt::get(Idx->getType(), 1));4158 CGF.EmitStoreOfScalar(Idx, PosLVal);4159 }4160 }4161}4162 4163SmallVector<llvm::Value *, 4> CGOpenMPRuntime::emitDepobjElementsSizes(4164 CodeGenFunction &CGF, QualType &KmpDependInfoTy,4165 const OMPTaskDataTy::DependData &Data) {4166 assert(Data.DepKind == OMPC_DEPEND_depobj &&4167 "Expected depobj dependency kind.");4168 SmallVector<llvm::Value *, 4> Sizes;4169 SmallVector<LValue, 4> SizeLVals;4170 ASTContext &C = CGF.getContext();4171 {4172 OMPIteratorGeneratorScope IteratorScope(4173 CGF, cast_or_null<OMPIteratorExpr>(4174 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()4175 : nullptr));4176 for (const Expr *E : Data.DepExprs) {4177 llvm::Value *NumDeps;4178 LValue Base;4179 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());4180 std::tie(NumDeps, Base) =4181 getDepobjElements(CGF, DepobjLVal, E->getExprLoc());4182 LValue NumLVal = CGF.MakeAddrLValue(4183 CGF.CreateMemTemp(C.getUIntPtrType(), "depobj.size.addr"),4184 C.getUIntPtrType());4185 CGF.Builder.CreateStore(llvm::ConstantInt::get(CGF.IntPtrTy, 0),4186 NumLVal.getAddress());4187 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(NumLVal, E->getExprLoc());4188 llvm::Value *Add = CGF.Builder.CreateNUWAdd(PrevVal, NumDeps);4189 CGF.EmitStoreOfScalar(Add, NumLVal);4190 SizeLVals.push_back(NumLVal);4191 }4192 }4193 for (unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {4194 llvm::Value *Size =4195 CGF.EmitLoadOfScalar(SizeLVals[I], Data.DepExprs[I]->getExprLoc());4196 Sizes.push_back(Size);4197 }4198 return Sizes;4199}4200 4201void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF,4202 QualType &KmpDependInfoTy,4203 LValue PosLVal,4204 const OMPTaskDataTy::DependData &Data,4205 Address DependenciesArray) {4206 assert(Data.DepKind == OMPC_DEPEND_depobj &&4207 "Expected depobj dependency kind.");4208 llvm::Value *ElSize = CGF.getTypeSize(KmpDependInfoTy);4209 {4210 OMPIteratorGeneratorScope IteratorScope(4211 CGF, cast_or_null<OMPIteratorExpr>(4212 Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()4213 : nullptr));4214 for (const Expr *E : Data.DepExprs) {4215 llvm::Value *NumDeps;4216 LValue Base;4217 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts());4218 std::tie(NumDeps, Base) =4219 getDepobjElements(CGF, DepobjLVal, E->getExprLoc());4220 4221 // memcopy dependency data.4222 llvm::Value *Size = CGF.Builder.CreateNUWMul(4223 ElSize,4224 CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false));4225 llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());4226 Address DepAddr = CGF.Builder.CreateGEP(CGF, DependenciesArray, Pos);4227 CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(), Size);4228 4229 // Increase pos.4230 // pos += size;4231 llvm::Value *Add = CGF.Builder.CreateNUWAdd(Pos, NumDeps);4232 CGF.EmitStoreOfScalar(Add, PosLVal);4233 }4234 }4235}4236 4237std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause(4238 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies,4239 SourceLocation Loc) {4240 if (llvm::all_of(Dependencies, [](const OMPTaskDataTy::DependData &D) {4241 return D.DepExprs.empty();4242 }))4243 return std::make_pair(nullptr, Address::invalid());4244 // Process list of dependencies.4245 ASTContext &C = CGM.getContext();4246 Address DependenciesArray = Address::invalid();4247 llvm::Value *NumOfElements = nullptr;4248 unsigned NumDependencies = std::accumulate(4249 Dependencies.begin(), Dependencies.end(), 0,4250 [](unsigned V, const OMPTaskDataTy::DependData &D) {4251 return D.DepKind == OMPC_DEPEND_depobj4252 ? V4253 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));4254 });4255 QualType FlagsTy;4256 getDependTypes(C, KmpDependInfoTy, FlagsTy);4257 bool HasDepobjDeps = false;4258 bool HasRegularWithIterators = false;4259 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.IntPtrTy, 0);4260 llvm::Value *NumOfRegularWithIterators =4261 llvm::ConstantInt::get(CGF.IntPtrTy, 0);4262 // Calculate number of depobj dependencies and regular deps with the4263 // iterators.4264 for (const OMPTaskDataTy::DependData &D : Dependencies) {4265 if (D.DepKind == OMPC_DEPEND_depobj) {4266 SmallVector<llvm::Value *, 4> Sizes =4267 emitDepobjElementsSizes(CGF, KmpDependInfoTy, D);4268 for (llvm::Value *Size : Sizes) {4269 NumOfDepobjElements =4270 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, Size);4271 }4272 HasDepobjDeps = true;4273 continue;4274 }4275 // Include number of iterations, if any.4276 4277 if (const auto *IE = cast_or_null<OMPIteratorExpr>(D.IteratorExpr)) {4278 llvm::Value *ClauseIteratorSpace =4279 llvm::ConstantInt::get(CGF.IntPtrTy, 1);4280 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {4281 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);4282 Sz = CGF.Builder.CreateIntCast(Sz, CGF.IntPtrTy, /*isSigned=*/false);4283 ClauseIteratorSpace = CGF.Builder.CreateNUWMul(Sz, ClauseIteratorSpace);4284 }4285 llvm::Value *NumClauseDeps = CGF.Builder.CreateNUWMul(4286 ClauseIteratorSpace,4287 llvm::ConstantInt::get(CGF.IntPtrTy, D.DepExprs.size()));4288 NumOfRegularWithIterators =4289 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps);4290 HasRegularWithIterators = true;4291 continue;4292 }4293 }4294 4295 QualType KmpDependInfoArrayTy;4296 if (HasDepobjDeps || HasRegularWithIterators) {4297 NumOfElements = llvm::ConstantInt::get(CGM.IntPtrTy, NumDependencies,4298 /*isSigned=*/false);4299 if (HasDepobjDeps) {4300 NumOfElements =4301 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements);4302 }4303 if (HasRegularWithIterators) {4304 NumOfElements =4305 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);4306 }4307 auto *OVE = new (C) OpaqueValueExpr(4308 Loc, C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0),4309 VK_PRValue);4310 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,4311 RValue::get(NumOfElements));4312 KmpDependInfoArrayTy =4313 C.getVariableArrayType(KmpDependInfoTy, OVE, ArraySizeModifier::Normal,4314 /*IndexTypeQuals=*/0);4315 // CGF.EmitVariablyModifiedType(KmpDependInfoArrayTy);4316 // Properly emit variable-sized array.4317 auto *PD = ImplicitParamDecl::Create(C, KmpDependInfoArrayTy,4318 ImplicitParamKind::Other);4319 CGF.EmitVarDecl(*PD);4320 DependenciesArray = CGF.GetAddrOfLocalVar(PD);4321 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty,4322 /*isSigned=*/false);4323 } else {4324 KmpDependInfoArrayTy = C.getConstantArrayType(4325 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies), nullptr,4326 ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);4327 DependenciesArray =4328 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");4329 DependenciesArray = CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0);4330 NumOfElements = llvm::ConstantInt::get(CGM.Int32Ty, NumDependencies,4331 /*isSigned=*/false);4332 }4333 unsigned Pos = 0;4334 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {4335 if (Dep.DepKind == OMPC_DEPEND_depobj || Dep.IteratorExpr)4336 continue;4337 emitDependData(CGF, KmpDependInfoTy, &Pos, Dep, DependenciesArray);4338 }4339 // Copy regular dependencies with iterators.4340 LValue PosLVal = CGF.MakeAddrLValue(4341 CGF.CreateMemTemp(C.getSizeType(), "dep.counter.addr"), C.getSizeType());4342 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal);4343 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {4344 if (Dep.DepKind == OMPC_DEPEND_depobj || !Dep.IteratorExpr)4345 continue;4346 emitDependData(CGF, KmpDependInfoTy, &PosLVal, Dep, DependenciesArray);4347 }4348 // Copy final depobj arrays without iterators.4349 if (HasDepobjDeps) {4350 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {4351 if (Dep.DepKind != OMPC_DEPEND_depobj)4352 continue;4353 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Dep, DependenciesArray);4354 }4355 }4356 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(4357 DependenciesArray, CGF.VoidPtrTy, CGF.Int8Ty);4358 return std::make_pair(NumOfElements, DependenciesArray);4359}4360 4361Address CGOpenMPRuntime::emitDepobjDependClause(4362 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies,4363 SourceLocation Loc) {4364 if (Dependencies.DepExprs.empty())4365 return Address::invalid();4366 // Process list of dependencies.4367 ASTContext &C = CGM.getContext();4368 Address DependenciesArray = Address::invalid();4369 unsigned NumDependencies = Dependencies.DepExprs.size();4370 QualType FlagsTy;4371 getDependTypes(C, KmpDependInfoTy, FlagsTy);4372 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();4373 4374 llvm::Value *Size;4375 // Define type kmp_depend_info[<Dependencies.size()>];4376 // For depobj reserve one extra element to store the number of elements.4377 // It is required to handle depobj(x) update(in) construct.4378 // kmp_depend_info[<Dependencies.size()>] deps;4379 llvm::Value *NumDepsVal;4380 CharUnits Align = C.getTypeAlignInChars(KmpDependInfoTy);4381 if (const auto *IE =4382 cast_or_null<OMPIteratorExpr>(Dependencies.IteratorExpr)) {4383 NumDepsVal = llvm::ConstantInt::get(CGF.SizeTy, 1);4384 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {4385 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper);4386 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false);4387 NumDepsVal = CGF.Builder.CreateNUWMul(NumDepsVal, Sz);4388 }4389 Size = CGF.Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.SizeTy, 1),4390 NumDepsVal);4391 CharUnits SizeInBytes =4392 C.getTypeSizeInChars(KmpDependInfoTy).alignTo(Align);4393 llvm::Value *RecSize = CGM.getSize(SizeInBytes);4394 Size = CGF.Builder.CreateNUWMul(Size, RecSize);4395 NumDepsVal =4396 CGF.Builder.CreateIntCast(NumDepsVal, CGF.IntPtrTy, /*isSigned=*/false);4397 } else {4398 QualType KmpDependInfoArrayTy = C.getConstantArrayType(4399 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies + 1),4400 nullptr, ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);4401 CharUnits Sz = C.getTypeSizeInChars(KmpDependInfoArrayTy);4402 Size = CGM.getSize(Sz.alignTo(Align));4403 NumDepsVal = llvm::ConstantInt::get(CGF.IntPtrTy, NumDependencies);4404 }4405 // Need to allocate on the dynamic memory.4406 llvm::Value *ThreadID = getThreadID(CGF, Loc);4407 // Use default allocator.4408 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);4409 llvm::Value *Args[] = {ThreadID, Size, Allocator};4410 4411 llvm::Value *Addr =4412 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(4413 CGM.getModule(), OMPRTL___kmpc_alloc),4414 Args, ".dep.arr.addr");4415 llvm::Type *KmpDependInfoLlvmTy = CGF.ConvertTypeForMem(KmpDependInfoTy);4416 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(4417 Addr, CGF.Builder.getPtrTy(0));4418 DependenciesArray = Address(Addr, KmpDependInfoLlvmTy, Align);4419 // Write number of elements in the first element of array for depobj.4420 LValue Base = CGF.MakeAddrLValue(DependenciesArray, KmpDependInfoTy);4421 // deps[i].base_addr = NumDependencies;4422 LValue BaseAddrLVal = CGF.EmitLValueForField(4423 Base,4424 *std::next(KmpDependInfoRD->field_begin(),4425 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));4426 CGF.EmitStoreOfScalar(NumDepsVal, BaseAddrLVal);4427 llvm::PointerUnion<unsigned *, LValue *> Pos;4428 unsigned Idx = 1;4429 LValue PosLVal;4430 if (Dependencies.IteratorExpr) {4431 PosLVal = CGF.MakeAddrLValue(4432 CGF.CreateMemTemp(C.getSizeType(), "iterator.counter.addr"),4433 C.getSizeType());4434 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Idx), PosLVal,4435 /*IsInit=*/true);4436 Pos = &PosLVal;4437 } else {4438 Pos = &Idx;4439 }4440 emitDependData(CGF, KmpDependInfoTy, Pos, Dependencies, DependenciesArray);4441 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(4442 CGF.Builder.CreateConstGEP(DependenciesArray, 1), CGF.VoidPtrTy,4443 CGF.Int8Ty);4444 return DependenciesArray;4445}4446 4447void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,4448 SourceLocation Loc) {4449 ASTContext &C = CGM.getContext();4450 QualType FlagsTy;4451 getDependTypes(C, KmpDependInfoTy, FlagsTy);4452 LValue Base = CGF.EmitLoadOfPointerLValue(DepobjLVal.getAddress(),4453 C.VoidPtrTy.castAs<PointerType>());4454 QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy);4455 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(4456 Base.getAddress(), CGF.ConvertTypeForMem(KmpDependInfoPtrTy),4457 CGF.ConvertTypeForMem(KmpDependInfoTy));4458 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(4459 Addr.getElementType(), Addr.emitRawPointer(CGF),4460 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));4461 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr,4462 CGF.VoidPtrTy);4463 llvm::Value *ThreadID = getThreadID(CGF, Loc);4464 // Use default allocator.4465 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);4466 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};4467 4468 // _kmpc_free(gtid, addr, nullptr);4469 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(4470 CGM.getModule(), OMPRTL___kmpc_free),4471 Args);4472}4473 4474void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,4475 OpenMPDependClauseKind NewDepKind,4476 SourceLocation Loc) {4477 ASTContext &C = CGM.getContext();4478 QualType FlagsTy;4479 getDependTypes(C, KmpDependInfoTy, FlagsTy);4480 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();4481 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);4482 llvm::Value *NumDeps;4483 LValue Base;4484 std::tie(NumDeps, Base) = getDepobjElements(CGF, DepobjLVal, Loc);4485 4486 Address Begin = Base.getAddress();4487 // Cast from pointer to array type to pointer to single element.4488 llvm::Value *End = CGF.Builder.CreateGEP(Begin.getElementType(),4489 Begin.emitRawPointer(CGF), NumDeps);4490 // The basic structure here is a while-do loop.4491 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body");4492 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done");4493 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();4494 CGF.EmitBlock(BodyBB);4495 llvm::PHINode *ElementPHI =4496 CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast");4497 ElementPHI->addIncoming(Begin.emitRawPointer(CGF), EntryBB);4498 Begin = Begin.withPointer(ElementPHI, KnownNonNull);4499 Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(),4500 Base.getTBAAInfo());4501 // deps[i].flags = NewDepKind;4502 RTLDependenceKindTy DepKind = translateDependencyKind(NewDepKind);4503 LValue FlagsLVal = CGF.EmitLValueForField(4504 Base, *std::next(KmpDependInfoRD->field_begin(),4505 static_cast<unsigned int>(RTLDependInfoFields::Flags)));4506 CGF.EmitStoreOfScalar(4507 llvm::ConstantInt::get(LLVMFlagsTy, static_cast<unsigned int>(DepKind)),4508 FlagsLVal);4509 4510 // Shift the address forward by one element.4511 llvm::Value *ElementNext =4512 CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext")4513 .emitRawPointer(CGF);4514 ElementPHI->addIncoming(ElementNext, CGF.Builder.GetInsertBlock());4515 llvm::Value *IsEmpty =4516 CGF.Builder.CreateICmpEQ(ElementNext, End, "omp.isempty");4517 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);4518 // Done.4519 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);4520}4521 4522void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,4523 const OMPExecutableDirective &D,4524 llvm::Function *TaskFunction,4525 QualType SharedsTy, Address Shareds,4526 const Expr *IfCond,4527 const OMPTaskDataTy &Data) {4528 if (!CGF.HaveInsertPoint())4529 return;4530 4531 TaskResultTy Result =4532 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);4533 llvm::Value *NewTask = Result.NewTask;4534 llvm::Function *TaskEntry = Result.TaskEntry;4535 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;4536 LValue TDBase = Result.TDBase;4537 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;4538 // Process list of dependences.4539 Address DependenciesArray = Address::invalid();4540 llvm::Value *NumOfElements;4541 std::tie(NumOfElements, DependenciesArray) =4542 emitDependClause(CGF, Data.Dependences, Loc);4543 4544 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()4545 // libcall.4546 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,4547 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,4548 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence4549 // list is not empty4550 llvm::Value *ThreadID = getThreadID(CGF, Loc);4551 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);4552 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };4553 llvm::Value *DepTaskArgs[7];4554 if (!Data.Dependences.empty()) {4555 DepTaskArgs[0] = UpLoc;4556 DepTaskArgs[1] = ThreadID;4557 DepTaskArgs[2] = NewTask;4558 DepTaskArgs[3] = NumOfElements;4559 DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF);4560 DepTaskArgs[5] = CGF.Builder.getInt32(0);4561 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);4562 }4563 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, &TaskArgs,4564 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {4565 if (!Data.Tied) {4566 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);4567 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);4568 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);4569 }4570 if (!Data.Dependences.empty()) {4571 CGF.EmitRuntimeCall(4572 OMPBuilder.getOrCreateRuntimeFunction(4573 CGM.getModule(), OMPRTL___kmpc_omp_task_with_deps),4574 DepTaskArgs);4575 } else {4576 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(4577 CGM.getModule(), OMPRTL___kmpc_omp_task),4578 TaskArgs);4579 }4580 // Check if parent region is untied and build return for untied task;4581 if (auto *Region =4582 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))4583 Region->emitUntiedSwitch(CGF);4584 };4585 4586 llvm::Value *DepWaitTaskArgs[7];4587 if (!Data.Dependences.empty()) {4588 DepWaitTaskArgs[0] = UpLoc;4589 DepWaitTaskArgs[1] = ThreadID;4590 DepWaitTaskArgs[2] = NumOfElements;4591 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);4592 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);4593 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);4594 DepWaitTaskArgs[6] =4595 llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);4596 }4597 auto &M = CGM.getModule();4598 auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,4599 TaskEntry, &Data, &DepWaitTaskArgs,4600 Loc](CodeGenFunction &CGF, PrePostActionTy &) {4601 CodeGenFunction::RunCleanupsScope LocalScope(CGF);4602 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,4603 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int324604 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info4605 // is specified.4606 if (!Data.Dependences.empty())4607 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(4608 M, OMPRTL___kmpc_omp_taskwait_deps_51),4609 DepWaitTaskArgs);4610 // Call proxy_task_entry(gtid, new_task);4611 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,4612 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {4613 Action.Enter(CGF);4614 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};4615 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,4616 OutlinedFnArgs);4617 };4618 4619 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,4620 // kmp_task_t *new_task);4621 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,4622 // kmp_task_t *new_task);4623 RegionCodeGenTy RCG(CodeGen);4624 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(4625 M, OMPRTL___kmpc_omp_task_begin_if0),4626 TaskArgs,4627 OMPBuilder.getOrCreateRuntimeFunction(4628 M, OMPRTL___kmpc_omp_task_complete_if0),4629 TaskArgs);4630 RCG.setAction(Action);4631 RCG(CGF);4632 };4633 4634 if (IfCond) {4635 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);4636 } else {4637 RegionCodeGenTy ThenRCG(ThenCodeGen);4638 ThenRCG(CGF);4639 }4640}4641 4642void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,4643 const OMPLoopDirective &D,4644 llvm::Function *TaskFunction,4645 QualType SharedsTy, Address Shareds,4646 const Expr *IfCond,4647 const OMPTaskDataTy &Data) {4648 if (!CGF.HaveInsertPoint())4649 return;4650 TaskResultTy Result =4651 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);4652 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()4653 // libcall.4654 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int4655 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int4656 // sched, kmp_uint64 grainsize, void *task_dup);4657 llvm::Value *ThreadID = getThreadID(CGF, Loc);4658 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);4659 llvm::Value *IfVal;4660 if (IfCond) {4661 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,4662 /*isSigned=*/true);4663 } else {4664 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);4665 }4666 4667 LValue LBLVal = CGF.EmitLValueForField(4668 Result.TDBase,4669 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));4670 const auto *LBVar =4671 cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());4672 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(),4673 /*IsInitializer=*/true);4674 LValue UBLVal = CGF.EmitLValueForField(4675 Result.TDBase,4676 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));4677 const auto *UBVar =4678 cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());4679 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(),4680 /*IsInitializer=*/true);4681 LValue StLVal = CGF.EmitLValueForField(4682 Result.TDBase,4683 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));4684 const auto *StVar =4685 cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());4686 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(),4687 /*IsInitializer=*/true);4688 // Store reductions address.4689 LValue RedLVal = CGF.EmitLValueForField(4690 Result.TDBase,4691 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));4692 if (Data.Reductions) {4693 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);4694 } else {4695 CGF.EmitNullInitialization(RedLVal.getAddress(),4696 CGF.getContext().VoidPtrTy);4697 }4698 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };4699 llvm::SmallVector<llvm::Value *, 12> TaskArgs{4700 UpLoc,4701 ThreadID,4702 Result.NewTask,4703 IfVal,4704 LBLVal.getPointer(CGF),4705 UBLVal.getPointer(CGF),4706 CGF.EmitLoadOfScalar(StLVal, Loc),4707 llvm::ConstantInt::getSigned(4708 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler4709 llvm::ConstantInt::getSigned(4710 CGF.IntTy, Data.Schedule.getPointer()4711 ? Data.Schedule.getInt() ? NumTasks : Grainsize4712 : NoSchedule),4713 Data.Schedule.getPointer()4714 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,4715 /*isSigned=*/false)4716 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0)};4717 if (Data.HasModifier)4718 TaskArgs.push_back(llvm::ConstantInt::get(CGF.Int32Ty, 1));4719 4720 TaskArgs.push_back(Result.TaskDupFn4721 ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(4722 Result.TaskDupFn, CGF.VoidPtrTy)4723 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy));4724 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(4725 CGM.getModule(), Data.HasModifier4726 ? OMPRTL___kmpc_taskloop_54727 : OMPRTL___kmpc_taskloop),4728 TaskArgs);4729}4730 4731/// Emit reduction operation for each element of array (required for4732/// array sections) LHS op = RHS.4733/// \param Type Type of array.4734/// \param LHSVar Variable on the left side of the reduction operation4735/// (references element of array in original variable).4736/// \param RHSVar Variable on the right side of the reduction operation4737/// (references element of array in original variable).4738/// \param RedOpGen Generator of reduction operation with use of LHSVar and4739/// RHSVar.4740static void EmitOMPAggregateReduction(4741 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,4742 const VarDecl *RHSVar,4743 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,4744 const Expr *, const Expr *)> &RedOpGen,4745 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,4746 const Expr *UpExpr = nullptr) {4747 // Perform element-by-element initialization.4748 QualType ElementTy;4749 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);4750 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);4751 4752 // Drill down to the base element type on both arrays.4753 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();4754 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);4755 4756 llvm::Value *RHSBegin = RHSAddr.emitRawPointer(CGF);4757 llvm::Value *LHSBegin = LHSAddr.emitRawPointer(CGF);4758 // Cast from pointer to array type to pointer to single element.4759 llvm::Value *LHSEnd =4760 CGF.Builder.CreateGEP(LHSAddr.getElementType(), LHSBegin, NumElements);4761 // The basic structure here is a while-do loop.4762 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");4763 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");4764 llvm::Value *IsEmpty =4765 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");4766 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);4767 4768 // Enter the loop body, making that address the current address.4769 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();4770 CGF.EmitBlock(BodyBB);4771 4772 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);4773 4774 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(4775 RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");4776 RHSElementPHI->addIncoming(RHSBegin, EntryBB);4777 Address RHSElementCurrent(4778 RHSElementPHI, RHSAddr.getElementType(),4779 RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));4780 4781 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(4782 LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");4783 LHSElementPHI->addIncoming(LHSBegin, EntryBB);4784 Address LHSElementCurrent(4785 LHSElementPHI, LHSAddr.getElementType(),4786 LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));4787 4788 // Emit copy.4789 CodeGenFunction::OMPPrivateScope Scope(CGF);4790 Scope.addPrivate(LHSVar, LHSElementCurrent);4791 Scope.addPrivate(RHSVar, RHSElementCurrent);4792 Scope.Privatize();4793 RedOpGen(CGF, XExpr, EExpr, UpExpr);4794 Scope.ForceCleanup();4795 4796 // Shift the address forward by one element.4797 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(4798 LHSAddr.getElementType(), LHSElementPHI, /*Idx0=*/1,4799 "omp.arraycpy.dest.element");4800 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(4801 RHSAddr.getElementType(), RHSElementPHI, /*Idx0=*/1,4802 "omp.arraycpy.src.element");4803 // Check whether we've reached the end.4804 llvm::Value *Done =4805 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");4806 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);4807 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());4808 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());4809 4810 // Done.4811 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);4812}4813 4814/// Emit reduction combiner. If the combiner is a simple expression emit it as4815/// is, otherwise consider it as combiner of UDR decl and emit it as a call of4816/// UDR combiner function.4817static void emitReductionCombiner(CodeGenFunction &CGF,4818 const Expr *ReductionOp) {4819 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))4820 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))4821 if (const auto *DRE =4822 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))4823 if (const auto *DRD =4824 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {4825 std::pair<llvm::Function *, llvm::Function *> Reduction =4826 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);4827 RValue Func = RValue::get(Reduction.first);4828 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);4829 CGF.EmitIgnoredExpr(ReductionOp);4830 return;4831 }4832 CGF.EmitIgnoredExpr(ReductionOp);4833}4834 4835llvm::Function *CGOpenMPRuntime::emitReductionFunction(4836 StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType,4837 ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,4838 ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps) {4839 ASTContext &C = CGM.getContext();4840 4841 // void reduction_func(void *LHSArg, void *RHSArg);4842 FunctionArgList Args;4843 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,4844 ImplicitParamKind::Other);4845 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,4846 ImplicitParamKind::Other);4847 Args.push_back(&LHSArg);4848 Args.push_back(&RHSArg);4849 const auto &CGFI =4850 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);4851 std::string Name = getReductionFuncName(ReducerName);4852 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),4853 llvm::GlobalValue::InternalLinkage, Name,4854 &CGM.getModule());4855 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);4856 Fn->setDoesNotRecurse();4857 CodeGenFunction CGF(CGM);4858 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);4859 4860 // Dst = (void*[n])(LHSArg);4861 // Src = (void*[n])(RHSArg);4862 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(4863 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),4864 CGF.Builder.getPtrTy(0)),4865 ArgsElemType, CGF.getPointerAlign());4866 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(4867 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),4868 CGF.Builder.getPtrTy(0)),4869 ArgsElemType, CGF.getPointerAlign());4870 4871 // ...4872 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);4873 // ...4874 CodeGenFunction::OMPPrivateScope Scope(CGF);4875 const auto *IPriv = Privates.begin();4876 unsigned Idx = 0;4877 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {4878 const auto *RHSVar =4879 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());4880 Scope.addPrivate(RHSVar, emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar));4881 const auto *LHSVar =4882 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());4883 Scope.addPrivate(LHSVar, emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar));4884 QualType PrivTy = (*IPriv)->getType();4885 if (PrivTy->isVariablyModifiedType()) {4886 // Get array size and emit VLA type.4887 ++Idx;4888 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx);4889 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);4890 const VariableArrayType *VLA =4891 CGF.getContext().getAsVariableArrayType(PrivTy);4892 const auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());4893 CodeGenFunction::OpaqueValueMapping OpaqueMap(4894 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));4895 CGF.EmitVariablyModifiedType(PrivTy);4896 }4897 }4898 Scope.Privatize();4899 IPriv = Privates.begin();4900 const auto *ILHS = LHSExprs.begin();4901 const auto *IRHS = RHSExprs.begin();4902 for (const Expr *E : ReductionOps) {4903 if ((*IPriv)->getType()->isArrayType()) {4904 // Emit reduction for array section.4905 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());4906 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());4907 EmitOMPAggregateReduction(4908 CGF, (*IPriv)->getType(), LHSVar, RHSVar,4909 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {4910 emitReductionCombiner(CGF, E);4911 });4912 } else {4913 // Emit reduction for array subscript or single variable.4914 emitReductionCombiner(CGF, E);4915 }4916 ++IPriv;4917 ++ILHS;4918 ++IRHS;4919 }4920 Scope.ForceCleanup();4921 CGF.FinishFunction();4922 return Fn;4923}4924 4925void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,4926 const Expr *ReductionOp,4927 const Expr *PrivateRef,4928 const DeclRefExpr *LHS,4929 const DeclRefExpr *RHS) {4930 if (PrivateRef->getType()->isArrayType()) {4931 // Emit reduction for array section.4932 const auto *LHSVar = cast<VarDecl>(LHS->getDecl());4933 const auto *RHSVar = cast<VarDecl>(RHS->getDecl());4934 EmitOMPAggregateReduction(4935 CGF, PrivateRef->getType(), LHSVar, RHSVar,4936 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {4937 emitReductionCombiner(CGF, ReductionOp);4938 });4939 } else {4940 // Emit reduction for array subscript or single variable.4941 emitReductionCombiner(CGF, ReductionOp);4942 }4943}4944 4945static std::string generateUniqueName(CodeGenModule &CGM,4946 llvm::StringRef Prefix, const Expr *Ref);4947 4948void CGOpenMPRuntime::emitPrivateReduction(4949 CodeGenFunction &CGF, SourceLocation Loc, const Expr *Privates,4950 const Expr *LHSExprs, const Expr *RHSExprs, const Expr *ReductionOps) {4951 4952 // Create a shared global variable (__shared_reduction_var) to accumulate the4953 // final result.4954 //4955 // Call __kmpc_barrier to synchronize threads before initialization.4956 //4957 // The master thread (thread_id == 0) initializes __shared_reduction_var4958 // with the identity value or initializer.4959 //4960 // Call __kmpc_barrier to synchronize before combining.4961 // For each i:4962 // - Thread enters critical section.4963 // - Reads its private value from LHSExprs[i].4964 // - Updates __shared_reduction_var[i] = RedOp_i(__shared_reduction_var[i],4965 // Privates[i]).4966 // - Exits critical section.4967 //4968 // Call __kmpc_barrier after combining.4969 //4970 // Each thread copies __shared_reduction_var[i] back to RHSExprs[i].4971 //4972 // Final __kmpc_barrier to synchronize after broadcasting4973 QualType PrivateType = Privates->getType();4974 llvm::Type *LLVMType = CGF.ConvertTypeForMem(PrivateType);4975 4976 const OMPDeclareReductionDecl *UDR = getReductionInit(ReductionOps);4977 std::string ReductionVarNameStr;4978 if (const auto *DRE = dyn_cast<DeclRefExpr>(Privates->IgnoreParenCasts()))4979 ReductionVarNameStr =4980 generateUniqueName(CGM, DRE->getDecl()->getNameAsString(), Privates);4981 else4982 ReductionVarNameStr = "unnamed_priv_var";4983 4984 // Create an internal shared variable4985 std::string SharedName =4986 CGM.getOpenMPRuntime().getName({"internal_pivate_", ReductionVarNameStr});4987 llvm::GlobalVariable *SharedVar = OMPBuilder.getOrCreateInternalVariable(4988 LLVMType, ".omp.reduction." + SharedName);4989 4990 SharedVar->setAlignment(4991 llvm::MaybeAlign(CGF.getContext().getTypeAlign(PrivateType) / 8));4992 4993 Address SharedResult =4994 CGF.MakeNaturalAlignRawAddrLValue(SharedVar, PrivateType).getAddress();4995 4996 llvm::Value *ThreadId = getThreadID(CGF, Loc);4997 llvm::Value *BarrierLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);4998 llvm::Value *BarrierArgs[] = {BarrierLoc, ThreadId};4999 5000 llvm::BasicBlock *InitBB = CGF.createBasicBlock("init");5001 llvm::BasicBlock *InitEndBB = CGF.createBasicBlock("init.end");5002 5003 llvm::Value *IsWorker = CGF.Builder.CreateICmpEQ(5004 ThreadId, llvm::ConstantInt::get(ThreadId->getType(), 0));5005 CGF.Builder.CreateCondBr(IsWorker, InitBB, InitEndBB);5006 5007 CGF.EmitBlock(InitBB);5008 5009 auto EmitSharedInit = [&]() {5010 if (UDR) { // Check if it's a User-Defined Reduction5011 if (const Expr *UDRInitExpr = UDR->getInitializer()) {5012 std::pair<llvm::Function *, llvm::Function *> FnPair =5013 getUserDefinedReduction(UDR);5014 llvm::Function *InitializerFn = FnPair.second;5015 if (InitializerFn) {5016 if (const auto *CE =5017 dyn_cast<CallExpr>(UDRInitExpr->IgnoreParenImpCasts())) {5018 const auto *OutDRE = cast<DeclRefExpr>(5019 cast<UnaryOperator>(CE->getArg(0)->IgnoreParenImpCasts())5020 ->getSubExpr());5021 const VarDecl *OutVD = cast<VarDecl>(OutDRE->getDecl());5022 5023 CodeGenFunction::OMPPrivateScope LocalScope(CGF);5024 LocalScope.addPrivate(OutVD, SharedResult);5025 5026 (void)LocalScope.Privatize();5027 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(5028 CE->getCallee()->IgnoreParenImpCasts())) {5029 CodeGenFunction::OpaqueValueMapping OpaqueMap(5030 CGF, OVE, RValue::get(InitializerFn));5031 CGF.EmitIgnoredExpr(CE);5032 } else {5033 CGF.EmitAnyExprToMem(UDRInitExpr, SharedResult,5034 PrivateType.getQualifiers(),5035 /*IsInitializer=*/true);5036 }5037 } else {5038 CGF.EmitAnyExprToMem(UDRInitExpr, SharedResult,5039 PrivateType.getQualifiers(),5040 /*IsInitializer=*/true);5041 }5042 } else {5043 CGF.EmitAnyExprToMem(UDRInitExpr, SharedResult,5044 PrivateType.getQualifiers(),5045 /*IsInitializer=*/true);5046 }5047 } else {5048 // EmitNullInitialization handles default construction for C++ classes5049 // and zeroing for scalars, which is a reasonable default.5050 CGF.EmitNullInitialization(SharedResult, PrivateType);5051 }5052 return; // UDR initialization handled5053 }5054 if (const auto *DRE = dyn_cast<DeclRefExpr>(Privates)) {5055 if (const auto *VD = dyn_cast<VarDecl>(DRE->getDecl())) {5056 if (const Expr *InitExpr = VD->getInit()) {5057 CGF.EmitAnyExprToMem(InitExpr, SharedResult,5058 PrivateType.getQualifiers(), true);5059 return;5060 }5061 }5062 }5063 CGF.EmitNullInitialization(SharedResult, PrivateType);5064 };5065 EmitSharedInit();5066 CGF.Builder.CreateBr(InitEndBB);5067 CGF.EmitBlock(InitEndBB);5068 5069 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(5070 CGM.getModule(), OMPRTL___kmpc_barrier),5071 BarrierArgs);5072 5073 const Expr *ReductionOp = ReductionOps;5074 const OMPDeclareReductionDecl *CurrentUDR = getReductionInit(ReductionOp);5075 LValue SharedLV = CGF.MakeAddrLValue(SharedResult, PrivateType);5076 LValue LHSLV = CGF.EmitLValue(Privates);5077 5078 auto EmitCriticalReduction = [&](auto ReductionGen) {5079 std::string CriticalName = getName({"reduction_critical"});5080 emitCriticalRegion(CGF, CriticalName, ReductionGen, Loc);5081 };5082 5083 if (CurrentUDR) {5084 // Handle user-defined reduction.5085 auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {5086 Action.Enter(CGF);5087 std::pair<llvm::Function *, llvm::Function *> FnPair =5088 getUserDefinedReduction(CurrentUDR);5089 if (FnPair.first) {5090 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp)) {5091 const auto *OutDRE = cast<DeclRefExpr>(5092 cast<UnaryOperator>(CE->getArg(0)->IgnoreParenImpCasts())5093 ->getSubExpr());5094 const auto *InDRE = cast<DeclRefExpr>(5095 cast<UnaryOperator>(CE->getArg(1)->IgnoreParenImpCasts())5096 ->getSubExpr());5097 CodeGenFunction::OMPPrivateScope LocalScope(CGF);5098 LocalScope.addPrivate(cast<VarDecl>(OutDRE->getDecl()),5099 SharedLV.getAddress());5100 LocalScope.addPrivate(cast<VarDecl>(InDRE->getDecl()),5101 LHSLV.getAddress());5102 (void)LocalScope.Privatize();5103 emitReductionCombiner(CGF, ReductionOp);5104 }5105 }5106 };5107 EmitCriticalReduction(ReductionGen);5108 } else {5109 // Handle built-in reduction operations.5110#ifndef NDEBUG5111 const Expr *ReductionClauseExpr = ReductionOp->IgnoreParenCasts();5112 if (const auto *Cleanup = dyn_cast<ExprWithCleanups>(ReductionClauseExpr))5113 ReductionClauseExpr = Cleanup->getSubExpr()->IgnoreParenCasts();5114 5115 const Expr *AssignRHS = nullptr;5116 if (const auto *BinOp = dyn_cast<BinaryOperator>(ReductionClauseExpr)) {5117 if (BinOp->getOpcode() == BO_Assign)5118 AssignRHS = BinOp->getRHS();5119 } else if (const auto *OpCall =5120 dyn_cast<CXXOperatorCallExpr>(ReductionClauseExpr)) {5121 if (OpCall->getOperator() == OO_Equal)5122 AssignRHS = OpCall->getArg(1);5123 }5124 5125 assert(AssignRHS &&5126 "Private Variable Reduction : Invalid ReductionOp expression");5127#endif5128 5129 auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {5130 Action.Enter(CGF);5131 const auto *OmpOutDRE =5132 dyn_cast<DeclRefExpr>(LHSExprs->IgnoreParenImpCasts());5133 const auto *OmpInDRE =5134 dyn_cast<DeclRefExpr>(RHSExprs->IgnoreParenImpCasts());5135 assert(5136 OmpOutDRE && OmpInDRE &&5137 "Private Variable Reduction : LHSExpr/RHSExpr must be DeclRefExprs");5138 const VarDecl *OmpOutVD = cast<VarDecl>(OmpOutDRE->getDecl());5139 const VarDecl *OmpInVD = cast<VarDecl>(OmpInDRE->getDecl());5140 CodeGenFunction::OMPPrivateScope LocalScope(CGF);5141 LocalScope.addPrivate(OmpOutVD, SharedLV.getAddress());5142 LocalScope.addPrivate(OmpInVD, LHSLV.getAddress());5143 (void)LocalScope.Privatize();5144 // Emit the actual reduction operation5145 CGF.EmitIgnoredExpr(ReductionOp);5146 };5147 EmitCriticalReduction(ReductionGen);5148 }5149 5150 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(5151 CGM.getModule(), OMPRTL___kmpc_barrier),5152 BarrierArgs);5153 5154 // Broadcast final result5155 bool IsAggregate = PrivateType->isAggregateType();5156 LValue SharedLV1 = CGF.MakeAddrLValue(SharedResult, PrivateType);5157 llvm::Value *FinalResultVal = nullptr;5158 Address FinalResultAddr = Address::invalid();5159 5160 if (IsAggregate)5161 FinalResultAddr = SharedResult;5162 else5163 FinalResultVal = CGF.EmitLoadOfScalar(SharedLV1, Loc);5164 5165 LValue TargetLHSLV = CGF.EmitLValue(RHSExprs);5166 if (IsAggregate) {5167 CGF.EmitAggregateCopy(TargetLHSLV,5168 CGF.MakeAddrLValue(FinalResultAddr, PrivateType),5169 PrivateType, AggValueSlot::DoesNotOverlap, false);5170 } else {5171 CGF.EmitStoreOfScalar(FinalResultVal, TargetLHSLV);5172 }5173 // Final synchronization barrier5174 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(5175 CGM.getModule(), OMPRTL___kmpc_barrier),5176 BarrierArgs);5177 5178 // Combiner with original list item5179 auto OriginalListCombiner = [&](CodeGenFunction &CGF,5180 PrePostActionTy &Action) {5181 Action.Enter(CGF);5182 emitSingleReductionCombiner(CGF, ReductionOps, Privates,5183 cast<DeclRefExpr>(LHSExprs),5184 cast<DeclRefExpr>(RHSExprs));5185 };5186 EmitCriticalReduction(OriginalListCombiner);5187}5188 5189void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,5190 ArrayRef<const Expr *> OrgPrivates,5191 ArrayRef<const Expr *> OrgLHSExprs,5192 ArrayRef<const Expr *> OrgRHSExprs,5193 ArrayRef<const Expr *> OrgReductionOps,5194 ReductionOptionsTy Options) {5195 if (!CGF.HaveInsertPoint())5196 return;5197 5198 bool WithNowait = Options.WithNowait;5199 bool SimpleReduction = Options.SimpleReduction;5200 5201 // Next code should be emitted for reduction:5202 //5203 // static kmp_critical_name lock = { 0 };5204 //5205 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {5206 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);5207 // ...5208 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],5209 // *(Type<n>-1*)rhs[<n>-1]);5210 // }5211 //5212 // ...5213 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};5214 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),5215 // RedList, reduce_func, &<lock>)) {5216 // case 1:5217 // ...5218 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);5219 // ...5220 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);5221 // break;5222 // case 2:5223 // ...5224 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));5225 // ...5226 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]5227 // break;5228 // default:;5229 // }5230 //5231 // if SimpleReduction is true, only the next code is generated:5232 // ...5233 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);5234 // ...5235 5236 ASTContext &C = CGM.getContext();5237 5238 if (SimpleReduction) {5239 CodeGenFunction::RunCleanupsScope Scope(CGF);5240 const auto *IPriv = OrgPrivates.begin();5241 const auto *ILHS = OrgLHSExprs.begin();5242 const auto *IRHS = OrgRHSExprs.begin();5243 for (const Expr *E : OrgReductionOps) {5244 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),5245 cast<DeclRefExpr>(*IRHS));5246 ++IPriv;5247 ++ILHS;5248 ++IRHS;5249 }5250 return;5251 }5252 5253 // Filter out shared reduction variables based on IsPrivateVarReduction flag.5254 // Only keep entries where the corresponding variable is not private.5255 SmallVector<const Expr *> FilteredPrivates, FilteredLHSExprs,5256 FilteredRHSExprs, FilteredReductionOps;5257 for (unsigned I : llvm::seq<unsigned>(5258 std::min(OrgReductionOps.size(), OrgLHSExprs.size()))) {5259 if (!Options.IsPrivateVarReduction[I]) {5260 FilteredPrivates.emplace_back(OrgPrivates[I]);5261 FilteredLHSExprs.emplace_back(OrgLHSExprs[I]);5262 FilteredRHSExprs.emplace_back(OrgRHSExprs[I]);5263 FilteredReductionOps.emplace_back(OrgReductionOps[I]);5264 }5265 }5266 // Wrap filtered vectors in ArrayRef for downstream shared reduction5267 // processing.5268 ArrayRef<const Expr *> Privates = FilteredPrivates;5269 ArrayRef<const Expr *> LHSExprs = FilteredLHSExprs;5270 ArrayRef<const Expr *> RHSExprs = FilteredRHSExprs;5271 ArrayRef<const Expr *> ReductionOps = FilteredReductionOps;5272 5273 // 1. Build a list of reduction variables.5274 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};5275 auto Size = RHSExprs.size();5276 for (const Expr *E : Privates) {5277 if (E->getType()->isVariablyModifiedType())5278 // Reserve place for array size.5279 ++Size;5280 }5281 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);5282 QualType ReductionArrayTy = C.getConstantArrayType(5283 C.VoidPtrTy, ArraySize, nullptr, ArraySizeModifier::Normal,5284 /*IndexTypeQuals=*/0);5285 RawAddress ReductionList =5286 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");5287 const auto *IPriv = Privates.begin();5288 unsigned Idx = 0;5289 for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {5290 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);5291 CGF.Builder.CreateStore(5292 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(5293 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy),5294 Elem);5295 if ((*IPriv)->getType()->isVariablyModifiedType()) {5296 // Store array size.5297 ++Idx;5298 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);5299 llvm::Value *Size = CGF.Builder.CreateIntCast(5300 CGF.getVLASize(5301 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))5302 .NumElts,5303 CGF.SizeTy, /*isSigned=*/false);5304 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),5305 Elem);5306 }5307 }5308 5309 // 2. Emit reduce_func().5310 llvm::Function *ReductionFn = emitReductionFunction(5311 CGF.CurFn->getName(), Loc, CGF.ConvertTypeForMem(ReductionArrayTy),5312 Privates, LHSExprs, RHSExprs, ReductionOps);5313 5314 // 3. Create static kmp_critical_name lock = { 0 };5315 std::string Name = getName({"reduction"});5316 llvm::Value *Lock = getCriticalRegionLock(Name);5317 5318 // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),5319 // RedList, reduce_func, &<lock>);5320 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);5321 llvm::Value *ThreadId = getThreadID(CGF, Loc);5322 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);5323 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(5324 ReductionList.getPointer(), CGF.VoidPtrTy);5325 llvm::Value *Args[] = {5326 IdentTLoc, // ident_t *<loc>5327 ThreadId, // i32 <gtid>5328 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>5329 ReductionArrayTySize, // size_type sizeof(RedList)5330 RL, // void *RedList5331 ReductionFn, // void (*) (void *, void *) <reduce_func>5332 Lock // kmp_critical_name *&<lock>5333 };5334 llvm::Value *Res = CGF.EmitRuntimeCall(5335 OMPBuilder.getOrCreateRuntimeFunction(5336 CGM.getModule(),5337 WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),5338 Args);5339 5340 // 5. Build switch(res)5341 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");5342 llvm::SwitchInst *SwInst =5343 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);5344 5345 // 6. Build case 1:5346 // ...5347 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);5348 // ...5349 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);5350 // break;5351 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");5352 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);5353 CGF.EmitBlock(Case1BB);5354 5355 // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);5356 llvm::Value *EndArgs[] = {5357 IdentTLoc, // ident_t *<loc>5358 ThreadId, // i32 <gtid>5359 Lock // kmp_critical_name *&<lock>5360 };5361 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](5362 CodeGenFunction &CGF, PrePostActionTy &Action) {5363 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();5364 const auto *IPriv = Privates.begin();5365 const auto *ILHS = LHSExprs.begin();5366 const auto *IRHS = RHSExprs.begin();5367 for (const Expr *E : ReductionOps) {5368 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),5369 cast<DeclRefExpr>(*IRHS));5370 ++IPriv;5371 ++ILHS;5372 ++IRHS;5373 }5374 };5375 RegionCodeGenTy RCG(CodeGen);5376 CommonActionTy Action(5377 nullptr, {},5378 OMPBuilder.getOrCreateRuntimeFunction(5379 CGM.getModule(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait5380 : OMPRTL___kmpc_end_reduce),5381 EndArgs);5382 RCG.setAction(Action);5383 RCG(CGF);5384 5385 CGF.EmitBranch(DefaultBB);5386 5387 // 7. Build case 2:5388 // ...5389 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));5390 // ...5391 // break;5392 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2");5393 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);5394 CGF.EmitBlock(Case2BB);5395 5396 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](5397 CodeGenFunction &CGF, PrePostActionTy &Action) {5398 const auto *ILHS = LHSExprs.begin();5399 const auto *IRHS = RHSExprs.begin();5400 const auto *IPriv = Privates.begin();5401 for (const Expr *E : ReductionOps) {5402 const Expr *XExpr = nullptr;5403 const Expr *EExpr = nullptr;5404 const Expr *UpExpr = nullptr;5405 BinaryOperatorKind BO = BO_Comma;5406 if (const auto *BO = dyn_cast<BinaryOperator>(E)) {5407 if (BO->getOpcode() == BO_Assign) {5408 XExpr = BO->getLHS();5409 UpExpr = BO->getRHS();5410 }5411 }5412 // Try to emit update expression as a simple atomic.5413 const Expr *RHSExpr = UpExpr;5414 if (RHSExpr) {5415 // Analyze RHS part of the whole expression.5416 if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(5417 RHSExpr->IgnoreParenImpCasts())) {5418 // If this is a conditional operator, analyze its condition for5419 // min/max reduction operator.5420 RHSExpr = ACO->getCond();5421 }5422 if (const auto *BORHS =5423 dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {5424 EExpr = BORHS->getRHS();5425 BO = BORHS->getOpcode();5426 }5427 }5428 if (XExpr) {5429 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());5430 auto &&AtomicRedGen = [BO, VD,5431 Loc](CodeGenFunction &CGF, const Expr *XExpr,5432 const Expr *EExpr, const Expr *UpExpr) {5433 LValue X = CGF.EmitLValue(XExpr);5434 RValue E;5435 if (EExpr)5436 E = CGF.EmitAnyExpr(EExpr);5437 CGF.EmitOMPAtomicSimpleUpdateExpr(5438 X, E, BO, /*IsXLHSInRHSPart=*/true,5439 llvm::AtomicOrdering::Monotonic, Loc,5440 [&CGF, UpExpr, VD, Loc](RValue XRValue) {5441 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);5442 Address LHSTemp = CGF.CreateMemTemp(VD->getType());5443 CGF.emitOMPSimpleStore(5444 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,5445 VD->getType().getNonReferenceType(), Loc);5446 PrivateScope.addPrivate(VD, LHSTemp);5447 (void)PrivateScope.Privatize();5448 return CGF.EmitAnyExpr(UpExpr);5449 });5450 };5451 if ((*IPriv)->getType()->isArrayType()) {5452 // Emit atomic reduction for array section.5453 const auto *RHSVar =5454 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());5455 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,5456 AtomicRedGen, XExpr, EExpr, UpExpr);5457 } else {5458 // Emit atomic reduction for array subscript or single variable.5459 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);5460 }5461 } else {5462 // Emit as a critical region.5463 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,5464 const Expr *, const Expr *) {5465 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();5466 std::string Name = RT.getName({"atomic_reduction"});5467 RT.emitCriticalRegion(5468 CGF, Name,5469 [=](CodeGenFunction &CGF, PrePostActionTy &Action) {5470 Action.Enter(CGF);5471 emitReductionCombiner(CGF, E);5472 },5473 Loc);5474 };5475 if ((*IPriv)->getType()->isArrayType()) {5476 const auto *LHSVar =5477 cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());5478 const auto *RHSVar =5479 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());5480 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,5481 CritRedGen);5482 } else {5483 CritRedGen(CGF, nullptr, nullptr, nullptr);5484 }5485 }5486 ++ILHS;5487 ++IRHS;5488 ++IPriv;5489 }5490 };5491 RegionCodeGenTy AtomicRCG(AtomicCodeGen);5492 if (!WithNowait) {5493 // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);5494 llvm::Value *EndArgs[] = {5495 IdentTLoc, // ident_t *<loc>5496 ThreadId, // i32 <gtid>5497 Lock // kmp_critical_name *&<lock>5498 };5499 CommonActionTy Action(nullptr, {},5500 OMPBuilder.getOrCreateRuntimeFunction(5501 CGM.getModule(), OMPRTL___kmpc_end_reduce),5502 EndArgs);5503 AtomicRCG.setAction(Action);5504 AtomicRCG(CGF);5505 } else {5506 AtomicRCG(CGF);5507 }5508 5509 CGF.EmitBranch(DefaultBB);5510 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);5511 assert(OrgLHSExprs.size() == OrgPrivates.size() &&5512 "PrivateVarReduction: Privates size mismatch");5513 assert(OrgLHSExprs.size() == OrgReductionOps.size() &&5514 "PrivateVarReduction: ReductionOps size mismatch");5515 for (unsigned I : llvm::seq<unsigned>(5516 std::min(OrgReductionOps.size(), OrgLHSExprs.size()))) {5517 if (Options.IsPrivateVarReduction[I])5518 emitPrivateReduction(CGF, Loc, OrgPrivates[I], OrgLHSExprs[I],5519 OrgRHSExprs[I], OrgReductionOps[I]);5520 }5521}5522 5523/// Generates unique name for artificial threadprivate variables.5524/// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>"5525static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix,5526 const Expr *Ref) {5527 SmallString<256> Buffer;5528 llvm::raw_svector_ostream Out(Buffer);5529 const clang::DeclRefExpr *DE;5530 const VarDecl *D = ::getBaseDecl(Ref, DE);5531 if (!D)5532 D = cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl());5533 D = D->getCanonicalDecl();5534 std::string Name = CGM.getOpenMPRuntime().getName(5535 {D->isLocalVarDeclOrParm() ? D->getName() : CGM.getMangledName(D)});5536 Out << Prefix << Name << "_"5537 << D->getCanonicalDecl()->getBeginLoc().getRawEncoding();5538 return std::string(Out.str());5539}5540 5541/// Emits reduction initializer function:5542/// \code5543/// void @.red_init(void* %arg, void* %orig) {5544/// %0 = bitcast void* %arg to <type>*5545/// store <type> <init>, <type>* %05546/// ret void5547/// }5548/// \endcode5549static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,5550 SourceLocation Loc,5551 ReductionCodeGen &RCG, unsigned N) {5552 ASTContext &C = CGM.getContext();5553 QualType VoidPtrTy = C.VoidPtrTy;5554 VoidPtrTy.addRestrict();5555 FunctionArgList Args;5556 ImplicitParamDecl Param(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, VoidPtrTy,5557 ImplicitParamKind::Other);5558 ImplicitParamDecl ParamOrig(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, VoidPtrTy,5559 ImplicitParamKind::Other);5560 Args.emplace_back(&Param);5561 Args.emplace_back(&ParamOrig);5562 const auto &FnInfo =5563 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);5564 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);5565 std::string Name = CGM.getOpenMPRuntime().getName({"red_init", ""});5566 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,5567 Name, &CGM.getModule());5568 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);5569 Fn->setDoesNotRecurse();5570 CodeGenFunction CGF(CGM);5571 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);5572 QualType PrivateType = RCG.getPrivateType(N);5573 Address PrivateAddr = CGF.EmitLoadOfPointer(5574 CGF.GetAddrOfLocalVar(&Param).withElementType(CGF.Builder.getPtrTy(0)),5575 C.getPointerType(PrivateType)->castAs<PointerType>());5576 llvm::Value *Size = nullptr;5577 // If the size of the reduction item is non-constant, load it from global5578 // threadprivate variable.5579 if (RCG.getSizes(N).second) {5580 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(5581 CGF, CGM.getContext().getSizeType(),5582 generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));5583 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,5584 CGM.getContext().getSizeType(), Loc);5585 }5586 RCG.emitAggregateType(CGF, N, Size);5587 Address OrigAddr = Address::invalid();5588 // If initializer uses initializer from declare reduction construct, emit a5589 // pointer to the address of the original reduction item (reuired by reduction5590 // initializer)5591 if (RCG.usesReductionInitializer(N)) {5592 Address SharedAddr = CGF.GetAddrOfLocalVar(&ParamOrig);5593 OrigAddr = CGF.EmitLoadOfPointer(5594 SharedAddr,5595 CGM.getContext().VoidPtrTy.castAs<PointerType>()->getTypePtr());5596 }5597 // Emit the initializer:5598 // %0 = bitcast void* %arg to <type>*5599 // store <type> <init>, <type>* %05600 RCG.emitInitialization(CGF, N, PrivateAddr, OrigAddr,5601 [](CodeGenFunction &) { return false; });5602 CGF.FinishFunction();5603 return Fn;5604}5605 5606/// Emits reduction combiner function:5607/// \code5608/// void @.red_comb(void* %arg0, void* %arg1) {5609/// %lhs = bitcast void* %arg0 to <type>*5610/// %rhs = bitcast void* %arg1 to <type>*5611/// %2 = <ReductionOp>(<type>* %lhs, <type>* %rhs)5612/// store <type> %2, <type>* %lhs5613/// ret void5614/// }5615/// \endcode5616static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM,5617 SourceLocation Loc,5618 ReductionCodeGen &RCG, unsigned N,5619 const Expr *ReductionOp,5620 const Expr *LHS, const Expr *RHS,5621 const Expr *PrivateRef) {5622 ASTContext &C = CGM.getContext();5623 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl());5624 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl());5625 FunctionArgList Args;5626 ImplicitParamDecl ParamInOut(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,5627 C.VoidPtrTy, ImplicitParamKind::Other);5628 ImplicitParamDecl ParamIn(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,5629 ImplicitParamKind::Other);5630 Args.emplace_back(&ParamInOut);5631 Args.emplace_back(&ParamIn);5632 const auto &FnInfo =5633 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);5634 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);5635 std::string Name = CGM.getOpenMPRuntime().getName({"red_comb", ""});5636 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,5637 Name, &CGM.getModule());5638 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);5639 Fn->setDoesNotRecurse();5640 CodeGenFunction CGF(CGM);5641 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);5642 llvm::Value *Size = nullptr;5643 // If the size of the reduction item is non-constant, load it from global5644 // threadprivate variable.5645 if (RCG.getSizes(N).second) {5646 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(5647 CGF, CGM.getContext().getSizeType(),5648 generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));5649 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,5650 CGM.getContext().getSizeType(), Loc);5651 }5652 RCG.emitAggregateType(CGF, N, Size);5653 // Remap lhs and rhs variables to the addresses of the function arguments.5654 // %lhs = bitcast void* %arg0 to <type>*5655 // %rhs = bitcast void* %arg1 to <type>*5656 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);5657 PrivateScope.addPrivate(5658 LHSVD,5659 // Pull out the pointer to the variable.5660 CGF.EmitLoadOfPointer(5661 CGF.GetAddrOfLocalVar(&ParamInOut)5662 .withElementType(CGF.Builder.getPtrTy(0)),5663 C.getPointerType(LHSVD->getType())->castAs<PointerType>()));5664 PrivateScope.addPrivate(5665 RHSVD,5666 // Pull out the pointer to the variable.5667 CGF.EmitLoadOfPointer(5668 CGF.GetAddrOfLocalVar(&ParamIn).withElementType(5669 CGF.Builder.getPtrTy(0)),5670 C.getPointerType(RHSVD->getType())->castAs<PointerType>()));5671 PrivateScope.Privatize();5672 // Emit the combiner body:5673 // %2 = <ReductionOp>(<type> *%lhs, <type> *%rhs)5674 // store <type> %2, <type>* %lhs5675 CGM.getOpenMPRuntime().emitSingleReductionCombiner(5676 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),5677 cast<DeclRefExpr>(RHS));5678 CGF.FinishFunction();5679 return Fn;5680}5681 5682/// Emits reduction finalizer function:5683/// \code5684/// void @.red_fini(void* %arg) {5685/// %0 = bitcast void* %arg to <type>*5686/// <destroy>(<type>* %0)5687/// ret void5688/// }5689/// \endcode5690static llvm::Value *emitReduceFiniFunction(CodeGenModule &CGM,5691 SourceLocation Loc,5692 ReductionCodeGen &RCG, unsigned N) {5693 if (!RCG.needCleanups(N))5694 return nullptr;5695 ASTContext &C = CGM.getContext();5696 FunctionArgList Args;5697 ImplicitParamDecl Param(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,5698 ImplicitParamKind::Other);5699 Args.emplace_back(&Param);5700 const auto &FnInfo =5701 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);5702 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);5703 std::string Name = CGM.getOpenMPRuntime().getName({"red_fini", ""});5704 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,5705 Name, &CGM.getModule());5706 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);5707 Fn->setDoesNotRecurse();5708 CodeGenFunction CGF(CGM);5709 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);5710 Address PrivateAddr = CGF.EmitLoadOfPointer(5711 CGF.GetAddrOfLocalVar(&Param), C.VoidPtrTy.castAs<PointerType>());5712 llvm::Value *Size = nullptr;5713 // If the size of the reduction item is non-constant, load it from global5714 // threadprivate variable.5715 if (RCG.getSizes(N).second) {5716 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(5717 CGF, CGM.getContext().getSizeType(),5718 generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));5719 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,5720 CGM.getContext().getSizeType(), Loc);5721 }5722 RCG.emitAggregateType(CGF, N, Size);5723 // Emit the finalizer body:5724 // <destroy>(<type>* %0)5725 RCG.emitCleanups(CGF, N, PrivateAddr);5726 CGF.FinishFunction(Loc);5727 return Fn;5728}5729 5730llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(5731 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,5732 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {5733 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty())5734 return nullptr;5735 5736 // Build typedef struct:5737 // kmp_taskred_input {5738 // void *reduce_shar; // shared reduction item5739 // void *reduce_orig; // original reduction item used for initialization5740 // size_t reduce_size; // size of data item5741 // void *reduce_init; // data initialization routine5742 // void *reduce_fini; // data finalization routine5743 // void *reduce_comb; // data combiner routine5744 // kmp_task_red_flags_t flags; // flags for additional info from compiler5745 // } kmp_taskred_input_t;5746 ASTContext &C = CGM.getContext();5747 RecordDecl *RD = C.buildImplicitRecord("kmp_taskred_input_t");5748 RD->startDefinition();5749 const FieldDecl *SharedFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);5750 const FieldDecl *OrigFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);5751 const FieldDecl *SizeFD = addFieldToRecordDecl(C, RD, C.getSizeType());5752 const FieldDecl *InitFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);5753 const FieldDecl *FiniFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);5754 const FieldDecl *CombFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);5755 const FieldDecl *FlagsFD = addFieldToRecordDecl(5756 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/false));5757 RD->completeDefinition();5758 CanQualType RDType = C.getCanonicalTagType(RD);5759 unsigned Size = Data.ReductionVars.size();5760 llvm::APInt ArraySize(/*numBits=*/64, Size);5761 QualType ArrayRDType =5762 C.getConstantArrayType(RDType, ArraySize, nullptr,5763 ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);5764 // kmp_task_red_input_t .rd_input.[Size];5765 RawAddress TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input.");5766 ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionOrigs,5767 Data.ReductionCopies, Data.ReductionOps);5768 for (unsigned Cnt = 0; Cnt < Size; ++Cnt) {5769 // kmp_task_red_input_t &ElemLVal = .rd_input.[Cnt];5770 llvm::Value *Idxs[] = {llvm::ConstantInt::get(CGM.SizeTy, /*V=*/0),5771 llvm::ConstantInt::get(CGM.SizeTy, Cnt)};5772 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP(5773 TaskRedInput.getElementType(), TaskRedInput.getPointer(), Idxs,5774 /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc,5775 ".rd_input.gep.");5776 LValue ElemLVal = CGF.MakeNaturalAlignRawAddrLValue(GEP, RDType);5777 // ElemLVal.reduce_shar = &Shareds[Cnt];5778 LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD);5779 RCG.emitSharedOrigLValue(CGF, Cnt);5780 llvm::Value *Shared = RCG.getSharedLValue(Cnt).getPointer(CGF);5781 CGF.EmitStoreOfScalar(Shared, SharedLVal);5782 // ElemLVal.reduce_orig = &Origs[Cnt];5783 LValue OrigLVal = CGF.EmitLValueForField(ElemLVal, OrigFD);5784 llvm::Value *Orig = RCG.getOrigLValue(Cnt).getPointer(CGF);5785 CGF.EmitStoreOfScalar(Orig, OrigLVal);5786 RCG.emitAggregateType(CGF, Cnt);5787 llvm::Value *SizeValInChars;5788 llvm::Value *SizeVal;5789 std::tie(SizeValInChars, SizeVal) = RCG.getSizes(Cnt);5790 // We use delayed creation/initialization for VLAs and array sections. It is5791 // required because runtime does not provide the way to pass the sizes of5792 // VLAs/array sections to initializer/combiner/finalizer functions. Instead5793 // threadprivate global variables are used to store these values and use5794 // them in the functions.5795 bool DelayedCreation = !!SizeVal;5796 SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy,5797 /*isSigned=*/false);5798 LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD);5799 CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal);5800 // ElemLVal.reduce_init = init;5801 LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD);5802 llvm::Value *InitAddr = emitReduceInitFunction(CGM, Loc, RCG, Cnt);5803 CGF.EmitStoreOfScalar(InitAddr, InitLVal);5804 // ElemLVal.reduce_fini = fini;5805 LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD);5806 llvm::Value *Fini = emitReduceFiniFunction(CGM, Loc, RCG, Cnt);5807 llvm::Value *FiniAddr =5808 Fini ? Fini : llvm::ConstantPointerNull::get(CGM.VoidPtrTy);5809 CGF.EmitStoreOfScalar(FiniAddr, FiniLVal);5810 // ElemLVal.reduce_comb = comb;5811 LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD);5812 llvm::Value *CombAddr = emitReduceCombFunction(5813 CGM, Loc, RCG, Cnt, Data.ReductionOps[Cnt], LHSExprs[Cnt],5814 RHSExprs[Cnt], Data.ReductionCopies[Cnt]);5815 CGF.EmitStoreOfScalar(CombAddr, CombLVal);5816 // ElemLVal.flags = 0;5817 LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD);5818 if (DelayedCreation) {5819 CGF.EmitStoreOfScalar(5820 llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/1, /*isSigned=*/true),5821 FlagsLVal);5822 } else5823 CGF.EmitNullInitialization(FlagsLVal.getAddress(), FlagsLVal.getType());5824 }5825 if (Data.IsReductionWithTaskMod) {5826 // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int5827 // is_ws, int num, void *data);5828 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);5829 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc),5830 CGM.IntTy, /*isSigned=*/true);5831 llvm::Value *Args[] = {5832 IdentTLoc, GTid,5833 llvm::ConstantInt::get(CGM.IntTy, Data.IsWorksharingReduction ? 1 : 0,5834 /*isSigned=*/true),5835 llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true),5836 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(5837 TaskRedInput.getPointer(), CGM.VoidPtrTy)};5838 return CGF.EmitRuntimeCall(5839 OMPBuilder.getOrCreateRuntimeFunction(5840 CGM.getModule(), OMPRTL___kmpc_taskred_modifier_init),5841 Args);5842 }5843 // Build call void *__kmpc_taskred_init(int gtid, int num_data, void *data);5844 llvm::Value *Args[] = {5845 CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy,5846 /*isSigned=*/true),5847 llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true),5848 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(),5849 CGM.VoidPtrTy)};5850 return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(5851 CGM.getModule(), OMPRTL___kmpc_taskred_init),5852 Args);5853}5854 5855void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF,5856 SourceLocation Loc,5857 bool IsWorksharingReduction) {5858 // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int5859 // is_ws, int num, void *data);5860 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);5861 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc),5862 CGM.IntTy, /*isSigned=*/true);5863 llvm::Value *Args[] = {IdentTLoc, GTid,5864 llvm::ConstantInt::get(CGM.IntTy,5865 IsWorksharingReduction ? 1 : 0,5866 /*isSigned=*/true)};5867 (void)CGF.EmitRuntimeCall(5868 OMPBuilder.getOrCreateRuntimeFunction(5869 CGM.getModule(), OMPRTL___kmpc_task_reduction_modifier_fini),5870 Args);5871}5872 5873void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,5874 SourceLocation Loc,5875 ReductionCodeGen &RCG,5876 unsigned N) {5877 auto Sizes = RCG.getSizes(N);5878 // Emit threadprivate global variable if the type is non-constant5879 // (Sizes.second = nullptr).5880 if (Sizes.second) {5881 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy,5882 /*isSigned=*/false);5883 Address SizeAddr = getAddrOfArtificialThreadPrivate(5884 CGF, CGM.getContext().getSizeType(),5885 generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));5886 CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false);5887 }5888}5889 5890Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,5891 SourceLocation Loc,5892 llvm::Value *ReductionsPtr,5893 LValue SharedLVal) {5894 // Build call void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void5895 // *d);5896 llvm::Value *Args[] = {CGF.Builder.CreateIntCast(getThreadID(CGF, Loc),5897 CGM.IntTy,5898 /*isSigned=*/true),5899 ReductionsPtr,5900 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(5901 SharedLVal.getPointer(CGF), CGM.VoidPtrTy)};5902 return Address(5903 CGF.EmitRuntimeCall(5904 OMPBuilder.getOrCreateRuntimeFunction(5905 CGM.getModule(), OMPRTL___kmpc_task_reduction_get_th_data),5906 Args),5907 CGF.Int8Ty, SharedLVal.getAlignment());5908}5909 5910void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,5911 const OMPTaskDataTy &Data) {5912 if (!CGF.HaveInsertPoint())5913 return;5914 5915 if (CGF.CGM.getLangOpts().OpenMPIRBuilder && Data.Dependences.empty()) {5916 // TODO: Need to support taskwait with dependences in the OpenMPIRBuilder.5917 OMPBuilder.createTaskwait(CGF.Builder);5918 } else {5919 llvm::Value *ThreadID = getThreadID(CGF, Loc);5920 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);5921 auto &M = CGM.getModule();5922 Address DependenciesArray = Address::invalid();5923 llvm::Value *NumOfElements;5924 std::tie(NumOfElements, DependenciesArray) =5925 emitDependClause(CGF, Data.Dependences, Loc);5926 if (!Data.Dependences.empty()) {5927 llvm::Value *DepWaitTaskArgs[7];5928 DepWaitTaskArgs[0] = UpLoc;5929 DepWaitTaskArgs[1] = ThreadID;5930 DepWaitTaskArgs[2] = NumOfElements;5931 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);5932 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);5933 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);5934 DepWaitTaskArgs[6] =5935 llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);5936 5937 CodeGenFunction::RunCleanupsScope LocalScope(CGF);5938 5939 // Build void __kmpc_omp_taskwait_deps_51(ident_t *, kmp_int32 gtid,5940 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int325941 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list,5942 // kmp_int32 has_no_wait); if dependence info is specified.5943 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(5944 M, OMPRTL___kmpc_omp_taskwait_deps_51),5945 DepWaitTaskArgs);5946 5947 } else {5948 5949 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int325950 // global_tid);5951 llvm::Value *Args[] = {UpLoc, ThreadID};5952 // Ignore return result until untied tasks are supported.5953 CGF.EmitRuntimeCall(5954 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_taskwait),5955 Args);5956 }5957 }5958 5959 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))5960 Region->emitUntiedSwitch(CGF);5961}5962 5963void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,5964 OpenMPDirectiveKind InnerKind,5965 const RegionCodeGenTy &CodeGen,5966 bool HasCancel) {5967 if (!CGF.HaveInsertPoint())5968 return;5969 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel,5970 InnerKind != OMPD_critical &&5971 InnerKind != OMPD_master &&5972 InnerKind != OMPD_masked);5973 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);5974}5975 5976namespace {5977enum RTCancelKind {5978 CancelNoreq = 0,5979 CancelParallel = 1,5980 CancelLoop = 2,5981 CancelSections = 3,5982 CancelTaskgroup = 45983};5984} // anonymous namespace5985 5986static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) {5987 RTCancelKind CancelKind = CancelNoreq;5988 if (CancelRegion == OMPD_parallel)5989 CancelKind = CancelParallel;5990 else if (CancelRegion == OMPD_for)5991 CancelKind = CancelLoop;5992 else if (CancelRegion == OMPD_sections)5993 CancelKind = CancelSections;5994 else {5995 assert(CancelRegion == OMPD_taskgroup);5996 CancelKind = CancelTaskgroup;5997 }5998 return CancelKind;5999}6000 6001void CGOpenMPRuntime::emitCancellationPointCall(6002 CodeGenFunction &CGF, SourceLocation Loc,6003 OpenMPDirectiveKind CancelRegion) {6004 if (!CGF.HaveInsertPoint())6005 return;6006 // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int326007 // global_tid, kmp_int32 cncl_kind);6008 if (auto *OMPRegionInfo =6009 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {6010 // For 'cancellation point taskgroup', the task region info may not have a6011 // cancel. This may instead happen in another adjacent task.6012 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {6013 llvm::Value *Args[] = {6014 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),6015 CGF.Builder.getInt32(getCancellationKind(CancelRegion))};6016 // Ignore return result until untied tasks are supported.6017 llvm::Value *Result = CGF.EmitRuntimeCall(6018 OMPBuilder.getOrCreateRuntimeFunction(6019 CGM.getModule(), OMPRTL___kmpc_cancellationpoint),6020 Args);6021 // if (__kmpc_cancellationpoint()) {6022 // call i32 @__kmpc_cancel_barrier( // for parallel cancellation only6023 // exit from construct;6024 // }6025 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");6026 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");6027 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);6028 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);6029 CGF.EmitBlock(ExitBB);6030 if (CancelRegion == OMPD_parallel)6031 emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false);6032 // exit from construct;6033 CodeGenFunction::JumpDest CancelDest =6034 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());6035 CGF.EmitBranchThroughCleanup(CancelDest);6036 CGF.EmitBlock(ContBB, /*IsFinished=*/true);6037 }6038 }6039}6040 6041void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,6042 const Expr *IfCond,6043 OpenMPDirectiveKind CancelRegion) {6044 if (!CGF.HaveInsertPoint())6045 return;6046 // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,6047 // kmp_int32 cncl_kind);6048 auto &M = CGM.getModule();6049 if (auto *OMPRegionInfo =6050 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {6051 auto &&ThenGen = [this, &M, Loc, CancelRegion,6052 OMPRegionInfo](CodeGenFunction &CGF, PrePostActionTy &) {6053 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();6054 llvm::Value *Args[] = {6055 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),6056 CGF.Builder.getInt32(getCancellationKind(CancelRegion))};6057 // Ignore return result until untied tasks are supported.6058 llvm::Value *Result = CGF.EmitRuntimeCall(6059 OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_cancel), Args);6060 // if (__kmpc_cancel()) {6061 // call i32 @__kmpc_cancel_barrier( // for parallel cancellation only6062 // exit from construct;6063 // }6064 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");6065 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");6066 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);6067 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);6068 CGF.EmitBlock(ExitBB);6069 if (CancelRegion == OMPD_parallel)6070 RT.emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false);6071 // exit from construct;6072 CodeGenFunction::JumpDest CancelDest =6073 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());6074 CGF.EmitBranchThroughCleanup(CancelDest);6075 CGF.EmitBlock(ContBB, /*IsFinished=*/true);6076 };6077 if (IfCond) {6078 emitIfClause(CGF, IfCond, ThenGen,6079 [](CodeGenFunction &, PrePostActionTy &) {});6080 } else {6081 RegionCodeGenTy ThenRCG(ThenGen);6082 ThenRCG(CGF);6083 }6084 }6085}6086 6087namespace {6088/// Cleanup action for uses_allocators support.6089class OMPUsesAllocatorsActionTy final : public PrePostActionTy {6090 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators;6091 6092public:6093 OMPUsesAllocatorsActionTy(6094 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators)6095 : Allocators(Allocators) {}6096 void Enter(CodeGenFunction &CGF) override {6097 if (!CGF.HaveInsertPoint())6098 return;6099 for (const auto &AllocatorData : Allocators) {6100 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsInit(6101 CGF, AllocatorData.first, AllocatorData.second);6102 }6103 }6104 void Exit(CodeGenFunction &CGF) override {6105 if (!CGF.HaveInsertPoint())6106 return;6107 for (const auto &AllocatorData : Allocators) {6108 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsFini(CGF,6109 AllocatorData.first);6110 }6111 }6112};6113} // namespace6114 6115void CGOpenMPRuntime::emitTargetOutlinedFunction(6116 const OMPExecutableDirective &D, StringRef ParentName,6117 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,6118 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {6119 assert(!ParentName.empty() && "Invalid target entry parent name!");6120 HasEmittedTargetRegion = true;6121 SmallVector<std::pair<const Expr *, const Expr *>, 4> Allocators;6122 for (const auto *C : D.getClausesOfKind<OMPUsesAllocatorsClause>()) {6123 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {6124 const OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);6125 if (!D.AllocatorTraits)6126 continue;6127 Allocators.emplace_back(D.Allocator, D.AllocatorTraits);6128 }6129 }6130 OMPUsesAllocatorsActionTy UsesAllocatorAction(Allocators);6131 CodeGen.setAction(UsesAllocatorAction);6132 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,6133 IsOffloadEntry, CodeGen);6134}6135 6136void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF,6137 const Expr *Allocator,6138 const Expr *AllocatorTraits) {6139 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc());6140 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true);6141 // Use default memspace handle.6142 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);6143 llvm::Value *NumTraits = llvm::ConstantInt::get(6144 CGF.IntTy, cast<ConstantArrayType>(6145 AllocatorTraits->getType()->getAsArrayTypeUnsafe())6146 ->getSize()6147 .getLimitedValue());6148 LValue AllocatorTraitsLVal = CGF.EmitLValue(AllocatorTraits);6149 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(6150 AllocatorTraitsLVal.getAddress(), CGF.VoidPtrPtrTy, CGF.VoidPtrTy);6151 AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy,6152 AllocatorTraitsLVal.getBaseInfo(),6153 AllocatorTraitsLVal.getTBAAInfo());6154 llvm::Value *Traits = Addr.emitRawPointer(CGF);6155 6156 llvm::Value *AllocatorVal =6157 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(6158 CGM.getModule(), OMPRTL___kmpc_init_allocator),6159 {ThreadId, MemSpaceHandle, NumTraits, Traits});6160 // Store to allocator.6161 CGF.EmitAutoVarAlloca(*cast<VarDecl>(6162 cast<DeclRefExpr>(Allocator->IgnoreParenImpCasts())->getDecl()));6163 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts());6164 AllocatorVal =6165 CGF.EmitScalarConversion(AllocatorVal, CGF.getContext().VoidPtrTy,6166 Allocator->getType(), Allocator->getExprLoc());6167 CGF.EmitStoreOfScalar(AllocatorVal, AllocatorLVal);6168}6169 6170void CGOpenMPRuntime::emitUsesAllocatorsFini(CodeGenFunction &CGF,6171 const Expr *Allocator) {6172 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc());6173 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true);6174 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts());6175 llvm::Value *AllocatorVal =6176 CGF.EmitLoadOfScalar(AllocatorLVal, Allocator->getExprLoc());6177 AllocatorVal = CGF.EmitScalarConversion(AllocatorVal, Allocator->getType(),6178 CGF.getContext().VoidPtrTy,6179 Allocator->getExprLoc());6180 (void)CGF.EmitRuntimeCall(6181 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),6182 OMPRTL___kmpc_destroy_allocator),6183 {ThreadId, AllocatorVal});6184}6185 6186void CGOpenMPRuntime::computeMinAndMaxThreadsAndTeams(6187 const OMPExecutableDirective &D, CodeGenFunction &CGF,6188 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {6189 assert(Attrs.MaxTeams.size() == 1 && Attrs.MaxThreads.size() == 1 &&6190 "invalid default attrs structure");6191 int32_t &MaxTeamsVal = Attrs.MaxTeams.front();6192 int32_t &MaxThreadsVal = Attrs.MaxThreads.front();6193 6194 getNumTeamsExprForTargetDirective(CGF, D, Attrs.MinTeams, MaxTeamsVal);6195 getNumThreadsExprForTargetDirective(CGF, D, MaxThreadsVal,6196 /*UpperBoundOnly=*/true);6197 6198 for (auto *C : D.getClausesOfKind<OMPXAttributeClause>()) {6199 for (auto *A : C->getAttrs()) {6200 int32_t AttrMinThreadsVal = 1, AttrMaxThreadsVal = -1;6201 int32_t AttrMinBlocksVal = 1, AttrMaxBlocksVal = -1;6202 if (auto *Attr = dyn_cast<CUDALaunchBoundsAttr>(A))6203 CGM.handleCUDALaunchBoundsAttr(nullptr, Attr, &AttrMaxThreadsVal,6204 &AttrMinBlocksVal, &AttrMaxBlocksVal);6205 else if (auto *Attr = dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(A))6206 CGM.handleAMDGPUFlatWorkGroupSizeAttr(6207 nullptr, Attr, /*ReqdWGS=*/nullptr, &AttrMinThreadsVal,6208 &AttrMaxThreadsVal);6209 else6210 continue;6211 6212 Attrs.MinThreads = std::max(Attrs.MinThreads, AttrMinThreadsVal);6213 if (AttrMaxThreadsVal > 0)6214 MaxThreadsVal = MaxThreadsVal > 06215 ? std::min(MaxThreadsVal, AttrMaxThreadsVal)6216 : AttrMaxThreadsVal;6217 Attrs.MinTeams = std::max(Attrs.MinTeams, AttrMinBlocksVal);6218 if (AttrMaxBlocksVal > 0)6219 MaxTeamsVal = MaxTeamsVal > 0 ? std::min(MaxTeamsVal, AttrMaxBlocksVal)6220 : AttrMaxBlocksVal;6221 }6222 }6223}6224 6225void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(6226 const OMPExecutableDirective &D, StringRef ParentName,6227 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,6228 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {6229 6230 llvm::TargetRegionEntryInfo EntryInfo =6231 getEntryInfoFromPresumedLoc(CGM, OMPBuilder, D.getBeginLoc(), ParentName);6232 6233 CodeGenFunction CGF(CGM, true);6234 llvm::OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =6235 [&CGF, &D, &CodeGen](StringRef EntryFnName) {6236 const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);6237 6238 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);6239 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);6240 return CGF.GenerateOpenMPCapturedStmtFunction(CS, D);6241 };6242 6243 cantFail(OMPBuilder.emitTargetRegionFunction(6244 EntryInfo, GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,6245 OutlinedFnID));6246 6247 if (!OutlinedFn)6248 return;6249 6250 CGM.getTargetCodeGenInfo().setTargetAttributes(nullptr, OutlinedFn, CGM);6251 6252 for (auto *C : D.getClausesOfKind<OMPXAttributeClause>()) {6253 for (auto *A : C->getAttrs()) {6254 if (auto *Attr = dyn_cast<AMDGPUWavesPerEUAttr>(A))6255 CGM.handleAMDGPUWavesPerEUAttr(OutlinedFn, Attr);6256 }6257 }6258}6259 6260/// Checks if the expression is constant or does not have non-trivial function6261/// calls.6262static bool isTrivial(ASTContext &Ctx, const Expr * E) {6263 // We can skip constant expressions.6264 // We can skip expressions with trivial calls or simple expressions.6265 return (E->isEvaluatable(Ctx, Expr::SE_AllowUndefinedBehavior) ||6266 !E->hasNonTrivialCall(Ctx)) &&6267 !E->HasSideEffects(Ctx, /*IncludePossibleEffects=*/true);6268}6269 6270const Stmt *CGOpenMPRuntime::getSingleCompoundChild(ASTContext &Ctx,6271 const Stmt *Body) {6272 const Stmt *Child = Body->IgnoreContainers();6273 while (const auto *C = dyn_cast_or_null<CompoundStmt>(Child)) {6274 Child = nullptr;6275 for (const Stmt *S : C->body()) {6276 if (const auto *E = dyn_cast<Expr>(S)) {6277 if (isTrivial(Ctx, E))6278 continue;6279 }6280 // Some of the statements can be ignored.6281 if (isa<AsmStmt>(S) || isa<NullStmt>(S) || isa<OMPFlushDirective>(S) ||6282 isa<OMPBarrierDirective>(S) || isa<OMPTaskyieldDirective>(S))6283 continue;6284 // Analyze declarations.6285 if (const auto *DS = dyn_cast<DeclStmt>(S)) {6286 if (llvm::all_of(DS->decls(), [](const Decl *D) {6287 if (isa<EmptyDecl>(D) || isa<DeclContext>(D) ||6288 isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) ||6289 isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) ||6290 isa<UsingDirectiveDecl>(D) ||6291 isa<OMPDeclareReductionDecl>(D) ||6292 isa<OMPThreadPrivateDecl>(D) || isa<OMPAllocateDecl>(D))6293 return true;6294 const auto *VD = dyn_cast<VarDecl>(D);6295 if (!VD)6296 return false;6297 return VD->hasGlobalStorage() || !VD->isUsed();6298 }))6299 continue;6300 }6301 // Found multiple children - cannot get the one child only.6302 if (Child)6303 return nullptr;6304 Child = S;6305 }6306 if (Child)6307 Child = Child->IgnoreContainers();6308 }6309 return Child;6310}6311 6312const Expr *CGOpenMPRuntime::getNumTeamsExprForTargetDirective(6313 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal,6314 int32_t &MaxTeamsVal) {6315 6316 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();6317 assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&6318 "Expected target-based executable directive.");6319 switch (DirectiveKind) {6320 case OMPD_target: {6321 const auto *CS = D.getInnermostCapturedStmt();6322 const auto *Body =6323 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);6324 const Stmt *ChildStmt =6325 CGOpenMPRuntime::getSingleCompoundChild(CGF.getContext(), Body);6326 if (const auto *NestedDir =6327 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {6328 if (isOpenMPTeamsDirective(NestedDir->getDirectiveKind())) {6329 if (NestedDir->hasClausesOfKind<OMPNumTeamsClause>()) {6330 const Expr *NumTeams = NestedDir->getSingleClause<OMPNumTeamsClause>()6331 ->getNumTeams()6332 .front();6333 if (NumTeams->isIntegerConstantExpr(CGF.getContext()))6334 if (auto Constant =6335 NumTeams->getIntegerConstantExpr(CGF.getContext()))6336 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();6337 return NumTeams;6338 }6339 MinTeamsVal = MaxTeamsVal = 0;6340 return nullptr;6341 }6342 MinTeamsVal = MaxTeamsVal = 1;6343 return nullptr;6344 }6345 // A value of -1 is used to check if we need to emit no teams region6346 MinTeamsVal = MaxTeamsVal = -1;6347 return nullptr;6348 }6349 case OMPD_target_teams_loop:6350 case OMPD_target_teams:6351 case OMPD_target_teams_distribute:6352 case OMPD_target_teams_distribute_simd:6353 case OMPD_target_teams_distribute_parallel_for:6354 case OMPD_target_teams_distribute_parallel_for_simd: {6355 if (D.hasClausesOfKind<OMPNumTeamsClause>()) {6356 const Expr *NumTeams =6357 D.getSingleClause<OMPNumTeamsClause>()->getNumTeams().front();6358 if (NumTeams->isIntegerConstantExpr(CGF.getContext()))6359 if (auto Constant = NumTeams->getIntegerConstantExpr(CGF.getContext()))6360 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();6361 return NumTeams;6362 }6363 MinTeamsVal = MaxTeamsVal = 0;6364 return nullptr;6365 }6366 case OMPD_target_parallel:6367 case OMPD_target_parallel_for:6368 case OMPD_target_parallel_for_simd:6369 case OMPD_target_parallel_loop:6370 case OMPD_target_simd:6371 MinTeamsVal = MaxTeamsVal = 1;6372 return nullptr;6373 case OMPD_parallel:6374 case OMPD_for:6375 case OMPD_parallel_for:6376 case OMPD_parallel_loop:6377 case OMPD_parallel_master:6378 case OMPD_parallel_sections:6379 case OMPD_for_simd:6380 case OMPD_parallel_for_simd:6381 case OMPD_cancel:6382 case OMPD_cancellation_point:6383 case OMPD_ordered:6384 case OMPD_threadprivate:6385 case OMPD_allocate:6386 case OMPD_task:6387 case OMPD_simd:6388 case OMPD_tile:6389 case OMPD_unroll:6390 case OMPD_sections:6391 case OMPD_section:6392 case OMPD_single:6393 case OMPD_master:6394 case OMPD_critical:6395 case OMPD_taskyield:6396 case OMPD_barrier:6397 case OMPD_taskwait:6398 case OMPD_taskgroup:6399 case OMPD_atomic:6400 case OMPD_flush:6401 case OMPD_depobj:6402 case OMPD_scan:6403 case OMPD_teams:6404 case OMPD_target_data:6405 case OMPD_target_exit_data:6406 case OMPD_target_enter_data:6407 case OMPD_distribute:6408 case OMPD_distribute_simd:6409 case OMPD_distribute_parallel_for:6410 case OMPD_distribute_parallel_for_simd:6411 case OMPD_teams_distribute:6412 case OMPD_teams_distribute_simd:6413 case OMPD_teams_distribute_parallel_for:6414 case OMPD_teams_distribute_parallel_for_simd:6415 case OMPD_target_update:6416 case OMPD_declare_simd:6417 case OMPD_declare_variant:6418 case OMPD_begin_declare_variant:6419 case OMPD_end_declare_variant:6420 case OMPD_declare_target:6421 case OMPD_end_declare_target:6422 case OMPD_declare_reduction:6423 case OMPD_declare_mapper:6424 case OMPD_taskloop:6425 case OMPD_taskloop_simd:6426 case OMPD_master_taskloop:6427 case OMPD_master_taskloop_simd:6428 case OMPD_parallel_master_taskloop:6429 case OMPD_parallel_master_taskloop_simd:6430 case OMPD_requires:6431 case OMPD_metadirective:6432 case OMPD_unknown:6433 break;6434 default:6435 break;6436 }6437 llvm_unreachable("Unexpected directive kind.");6438}6439 6440llvm::Value *CGOpenMPRuntime::emitNumTeamsForTargetDirective(6441 CodeGenFunction &CGF, const OMPExecutableDirective &D) {6442 assert(!CGF.getLangOpts().OpenMPIsTargetDevice &&6443 "Clauses associated with the teams directive expected to be emitted "6444 "only for the host!");6445 CGBuilderTy &Bld = CGF.Builder;6446 int32_t MinNT = -1, MaxNT = -1;6447 const Expr *NumTeams =6448 getNumTeamsExprForTargetDirective(CGF, D, MinNT, MaxNT);6449 if (NumTeams != nullptr) {6450 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();6451 6452 switch (DirectiveKind) {6453 case OMPD_target: {6454 const auto *CS = D.getInnermostCapturedStmt();6455 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);6456 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);6457 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams,6458 /*IgnoreResultAssign*/ true);6459 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty,6460 /*isSigned=*/true);6461 }6462 case OMPD_target_teams:6463 case OMPD_target_teams_distribute:6464 case OMPD_target_teams_distribute_simd:6465 case OMPD_target_teams_distribute_parallel_for:6466 case OMPD_target_teams_distribute_parallel_for_simd: {6467 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF);6468 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams,6469 /*IgnoreResultAssign*/ true);6470 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty,6471 /*isSigned=*/true);6472 }6473 default:6474 break;6475 }6476 }6477 6478 assert(MinNT == MaxNT && "Num threads ranges require handling here.");6479 return llvm::ConstantInt::get(CGF.Int32Ty, MinNT);6480}6481 6482/// Check for a num threads constant value (stored in \p DefaultVal), or6483/// expression (stored in \p E). If the value is conditional (via an if-clause),6484/// store the condition in \p CondVal. If \p E, and \p CondVal respectively, are6485/// nullptr, no expression evaluation is perfomed.6486static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS,6487 const Expr **E, int32_t &UpperBound,6488 bool UpperBoundOnly, llvm::Value **CondVal) {6489 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(6490 CGF.getContext(), CS->getCapturedStmt());6491 const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);6492 if (!Dir)6493 return;6494 6495 if (isOpenMPParallelDirective(Dir->getDirectiveKind())) {6496 // Handle if clause. If if clause present, the number of threads is6497 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.6498 if (CondVal && Dir->hasClausesOfKind<OMPIfClause>()) {6499 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);6500 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);6501 const OMPIfClause *IfClause = nullptr;6502 for (const auto *C : Dir->getClausesOfKind<OMPIfClause>()) {6503 if (C->getNameModifier() == OMPD_unknown ||6504 C->getNameModifier() == OMPD_parallel) {6505 IfClause = C;6506 break;6507 }6508 }6509 if (IfClause) {6510 const Expr *CondExpr = IfClause->getCondition();6511 bool Result;6512 if (CondExpr->EvaluateAsBooleanCondition(Result, CGF.getContext())) {6513 if (!Result) {6514 UpperBound = 1;6515 return;6516 }6517 } else {6518 CodeGenFunction::LexicalScope Scope(CGF, CondExpr->getSourceRange());6519 if (const auto *PreInit =6520 cast_or_null<DeclStmt>(IfClause->getPreInitStmt())) {6521 for (const auto *I : PreInit->decls()) {6522 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {6523 CGF.EmitVarDecl(cast<VarDecl>(*I));6524 } else {6525 CodeGenFunction::AutoVarEmission Emission =6526 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));6527 CGF.EmitAutoVarCleanups(Emission);6528 }6529 }6530 *CondVal = CGF.EvaluateExprAsBool(CondExpr);6531 }6532 }6533 }6534 }6535 // Check the value of num_threads clause iff if clause was not specified6536 // or is not evaluated to false.6537 if (Dir->hasClausesOfKind<OMPNumThreadsClause>()) {6538 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);6539 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);6540 const auto *NumThreadsClause =6541 Dir->getSingleClause<OMPNumThreadsClause>();6542 const Expr *NTExpr = NumThreadsClause->getNumThreads();6543 if (NTExpr->isIntegerConstantExpr(CGF.getContext()))6544 if (auto Constant = NTExpr->getIntegerConstantExpr(CGF.getContext()))6545 UpperBound =6546 UpperBound6547 ? Constant->getZExtValue()6548 : std::min(UpperBound,6549 static_cast<int32_t>(Constant->getZExtValue()));6550 // If we haven't found a upper bound, remember we saw a thread limiting6551 // clause.6552 if (UpperBound == -1)6553 UpperBound = 0;6554 if (!E)6555 return;6556 CodeGenFunction::LexicalScope Scope(CGF, NTExpr->getSourceRange());6557 if (const auto *PreInit =6558 cast_or_null<DeclStmt>(NumThreadsClause->getPreInitStmt())) {6559 for (const auto *I : PreInit->decls()) {6560 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {6561 CGF.EmitVarDecl(cast<VarDecl>(*I));6562 } else {6563 CodeGenFunction::AutoVarEmission Emission =6564 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));6565 CGF.EmitAutoVarCleanups(Emission);6566 }6567 }6568 }6569 *E = NTExpr;6570 }6571 return;6572 }6573 if (isOpenMPSimdDirective(Dir->getDirectiveKind()))6574 UpperBound = 1;6575}6576 6577const Expr *CGOpenMPRuntime::getNumThreadsExprForTargetDirective(6578 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound,6579 bool UpperBoundOnly, llvm::Value **CondVal, const Expr **ThreadLimitExpr) {6580 assert((!CGF.getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) &&6581 "Clauses associated with the teams directive expected to be emitted "6582 "only for the host!");6583 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();6584 assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&6585 "Expected target-based executable directive.");6586 6587 const Expr *NT = nullptr;6588 const Expr **NTPtr = UpperBoundOnly ? nullptr : &NT;6589 6590 auto CheckForConstExpr = [&](const Expr *E, const Expr **EPtr) {6591 if (E->isIntegerConstantExpr(CGF.getContext())) {6592 if (auto Constant = E->getIntegerConstantExpr(CGF.getContext()))6593 UpperBound = UpperBound ? Constant->getZExtValue()6594 : std::min(UpperBound,6595 int32_t(Constant->getZExtValue()));6596 }6597 // If we haven't found a upper bound, remember we saw a thread limiting6598 // clause.6599 if (UpperBound == -1)6600 UpperBound = 0;6601 if (EPtr)6602 *EPtr = E;6603 };6604 6605 auto ReturnSequential = [&]() {6606 UpperBound = 1;6607 return NT;6608 };6609 6610 switch (DirectiveKind) {6611 case OMPD_target: {6612 const CapturedStmt *CS = D.getInnermostCapturedStmt();6613 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);6614 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(6615 CGF.getContext(), CS->getCapturedStmt());6616 // TODO: The standard is not clear how to resolve two thread limit clauses,6617 // let's pick the teams one if it's present, otherwise the target one.6618 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();6619 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {6620 if (const auto *TLC = Dir->getSingleClause<OMPThreadLimitClause>()) {6621 ThreadLimitClause = TLC;6622 if (ThreadLimitExpr) {6623 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);6624 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);6625 CodeGenFunction::LexicalScope Scope(6626 CGF,6627 ThreadLimitClause->getThreadLimit().front()->getSourceRange());6628 if (const auto *PreInit =6629 cast_or_null<DeclStmt>(ThreadLimitClause->getPreInitStmt())) {6630 for (const auto *I : PreInit->decls()) {6631 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {6632 CGF.EmitVarDecl(cast<VarDecl>(*I));6633 } else {6634 CodeGenFunction::AutoVarEmission Emission =6635 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));6636 CGF.EmitAutoVarCleanups(Emission);6637 }6638 }6639 }6640 }6641 }6642 }6643 if (ThreadLimitClause)6644 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),6645 ThreadLimitExpr);6646 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {6647 if (isOpenMPTeamsDirective(Dir->getDirectiveKind()) &&6648 !isOpenMPDistributeDirective(Dir->getDirectiveKind())) {6649 CS = Dir->getInnermostCapturedStmt();6650 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(6651 CGF.getContext(), CS->getCapturedStmt());6652 Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);6653 }6654 if (Dir && isOpenMPParallelDirective(Dir->getDirectiveKind())) {6655 CS = Dir->getInnermostCapturedStmt();6656 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);6657 } else if (Dir && isOpenMPSimdDirective(Dir->getDirectiveKind()))6658 return ReturnSequential();6659 }6660 return NT;6661 }6662 case OMPD_target_teams: {6663 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {6664 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);6665 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();6666 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),6667 ThreadLimitExpr);6668 }6669 const CapturedStmt *CS = D.getInnermostCapturedStmt();6670 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);6671 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(6672 CGF.getContext(), CS->getCapturedStmt());6673 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {6674 if (Dir->getDirectiveKind() == OMPD_distribute) {6675 CS = Dir->getInnermostCapturedStmt();6676 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);6677 }6678 }6679 return NT;6680 }6681 case OMPD_target_teams_distribute:6682 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {6683 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);6684 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();6685 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),6686 ThreadLimitExpr);6687 }6688 getNumThreads(CGF, D.getInnermostCapturedStmt(), NTPtr, UpperBound,6689 UpperBoundOnly, CondVal);6690 return NT;6691 case OMPD_target_teams_loop:6692 case OMPD_target_parallel_loop:6693 case OMPD_target_parallel:6694 case OMPD_target_parallel_for:6695 case OMPD_target_parallel_for_simd:6696 case OMPD_target_teams_distribute_parallel_for:6697 case OMPD_target_teams_distribute_parallel_for_simd: {6698 if (CondVal && D.hasClausesOfKind<OMPIfClause>()) {6699 const OMPIfClause *IfClause = nullptr;6700 for (const auto *C : D.getClausesOfKind<OMPIfClause>()) {6701 if (C->getNameModifier() == OMPD_unknown ||6702 C->getNameModifier() == OMPD_parallel) {6703 IfClause = C;6704 break;6705 }6706 }6707 if (IfClause) {6708 const Expr *Cond = IfClause->getCondition();6709 bool Result;6710 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) {6711 if (!Result)6712 return ReturnSequential();6713 } else {6714 CodeGenFunction::RunCleanupsScope Scope(CGF);6715 *CondVal = CGF.EvaluateExprAsBool(Cond);6716 }6717 }6718 }6719 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {6720 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);6721 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();6722 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),6723 ThreadLimitExpr);6724 }6725 if (D.hasClausesOfKind<OMPNumThreadsClause>()) {6726 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);6727 const auto *NumThreadsClause = D.getSingleClause<OMPNumThreadsClause>();6728 CheckForConstExpr(NumThreadsClause->getNumThreads(), nullptr);6729 return NumThreadsClause->getNumThreads();6730 }6731 return NT;6732 }6733 case OMPD_target_teams_distribute_simd:6734 case OMPD_target_simd:6735 return ReturnSequential();6736 default:6737 break;6738 }6739 llvm_unreachable("Unsupported directive kind.");6740}6741 6742llvm::Value *CGOpenMPRuntime::emitNumThreadsForTargetDirective(6743 CodeGenFunction &CGF, const OMPExecutableDirective &D) {6744 llvm::Value *NumThreadsVal = nullptr;6745 llvm::Value *CondVal = nullptr;6746 llvm::Value *ThreadLimitVal = nullptr;6747 const Expr *ThreadLimitExpr = nullptr;6748 int32_t UpperBound = -1;6749 6750 const Expr *NT = getNumThreadsExprForTargetDirective(6751 CGF, D, UpperBound, /* UpperBoundOnly */ false, &CondVal,6752 &ThreadLimitExpr);6753 6754 // Thread limit expressions are used below, emit them.6755 if (ThreadLimitExpr) {6756 ThreadLimitVal =6757 CGF.EmitScalarExpr(ThreadLimitExpr, /*IgnoreResultAssign=*/true);6758 ThreadLimitVal = CGF.Builder.CreateIntCast(ThreadLimitVal, CGF.Int32Ty,6759 /*isSigned=*/false);6760 }6761 6762 // Generate the num teams expression.6763 if (UpperBound == 1) {6764 NumThreadsVal = CGF.Builder.getInt32(UpperBound);6765 } else if (NT) {6766 NumThreadsVal = CGF.EmitScalarExpr(NT, /*IgnoreResultAssign=*/true);6767 NumThreadsVal = CGF.Builder.CreateIntCast(NumThreadsVal, CGF.Int32Ty,6768 /*isSigned=*/false);6769 } else if (ThreadLimitVal) {6770 // If we do not have a num threads value but a thread limit, replace the6771 // former with the latter. We know handled the thread limit expression.6772 NumThreadsVal = ThreadLimitVal;6773 ThreadLimitVal = nullptr;6774 } else {6775 // Default to "0" which means runtime choice.6776 assert(!ThreadLimitVal && "Default not applicable with thread limit value");6777 NumThreadsVal = CGF.Builder.getInt32(0);6778 }6779 6780 // Handle if clause. If if clause present, the number of threads is6781 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.6782 if (CondVal) {6783 CodeGenFunction::RunCleanupsScope Scope(CGF);6784 NumThreadsVal = CGF.Builder.CreateSelect(CondVal, NumThreadsVal,6785 CGF.Builder.getInt32(1));6786 }6787 6788 // If the thread limit and num teams expression were present, take the6789 // minimum.6790 if (ThreadLimitVal) {6791 NumThreadsVal = CGF.Builder.CreateSelect(6792 CGF.Builder.CreateICmpULT(ThreadLimitVal, NumThreadsVal),6793 ThreadLimitVal, NumThreadsVal);6794 }6795 6796 return NumThreadsVal;6797}6798 6799namespace {6800LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();6801 6802// Utility to handle information from clauses associated with a given6803// construct that use mappable expressions (e.g. 'map' clause, 'to' clause).6804// It provides a convenient interface to obtain the information and generate6805// code for that information.6806class MappableExprsHandler {6807public:6808 /// Custom comparator for attach-pointer expressions that compares them by6809 /// complexity (i.e. their component-depth) first, then by the order in which6810 /// they were computed by collectAttachPtrExprInfo(), if they are semantically6811 /// different.6812 struct AttachPtrExprComparator {6813 const MappableExprsHandler &Handler;6814 // Cache of previous equality comparison results.6815 mutable llvm::DenseMap<std::pair<const Expr *, const Expr *>, bool>6816 CachedEqualityComparisons;6817 6818 AttachPtrExprComparator(const MappableExprsHandler &H) : Handler(H) {}6819 AttachPtrExprComparator() = delete;6820 6821 // Return true iff LHS is "less than" RHS.6822 bool operator()(const Expr *LHS, const Expr *RHS) const {6823 if (LHS == RHS)6824 return false;6825 6826 // First, compare by complexity (depth)6827 const auto ItLHS = Handler.AttachPtrComponentDepthMap.find(LHS);6828 const auto ItRHS = Handler.AttachPtrComponentDepthMap.find(RHS);6829 6830 std::optional<size_t> DepthLHS =6831 (ItLHS != Handler.AttachPtrComponentDepthMap.end()) ? ItLHS->second6832 : std::nullopt;6833 std::optional<size_t> DepthRHS =6834 (ItRHS != Handler.AttachPtrComponentDepthMap.end()) ? ItRHS->second6835 : std::nullopt;6836 6837 // std::nullopt (no attach pointer) has lowest complexity6838 if (!DepthLHS.has_value() && !DepthRHS.has_value()) {6839 // Both have same complexity, now check semantic equality6840 if (areEqual(LHS, RHS))6841 return false;6842 // Different semantically, compare by computation order6843 return wasComputedBefore(LHS, RHS);6844 }6845 if (!DepthLHS.has_value())6846 return true; // LHS has lower complexity6847 if (!DepthRHS.has_value())6848 return false; // RHS has lower complexity6849 6850 // Both have values, compare by depth (lower depth = lower complexity)6851 if (DepthLHS.value() != DepthRHS.value())6852 return DepthLHS.value() < DepthRHS.value();6853 6854 // Same complexity, now check semantic equality6855 if (areEqual(LHS, RHS))6856 return false;6857 // Different semantically, compare by computation order6858 return wasComputedBefore(LHS, RHS);6859 }6860 6861 public:6862 /// Return true if \p LHS and \p RHS are semantically equal. Uses pre-cached6863 /// results, if available, otherwise does a recursive semantic comparison.6864 bool areEqual(const Expr *LHS, const Expr *RHS) const {6865 // Check cache first for faster lookup6866 const auto CachedResultIt = CachedEqualityComparisons.find({LHS, RHS});6867 if (CachedResultIt != CachedEqualityComparisons.end())6868 return CachedResultIt->second;6869 6870 bool ComparisonResult = areSemanticallyEqual(LHS, RHS);6871 6872 // Cache the result for future lookups (both orders since semantic6873 // equality is commutative)6874 CachedEqualityComparisons[{LHS, RHS}] = ComparisonResult;6875 CachedEqualityComparisons[{RHS, LHS}] = ComparisonResult;6876 return ComparisonResult;6877 }6878 6879 /// Compare the two attach-ptr expressions by their computation order.6880 /// Returns true iff LHS was computed before RHS by6881 /// collectAttachPtrExprInfo().6882 bool wasComputedBefore(const Expr *LHS, const Expr *RHS) const {6883 const size_t &OrderLHS = Handler.AttachPtrComputationOrderMap.at(LHS);6884 const size_t &OrderRHS = Handler.AttachPtrComputationOrderMap.at(RHS);6885 6886 return OrderLHS < OrderRHS;6887 }6888 6889 private:6890 /// Helper function to compare attach-pointer expressions semantically.6891 /// This function handles various expression types that can be part of an6892 /// attach-pointer.6893 /// TODO: Not urgent, but we should ideally return true when comparing6894 /// `p[10]`, `*(p + 10)`, `*(p + 5 + 5)`, `p[10:1]` etc.6895 bool areSemanticallyEqual(const Expr *LHS, const Expr *RHS) const {6896 if (LHS == RHS)6897 return true;6898 6899 // If only one is null, they aren't equal6900 if (!LHS || !RHS)6901 return false;6902 6903 ASTContext &Ctx = Handler.CGF.getContext();6904 // Strip away parentheses and no-op casts to get to the core expression6905 LHS = LHS->IgnoreParenNoopCasts(Ctx);6906 RHS = RHS->IgnoreParenNoopCasts(Ctx);6907 6908 // Direct pointer comparison of the underlying expressions6909 if (LHS == RHS)6910 return true;6911 6912 // Check if the expression classes match6913 if (LHS->getStmtClass() != RHS->getStmtClass())6914 return false;6915 6916 // Handle DeclRefExpr (variable references)6917 if (const auto *LD = dyn_cast<DeclRefExpr>(LHS)) {6918 const auto *RD = dyn_cast<DeclRefExpr>(RHS);6919 if (!RD)6920 return false;6921 return LD->getDecl()->getCanonicalDecl() ==6922 RD->getDecl()->getCanonicalDecl();6923 }6924 6925 // Handle ArraySubscriptExpr (array indexing like a[i])6926 if (const auto *LA = dyn_cast<ArraySubscriptExpr>(LHS)) {6927 const auto *RA = dyn_cast<ArraySubscriptExpr>(RHS);6928 if (!RA)6929 return false;6930 return areSemanticallyEqual(LA->getBase(), RA->getBase()) &&6931 areSemanticallyEqual(LA->getIdx(), RA->getIdx());6932 }6933 6934 // Handle MemberExpr (member access like s.m or p->m)6935 if (const auto *LM = dyn_cast<MemberExpr>(LHS)) {6936 const auto *RM = dyn_cast<MemberExpr>(RHS);6937 if (!RM)6938 return false;6939 if (LM->getMemberDecl()->getCanonicalDecl() !=6940 RM->getMemberDecl()->getCanonicalDecl())6941 return false;6942 return areSemanticallyEqual(LM->getBase(), RM->getBase());6943 }6944 6945 // Handle UnaryOperator (unary operations like *p, &x, etc.)6946 if (const auto *LU = dyn_cast<UnaryOperator>(LHS)) {6947 const auto *RU = dyn_cast<UnaryOperator>(RHS);6948 if (!RU)6949 return false;6950 if (LU->getOpcode() != RU->getOpcode())6951 return false;6952 return areSemanticallyEqual(LU->getSubExpr(), RU->getSubExpr());6953 }6954 6955 // Handle BinaryOperator (binary operations like p + offset)6956 if (const auto *LB = dyn_cast<BinaryOperator>(LHS)) {6957 const auto *RB = dyn_cast<BinaryOperator>(RHS);6958 if (!RB)6959 return false;6960 if (LB->getOpcode() != RB->getOpcode())6961 return false;6962 return areSemanticallyEqual(LB->getLHS(), RB->getLHS()) &&6963 areSemanticallyEqual(LB->getRHS(), RB->getRHS());6964 }6965 6966 // Handle ArraySectionExpr (array sections like a[0:1])6967 // Attach pointers should not contain array-sections, but currently we6968 // don't emit an error.6969 if (const auto *LAS = dyn_cast<ArraySectionExpr>(LHS)) {6970 const auto *RAS = dyn_cast<ArraySectionExpr>(RHS);6971 if (!RAS)6972 return false;6973 return areSemanticallyEqual(LAS->getBase(), RAS->getBase()) &&6974 areSemanticallyEqual(LAS->getLowerBound(),6975 RAS->getLowerBound()) &&6976 areSemanticallyEqual(LAS->getLength(), RAS->getLength());6977 }6978 6979 // Handle CastExpr (explicit casts)6980 if (const auto *LC = dyn_cast<CastExpr>(LHS)) {6981 const auto *RC = dyn_cast<CastExpr>(RHS);6982 if (!RC)6983 return false;6984 if (LC->getCastKind() != RC->getCastKind())6985 return false;6986 return areSemanticallyEqual(LC->getSubExpr(), RC->getSubExpr());6987 }6988 6989 // Handle CXXThisExpr (this pointer)6990 if (isa<CXXThisExpr>(LHS) && isa<CXXThisExpr>(RHS))6991 return true;6992 6993 // Handle IntegerLiteral (integer constants)6994 if (const auto *LI = dyn_cast<IntegerLiteral>(LHS)) {6995 const auto *RI = dyn_cast<IntegerLiteral>(RHS);6996 if (!RI)6997 return false;6998 return LI->getValue() == RI->getValue();6999 }7000 7001 // Handle CharacterLiteral (character constants)7002 if (const auto *LC = dyn_cast<CharacterLiteral>(LHS)) {7003 const auto *RC = dyn_cast<CharacterLiteral>(RHS);7004 if (!RC)7005 return false;7006 return LC->getValue() == RC->getValue();7007 }7008 7009 // Handle FloatingLiteral (floating point constants)7010 if (const auto *LF = dyn_cast<FloatingLiteral>(LHS)) {7011 const auto *RF = dyn_cast<FloatingLiteral>(RHS);7012 if (!RF)7013 return false;7014 // Use bitwise comparison for floating point literals7015 return LF->getValue().bitwiseIsEqual(RF->getValue());7016 }7017 7018 // Handle StringLiteral (string constants)7019 if (const auto *LS = dyn_cast<StringLiteral>(LHS)) {7020 const auto *RS = dyn_cast<StringLiteral>(RHS);7021 if (!RS)7022 return false;7023 return LS->getString() == RS->getString();7024 }7025 7026 // Handle CXXNullPtrLiteralExpr (nullptr)7027 if (isa<CXXNullPtrLiteralExpr>(LHS) && isa<CXXNullPtrLiteralExpr>(RHS))7028 return true;7029 7030 // Handle CXXBoolLiteralExpr (true/false)7031 if (const auto *LB = dyn_cast<CXXBoolLiteralExpr>(LHS)) {7032 const auto *RB = dyn_cast<CXXBoolLiteralExpr>(RHS);7033 if (!RB)7034 return false;7035 return LB->getValue() == RB->getValue();7036 }7037 7038 // Fallback for other forms - use the existing comparison method7039 return Expr::isSameComparisonOperand(LHS, RHS);7040 }7041 };7042 7043 /// Get the offset of the OMP_MAP_MEMBER_OF field.7044 static unsigned getFlagMemberOffset() {7045 unsigned Offset = 0;7046 for (uint64_t Remain =7047 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(7048 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);7049 !(Remain & 1); Remain = Remain >> 1)7050 Offset++;7051 return Offset;7052 }7053 7054 /// Class that holds debugging information for a data mapping to be passed to7055 /// the runtime library.7056 class MappingExprInfo {7057 /// The variable declaration used for the data mapping.7058 const ValueDecl *MapDecl = nullptr;7059 /// The original expression used in the map clause, or null if there is7060 /// none.7061 const Expr *MapExpr = nullptr;7062 7063 public:7064 MappingExprInfo(const ValueDecl *MapDecl, const Expr *MapExpr = nullptr)7065 : MapDecl(MapDecl), MapExpr(MapExpr) {}7066 7067 const ValueDecl *getMapDecl() const { return MapDecl; }7068 const Expr *getMapExpr() const { return MapExpr; }7069 };7070 7071 using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;7072 using MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;7073 using MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;7074 using MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy;7075 using MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy;7076 using MapNonContiguousArrayTy =7077 llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;7078 using MapExprsArrayTy = SmallVector<MappingExprInfo, 4>;7079 using MapValueDeclsArrayTy = SmallVector<const ValueDecl *, 4>;7080 using MapData =7081 std::tuple<OMPClauseMappableExprCommon::MappableExprComponentListRef,7082 OpenMPMapClauseKind, ArrayRef<OpenMPMapModifierKind>,7083 bool /*IsImplicit*/, const ValueDecl *, const Expr *>;7084 using MapDataArrayTy = SmallVector<MapData, 4>;7085 7086 /// This structure contains combined information generated for mappable7087 /// clauses, including base pointers, pointers, sizes, map types, user-defined7088 /// mappers, and non-contiguous information.7089 struct MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy {7090 MapExprsArrayTy Exprs;7091 MapValueDeclsArrayTy Mappers;7092 MapValueDeclsArrayTy DevicePtrDecls;7093 7094 /// Append arrays in \a CurInfo.7095 void append(MapCombinedInfoTy &CurInfo) {7096 Exprs.append(CurInfo.Exprs.begin(), CurInfo.Exprs.end());7097 DevicePtrDecls.append(CurInfo.DevicePtrDecls.begin(),7098 CurInfo.DevicePtrDecls.end());7099 Mappers.append(CurInfo.Mappers.begin(), CurInfo.Mappers.end());7100 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);7101 }7102 };7103 7104 /// Map between a struct and the its lowest & highest elements which have been7105 /// mapped.7106 /// [ValueDecl *] --> {LE(FieldIndex, Pointer),7107 /// HE(FieldIndex, Pointer)}7108 struct StructRangeInfoTy {7109 MapCombinedInfoTy PreliminaryMapData;7110 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> LowestElem = {7111 0, Address::invalid()};7112 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> HighestElem = {7113 0, Address::invalid()};7114 Address Base = Address::invalid();7115 Address LB = Address::invalid();7116 bool IsArraySection = false;7117 bool HasCompleteRecord = false;7118 };7119 7120 /// A struct to store the attach pointer and pointee information, to be used7121 /// when emitting an attach entry.7122 struct AttachInfoTy {7123 Address AttachPtrAddr = Address::invalid();7124 Address AttachPteeAddr = Address::invalid();7125 const ValueDecl *AttachPtrDecl = nullptr;7126 const Expr *AttachMapExpr = nullptr;7127 7128 bool isValid() const {7129 return AttachPtrAddr.isValid() && AttachPteeAddr.isValid();7130 }7131 };7132 7133 /// Check if there's any component list where the attach pointer expression7134 /// matches the given captured variable.7135 bool hasAttachEntryForCapturedVar(const ValueDecl *VD) const {7136 for (const auto &AttachEntry : AttachPtrExprMap) {7137 if (AttachEntry.second) {7138 // Check if the attach pointer expression is a DeclRefExpr that7139 // references the captured variable7140 if (const auto *DRE = dyn_cast<DeclRefExpr>(AttachEntry.second))7141 if (DRE->getDecl() == VD)7142 return true;7143 }7144 }7145 return false;7146 }7147 7148 /// Get the previously-cached attach pointer for a component list, if-any.7149 const Expr *getAttachPtrExpr(7150 OMPClauseMappableExprCommon::MappableExprComponentListRef Components)7151 const {7152 const auto It = AttachPtrExprMap.find(Components);7153 if (It != AttachPtrExprMap.end())7154 return It->second;7155 7156 return nullptr;7157 }7158 7159private:7160 /// Kind that defines how a device pointer has to be returned.7161 struct MapInfo {7162 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;7163 OpenMPMapClauseKind MapType = OMPC_MAP_unknown;7164 ArrayRef<OpenMPMapModifierKind> MapModifiers;7165 ArrayRef<OpenMPMotionModifierKind> MotionModifiers;7166 bool ReturnDevicePointer = false;7167 bool IsImplicit = false;7168 const ValueDecl *Mapper = nullptr;7169 const Expr *VarRef = nullptr;7170 bool ForDeviceAddr = false;7171 7172 MapInfo() = default;7173 MapInfo(7174 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,7175 OpenMPMapClauseKind MapType,7176 ArrayRef<OpenMPMapModifierKind> MapModifiers,7177 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,7178 bool ReturnDevicePointer, bool IsImplicit,7179 const ValueDecl *Mapper = nullptr, const Expr *VarRef = nullptr,7180 bool ForDeviceAddr = false)7181 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),7182 MotionModifiers(MotionModifiers),7183 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit),7184 Mapper(Mapper), VarRef(VarRef), ForDeviceAddr(ForDeviceAddr) {}7185 };7186 7187 /// If use_device_ptr or use_device_addr is used on a decl which is a struct7188 /// member and there is no map information about it, then emission of that7189 /// entry is deferred until the whole struct has been processed.7190 struct DeferredDevicePtrEntryTy {7191 const Expr *IE = nullptr;7192 const ValueDecl *VD = nullptr;7193 bool ForDeviceAddr = false;7194 7195 DeferredDevicePtrEntryTy(const Expr *IE, const ValueDecl *VD,7196 bool ForDeviceAddr)7197 : IE(IE), VD(VD), ForDeviceAddr(ForDeviceAddr) {}7198 };7199 7200 /// The target directive from where the mappable clauses were extracted. It7201 /// is either a executable directive or a user-defined mapper directive.7202 llvm::PointerUnion<const OMPExecutableDirective *,7203 const OMPDeclareMapperDecl *>7204 CurDir;7205 7206 /// Function the directive is being generated for.7207 CodeGenFunction &CGF;7208 7209 /// Set of all first private variables in the current directive.7210 /// bool data is set to true if the variable is implicitly marked as7211 /// firstprivate, false otherwise.7212 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, bool> FirstPrivateDecls;7213 7214 /// Map between device pointer declarations and their expression components.7215 /// The key value for declarations in 'this' is null.7216 llvm::DenseMap<7217 const ValueDecl *,7218 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>7219 DevPointersMap;7220 7221 /// Map between device addr declarations and their expression components.7222 /// The key value for declarations in 'this' is null.7223 llvm::DenseMap<7224 const ValueDecl *,7225 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>7226 HasDevAddrsMap;7227 7228 /// Map between lambda declarations and their map type.7229 llvm::DenseMap<const ValueDecl *, const OMPMapClause *> LambdasMap;7230 7231 /// Map from component lists to their attach pointer expressions.7232 llvm::DenseMap<OMPClauseMappableExprCommon::MappableExprComponentListRef,7233 const Expr *>7234 AttachPtrExprMap;7235 7236 /// Map from attach pointer expressions to their component depth.7237 /// nullptr key has std::nullopt depth. This can be used to order attach-ptr7238 /// expressions with increasing/decreasing depth.7239 /// The component-depth of `nullptr` (i.e. no attach-ptr) is `std::nullopt`.7240 /// TODO: Not urgent, but we should ideally use the number of pointer7241 /// dereferences in an expr as an indicator of its complexity, instead of the7242 /// component-depth. That would be needed for us to treat `p[1]`, `*(p + 10)`,7243 /// `*(p + 5 + 5)` together.7244 llvm::DenseMap<const Expr *, std::optional<size_t>>7245 AttachPtrComponentDepthMap = {{nullptr, std::nullopt}};7246 7247 /// Map from attach pointer expressions to the order they were computed in, in7248 /// collectAttachPtrExprInfo().7249 llvm::DenseMap<const Expr *, size_t> AttachPtrComputationOrderMap = {7250 {nullptr, 0}};7251 7252 /// An instance of attach-ptr-expr comparator that can be used throughout the7253 /// lifetime of this handler.7254 AttachPtrExprComparator AttachPtrComparator;7255 7256 llvm::Value *getExprTypeSize(const Expr *E) const {7257 QualType ExprTy = E->getType().getCanonicalType();7258 7259 // Calculate the size for array shaping expression.7260 if (const auto *OAE = dyn_cast<OMPArrayShapingExpr>(E)) {7261 llvm::Value *Size =7262 CGF.getTypeSize(OAE->getBase()->getType()->getPointeeType());7263 for (const Expr *SE : OAE->getDimensions()) {7264 llvm::Value *Sz = CGF.EmitScalarExpr(SE);7265 Sz = CGF.EmitScalarConversion(Sz, SE->getType(),7266 CGF.getContext().getSizeType(),7267 SE->getExprLoc());7268 Size = CGF.Builder.CreateNUWMul(Size, Sz);7269 }7270 return Size;7271 }7272 7273 // Reference types are ignored for mapping purposes.7274 if (const auto *RefTy = ExprTy->getAs<ReferenceType>())7275 ExprTy = RefTy->getPointeeType().getCanonicalType();7276 7277 // Given that an array section is considered a built-in type, we need to7278 // do the calculation based on the length of the section instead of relying7279 // on CGF.getTypeSize(E->getType()).7280 if (const auto *OAE = dyn_cast<ArraySectionExpr>(E)) {7281 QualType BaseTy = ArraySectionExpr::getBaseOriginalType(7282 OAE->getBase()->IgnoreParenImpCasts())7283 .getCanonicalType();7284 7285 // If there is no length associated with the expression and lower bound is7286 // not specified too, that means we are using the whole length of the7287 // base.7288 if (!OAE->getLength() && OAE->getColonLocFirst().isValid() &&7289 !OAE->getLowerBound())7290 return CGF.getTypeSize(BaseTy);7291 7292 llvm::Value *ElemSize;7293 if (const auto *PTy = BaseTy->getAs<PointerType>()) {7294 ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType());7295 } else {7296 const auto *ATy = cast<ArrayType>(BaseTy.getTypePtr());7297 assert(ATy && "Expecting array type if not a pointer type.");7298 ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType());7299 }7300 7301 // If we don't have a length at this point, that is because we have an7302 // array section with a single element.7303 if (!OAE->getLength() && OAE->getColonLocFirst().isInvalid())7304 return ElemSize;7305 7306 if (const Expr *LenExpr = OAE->getLength()) {7307 llvm::Value *LengthVal = CGF.EmitScalarExpr(LenExpr);7308 LengthVal = CGF.EmitScalarConversion(LengthVal, LenExpr->getType(),7309 CGF.getContext().getSizeType(),7310 LenExpr->getExprLoc());7311 return CGF.Builder.CreateNUWMul(LengthVal, ElemSize);7312 }7313 assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&7314 OAE->getLowerBound() && "expected array_section[lb:].");7315 // Size = sizetype - lb * elemtype;7316 llvm::Value *LengthVal = CGF.getTypeSize(BaseTy);7317 llvm::Value *LBVal = CGF.EmitScalarExpr(OAE->getLowerBound());7318 LBVal = CGF.EmitScalarConversion(LBVal, OAE->getLowerBound()->getType(),7319 CGF.getContext().getSizeType(),7320 OAE->getLowerBound()->getExprLoc());7321 LBVal = CGF.Builder.CreateNUWMul(LBVal, ElemSize);7322 llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LengthVal, LBVal);7323 llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LengthVal, LBVal);7324 LengthVal = CGF.Builder.CreateSelect(7325 Cmp, TrueVal, llvm::ConstantInt::get(CGF.SizeTy, 0));7326 return LengthVal;7327 }7328 return CGF.getTypeSize(ExprTy);7329 }7330 7331 /// Return the corresponding bits for a given map clause modifier. Add7332 /// a flag marking the map as a pointer if requested. Add a flag marking the7333 /// map as the first one of a series of maps that relate to the same map7334 /// expression.7335 OpenMPOffloadMappingFlags getMapTypeBits(7336 OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,7337 ArrayRef<OpenMPMotionModifierKind> MotionModifiers, bool IsImplicit,7338 bool AddPtrFlag, bool AddIsTargetParamFlag, bool IsNonContiguous) const {7339 OpenMPOffloadMappingFlags Bits =7340 IsImplicit ? OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT7341 : OpenMPOffloadMappingFlags::OMP_MAP_NONE;7342 switch (MapType) {7343 case OMPC_MAP_alloc:7344 case OMPC_MAP_release:7345 // alloc and release is the default behavior in the runtime library, i.e.7346 // if we don't pass any bits alloc/release that is what the runtime is7347 // going to do. Therefore, we don't need to signal anything for these two7348 // type modifiers.7349 break;7350 case OMPC_MAP_to:7351 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO;7352 break;7353 case OMPC_MAP_from:7354 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_FROM;7355 break;7356 case OMPC_MAP_tofrom:7357 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO |7358 OpenMPOffloadMappingFlags::OMP_MAP_FROM;7359 break;7360 case OMPC_MAP_delete:7361 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_DELETE;7362 break;7363 case OMPC_MAP_unknown:7364 llvm_unreachable("Unexpected map type!");7365 }7366 if (AddPtrFlag)7367 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;7368 if (AddIsTargetParamFlag)7369 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;7370 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_always))7371 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;7372 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_close))7373 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;7374 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_present) ||7375 llvm::is_contained(MotionModifiers, OMPC_MOTION_MODIFIER_present))7376 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;7377 if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_ompx_hold))7378 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;7379 if (IsNonContiguous)7380 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG;7381 return Bits;7382 }7383 7384 /// Return true if the provided expression is a final array section. A7385 /// final array section, is one whose length can't be proved to be one.7386 bool isFinalArraySectionExpression(const Expr *E) const {7387 const auto *OASE = dyn_cast<ArraySectionExpr>(E);7388 7389 // It is not an array section and therefore not a unity-size one.7390 if (!OASE)7391 return false;7392 7393 // An array section with no colon always refer to a single element.7394 if (OASE->getColonLocFirst().isInvalid())7395 return false;7396 7397 const Expr *Length = OASE->getLength();7398 7399 // If we don't have a length we have to check if the array has size 17400 // for this dimension. Also, we should always expect a length if the7401 // base type is pointer.7402 if (!Length) {7403 QualType BaseQTy = ArraySectionExpr::getBaseOriginalType(7404 OASE->getBase()->IgnoreParenImpCasts())7405 .getCanonicalType();7406 if (const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.getTypePtr()))7407 return ATy->getSExtSize() != 1;7408 // If we don't have a constant dimension length, we have to consider7409 // the current section as having any size, so it is not necessarily7410 // unitary. If it happen to be unity size, that's user fault.7411 return true;7412 }7413 7414 // Check if the length evaluates to 1.7415 Expr::EvalResult Result;7416 if (!Length->EvaluateAsInt(Result, CGF.getContext()))7417 return true; // Can have more that size 1.7418 7419 llvm::APSInt ConstLength = Result.Val.getInt();7420 return ConstLength.getSExtValue() != 1;7421 }7422 7423 /// A helper class to copy structures with overlapped elements, i.e. those7424 /// which have mappings of both "s" and "s.mem". Consecutive elements that7425 /// are not explicitly copied have mapping nodes synthesized for them,7426 /// taking care to avoid generating zero-sized copies.7427 class CopyOverlappedEntryGaps {7428 CodeGenFunction &CGF;7429 MapCombinedInfoTy &CombinedInfo;7430 OpenMPOffloadMappingFlags Flags = OpenMPOffloadMappingFlags::OMP_MAP_NONE;7431 const ValueDecl *MapDecl = nullptr;7432 const Expr *MapExpr = nullptr;7433 Address BP = Address::invalid();7434 bool IsNonContiguous = false;7435 uint64_t DimSize = 0;7436 // These elements track the position as the struct is iterated over7437 // (in order of increasing element address).7438 const RecordDecl *LastParent = nullptr;7439 uint64_t Cursor = 0;7440 unsigned LastIndex = -1u;7441 Address LB = Address::invalid();7442 7443 public:7444 CopyOverlappedEntryGaps(CodeGenFunction &CGF,7445 MapCombinedInfoTy &CombinedInfo,7446 OpenMPOffloadMappingFlags Flags,7447 const ValueDecl *MapDecl, const Expr *MapExpr,7448 Address BP, Address LB, bool IsNonContiguous,7449 uint64_t DimSize)7450 : CGF(CGF), CombinedInfo(CombinedInfo), Flags(Flags), MapDecl(MapDecl),7451 MapExpr(MapExpr), BP(BP), IsNonContiguous(IsNonContiguous),7452 DimSize(DimSize), LB(LB) {}7453 7454 void processField(7455 const OMPClauseMappableExprCommon::MappableComponent &MC,7456 const FieldDecl *FD,7457 llvm::function_ref<LValue(CodeGenFunction &, const MemberExpr *)>7458 EmitMemberExprBase) {7459 const RecordDecl *RD = FD->getParent();7460 const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(RD);7461 uint64_t FieldOffset = RL.getFieldOffset(FD->getFieldIndex());7462 uint64_t FieldSize =7463 CGF.getContext().getTypeSize(FD->getType().getCanonicalType());7464 Address ComponentLB = Address::invalid();7465 7466 if (FD->getType()->isLValueReferenceType()) {7467 const auto *ME = cast<MemberExpr>(MC.getAssociatedExpression());7468 LValue BaseLVal = EmitMemberExprBase(CGF, ME);7469 ComponentLB =7470 CGF.EmitLValueForFieldInitialization(BaseLVal, FD).getAddress();7471 } else {7472 ComponentLB =7473 CGF.EmitOMPSharedLValue(MC.getAssociatedExpression()).getAddress();7474 }7475 7476 if (!LastParent)7477 LastParent = RD;7478 if (FD->getParent() == LastParent) {7479 if (FD->getFieldIndex() != LastIndex + 1)7480 copyUntilField(FD, ComponentLB);7481 } else {7482 LastParent = FD->getParent();7483 if (((int64_t)FieldOffset - (int64_t)Cursor) > 0)7484 copyUntilField(FD, ComponentLB);7485 }7486 Cursor = FieldOffset + FieldSize;7487 LastIndex = FD->getFieldIndex();7488 LB = CGF.Builder.CreateConstGEP(ComponentLB, 1);7489 }7490 7491 void copyUntilField(const FieldDecl *FD, Address ComponentLB) {7492 llvm::Value *ComponentLBPtr = ComponentLB.emitRawPointer(CGF);7493 llvm::Value *LBPtr = LB.emitRawPointer(CGF);7494 llvm::Value *Size =7495 CGF.Builder.CreatePtrDiff(CGF.Int8Ty, ComponentLBPtr, LBPtr);7496 copySizedChunk(LBPtr, Size);7497 }7498 7499 void copyUntilEnd(Address HB) {7500 if (LastParent) {7501 const ASTRecordLayout &RL =7502 CGF.getContext().getASTRecordLayout(LastParent);7503 if ((uint64_t)CGF.getContext().toBits(RL.getSize()) <= Cursor)7504 return;7505 }7506 llvm::Value *LBPtr = LB.emitRawPointer(CGF);7507 llvm::Value *Size = CGF.Builder.CreatePtrDiff(7508 CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).emitRawPointer(CGF),7509 LBPtr);7510 copySizedChunk(LBPtr, Size);7511 }7512 7513 void copySizedChunk(llvm::Value *Base, llvm::Value *Size) {7514 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);7515 CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF));7516 CombinedInfo.DevicePtrDecls.push_back(nullptr);7517 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);7518 CombinedInfo.Pointers.push_back(Base);7519 CombinedInfo.Sizes.push_back(7520 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true));7521 CombinedInfo.Types.push_back(Flags);7522 CombinedInfo.Mappers.push_back(nullptr);7523 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize : 1);7524 }7525 };7526 7527 /// Generate the base pointers, section pointers, sizes, map type bits, and7528 /// user-defined mappers (all included in \a CombinedInfo) for the provided7529 /// map type, map or motion modifiers, and expression components.7530 /// \a IsFirstComponent should be set to true if the provided set of7531 /// components is the first associated with a capture.7532 void generateInfoForComponentList(7533 OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,7534 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,7535 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,7536 MapCombinedInfoTy &CombinedInfo,7537 MapCombinedInfoTy &StructBaseCombinedInfo,7538 StructRangeInfoTy &PartialStruct, bool IsFirstComponentList,7539 bool IsImplicit, bool GenerateAllInfoForClauses,7540 const ValueDecl *Mapper = nullptr, bool ForDeviceAddr = false,7541 const ValueDecl *BaseDecl = nullptr, const Expr *MapExpr = nullptr,7542 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>7543 OverlappedElements = {},7544 bool AreBothBasePtrAndPteeMapped = false) const {7545 // The following summarizes what has to be generated for each map and the7546 // types below. The generated information is expressed in this order:7547 // base pointer, section pointer, size, flags7548 // (to add to the ones that come from the map type and modifier).7549 //7550 // double d;7551 // int i[100];7552 // float *p;7553 // int **a = &i;7554 //7555 // struct S1 {7556 // int i;7557 // float f[50];7558 // }7559 // struct S2 {7560 // int i;7561 // float f[50];7562 // S1 s;7563 // double *p;7564 // struct S2 *ps;7565 // int &ref;7566 // }7567 // S2 s;7568 // S2 *ps;7569 //7570 // map(d)7571 // &d, &d, sizeof(double), TARGET_PARAM | TO | FROM7572 //7573 // map(i)7574 // &i, &i, 100*sizeof(int), TARGET_PARAM | TO | FROM7575 //7576 // map(i[1:23])7577 // &i(=&i[0]), &i[1], 23*sizeof(int), TARGET_PARAM | TO | FROM7578 //7579 // map(p)7580 // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM7581 //7582 // map(p[1:24])7583 // &p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM | PTR_AND_OBJ7584 // in unified shared memory mode or for local pointers7585 // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM7586 //7587 // map((*a)[0:3])7588 // &(*a), &(*a), sizeof(pointer), TARGET_PARAM | TO | FROM7589 // &(*a), &(*a)[0], 3*sizeof(int), PTR_AND_OBJ | TO | FROM7590 //7591 // map(**a)7592 // &(*a), &(*a), sizeof(pointer), TARGET_PARAM | TO | FROM7593 // &(*a), &(**a), sizeof(int), PTR_AND_OBJ | TO | FROM7594 //7595 // map(s)7596 // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM7597 //7598 // map(s.i)7599 // &s, &(s.i), sizeof(int), TARGET_PARAM | TO | FROM7600 //7601 // map(s.s.f)7602 // &s, &(s.s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM7603 //7604 // map(s.p)7605 // &s, &(s.p), sizeof(double*), TARGET_PARAM | TO | FROM7606 //7607 // map(to: s.p[:22])7608 // &s, &(s.p), sizeof(double*), TARGET_PARAM (*)7609 // &s, &(s.p), sizeof(double*), MEMBER_OF(1) (**)7610 // &(s.p), &(s.p[0]), 22*sizeof(double),7611 // MEMBER_OF(1) | PTR_AND_OBJ | TO (***)7612 // (*) alloc space for struct members, only this is a target parameter7613 // (**) map the pointer (nothing to be mapped in this example) (the compiler7614 // optimizes this entry out, same in the examples below)7615 // (***) map the pointee (map: to)7616 //7617 // map(to: s.ref)7618 // &s, &(s.ref), sizeof(int*), TARGET_PARAM (*)7619 // &s, &(s.ref), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | TO (***)7620 // (*) alloc space for struct members, only this is a target parameter7621 // (**) map the pointer (nothing to be mapped in this example) (the compiler7622 // optimizes this entry out, same in the examples below)7623 // (***) map the pointee (map: to)7624 //7625 // map(s.ps)7626 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM7627 //7628 // map(from: s.ps->s.i)7629 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM7630 // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)7631 // &(s.ps), &(s.ps->s.i), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | FROM7632 //7633 // map(to: s.ps->ps)7634 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM7635 // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)7636 // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ | TO7637 //7638 // map(s.ps->ps->ps)7639 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM7640 // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)7641 // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ7642 // &(s.ps->ps), &(s.ps->ps->ps), sizeof(S2*), PTR_AND_OBJ | TO | FROM7643 //7644 // map(to: s.ps->ps->s.f[:22])7645 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM7646 // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)7647 // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ7648 // &(s.ps->ps), &(s.ps->ps->s.f[0]), 22*sizeof(float), PTR_AND_OBJ | TO7649 //7650 // map(ps)7651 // &ps, &ps, sizeof(S2*), TARGET_PARAM | TO | FROM7652 //7653 // map(ps->i)7654 // ps, &(ps->i), sizeof(int), TARGET_PARAM | TO | FROM7655 //7656 // map(ps->s.f)7657 // ps, &(ps->s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM7658 //7659 // map(from: ps->p)7660 // ps, &(ps->p), sizeof(double*), TARGET_PARAM | FROM7661 //7662 // map(to: ps->p[:22])7663 // ps, &(ps->p), sizeof(double*), TARGET_PARAM7664 // ps, &(ps->p), sizeof(double*), MEMBER_OF(1)7665 // &(ps->p), &(ps->p[0]), 22*sizeof(double), MEMBER_OF(1) | PTR_AND_OBJ | TO7666 //7667 // map(ps->ps)7668 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM | TO | FROM7669 //7670 // map(from: ps->ps->s.i)7671 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM7672 // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)7673 // &(ps->ps), &(ps->ps->s.i), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | FROM7674 //7675 // map(from: ps->ps->ps)7676 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM7677 // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)7678 // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ | FROM7679 //7680 // map(ps->ps->ps->ps)7681 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM7682 // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)7683 // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ7684 // &(ps->ps->ps), &(ps->ps->ps->ps), sizeof(S2*), PTR_AND_OBJ | TO | FROM7685 //7686 // map(to: ps->ps->ps->s.f[:22])7687 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM7688 // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)7689 // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ7690 // &(ps->ps->ps), &(ps->ps->ps->s.f[0]), 22*sizeof(float), PTR_AND_OBJ | TO7691 //7692 // map(to: s.f[:22]) map(from: s.p[:33])7693 // &s, &(s.f[0]), 50*sizeof(float) + sizeof(struct S1) +7694 // sizeof(double*) (**), TARGET_PARAM7695 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | TO7696 // &s, &(s.p), sizeof(double*), MEMBER_OF(1)7697 // &(s.p), &(s.p[0]), 33*sizeof(double), MEMBER_OF(1) | PTR_AND_OBJ | FROM7698 // (*) allocate contiguous space needed to fit all mapped members even if7699 // we allocate space for members not mapped (in this example,7700 // s.f[22..49] and s.s are not mapped, yet we must allocate space for7701 // them as well because they fall between &s.f[0] and &s.p)7702 //7703 // map(from: s.f[:22]) map(to: ps->p[:33])7704 // &s, &(s.f[0]), 22*sizeof(float), TARGET_PARAM | FROM7705 // ps, &(ps->p), sizeof(S2*), TARGET_PARAM7706 // ps, &(ps->p), sizeof(double*), MEMBER_OF(2) (*)7707 // &(ps->p), &(ps->p[0]), 33*sizeof(double), MEMBER_OF(2) | PTR_AND_OBJ | TO7708 // (*) the struct this entry pertains to is the 2nd element in the list of7709 // arguments, hence MEMBER_OF(2)7710 //7711 // map(from: s.f[:22], s.s) map(to: ps->p[:33])7712 // &s, &(s.f[0]), 50*sizeof(float) + sizeof(struct S1), TARGET_PARAM7713 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | FROM7714 // &s, &(s.s), sizeof(struct S1), MEMBER_OF(1) | FROM7715 // ps, &(ps->p), sizeof(S2*), TARGET_PARAM7716 // ps, &(ps->p), sizeof(double*), MEMBER_OF(4) (*)7717 // &(ps->p), &(ps->p[0]), 33*sizeof(double), MEMBER_OF(4) | PTR_AND_OBJ | TO7718 // (*) the struct this entry pertains to is the 4th element in the list7719 // of arguments, hence MEMBER_OF(4)7720 //7721 // map(p, p[:100])7722 // ===> map(p[:100])7723 // &p, &p[0], 100*sizeof(float), TARGET_PARAM | PTR_AND_OBJ | TO | FROM7724 7725 // Track if the map information being generated is the first for a capture.7726 bool IsCaptureFirstInfo = IsFirstComponentList;7727 // When the variable is on a declare target link or in a to clause with7728 // unified memory, a reference is needed to hold the host/device address7729 // of the variable.7730 bool RequiresReference = false;7731 7732 // Scan the components from the base to the complete expression.7733 auto CI = Components.rbegin();7734 auto CE = Components.rend();7735 auto I = CI;7736 7737 // Track if the map information being generated is the first for a list of7738 // components.7739 bool IsExpressionFirstInfo = true;7740 bool FirstPointerInComplexData = false;7741 Address BP = Address::invalid();7742 const Expr *AssocExpr = I->getAssociatedExpression();7743 const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);7744 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);7745 const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(AssocExpr);7746 7747 if (AreBothBasePtrAndPteeMapped && std::next(I) == CE)7748 return;7749 if (isa<MemberExpr>(AssocExpr)) {7750 // The base is the 'this' pointer. The content of the pointer is going7751 // to be the base of the field being mapped.7752 BP = CGF.LoadCXXThisAddress();7753 } else if ((AE && isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) ||7754 (OASE &&7755 isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts()))) {7756 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress();7757 } else if (OAShE &&7758 isa<CXXThisExpr>(OAShE->getBase()->IgnoreParenCasts())) {7759 BP = Address(7760 CGF.EmitScalarExpr(OAShE->getBase()),7761 CGF.ConvertTypeForMem(OAShE->getBase()->getType()->getPointeeType()),7762 CGF.getContext().getTypeAlignInChars(OAShE->getBase()->getType()));7763 } else {7764 // The base is the reference to the variable.7765 // BP = &Var.7766 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress();7767 if (const auto *VD =7768 dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) {7769 if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =7770 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {7771 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||7772 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||7773 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&7774 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) {7775 RequiresReference = true;7776 BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);7777 }7778 }7779 }7780 7781 // If the variable is a pointer and is being dereferenced (i.e. is not7782 // the last component), the base has to be the pointer itself, not its7783 // reference. References are ignored for mapping purposes.7784 QualType Ty =7785 I->getAssociatedDeclaration()->getType().getNonReferenceType();7786 if (Ty->isAnyPointerType() && std::next(I) != CE) {7787 // No need to generate individual map information for the pointer, it7788 // can be associated with the combined storage if shared memory mode is7789 // active or the base declaration is not global variable.7790 const auto *VD = dyn_cast<VarDecl>(I->getAssociatedDeclaration());7791 if (!AreBothBasePtrAndPteeMapped &&7792 (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||7793 !VD || VD->hasLocalStorage()))7794 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());7795 else7796 FirstPointerInComplexData = true;7797 ++I;7798 }7799 }7800 7801 // Track whether a component of the list should be marked as MEMBER_OF some7802 // combined entry (for partial structs). Only the first PTR_AND_OBJ entry7803 // in a component list should be marked as MEMBER_OF, all subsequent entries7804 // do not belong to the base struct. E.g.7805 // struct S2 s;7806 // s.ps->ps->ps->f[:]7807 // (1) (2) (3) (4)7808 // ps(1) is a member pointer, ps(2) is a pointee of ps(1), so it is a7809 // PTR_AND_OBJ entry; the PTR is ps(1), so MEMBER_OF the base struct. ps(3)7810 // is the pointee of ps(2) which is not member of struct s, so it should not7811 // be marked as such (it is still PTR_AND_OBJ).7812 // The variable is initialized to false so that PTR_AND_OBJ entries which7813 // are not struct members are not considered (e.g. array of pointers to7814 // data).7815 bool ShouldBeMemberOf = false;7816 7817 // Variable keeping track of whether or not we have encountered a component7818 // in the component list which is a member expression. Useful when we have a7819 // pointer or a final array section, in which case it is the previous7820 // component in the list which tells us whether we have a member expression.7821 // E.g. X.f[:]7822 // While processing the final array section "[:]" it is "f" which tells us7823 // whether we are dealing with a member of a declared struct.7824 const MemberExpr *EncounteredME = nullptr;7825 7826 // Track for the total number of dimension. Start from one for the dummy7827 // dimension.7828 uint64_t DimSize = 1;7829 7830 // Detects non-contiguous updates due to strided accesses.7831 // Sets the 'IsNonContiguous' flag so that the 'MapType' bits are set7832 // correctly when generating information to be passed to the runtime. The7833 // flag is set to true if any array section has a stride not equal to 1, or7834 // if the stride is not a constant expression (conservatively assumed7835 // non-contiguous).7836 bool IsNonContiguous =7837 CombinedInfo.NonContigInfo.IsNonContiguous ||7838 any_of(Components, [&](const auto &Component) {7839 const auto *OASE =7840 dyn_cast<ArraySectionExpr>(Component.getAssociatedExpression());7841 if (!OASE)7842 return false;7843 7844 const Expr *StrideExpr = OASE->getStride();7845 if (!StrideExpr)7846 return false;7847 7848 const auto Constant =7849 StrideExpr->getIntegerConstantExpr(CGF.getContext());7850 if (!Constant)7851 return false;7852 7853 return !Constant->isOne();7854 });7855 7856 bool IsPrevMemberReference = false;7857 7858 bool IsPartialMapped =7859 !PartialStruct.PreliminaryMapData.BasePointers.empty();7860 7861 // We need to check if we will be encountering any MEs. If we do not7862 // encounter any ME expression it means we will be mapping the whole struct.7863 // In that case we need to skip adding an entry for the struct to the7864 // CombinedInfo list and instead add an entry to the StructBaseCombinedInfo7865 // list only when generating all info for clauses.7866 bool IsMappingWholeStruct = true;7867 if (!GenerateAllInfoForClauses) {7868 IsMappingWholeStruct = false;7869 } else {7870 for (auto TempI = I; TempI != CE; ++TempI) {7871 const MemberExpr *PossibleME =7872 dyn_cast<MemberExpr>(TempI->getAssociatedExpression());7873 if (PossibleME) {7874 IsMappingWholeStruct = false;7875 break;7876 }7877 }7878 }7879 7880 for (; I != CE; ++I) {7881 // If the current component is member of a struct (parent struct) mark it.7882 if (!EncounteredME) {7883 EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());7884 // If we encounter a PTR_AND_OBJ entry from now on it should be marked7885 // as MEMBER_OF the parent struct.7886 if (EncounteredME) {7887 ShouldBeMemberOf = true;7888 // Do not emit as complex pointer if this is actually not array-like7889 // expression.7890 if (FirstPointerInComplexData) {7891 QualType Ty = std::prev(I)7892 ->getAssociatedDeclaration()7893 ->getType()7894 .getNonReferenceType();7895 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());7896 FirstPointerInComplexData = false;7897 }7898 }7899 }7900 7901 auto Next = std::next(I);7902 7903 // We need to generate the addresses and sizes if this is the last7904 // component, if the component is a pointer or if it is an array section7905 // whose length can't be proved to be one. If this is a pointer, it7906 // becomes the base address for the following components.7907 7908 // A final array section, is one whose length can't be proved to be one.7909 // If the map item is non-contiguous then we don't treat any array section7910 // as final array section.7911 bool IsFinalArraySection =7912 !IsNonContiguous &&7913 isFinalArraySectionExpression(I->getAssociatedExpression());7914 7915 // If we have a declaration for the mapping use that, otherwise use7916 // the base declaration of the map clause.7917 const ValueDecl *MapDecl = (I->getAssociatedDeclaration())7918 ? I->getAssociatedDeclaration()7919 : BaseDecl;7920 MapExpr = (I->getAssociatedExpression()) ? I->getAssociatedExpression()7921 : MapExpr;7922 7923 // Get information on whether the element is a pointer. Have to do a7924 // special treatment for array sections given that they are built-in7925 // types.7926 const auto *OASE =7927 dyn_cast<ArraySectionExpr>(I->getAssociatedExpression());7928 const auto *OAShE =7929 dyn_cast<OMPArrayShapingExpr>(I->getAssociatedExpression());7930 const auto *UO = dyn_cast<UnaryOperator>(I->getAssociatedExpression());7931 const auto *BO = dyn_cast<BinaryOperator>(I->getAssociatedExpression());7932 bool IsPointer =7933 OAShE ||7934 (OASE && ArraySectionExpr::getBaseOriginalType(OASE)7935 .getCanonicalType()7936 ->isAnyPointerType()) ||7937 I->getAssociatedExpression()->getType()->isAnyPointerType();7938 bool IsMemberReference = isa<MemberExpr>(I->getAssociatedExpression()) &&7939 MapDecl &&7940 MapDecl->getType()->isLValueReferenceType();7941 bool IsNonDerefPointer = IsPointer &&7942 !(UO && UO->getOpcode() != UO_Deref) && !BO &&7943 !IsNonContiguous;7944 7945 if (OASE)7946 ++DimSize;7947 7948 if (Next == CE || IsMemberReference || IsNonDerefPointer ||7949 IsFinalArraySection) {7950 // If this is not the last component, we expect the pointer to be7951 // associated with an array expression or member expression.7952 assert((Next == CE ||7953 isa<MemberExpr>(Next->getAssociatedExpression()) ||7954 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||7955 isa<ArraySectionExpr>(Next->getAssociatedExpression()) ||7956 isa<OMPArrayShapingExpr>(Next->getAssociatedExpression()) ||7957 isa<UnaryOperator>(Next->getAssociatedExpression()) ||7958 isa<BinaryOperator>(Next->getAssociatedExpression())) &&7959 "Unexpected expression");7960 7961 Address LB = Address::invalid();7962 Address LowestElem = Address::invalid();7963 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF,7964 const MemberExpr *E) {7965 const Expr *BaseExpr = E->getBase();7966 // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a7967 // scalar.7968 LValue BaseLV;7969 if (E->isArrow()) {7970 LValueBaseInfo BaseInfo;7971 TBAAAccessInfo TBAAInfo;7972 Address Addr =7973 CGF.EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo);7974 QualType PtrTy = BaseExpr->getType()->getPointeeType();7975 BaseLV = CGF.MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo);7976 } else {7977 BaseLV = CGF.EmitOMPSharedLValue(BaseExpr);7978 }7979 return BaseLV;7980 };7981 if (OAShE) {7982 LowestElem = LB =7983 Address(CGF.EmitScalarExpr(OAShE->getBase()),7984 CGF.ConvertTypeForMem(7985 OAShE->getBase()->getType()->getPointeeType()),7986 CGF.getContext().getTypeAlignInChars(7987 OAShE->getBase()->getType()));7988 } else if (IsMemberReference) {7989 const auto *ME = cast<MemberExpr>(I->getAssociatedExpression());7990 LValue BaseLVal = EmitMemberExprBase(CGF, ME);7991 LowestElem = CGF.EmitLValueForFieldInitialization(7992 BaseLVal, cast<FieldDecl>(MapDecl))7993 .getAddress();7994 LB = CGF.EmitLoadOfReferenceLValue(LowestElem, MapDecl->getType())7995 .getAddress();7996 } else {7997 LowestElem = LB =7998 CGF.EmitOMPSharedLValue(I->getAssociatedExpression())7999 .getAddress();8000 }8001 8002 // If this component is a pointer inside the base struct then we don't8003 // need to create any entry for it - it will be combined with the object8004 // it is pointing to into a single PTR_AND_OBJ entry.8005 bool IsMemberPointerOrAddr =8006 EncounteredME &&8007 (((IsPointer || ForDeviceAddr) &&8008 I->getAssociatedExpression() == EncounteredME) ||8009 (IsPrevMemberReference && !IsPointer) ||8010 (IsMemberReference && Next != CE &&8011 !Next->getAssociatedExpression()->getType()->isPointerType()));8012 if (!OverlappedElements.empty() && Next == CE) {8013 // Handle base element with the info for overlapped elements.8014 assert(!PartialStruct.Base.isValid() && "The base element is set.");8015 assert(!IsPointer &&8016 "Unexpected base element with the pointer type.");8017 // Mark the whole struct as the struct that requires allocation on the8018 // device.8019 PartialStruct.LowestElem = {0, LowestElem};8020 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars(8021 I->getAssociatedExpression()->getType());8022 Address HB = CGF.Builder.CreateConstGEP(8023 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(8024 LowestElem, CGF.VoidPtrTy, CGF.Int8Ty),8025 TypeSize.getQuantity() - 1);8026 PartialStruct.HighestElem = {8027 std::numeric_limits<decltype(8028 PartialStruct.HighestElem.first)>::max(),8029 HB};8030 PartialStruct.Base = BP;8031 PartialStruct.LB = LB;8032 assert(8033 PartialStruct.PreliminaryMapData.BasePointers.empty() &&8034 "Overlapped elements must be used only once for the variable.");8035 std::swap(PartialStruct.PreliminaryMapData, CombinedInfo);8036 // Emit data for non-overlapped data.8037 OpenMPOffloadMappingFlags Flags =8038 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |8039 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,8040 /*AddPtrFlag=*/false,8041 /*AddIsTargetParamFlag=*/false, IsNonContiguous);8042 CopyOverlappedEntryGaps CopyGaps(CGF, CombinedInfo, Flags, MapDecl,8043 MapExpr, BP, LB, IsNonContiguous,8044 DimSize);8045 // Do bitcopy of all non-overlapped structure elements.8046 for (OMPClauseMappableExprCommon::MappableExprComponentListRef8047 Component : OverlappedElements) {8048 for (const OMPClauseMappableExprCommon::MappableComponent &MC :8049 Component) {8050 if (const ValueDecl *VD = MC.getAssociatedDeclaration()) {8051 if (const auto *FD = dyn_cast<FieldDecl>(VD)) {8052 CopyGaps.processField(MC, FD, EmitMemberExprBase);8053 }8054 }8055 }8056 }8057 CopyGaps.copyUntilEnd(HB);8058 break;8059 }8060 llvm::Value *Size = getExprTypeSize(I->getAssociatedExpression());8061 // Skip adding an entry in the CurInfo of this combined entry if the8062 // whole struct is currently being mapped. The struct needs to be added8063 // in the first position before any data internal to the struct is being8064 // mapped.8065 // Skip adding an entry in the CurInfo of this combined entry if the8066 // PartialStruct.PreliminaryMapData.BasePointers has been mapped.8067 if ((!IsMemberPointerOrAddr && !IsPartialMapped) ||8068 (Next == CE && MapType != OMPC_MAP_unknown)) {8069 if (!IsMappingWholeStruct) {8070 CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);8071 CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF));8072 CombinedInfo.DevicePtrDecls.push_back(nullptr);8073 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);8074 CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));8075 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(8076 Size, CGF.Int64Ty, /*isSigned=*/true));8077 CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize8078 : 1);8079 } else {8080 StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);8081 StructBaseCombinedInfo.BasePointers.push_back(8082 BP.emitRawPointer(CGF));8083 StructBaseCombinedInfo.DevicePtrDecls.push_back(nullptr);8084 StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);8085 StructBaseCombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));8086 StructBaseCombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(8087 Size, CGF.Int64Ty, /*isSigned=*/true));8088 StructBaseCombinedInfo.NonContigInfo.Dims.push_back(8089 IsNonContiguous ? DimSize : 1);8090 }8091 8092 // If Mapper is valid, the last component inherits the mapper.8093 bool HasMapper = Mapper && Next == CE;8094 if (!IsMappingWholeStruct)8095 CombinedInfo.Mappers.push_back(HasMapper ? Mapper : nullptr);8096 else8097 StructBaseCombinedInfo.Mappers.push_back(HasMapper ? Mapper8098 : nullptr);8099 8100 // We need to add a pointer flag for each map that comes from the8101 // same expression except for the first one. We also need to signal8102 // this map is the first one that relates with the current capture8103 // (there is a set of entries for each capture).8104 OpenMPOffloadMappingFlags Flags =8105 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,8106 !IsExpressionFirstInfo || RequiresReference ||8107 FirstPointerInComplexData || IsMemberReference,8108 AreBothBasePtrAndPteeMapped ||8109 (IsCaptureFirstInfo && !RequiresReference),8110 IsNonContiguous);8111 8112 if (!IsExpressionFirstInfo || IsMemberReference) {8113 // If we have a PTR_AND_OBJ pair where the OBJ is a pointer as well,8114 // then we reset the TO/FROM/ALWAYS/DELETE/CLOSE flags.8115 if (IsPointer || (IsMemberReference && Next != CE))8116 Flags &= ~(OpenMPOffloadMappingFlags::OMP_MAP_TO |8117 OpenMPOffloadMappingFlags::OMP_MAP_FROM |8118 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |8119 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |8120 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);8121 8122 if (ShouldBeMemberOf) {8123 // Set placeholder value MEMBER_OF=FFFF to indicate that the flag8124 // should be later updated with the correct value of MEMBER_OF.8125 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;8126 // From now on, all subsequent PTR_AND_OBJ entries should not be8127 // marked as MEMBER_OF.8128 ShouldBeMemberOf = false;8129 }8130 }8131 8132 if (!IsMappingWholeStruct)8133 CombinedInfo.Types.push_back(Flags);8134 else8135 StructBaseCombinedInfo.Types.push_back(Flags);8136 }8137 8138 // If we have encountered a member expression so far, keep track of the8139 // mapped member. If the parent is "*this", then the value declaration8140 // is nullptr.8141 if (EncounteredME) {8142 const auto *FD = cast<FieldDecl>(EncounteredME->getMemberDecl());8143 unsigned FieldIndex = FD->getFieldIndex();8144 8145 // Update info about the lowest and highest elements for this struct8146 if (!PartialStruct.Base.isValid()) {8147 PartialStruct.LowestElem = {FieldIndex, LowestElem};8148 if (IsFinalArraySection && OASE) {8149 Address HB =8150 CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false)8151 .getAddress();8152 PartialStruct.HighestElem = {FieldIndex, HB};8153 } else {8154 PartialStruct.HighestElem = {FieldIndex, LowestElem};8155 }8156 PartialStruct.Base = BP;8157 PartialStruct.LB = BP;8158 } else if (FieldIndex < PartialStruct.LowestElem.first) {8159 PartialStruct.LowestElem = {FieldIndex, LowestElem};8160 } else if (FieldIndex > PartialStruct.HighestElem.first) {8161 if (IsFinalArraySection && OASE) {8162 Address HB =8163 CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false)8164 .getAddress();8165 PartialStruct.HighestElem = {FieldIndex, HB};8166 } else {8167 PartialStruct.HighestElem = {FieldIndex, LowestElem};8168 }8169 }8170 }8171 8172 // Need to emit combined struct for array sections.8173 if (IsFinalArraySection || IsNonContiguous)8174 PartialStruct.IsArraySection = true;8175 8176 // If we have a final array section, we are done with this expression.8177 if (IsFinalArraySection)8178 break;8179 8180 // The pointer becomes the base for the next element.8181 if (Next != CE)8182 BP = IsMemberReference ? LowestElem : LB;8183 if (!IsPartialMapped)8184 IsExpressionFirstInfo = false;8185 IsCaptureFirstInfo = false;8186 FirstPointerInComplexData = false;8187 IsPrevMemberReference = IsMemberReference;8188 } else if (FirstPointerInComplexData) {8189 QualType Ty = Components.rbegin()8190 ->getAssociatedDeclaration()8191 ->getType()8192 .getNonReferenceType();8193 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());8194 FirstPointerInComplexData = false;8195 }8196 }8197 // If ran into the whole component - allocate the space for the whole8198 // record.8199 if (!EncounteredME)8200 PartialStruct.HasCompleteRecord = true;8201 8202 if (!IsNonContiguous)8203 return;8204 8205 const ASTContext &Context = CGF.getContext();8206 8207 // For supporting stride in array section, we need to initialize the first8208 // dimension size as 1, first offset as 0, and first count as 18209 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 0)};8210 MapValuesArrayTy CurCounts = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)};8211 MapValuesArrayTy CurStrides;8212 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)};8213 uint64_t ElementTypeSize;8214 8215 // Collect Size information for each dimension and get the element size as8216 // the first Stride. For example, for `int arr[10][10]`, the DimSizes8217 // should be [10, 10] and the first stride is 4 btyes.8218 for (const OMPClauseMappableExprCommon::MappableComponent &Component :8219 Components) {8220 const Expr *AssocExpr = Component.getAssociatedExpression();8221 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);8222 8223 if (!OASE)8224 continue;8225 8226 QualType Ty = ArraySectionExpr::getBaseOriginalType(OASE->getBase());8227 auto *CAT = Context.getAsConstantArrayType(Ty);8228 auto *VAT = Context.getAsVariableArrayType(Ty);8229 8230 // We need all the dimension size except for the last dimension.8231 assert((VAT || CAT || &Component == &*Components.begin()) &&8232 "Should be either ConstantArray or VariableArray if not the "8233 "first Component");8234 8235 // Get element size if CurStrides is empty.8236 if (CurStrides.empty()) {8237 const Type *ElementType = nullptr;8238 if (CAT)8239 ElementType = CAT->getElementType().getTypePtr();8240 else if (VAT)8241 ElementType = VAT->getElementType().getTypePtr();8242 else8243 assert(&Component == &*Components.begin() &&8244 "Only expect pointer (non CAT or VAT) when this is the "8245 "first Component");8246 // If ElementType is null, then it means the base is a pointer8247 // (neither CAT nor VAT) and we'll attempt to get ElementType again8248 // for next iteration.8249 if (ElementType) {8250 // For the case that having pointer as base, we need to remove one8251 // level of indirection.8252 if (&Component != &*Components.begin())8253 ElementType = ElementType->getPointeeOrArrayElementType();8254 ElementTypeSize =8255 Context.getTypeSizeInChars(ElementType).getQuantity();8256 CurStrides.push_back(8257 llvm::ConstantInt::get(CGF.Int64Ty, ElementTypeSize));8258 }8259 }8260 // Get dimension value except for the last dimension since we don't need8261 // it.8262 if (DimSizes.size() < Components.size() - 1) {8263 if (CAT)8264 DimSizes.push_back(8265 llvm::ConstantInt::get(CGF.Int64Ty, CAT->getZExtSize()));8266 else if (VAT)8267 DimSizes.push_back(CGF.Builder.CreateIntCast(8268 CGF.EmitScalarExpr(VAT->getSizeExpr()), CGF.Int64Ty,8269 /*IsSigned=*/false));8270 }8271 }8272 8273 // Skip the dummy dimension since we have already have its information.8274 auto *DI = DimSizes.begin() + 1;8275 // Product of dimension.8276 llvm::Value *DimProd =8277 llvm::ConstantInt::get(CGF.CGM.Int64Ty, ElementTypeSize);8278 8279 // Collect info for non-contiguous. Notice that offset, count, and stride8280 // are only meaningful for array-section, so we insert a null for anything8281 // other than array-section.8282 // Also, the size of offset, count, and stride are not the same as8283 // pointers, base_pointers, sizes, or dims. Instead, the size of offset,8284 // count, and stride are the same as the number of non-contiguous8285 // declaration in target update to/from clause.8286 for (const OMPClauseMappableExprCommon::MappableComponent &Component :8287 Components) {8288 const Expr *AssocExpr = Component.getAssociatedExpression();8289 8290 if (const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr)) {8291 llvm::Value *Offset = CGF.Builder.CreateIntCast(8292 CGF.EmitScalarExpr(AE->getIdx()), CGF.Int64Ty,8293 /*isSigned=*/false);8294 CurOffsets.push_back(Offset);8295 CurCounts.push_back(llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/1));8296 CurStrides.push_back(CurStrides.back());8297 continue;8298 }8299 8300 const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);8301 8302 if (!OASE)8303 continue;8304 8305 // Offset8306 const Expr *OffsetExpr = OASE->getLowerBound();8307 llvm::Value *Offset = nullptr;8308 if (!OffsetExpr) {8309 // If offset is absent, then we just set it to zero.8310 Offset = llvm::ConstantInt::get(CGF.Int64Ty, 0);8311 } else {8312 Offset = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(OffsetExpr),8313 CGF.Int64Ty,8314 /*isSigned=*/false);8315 }8316 CurOffsets.push_back(Offset);8317 8318 // Count8319 const Expr *CountExpr = OASE->getLength();8320 llvm::Value *Count = nullptr;8321 if (!CountExpr) {8322 // In Clang, once a high dimension is an array section, we construct all8323 // the lower dimension as array section, however, for case like8324 // arr[0:2][2], Clang construct the inner dimension as an array section8325 // but it actually is not in an array section form according to spec.8326 if (!OASE->getColonLocFirst().isValid() &&8327 !OASE->getColonLocSecond().isValid()) {8328 Count = llvm::ConstantInt::get(CGF.Int64Ty, 1);8329 } else {8330 // OpenMP 5.0, 2.1.5 Array Sections, Description.8331 // When the length is absent it defaults to ⌈(size −8332 // lower-bound)/stride⌉, where size is the size of the array8333 // dimension.8334 const Expr *StrideExpr = OASE->getStride();8335 llvm::Value *Stride =8336 StrideExpr8337 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr),8338 CGF.Int64Ty, /*isSigned=*/false)8339 : nullptr;8340 if (Stride)8341 Count = CGF.Builder.CreateUDiv(8342 CGF.Builder.CreateNUWSub(*DI, Offset), Stride);8343 else8344 Count = CGF.Builder.CreateNUWSub(*DI, Offset);8345 }8346 } else {8347 Count = CGF.EmitScalarExpr(CountExpr);8348 }8349 Count = CGF.Builder.CreateIntCast(Count, CGF.Int64Ty, /*isSigned=*/false);8350 CurCounts.push_back(Count);8351 8352 // Stride_n' = Stride_n * (D_0 * D_1 ... * D_n-1) * Unit size8353 // Take `int arr[5][5][5]` and `arr[0:2:2][1:2:1][0:2:2]` as an example:8354 // Offset Count Stride8355 // D0 0 1 4 (int) <- dummy dimension8356 // D1 0 2 8 (2 * (1) * 4)8357 // D2 1 2 20 (1 * (1 * 5) * 4)8358 // D3 0 2 200 (2 * (1 * 5 * 4) * 4)8359 const Expr *StrideExpr = OASE->getStride();8360 llvm::Value *Stride =8361 StrideExpr8362 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr),8363 CGF.Int64Ty, /*isSigned=*/false)8364 : nullptr;8365 DimProd = CGF.Builder.CreateNUWMul(DimProd, *(DI - 1));8366 if (Stride)8367 CurStrides.push_back(CGF.Builder.CreateNUWMul(DimProd, Stride));8368 else8369 CurStrides.push_back(DimProd);8370 if (DI != DimSizes.end())8371 ++DI;8372 }8373 8374 CombinedInfo.NonContigInfo.Offsets.push_back(CurOffsets);8375 CombinedInfo.NonContigInfo.Counts.push_back(CurCounts);8376 CombinedInfo.NonContigInfo.Strides.push_back(CurStrides);8377 }8378 8379 /// Return the adjusted map modifiers if the declaration a capture refers to8380 /// appears in a first-private clause. This is expected to be used only with8381 /// directives that start with 'target'.8382 OpenMPOffloadMappingFlags8383 getMapModifiersForPrivateClauses(const CapturedStmt::Capture &Cap) const {8384 assert(Cap.capturesVariable() && "Expected capture by reference only!");8385 8386 // A first private variable captured by reference will use only the8387 // 'private ptr' and 'map to' flag. Return the right flags if the captured8388 // declaration is known as first-private in this handler.8389 if (FirstPrivateDecls.count(Cap.getCapturedVar())) {8390 if (Cap.getCapturedVar()->getType()->isAnyPointerType())8391 return OpenMPOffloadMappingFlags::OMP_MAP_TO |8392 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;8393 return OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |8394 OpenMPOffloadMappingFlags::OMP_MAP_TO;8395 }8396 auto I = LambdasMap.find(Cap.getCapturedVar()->getCanonicalDecl());8397 if (I != LambdasMap.end())8398 // for map(to: lambda): using user specified map type.8399 return getMapTypeBits(8400 I->getSecond()->getMapType(), I->getSecond()->getMapTypeModifiers(),8401 /*MotionModifiers=*/{}, I->getSecond()->isImplicit(),8402 /*AddPtrFlag=*/false,8403 /*AddIsTargetParamFlag=*/false,8404 /*isNonContiguous=*/false);8405 return OpenMPOffloadMappingFlags::OMP_MAP_TO |8406 OpenMPOffloadMappingFlags::OMP_MAP_FROM;8407 }8408 8409 void getPlainLayout(const CXXRecordDecl *RD,8410 llvm::SmallVectorImpl<const FieldDecl *> &Layout,8411 bool AsBase) const {8412 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD);8413 8414 llvm::StructType *St =8415 AsBase ? RL.getBaseSubobjectLLVMType() : RL.getLLVMType();8416 8417 unsigned NumElements = St->getNumElements();8418 llvm::SmallVector<8419 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>8420 RecordLayout(NumElements);8421 8422 // Fill bases.8423 for (const auto &I : RD->bases()) {8424 if (I.isVirtual())8425 continue;8426 8427 QualType BaseTy = I.getType();8428 const auto *Base = BaseTy->getAsCXXRecordDecl();8429 // Ignore empty bases.8430 if (isEmptyRecordForLayout(CGF.getContext(), BaseTy) ||8431 CGF.getContext()8432 .getASTRecordLayout(Base)8433 .getNonVirtualSize()8434 .isZero())8435 continue;8436 8437 unsigned FieldIndex = RL.getNonVirtualBaseLLVMFieldNo(Base);8438 RecordLayout[FieldIndex] = Base;8439 }8440 // Fill in virtual bases.8441 for (const auto &I : RD->vbases()) {8442 QualType BaseTy = I.getType();8443 // Ignore empty bases.8444 if (isEmptyRecordForLayout(CGF.getContext(), BaseTy))8445 continue;8446 8447 const auto *Base = BaseTy->getAsCXXRecordDecl();8448 unsigned FieldIndex = RL.getVirtualBaseIndex(Base);8449 if (RecordLayout[FieldIndex])8450 continue;8451 RecordLayout[FieldIndex] = Base;8452 }8453 // Fill in all the fields.8454 assert(!RD->isUnion() && "Unexpected union.");8455 for (const auto *Field : RD->fields()) {8456 // Fill in non-bitfields. (Bitfields always use a zero pattern, which we8457 // will fill in later.)8458 if (!Field->isBitField() &&8459 !isEmptyFieldForLayout(CGF.getContext(), Field)) {8460 unsigned FieldIndex = RL.getLLVMFieldNo(Field);8461 RecordLayout[FieldIndex] = Field;8462 }8463 }8464 for (const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>8465 &Data : RecordLayout) {8466 if (Data.isNull())8467 continue;8468 if (const auto *Base = dyn_cast<const CXXRecordDecl *>(Data))8469 getPlainLayout(Base, Layout, /*AsBase=*/true);8470 else8471 Layout.push_back(cast<const FieldDecl *>(Data));8472 }8473 }8474 8475 /// Returns the address corresponding to \p PointerExpr.8476 static Address getAttachPtrAddr(const Expr *PointerExpr,8477 CodeGenFunction &CGF) {8478 assert(PointerExpr && "Cannot get addr from null attach-ptr expr");8479 Address AttachPtrAddr = Address::invalid();8480 8481 if (auto *DRE = dyn_cast<DeclRefExpr>(PointerExpr)) {8482 // If the pointer is a variable, we can use its address directly.8483 AttachPtrAddr = CGF.EmitLValue(DRE).getAddress();8484 } else if (auto *OASE = dyn_cast<ArraySectionExpr>(PointerExpr)) {8485 AttachPtrAddr =8486 CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/true).getAddress();8487 } else if (auto *ASE = dyn_cast<ArraySubscriptExpr>(PointerExpr)) {8488 AttachPtrAddr = CGF.EmitLValue(ASE).getAddress();8489 } else if (auto *ME = dyn_cast<MemberExpr>(PointerExpr)) {8490 AttachPtrAddr = CGF.EmitMemberExpr(ME).getAddress();8491 } else if (auto *UO = dyn_cast<UnaryOperator>(PointerExpr)) {8492 assert(UO->getOpcode() == UO_Deref &&8493 "Unexpected unary-operator on attach-ptr-expr");8494 AttachPtrAddr = CGF.EmitLValue(UO).getAddress();8495 }8496 assert(AttachPtrAddr.isValid() &&8497 "Failed to get address for attach pointer expression");8498 return AttachPtrAddr;8499 }8500 8501 /// Get the address of the attach pointer, and a load from it, to get the8502 /// pointee base address.8503 /// \return A pair containing AttachPtrAddr and AttachPteeBaseAddr. The pair8504 /// contains invalid addresses if \p AttachPtrExpr is null.8505 static std::pair<Address, Address>8506 getAttachPtrAddrAndPteeBaseAddr(const Expr *AttachPtrExpr,8507 CodeGenFunction &CGF) {8508 8509 if (!AttachPtrExpr)8510 return {Address::invalid(), Address::invalid()};8511 8512 Address AttachPtrAddr = getAttachPtrAddr(AttachPtrExpr, CGF);8513 assert(AttachPtrAddr.isValid() && "Invalid attach pointer addr");8514 8515 QualType AttachPtrType =8516 OMPClauseMappableExprCommon::getComponentExprElementType(AttachPtrExpr)8517 .getCanonicalType();8518 8519 Address AttachPteeBaseAddr = CGF.EmitLoadOfPointer(8520 AttachPtrAddr, AttachPtrType->castAs<PointerType>());8521 assert(AttachPteeBaseAddr.isValid() && "Invalid attach pointee base addr");8522 8523 return {AttachPtrAddr, AttachPteeBaseAddr};8524 }8525 8526 /// Returns whether an attach entry should be emitted for a map on8527 /// \p MapBaseDecl on the directive \p CurDir.8528 static bool8529 shouldEmitAttachEntry(const Expr *PointerExpr, const ValueDecl *MapBaseDecl,8530 CodeGenFunction &CGF,8531 llvm::PointerUnion<const OMPExecutableDirective *,8532 const OMPDeclareMapperDecl *>8533 CurDir) {8534 if (!PointerExpr)8535 return false;8536 8537 // Pointer attachment is needed at map-entering time or for declare8538 // mappers.8539 return isa<const OMPDeclareMapperDecl *>(CurDir) ||8540 isOpenMPTargetMapEnteringDirective(8541 cast<const OMPExecutableDirective *>(CurDir)8542 ->getDirectiveKind());8543 }8544 8545 /// Computes the attach-ptr expr for \p Components, and updates various maps8546 /// with the information.8547 /// It internally calls OMPClauseMappableExprCommon::findAttachPtrExpr()8548 /// with the OpenMPDirectiveKind extracted from \p CurDir.8549 /// It updates AttachPtrComputationOrderMap, AttachPtrComponentDepthMap, and8550 /// AttachPtrExprMap.8551 void collectAttachPtrExprInfo(8552 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,8553 llvm::PointerUnion<const OMPExecutableDirective *,8554 const OMPDeclareMapperDecl *>8555 CurDir) {8556 8557 OpenMPDirectiveKind CurDirectiveID =8558 isa<const OMPDeclareMapperDecl *>(CurDir)8559 ? OMPD_declare_mapper8560 : cast<const OMPExecutableDirective *>(CurDir)->getDirectiveKind();8561 8562 const auto &[AttachPtrExpr, Depth] =8563 OMPClauseMappableExprCommon::findAttachPtrExpr(Components,8564 CurDirectiveID);8565 8566 AttachPtrComputationOrderMap.try_emplace(8567 AttachPtrExpr, AttachPtrComputationOrderMap.size());8568 AttachPtrComponentDepthMap.try_emplace(AttachPtrExpr, Depth);8569 AttachPtrExprMap.try_emplace(Components, AttachPtrExpr);8570 }8571 8572 /// Generate all the base pointers, section pointers, sizes, map types, and8573 /// mappers for the extracted mappable expressions (all included in \a8574 /// CombinedInfo). Also, for each item that relates with a device pointer, a8575 /// pair of the relevant declaration and index where it occurs is appended to8576 /// the device pointers info array.8577 void generateAllInfoForClauses(8578 ArrayRef<const OMPClause *> Clauses, MapCombinedInfoTy &CombinedInfo,8579 llvm::OpenMPIRBuilder &OMPBuilder,8580 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =8581 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const {8582 // We have to process the component lists that relate with the same8583 // declaration in a single chunk so that we can generate the map flags8584 // correctly. Therefore, we organize all lists in a map.8585 enum MapKind { Present, Allocs, Other, Total };8586 llvm::MapVector<CanonicalDeclPtr<const Decl>,8587 SmallVector<SmallVector<MapInfo, 8>, 4>>8588 Info;8589 8590 // Helper function to fill the information map for the different supported8591 // clauses.8592 auto &&InfoGen =8593 [&Info, &SkipVarSet](8594 const ValueDecl *D, MapKind Kind,8595 OMPClauseMappableExprCommon::MappableExprComponentListRef L,8596 OpenMPMapClauseKind MapType,8597 ArrayRef<OpenMPMapModifierKind> MapModifiers,8598 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,8599 bool ReturnDevicePointer, bool IsImplicit, const ValueDecl *Mapper,8600 const Expr *VarRef = nullptr, bool ForDeviceAddr = false) {8601 if (SkipVarSet.contains(D))8602 return;8603 auto It = Info.try_emplace(D, Total).first;8604 It->second[Kind].emplace_back(8605 L, MapType, MapModifiers, MotionModifiers, ReturnDevicePointer,8606 IsImplicit, Mapper, VarRef, ForDeviceAddr);8607 };8608 8609 for (const auto *Cl : Clauses) {8610 const auto *C = dyn_cast<OMPMapClause>(Cl);8611 if (!C)8612 continue;8613 MapKind Kind = Other;8614 if (llvm::is_contained(C->getMapTypeModifiers(),8615 OMPC_MAP_MODIFIER_present))8616 Kind = Present;8617 else if (C->getMapType() == OMPC_MAP_alloc)8618 Kind = Allocs;8619 const auto *EI = C->getVarRefs().begin();8620 for (const auto L : C->component_lists()) {8621 const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr;8622 InfoGen(std::get<0>(L), Kind, std::get<1>(L), C->getMapType(),8623 C->getMapTypeModifiers(), {},8624 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(L),8625 E);8626 ++EI;8627 }8628 }8629 for (const auto *Cl : Clauses) {8630 const auto *C = dyn_cast<OMPToClause>(Cl);8631 if (!C)8632 continue;8633 MapKind Kind = Other;8634 if (llvm::is_contained(C->getMotionModifiers(),8635 OMPC_MOTION_MODIFIER_present))8636 Kind = Present;8637 if (llvm::is_contained(C->getMotionModifiers(),8638 OMPC_MOTION_MODIFIER_iterator)) {8639 if (auto *IteratorExpr = dyn_cast<OMPIteratorExpr>(8640 C->getIteratorModifier()->IgnoreParenImpCasts())) {8641 const auto *VD = cast<VarDecl>(IteratorExpr->getIteratorDecl(0));8642 CGF.EmitVarDecl(*VD);8643 }8644 }8645 8646 const auto *EI = C->getVarRefs().begin();8647 for (const auto L : C->component_lists()) {8648 InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_to, {},8649 C->getMotionModifiers(), /*ReturnDevicePointer=*/false,8650 C->isImplicit(), std::get<2>(L), *EI);8651 ++EI;8652 }8653 }8654 for (const auto *Cl : Clauses) {8655 const auto *C = dyn_cast<OMPFromClause>(Cl);8656 if (!C)8657 continue;8658 MapKind Kind = Other;8659 if (llvm::is_contained(C->getMotionModifiers(),8660 OMPC_MOTION_MODIFIER_present))8661 Kind = Present;8662 if (llvm::is_contained(C->getMotionModifiers(),8663 OMPC_MOTION_MODIFIER_iterator)) {8664 if (auto *IteratorExpr = dyn_cast<OMPIteratorExpr>(8665 C->getIteratorModifier()->IgnoreParenImpCasts())) {8666 const auto *VD = cast<VarDecl>(IteratorExpr->getIteratorDecl(0));8667 CGF.EmitVarDecl(*VD);8668 }8669 }8670 8671 const auto *EI = C->getVarRefs().begin();8672 for (const auto L : C->component_lists()) {8673 InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_from, {},8674 C->getMotionModifiers(),8675 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(L),8676 *EI);8677 ++EI;8678 }8679 }8680 8681 // Look at the use_device_ptr and use_device_addr clauses information and8682 // mark the existing map entries as such. If there is no map information for8683 // an entry in the use_device_ptr and use_device_addr list, we create one8684 // with map type 'alloc' and zero size section. It is the user fault if that8685 // was not mapped before. If there is no map information and the pointer is8686 // a struct member, then we defer the emission of that entry until the whole8687 // struct has been processed.8688 llvm::MapVector<CanonicalDeclPtr<const Decl>,8689 SmallVector<DeferredDevicePtrEntryTy, 4>>8690 DeferredInfo;8691 MapCombinedInfoTy UseDeviceDataCombinedInfo;8692 8693 auto &&UseDeviceDataCombinedInfoGen =8694 [&UseDeviceDataCombinedInfo](const ValueDecl *VD, llvm::Value *Ptr,8695 CodeGenFunction &CGF, bool IsDevAddr) {8696 UseDeviceDataCombinedInfo.Exprs.push_back(VD);8697 UseDeviceDataCombinedInfo.BasePointers.emplace_back(Ptr);8698 UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(VD);8699 UseDeviceDataCombinedInfo.DevicePointers.emplace_back(8700 IsDevAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);8701 UseDeviceDataCombinedInfo.Pointers.push_back(Ptr);8702 UseDeviceDataCombinedInfo.Sizes.push_back(8703 llvm::Constant::getNullValue(CGF.Int64Ty));8704 UseDeviceDataCombinedInfo.Types.push_back(8705 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM);8706 UseDeviceDataCombinedInfo.Mappers.push_back(nullptr);8707 };8708 8709 auto &&MapInfoGen =8710 [&DeferredInfo, &UseDeviceDataCombinedInfoGen,8711 &InfoGen](CodeGenFunction &CGF, const Expr *IE, const ValueDecl *VD,8712 OMPClauseMappableExprCommon::MappableExprComponentListRef8713 Components,8714 bool IsImplicit, bool IsDevAddr) {8715 // We didn't find any match in our map information - generate a zero8716 // size array section - if the pointer is a struct member we defer8717 // this action until the whole struct has been processed.8718 if (isa<MemberExpr>(IE)) {8719 // Insert the pointer into Info to be processed by8720 // generateInfoForComponentList. Because it is a member pointer8721 // without a pointee, no entry will be generated for it, therefore8722 // we need to generate one after the whole struct has been8723 // processed. Nonetheless, generateInfoForComponentList must be8724 // called to take the pointer into account for the calculation of8725 // the range of the partial struct.8726 InfoGen(nullptr, Other, Components, OMPC_MAP_unknown, {}, {},8727 /*ReturnDevicePointer=*/false, IsImplicit, nullptr, nullptr,8728 IsDevAddr);8729 DeferredInfo[nullptr].emplace_back(IE, VD, IsDevAddr);8730 } else {8731 llvm::Value *Ptr;8732 if (IsDevAddr) {8733 if (IE->isGLValue())8734 Ptr = CGF.EmitLValue(IE).getPointer(CGF);8735 else8736 Ptr = CGF.EmitScalarExpr(IE);8737 } else {8738 Ptr = CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc());8739 }8740 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, IsDevAddr);8741 }8742 };8743 8744 auto &&IsMapInfoExist = [&Info](CodeGenFunction &CGF, const ValueDecl *VD,8745 const Expr *IE, bool IsDevAddr) -> bool {8746 // We potentially have map information for this declaration already.8747 // Look for the first set of components that refer to it. If found,8748 // return true.8749 // If the first component is a member expression, we have to look into8750 // 'this', which maps to null in the map of map information. Otherwise8751 // look directly for the information.8752 auto It = Info.find(isa<MemberExpr>(IE) ? nullptr : VD);8753 if (It != Info.end()) {8754 bool Found = false;8755 for (auto &Data : It->second) {8756 auto *CI = llvm::find_if(Data, [VD](const MapInfo &MI) {8757 return MI.Components.back().getAssociatedDeclaration() == VD;8758 });8759 // If we found a map entry, signal that the pointer has to be8760 // returned and move on to the next declaration. Exclude cases where8761 // the base pointer is mapped as array subscript, array section or8762 // array shaping. The base address is passed as a pointer to base in8763 // this case and cannot be used as a base for use_device_ptr list8764 // item.8765 if (CI != Data.end()) {8766 if (IsDevAddr) {8767 CI->ForDeviceAddr = IsDevAddr;8768 CI->ReturnDevicePointer = true;8769 Found = true;8770 break;8771 } else {8772 auto PrevCI = std::next(CI->Components.rbegin());8773 const auto *VarD = dyn_cast<VarDecl>(VD);8774 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||8775 isa<MemberExpr>(IE) ||8776 !VD->getType().getNonReferenceType()->isPointerType() ||8777 PrevCI == CI->Components.rend() ||8778 isa<MemberExpr>(PrevCI->getAssociatedExpression()) || !VarD ||8779 VarD->hasLocalStorage()) {8780 CI->ForDeviceAddr = IsDevAddr;8781 CI->ReturnDevicePointer = true;8782 Found = true;8783 break;8784 }8785 }8786 }8787 }8788 return Found;8789 }8790 return false;8791 };8792 8793 // Look at the use_device_ptr clause information and mark the existing map8794 // entries as such. If there is no map information for an entry in the8795 // use_device_ptr list, we create one with map type 'alloc' and zero size8796 // section. It is the user fault if that was not mapped before. If there is8797 // no map information and the pointer is a struct member, then we defer the8798 // emission of that entry until the whole struct has been processed.8799 for (const auto *Cl : Clauses) {8800 const auto *C = dyn_cast<OMPUseDevicePtrClause>(Cl);8801 if (!C)8802 continue;8803 for (const auto L : C->component_lists()) {8804 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =8805 std::get<1>(L);8806 assert(!Components.empty() &&8807 "Not expecting empty list of components!");8808 const ValueDecl *VD = Components.back().getAssociatedDeclaration();8809 VD = cast<ValueDecl>(VD->getCanonicalDecl());8810 const Expr *IE = Components.back().getAssociatedExpression();8811 if (IsMapInfoExist(CGF, VD, IE, /*IsDevAddr=*/false))8812 continue;8813 MapInfoGen(CGF, IE, VD, Components, C->isImplicit(),8814 /*IsDevAddr=*/false);8815 }8816 }8817 8818 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;8819 for (const auto *Cl : Clauses) {8820 const auto *C = dyn_cast<OMPUseDeviceAddrClause>(Cl);8821 if (!C)8822 continue;8823 for (const auto L : C->component_lists()) {8824 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =8825 std::get<1>(L);8826 assert(!std::get<1>(L).empty() &&8827 "Not expecting empty list of components!");8828 const ValueDecl *VD = std::get<1>(L).back().getAssociatedDeclaration();8829 if (!Processed.insert(VD).second)8830 continue;8831 VD = cast<ValueDecl>(VD->getCanonicalDecl());8832 const Expr *IE = std::get<1>(L).back().getAssociatedExpression();8833 if (IsMapInfoExist(CGF, VD, IE, /*IsDevAddr=*/true))8834 continue;8835 MapInfoGen(CGF, IE, VD, Components, C->isImplicit(),8836 /*IsDevAddr=*/true);8837 }8838 }8839 8840 for (const auto &Data : Info) {8841 StructRangeInfoTy PartialStruct;8842 // Current struct information:8843 MapCombinedInfoTy CurInfo;8844 // Current struct base information:8845 MapCombinedInfoTy StructBaseCurInfo;8846 const Decl *D = Data.first;8847 const ValueDecl *VD = cast_or_null<ValueDecl>(D);8848 bool HasMapBasePtr = false;8849 bool HasMapArraySec = false;8850 if (VD && VD->getType()->isAnyPointerType()) {8851 for (const auto &M : Data.second) {8852 HasMapBasePtr = any_of(M, [](const MapInfo &L) {8853 return isa_and_present<DeclRefExpr>(L.VarRef);8854 });8855 HasMapArraySec = any_of(M, [](const MapInfo &L) {8856 return isa_and_present<ArraySectionExpr, ArraySubscriptExpr>(8857 L.VarRef);8858 });8859 if (HasMapBasePtr && HasMapArraySec)8860 break;8861 }8862 }8863 for (const auto &M : Data.second) {8864 for (const MapInfo &L : M) {8865 assert(!L.Components.empty() &&8866 "Not expecting declaration with no component lists.");8867 8868 // Remember the current base pointer index.8869 unsigned CurrentBasePointersIdx = CurInfo.BasePointers.size();8870 unsigned StructBasePointersIdx =8871 StructBaseCurInfo.BasePointers.size();8872 CurInfo.NonContigInfo.IsNonContiguous =8873 L.Components.back().isNonContiguous();8874 generateInfoForComponentList(8875 L.MapType, L.MapModifiers, L.MotionModifiers, L.Components,8876 CurInfo, StructBaseCurInfo, PartialStruct,8877 /*IsFirstComponentList=*/false, L.IsImplicit,8878 /*GenerateAllInfoForClauses*/ true, L.Mapper, L.ForDeviceAddr, VD,8879 L.VarRef, /*OverlappedElements*/ {},8880 HasMapBasePtr && HasMapArraySec);8881 8882 // If this entry relates to a device pointer, set the relevant8883 // declaration and add the 'return pointer' flag.8884 if (L.ReturnDevicePointer) {8885 // Check whether a value was added to either CurInfo or8886 // StructBaseCurInfo and error if no value was added to either of8887 // them:8888 assert((CurrentBasePointersIdx < CurInfo.BasePointers.size() ||8889 StructBasePointersIdx <8890 StructBaseCurInfo.BasePointers.size()) &&8891 "Unexpected number of mapped base pointers.");8892 8893 // Choose a base pointer index which is always valid:8894 const ValueDecl *RelevantVD =8895 L.Components.back().getAssociatedDeclaration();8896 assert(RelevantVD &&8897 "No relevant declaration related with device pointer??");8898 8899 // If StructBaseCurInfo has been updated this iteration then work on8900 // the first new entry added to it i.e. make sure that when multiple8901 // values are added to any of the lists, the first value added is8902 // being modified by the assignments below (not the last value8903 // added).8904 if (StructBasePointersIdx < StructBaseCurInfo.BasePointers.size()) {8905 StructBaseCurInfo.DevicePtrDecls[StructBasePointersIdx] =8906 RelevantVD;8907 StructBaseCurInfo.DevicePointers[StructBasePointersIdx] =8908 L.ForDeviceAddr ? DeviceInfoTy::Address8909 : DeviceInfoTy::Pointer;8910 StructBaseCurInfo.Types[StructBasePointersIdx] |=8911 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;8912 } else {8913 CurInfo.DevicePtrDecls[CurrentBasePointersIdx] = RelevantVD;8914 CurInfo.DevicePointers[CurrentBasePointersIdx] =8915 L.ForDeviceAddr ? DeviceInfoTy::Address8916 : DeviceInfoTy::Pointer;8917 CurInfo.Types[CurrentBasePointersIdx] |=8918 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;8919 }8920 }8921 }8922 }8923 8924 // Append any pending zero-length pointers which are struct members and8925 // used with use_device_ptr or use_device_addr.8926 auto CI = DeferredInfo.find(Data.first);8927 if (CI != DeferredInfo.end()) {8928 for (const DeferredDevicePtrEntryTy &L : CI->second) {8929 llvm::Value *BasePtr;8930 llvm::Value *Ptr;8931 if (L.ForDeviceAddr) {8932 if (L.IE->isGLValue())8933 Ptr = this->CGF.EmitLValue(L.IE).getPointer(CGF);8934 else8935 Ptr = this->CGF.EmitScalarExpr(L.IE);8936 BasePtr = Ptr;8937 // Entry is RETURN_PARAM. Also, set the placeholder value8938 // MEMBER_OF=FFFF so that the entry is later updated with the8939 // correct value of MEMBER_OF.8940 CurInfo.Types.push_back(8941 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM |8942 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);8943 } else {8944 BasePtr = this->CGF.EmitLValue(L.IE).getPointer(CGF);8945 Ptr = this->CGF.EmitLoadOfScalar(this->CGF.EmitLValue(L.IE),8946 L.IE->getExprLoc());8947 // Entry is PTR_AND_OBJ and RETURN_PARAM. Also, set the8948 // placeholder value MEMBER_OF=FFFF so that the entry is later8949 // updated with the correct value of MEMBER_OF.8950 CurInfo.Types.push_back(8951 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |8952 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM |8953 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);8954 }8955 CurInfo.Exprs.push_back(L.VD);8956 CurInfo.BasePointers.emplace_back(BasePtr);8957 CurInfo.DevicePtrDecls.emplace_back(L.VD);8958 CurInfo.DevicePointers.emplace_back(8959 L.ForDeviceAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);8960 CurInfo.Pointers.push_back(Ptr);8961 CurInfo.Sizes.push_back(8962 llvm::Constant::getNullValue(this->CGF.Int64Ty));8963 CurInfo.Mappers.push_back(nullptr);8964 }8965 }8966 8967 // Unify entries in one list making sure the struct mapping precedes the8968 // individual fields:8969 MapCombinedInfoTy UnionCurInfo;8970 UnionCurInfo.append(StructBaseCurInfo);8971 UnionCurInfo.append(CurInfo);8972 8973 // If there is an entry in PartialStruct it means we have a struct with8974 // individual members mapped. Emit an extra combined entry.8975 if (PartialStruct.Base.isValid()) {8976 UnionCurInfo.NonContigInfo.Dims.push_back(0);8977 // Emit a combined entry:8978 emitCombinedEntry(CombinedInfo, UnionCurInfo.Types, PartialStruct,8979 /*IsMapThis*/ !VD, OMPBuilder, VD);8980 }8981 8982 // We need to append the results of this capture to what we already have.8983 CombinedInfo.append(UnionCurInfo);8984 }8985 // Append data for use_device_ptr clauses.8986 CombinedInfo.append(UseDeviceDataCombinedInfo);8987 }8988 8989public:8990 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)8991 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {8992 // Extract firstprivate clause information.8993 for (const auto *C : Dir.getClausesOfKind<OMPFirstprivateClause>())8994 for (const auto *D : C->varlist())8995 FirstPrivateDecls.try_emplace(8996 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl()), C->isImplicit());8997 // Extract implicit firstprivates from uses_allocators clauses.8998 for (const auto *C : Dir.getClausesOfKind<OMPUsesAllocatorsClause>()) {8999 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {9000 OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);9001 if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(D.AllocatorTraits))9002 FirstPrivateDecls.try_emplace(cast<VarDecl>(DRE->getDecl()),9003 /*Implicit=*/true);9004 else if (const auto *VD = dyn_cast<VarDecl>(9005 cast<DeclRefExpr>(D.Allocator->IgnoreParenImpCasts())9006 ->getDecl()))9007 FirstPrivateDecls.try_emplace(VD, /*Implicit=*/true);9008 }9009 }9010 // Extract device pointer clause information.9011 for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())9012 for (auto L : C->component_lists())9013 DevPointersMap[std::get<0>(L)].push_back(std::get<1>(L));9014 // Extract device addr clause information.9015 for (const auto *C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())9016 for (auto L : C->component_lists())9017 HasDevAddrsMap[std::get<0>(L)].push_back(std::get<1>(L));9018 // Extract map information.9019 for (const auto *C : Dir.getClausesOfKind<OMPMapClause>()) {9020 if (C->getMapType() != OMPC_MAP_to)9021 continue;9022 for (auto L : C->component_lists()) {9023 const ValueDecl *VD = std::get<0>(L);9024 const auto *RD = VD ? VD->getType()9025 .getCanonicalType()9026 .getNonReferenceType()9027 ->getAsCXXRecordDecl()9028 : nullptr;9029 if (RD && RD->isLambda())9030 LambdasMap.try_emplace(std::get<0>(L), C);9031 }9032 }9033 }9034 9035 /// Constructor for the declare mapper directive.9036 MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF)9037 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {}9038 9039 /// Generate code for the combined entry if we have a partially mapped struct9040 /// and take care of the mapping flags of the arguments corresponding to9041 /// individual struct members.9042 void emitCombinedEntry(MapCombinedInfoTy &CombinedInfo,9043 MapFlagsArrayTy &CurTypes,9044 const StructRangeInfoTy &PartialStruct, bool IsMapThis,9045 llvm::OpenMPIRBuilder &OMPBuilder,9046 const ValueDecl *VD = nullptr,9047 unsigned OffsetForMemberOfFlag = 0,9048 bool NotTargetParams = true) const {9049 if (CurTypes.size() == 1 &&9050 ((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=9051 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) &&9052 !PartialStruct.IsArraySection)9053 return;9054 Address LBAddr = PartialStruct.LowestElem.second;9055 Address HBAddr = PartialStruct.HighestElem.second;9056 if (PartialStruct.HasCompleteRecord) {9057 LBAddr = PartialStruct.LB;9058 HBAddr = PartialStruct.LB;9059 }9060 CombinedInfo.Exprs.push_back(VD);9061 // Base is the base of the struct9062 CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF));9063 CombinedInfo.DevicePtrDecls.push_back(nullptr);9064 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);9065 // Pointer is the address of the lowest element9066 llvm::Value *LB = LBAddr.emitRawPointer(CGF);9067 const CXXMethodDecl *MD =9068 CGF.CurFuncDecl ? dyn_cast<CXXMethodDecl>(CGF.CurFuncDecl) : nullptr;9069 const CXXRecordDecl *RD = MD ? MD->getParent() : nullptr;9070 bool HasBaseClass = RD && IsMapThis ? RD->getNumBases() > 0 : false;9071 // There should not be a mapper for a combined entry.9072 if (HasBaseClass) {9073 // OpenMP 5.2 148:21:9074 // If the target construct is within a class non-static member function,9075 // and a variable is an accessible data member of the object for which the9076 // non-static data member function is invoked, the variable is treated as9077 // if the this[:1] expression had appeared in a map clause with a map-type9078 // of tofrom.9079 // Emit this[:1]9080 CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF));9081 QualType Ty = MD->getFunctionObjectParameterType();9082 llvm::Value *Size =9083 CGF.Builder.CreateIntCast(CGF.getTypeSize(Ty), CGF.Int64Ty,9084 /*isSigned=*/true);9085 CombinedInfo.Sizes.push_back(Size);9086 } else {9087 CombinedInfo.Pointers.push_back(LB);9088 // Size is (addr of {highest+1} element) - (addr of lowest element)9089 llvm::Value *HB = HBAddr.emitRawPointer(CGF);9090 llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(9091 HBAddr.getElementType(), HB, /*Idx0=*/1);9092 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy);9093 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(HAddr, CGF.VoidPtrTy);9094 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(CGF.Int8Ty, CHAddr, CLAddr);9095 llvm::Value *Size = CGF.Builder.CreateIntCast(Diff, CGF.Int64Ty,9096 /*isSigned=*/false);9097 CombinedInfo.Sizes.push_back(Size);9098 }9099 CombinedInfo.Mappers.push_back(nullptr);9100 // Map type is always TARGET_PARAM, if generate info for captures.9101 CombinedInfo.Types.push_back(9102 NotTargetParams ? OpenMPOffloadMappingFlags::OMP_MAP_NONE9103 : !PartialStruct.PreliminaryMapData.BasePointers.empty()9104 ? OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ9105 : OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);9106 // If any element has the present modifier, then make sure the runtime9107 // doesn't attempt to allocate the struct.9108 if (CurTypes.end() !=9109 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags Type) {9110 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(9111 Type & OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);9112 }))9113 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;9114 // Remove TARGET_PARAM flag from the first element9115 (*CurTypes.begin()) &= ~OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;9116 // If any element has the ompx_hold modifier, then make sure the runtime9117 // uses the hold reference count for the struct as a whole so that it won't9118 // be unmapped by an extra dynamic reference count decrement. Add it to all9119 // elements as well so the runtime knows which reference count to check9120 // when determining whether it's time for device-to-host transfers of9121 // individual elements.9122 if (CurTypes.end() !=9123 llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags Type) {9124 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(9125 Type & OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD);9126 })) {9127 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;9128 for (auto &M : CurTypes)9129 M |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;9130 }9131 9132 // All other current entries will be MEMBER_OF the combined entry9133 // (except for PTR_AND_OBJ entries which do not have a placeholder value9134 // 0xFFFF in the MEMBER_OF field).9135 OpenMPOffloadMappingFlags MemberOfFlag = OMPBuilder.getMemberOfFlag(9136 OffsetForMemberOfFlag + CombinedInfo.BasePointers.size() - 1);9137 for (auto &M : CurTypes)9138 OMPBuilder.setCorrectMemberOfFlag(M, MemberOfFlag);9139 }9140 9141 /// Generate all the base pointers, section pointers, sizes, map types, and9142 /// mappers for the extracted mappable expressions (all included in \a9143 /// CombinedInfo). Also, for each item that relates with a device pointer, a9144 /// pair of the relevant declaration and index where it occurs is appended to9145 /// the device pointers info array.9146 void generateAllInfo(9147 MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder,9148 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =9149 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const {9150 assert(isa<const OMPExecutableDirective *>(CurDir) &&9151 "Expect a executable directive");9152 const auto *CurExecDir = cast<const OMPExecutableDirective *>(CurDir);9153 generateAllInfoForClauses(CurExecDir->clauses(), CombinedInfo, OMPBuilder,9154 SkipVarSet);9155 }9156 9157 /// Generate all the base pointers, section pointers, sizes, map types, and9158 /// mappers for the extracted map clauses of user-defined mapper (all included9159 /// in \a CombinedInfo).9160 void generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo,9161 llvm::OpenMPIRBuilder &OMPBuilder) const {9162 assert(isa<const OMPDeclareMapperDecl *>(CurDir) &&9163 "Expect a declare mapper directive");9164 const auto *CurMapperDir = cast<const OMPDeclareMapperDecl *>(CurDir);9165 generateAllInfoForClauses(CurMapperDir->clauses(), CombinedInfo,9166 OMPBuilder);9167 }9168 9169 /// Emit capture info for lambdas for variables captured by reference.9170 void generateInfoForLambdaCaptures(9171 const ValueDecl *VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,9172 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers) const {9173 QualType VDType = VD->getType().getCanonicalType().getNonReferenceType();9174 const auto *RD = VDType->getAsCXXRecordDecl();9175 if (!RD || !RD->isLambda())9176 return;9177 Address VDAddr(Arg, CGF.ConvertTypeForMem(VDType),9178 CGF.getContext().getDeclAlign(VD));9179 LValue VDLVal = CGF.MakeAddrLValue(VDAddr, VDType);9180 llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures;9181 FieldDecl *ThisCapture = nullptr;9182 RD->getCaptureFields(Captures, ThisCapture);9183 if (ThisCapture) {9184 LValue ThisLVal =9185 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture);9186 LValue ThisLValVal = CGF.EmitLValueForField(VDLVal, ThisCapture);9187 LambdaPointers.try_emplace(ThisLVal.getPointer(CGF),9188 VDLVal.getPointer(CGF));9189 CombinedInfo.Exprs.push_back(VD);9190 CombinedInfo.BasePointers.push_back(ThisLVal.getPointer(CGF));9191 CombinedInfo.DevicePtrDecls.push_back(nullptr);9192 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);9193 CombinedInfo.Pointers.push_back(ThisLValVal.getPointer(CGF));9194 CombinedInfo.Sizes.push_back(9195 CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy),9196 CGF.Int64Ty, /*isSigned=*/true));9197 CombinedInfo.Types.push_back(9198 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |9199 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |9200 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |9201 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);9202 CombinedInfo.Mappers.push_back(nullptr);9203 }9204 for (const LambdaCapture &LC : RD->captures()) {9205 if (!LC.capturesVariable())9206 continue;9207 const VarDecl *VD = cast<VarDecl>(LC.getCapturedVar());9208 if (LC.getCaptureKind() != LCK_ByRef && !VD->getType()->isPointerType())9209 continue;9210 auto It = Captures.find(VD);9211 assert(It != Captures.end() && "Found lambda capture without field.");9212 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second);9213 if (LC.getCaptureKind() == LCK_ByRef) {9214 LValue VarLValVal = CGF.EmitLValueForField(VDLVal, It->second);9215 LambdaPointers.try_emplace(VarLVal.getPointer(CGF),9216 VDLVal.getPointer(CGF));9217 CombinedInfo.Exprs.push_back(VD);9218 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF));9219 CombinedInfo.DevicePtrDecls.push_back(nullptr);9220 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);9221 CombinedInfo.Pointers.push_back(VarLValVal.getPointer(CGF));9222 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(9223 CGF.getTypeSize(9224 VD->getType().getCanonicalType().getNonReferenceType()),9225 CGF.Int64Ty, /*isSigned=*/true));9226 } else {9227 RValue VarRVal = CGF.EmitLoadOfLValue(VarLVal, RD->getLocation());9228 LambdaPointers.try_emplace(VarLVal.getPointer(CGF),9229 VDLVal.getPointer(CGF));9230 CombinedInfo.Exprs.push_back(VD);9231 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF));9232 CombinedInfo.DevicePtrDecls.push_back(nullptr);9233 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);9234 CombinedInfo.Pointers.push_back(VarRVal.getScalarVal());9235 CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.Int64Ty, 0));9236 }9237 CombinedInfo.Types.push_back(9238 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |9239 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |9240 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |9241 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);9242 CombinedInfo.Mappers.push_back(nullptr);9243 }9244 }9245 9246 /// Set correct indices for lambdas captures.9247 void adjustMemberOfForLambdaCaptures(9248 llvm::OpenMPIRBuilder &OMPBuilder,9249 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,9250 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,9251 MapFlagsArrayTy &Types) const {9252 for (unsigned I = 0, E = Types.size(); I < E; ++I) {9253 // Set correct member_of idx for all implicit lambda captures.9254 if (Types[I] != (OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |9255 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |9256 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |9257 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))9258 continue;9259 llvm::Value *BasePtr = LambdaPointers.lookup(BasePointers[I]);9260 assert(BasePtr && "Unable to find base lambda address.");9261 int TgtIdx = -1;9262 for (unsigned J = I; J > 0; --J) {9263 unsigned Idx = J - 1;9264 if (Pointers[Idx] != BasePtr)9265 continue;9266 TgtIdx = Idx;9267 break;9268 }9269 assert(TgtIdx != -1 && "Unable to find parent lambda.");9270 // All other current entries will be MEMBER_OF the combined entry9271 // (except for PTR_AND_OBJ entries which do not have a placeholder value9272 // 0xFFFF in the MEMBER_OF field).9273 OpenMPOffloadMappingFlags MemberOfFlag =9274 OMPBuilder.getMemberOfFlag(TgtIdx);9275 OMPBuilder.setCorrectMemberOfFlag(Types[I], MemberOfFlag);9276 }9277 }9278 9279 /// For a capture that has an associated clause, generate the base pointers,9280 /// section pointers, sizes, map types, and mappers (all included in9281 /// \a CurCaptureVarInfo).9282 void generateInfoForCaptureFromClauseInfo(9283 const CapturedStmt::Capture *Cap, llvm::Value *Arg,9284 MapCombinedInfoTy &CurCaptureVarInfo, llvm::OpenMPIRBuilder &OMPBuilder,9285 unsigned OffsetForMemberOfFlag) const {9286 assert(!Cap->capturesVariableArrayType() &&9287 "Not expecting to generate map info for a variable array type!");9288 9289 // We need to know when we generating information for the first component9290 const ValueDecl *VD = Cap->capturesThis()9291 ? nullptr9292 : Cap->getCapturedVar()->getCanonicalDecl();9293 9294 // for map(to: lambda): skip here, processing it in9295 // generateDefaultMapInfo9296 if (LambdasMap.count(VD))9297 return;9298 9299 // If this declaration appears in a is_device_ptr clause we just have to9300 // pass the pointer by value. If it is a reference to a declaration, we just9301 // pass its value.9302 if (VD && (DevPointersMap.count(VD) || HasDevAddrsMap.count(VD))) {9303 CurCaptureVarInfo.Exprs.push_back(VD);9304 CurCaptureVarInfo.BasePointers.emplace_back(Arg);9305 CurCaptureVarInfo.DevicePtrDecls.emplace_back(VD);9306 CurCaptureVarInfo.DevicePointers.emplace_back(DeviceInfoTy::Pointer);9307 CurCaptureVarInfo.Pointers.push_back(Arg);9308 CurCaptureVarInfo.Sizes.push_back(CGF.Builder.CreateIntCast(9309 CGF.getTypeSize(CGF.getContext().VoidPtrTy), CGF.Int64Ty,9310 /*isSigned=*/true));9311 CurCaptureVarInfo.Types.push_back(9312 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |9313 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);9314 CurCaptureVarInfo.Mappers.push_back(nullptr);9315 return;9316 }9317 9318 MapDataArrayTy DeclComponentLists;9319 // For member fields list in is_device_ptr, store it in9320 // DeclComponentLists for generating components info.9321 static const OpenMPMapModifierKind Unknown = OMPC_MAP_MODIFIER_unknown;9322 auto It = DevPointersMap.find(VD);9323 if (It != DevPointersMap.end())9324 for (const auto &MCL : It->second)9325 DeclComponentLists.emplace_back(MCL, OMPC_MAP_to, Unknown,9326 /*IsImpicit = */ true, nullptr,9327 nullptr);9328 auto I = HasDevAddrsMap.find(VD);9329 if (I != HasDevAddrsMap.end())9330 for (const auto &MCL : I->second)9331 DeclComponentLists.emplace_back(MCL, OMPC_MAP_tofrom, Unknown,9332 /*IsImpicit = */ true, nullptr,9333 nullptr);9334 assert(isa<const OMPExecutableDirective *>(CurDir) &&9335 "Expect a executable directive");9336 const auto *CurExecDir = cast<const OMPExecutableDirective *>(CurDir);9337 bool HasMapBasePtr = false;9338 bool HasMapArraySec = false;9339 for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) {9340 const auto *EI = C->getVarRefs().begin();9341 for (const auto L : C->decl_component_lists(VD)) {9342 const ValueDecl *VDecl, *Mapper;9343 // The Expression is not correct if the mapping is implicit9344 const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr;9345 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;9346 std::tie(VDecl, Components, Mapper) = L;9347 assert(VDecl == VD && "We got information for the wrong declaration??");9348 assert(!Components.empty() &&9349 "Not expecting declaration with no component lists.");9350 if (VD && E && VD->getType()->isAnyPointerType() && isa<DeclRefExpr>(E))9351 HasMapBasePtr = true;9352 if (VD && E && VD->getType()->isAnyPointerType() &&9353 (isa<ArraySectionExpr>(E) || isa<ArraySubscriptExpr>(E)))9354 HasMapArraySec = true;9355 DeclComponentLists.emplace_back(Components, C->getMapType(),9356 C->getMapTypeModifiers(),9357 C->isImplicit(), Mapper, E);9358 ++EI;9359 }9360 }9361 llvm::stable_sort(DeclComponentLists, [](const MapData &LHS,9362 const MapData &RHS) {9363 ArrayRef<OpenMPMapModifierKind> MapModifiers = std::get<2>(LHS);9364 OpenMPMapClauseKind MapType = std::get<1>(RHS);9365 bool HasPresent =9366 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);9367 bool HasAllocs = MapType == OMPC_MAP_alloc;9368 MapModifiers = std::get<2>(RHS);9369 MapType = std::get<1>(LHS);9370 bool HasPresentR =9371 llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);9372 bool HasAllocsR = MapType == OMPC_MAP_alloc;9373 return (HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR);9374 });9375 9376 auto GenerateInfoForComponentLists =9377 [&](ArrayRef<MapData> DeclComponentLists,9378 bool IsEligibleForTargetParamFlag) {9379 MapCombinedInfoTy CurInfoForComponentLists;9380 StructRangeInfoTy PartialStruct;9381 9382 if (DeclComponentLists.empty())9383 return;9384 9385 generateInfoForCaptureFromComponentLists(9386 VD, DeclComponentLists, CurInfoForComponentLists, PartialStruct,9387 IsEligibleForTargetParamFlag,9388 /*AreBothBasePtrAndPteeMapped=*/HasMapBasePtr && HasMapArraySec);9389 9390 // If there is an entry in PartialStruct it means we have a9391 // struct with individual members mapped. Emit an extra combined9392 // entry.9393 if (PartialStruct.Base.isValid()) {9394 CurCaptureVarInfo.append(PartialStruct.PreliminaryMapData);9395 emitCombinedEntry(9396 CurCaptureVarInfo, CurInfoForComponentLists.Types,9397 PartialStruct, Cap->capturesThis(), OMPBuilder, nullptr,9398 OffsetForMemberOfFlag,9399 /*NotTargetParams*/ !IsEligibleForTargetParamFlag);9400 }9401 9402 // Return if we didn't add any entries.9403 if (CurInfoForComponentLists.BasePointers.empty())9404 return;9405 9406 CurCaptureVarInfo.append(CurInfoForComponentLists);9407 };9408 9409 GenerateInfoForComponentLists(DeclComponentLists,9410 /*IsEligibleForTargetParamFlag=*/true);9411 }9412 9413 /// Generate the base pointers, section pointers, sizes, map types, and9414 /// mappers associated to \a DeclComponentLists for a given capture9415 /// \a VD (all included in \a CurComponentListInfo).9416 void generateInfoForCaptureFromComponentLists(9417 const ValueDecl *VD, ArrayRef<MapData> DeclComponentLists,9418 MapCombinedInfoTy &CurComponentListInfo, StructRangeInfoTy &PartialStruct,9419 bool IsListEligibleForTargetParamFlag,9420 bool AreBothBasePtrAndPteeMapped = false) const {9421 // Find overlapping elements (including the offset from the base element).9422 llvm::SmallDenseMap<9423 const MapData *,9424 llvm::SmallVector<9425 OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>,9426 4>9427 OverlappedData;9428 size_t Count = 0;9429 for (const MapData &L : DeclComponentLists) {9430 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;9431 OpenMPMapClauseKind MapType;9432 ArrayRef<OpenMPMapModifierKind> MapModifiers;9433 bool IsImplicit;9434 const ValueDecl *Mapper;9435 const Expr *VarRef;9436 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =9437 L;9438 ++Count;9439 for (const MapData &L1 : ArrayRef(DeclComponentLists).slice(Count)) {9440 OMPClauseMappableExprCommon::MappableExprComponentListRef Components1;9441 std::tie(Components1, MapType, MapModifiers, IsImplicit, Mapper,9442 VarRef) = L1;9443 auto CI = Components.rbegin();9444 auto CE = Components.rend();9445 auto SI = Components1.rbegin();9446 auto SE = Components1.rend();9447 for (; CI != CE && SI != SE; ++CI, ++SI) {9448 if (CI->getAssociatedExpression()->getStmtClass() !=9449 SI->getAssociatedExpression()->getStmtClass())9450 break;9451 // Are we dealing with different variables/fields?9452 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())9453 break;9454 }9455 // Found overlapping if, at least for one component, reached the head9456 // of the components list.9457 if (CI == CE || SI == SE) {9458 // Ignore it if it is the same component.9459 if (CI == CE && SI == SE)9460 continue;9461 const auto It = (SI == SE) ? CI : SI;9462 // If one component is a pointer and another one is a kind of9463 // dereference of this pointer (array subscript, section, dereference,9464 // etc.), it is not an overlapping.9465 // Same, if one component is a base and another component is a9466 // dereferenced pointer memberexpr with the same base.9467 if (!isa<MemberExpr>(It->getAssociatedExpression()) ||9468 (std::prev(It)->getAssociatedDeclaration() &&9469 std::prev(It)9470 ->getAssociatedDeclaration()9471 ->getType()9472 ->isPointerType()) ||9473 (It->getAssociatedDeclaration() &&9474 It->getAssociatedDeclaration()->getType()->isPointerType() &&9475 std::next(It) != CE && std::next(It) != SE))9476 continue;9477 const MapData &BaseData = CI == CE ? L : L1;9478 OMPClauseMappableExprCommon::MappableExprComponentListRef SubData =9479 SI == SE ? Components : Components1;9480 OverlappedData[&BaseData].push_back(SubData);9481 }9482 }9483 }9484 // Sort the overlapped elements for each item.9485 llvm::SmallVector<const FieldDecl *, 4> Layout;9486 if (!OverlappedData.empty()) {9487 const Type *BaseType = VD->getType().getCanonicalType().getTypePtr();9488 const Type *OrigType = BaseType->getPointeeOrArrayElementType();9489 while (BaseType != OrigType) {9490 BaseType = OrigType->getCanonicalTypeInternal().getTypePtr();9491 OrigType = BaseType->getPointeeOrArrayElementType();9492 }9493 9494 if (const auto *CRD = BaseType->getAsCXXRecordDecl())9495 getPlainLayout(CRD, Layout, /*AsBase=*/false);9496 else {9497 const auto *RD = BaseType->getAsRecordDecl();9498 Layout.append(RD->field_begin(), RD->field_end());9499 }9500 }9501 for (auto &Pair : OverlappedData) {9502 llvm::stable_sort(9503 Pair.getSecond(),9504 [&Layout](9505 OMPClauseMappableExprCommon::MappableExprComponentListRef First,9506 OMPClauseMappableExprCommon::MappableExprComponentListRef9507 Second) {9508 auto CI = First.rbegin();9509 auto CE = First.rend();9510 auto SI = Second.rbegin();9511 auto SE = Second.rend();9512 for (; CI != CE && SI != SE; ++CI, ++SI) {9513 if (CI->getAssociatedExpression()->getStmtClass() !=9514 SI->getAssociatedExpression()->getStmtClass())9515 break;9516 // Are we dealing with different variables/fields?9517 if (CI->getAssociatedDeclaration() !=9518 SI->getAssociatedDeclaration())9519 break;9520 }9521 9522 // Lists contain the same elements.9523 if (CI == CE && SI == SE)9524 return false;9525 9526 // List with less elements is less than list with more elements.9527 if (CI == CE || SI == SE)9528 return CI == CE;9529 9530 const auto *FD1 = cast<FieldDecl>(CI->getAssociatedDeclaration());9531 const auto *FD2 = cast<FieldDecl>(SI->getAssociatedDeclaration());9532 if (FD1->getParent() == FD2->getParent())9533 return FD1->getFieldIndex() < FD2->getFieldIndex();9534 const auto *It =9535 llvm::find_if(Layout, [FD1, FD2](const FieldDecl *FD) {9536 return FD == FD1 || FD == FD2;9537 });9538 return *It == FD1;9539 });9540 }9541 9542 // Associated with a capture, because the mapping flags depend on it.9543 // Go through all of the elements with the overlapped elements.9544 bool AddTargetParamFlag = IsListEligibleForTargetParamFlag;9545 MapCombinedInfoTy StructBaseCombinedInfo;9546 for (const auto &Pair : OverlappedData) {9547 const MapData &L = *Pair.getFirst();9548 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;9549 OpenMPMapClauseKind MapType;9550 ArrayRef<OpenMPMapModifierKind> MapModifiers;9551 bool IsImplicit;9552 const ValueDecl *Mapper;9553 const Expr *VarRef;9554 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =9555 L;9556 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>9557 OverlappedComponents = Pair.getSecond();9558 generateInfoForComponentList(9559 MapType, MapModifiers, {}, Components, CurComponentListInfo,9560 StructBaseCombinedInfo, PartialStruct, AddTargetParamFlag, IsImplicit,9561 /*GenerateAllInfoForClauses*/ false, Mapper,9562 /*ForDeviceAddr=*/false, VD, VarRef, OverlappedComponents);9563 AddTargetParamFlag = false;9564 }9565 // Go through other elements without overlapped elements.9566 for (const MapData &L : DeclComponentLists) {9567 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;9568 OpenMPMapClauseKind MapType;9569 ArrayRef<OpenMPMapModifierKind> MapModifiers;9570 bool IsImplicit;9571 const ValueDecl *Mapper;9572 const Expr *VarRef;9573 std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =9574 L;9575 auto It = OverlappedData.find(&L);9576 if (It == OverlappedData.end())9577 generateInfoForComponentList(9578 MapType, MapModifiers, {}, Components, CurComponentListInfo,9579 StructBaseCombinedInfo, PartialStruct, AddTargetParamFlag,9580 IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper,9581 /*ForDeviceAddr=*/false, VD, VarRef,9582 /*OverlappedElements*/ {}, AreBothBasePtrAndPteeMapped);9583 AddTargetParamFlag = false;9584 }9585 }9586 9587 /// Generate the default map information for a given capture \a CI,9588 /// record field declaration \a RI and captured value \a CV.9589 void generateDefaultMapInfo(const CapturedStmt::Capture &CI,9590 const FieldDecl &RI, llvm::Value *CV,9591 MapCombinedInfoTy &CombinedInfo) const {9592 bool IsImplicit = true;9593 // Do the default mapping.9594 if (CI.capturesThis()) {9595 CombinedInfo.Exprs.push_back(nullptr);9596 CombinedInfo.BasePointers.push_back(CV);9597 CombinedInfo.DevicePtrDecls.push_back(nullptr);9598 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);9599 CombinedInfo.Pointers.push_back(CV);9600 const auto *PtrTy = cast<PointerType>(RI.getType().getTypePtr());9601 CombinedInfo.Sizes.push_back(9602 CGF.Builder.CreateIntCast(CGF.getTypeSize(PtrTy->getPointeeType()),9603 CGF.Int64Ty, /*isSigned=*/true));9604 // Default map type.9605 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_TO |9606 OpenMPOffloadMappingFlags::OMP_MAP_FROM);9607 } else if (CI.capturesVariableByCopy()) {9608 const VarDecl *VD = CI.getCapturedVar();9609 CombinedInfo.Exprs.push_back(VD->getCanonicalDecl());9610 CombinedInfo.BasePointers.push_back(CV);9611 CombinedInfo.DevicePtrDecls.push_back(nullptr);9612 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);9613 CombinedInfo.Pointers.push_back(CV);9614 if (!RI.getType()->isAnyPointerType()) {9615 // We have to signal to the runtime captures passed by value that are9616 // not pointers.9617 CombinedInfo.Types.push_back(9618 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);9619 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(9620 CGF.getTypeSize(RI.getType()), CGF.Int64Ty, /*isSigned=*/true));9621 } else {9622 // Pointers are implicitly mapped with a zero size and no flags9623 // (other than first map that is added for all implicit maps).9624 CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_NONE);9625 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty));9626 }9627 auto I = FirstPrivateDecls.find(VD);9628 if (I != FirstPrivateDecls.end())9629 IsImplicit = I->getSecond();9630 } else {9631 assert(CI.capturesVariable() && "Expected captured reference.");9632 const auto *PtrTy = cast<ReferenceType>(RI.getType().getTypePtr());9633 QualType ElementType = PtrTy->getPointeeType();9634 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(9635 CGF.getTypeSize(ElementType), CGF.Int64Ty, /*isSigned=*/true));9636 // The default map type for a scalar/complex type is 'to' because by9637 // default the value doesn't have to be retrieved. For an aggregate9638 // type, the default is 'tofrom'.9639 CombinedInfo.Types.push_back(getMapModifiersForPrivateClauses(CI));9640 const VarDecl *VD = CI.getCapturedVar();9641 auto I = FirstPrivateDecls.find(VD);9642 CombinedInfo.Exprs.push_back(VD->getCanonicalDecl());9643 CombinedInfo.BasePointers.push_back(CV);9644 CombinedInfo.DevicePtrDecls.push_back(nullptr);9645 CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);9646 if (I != FirstPrivateDecls.end() && ElementType->isAnyPointerType()) {9647 Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue(9648 CV, ElementType, CGF.getContext().getDeclAlign(VD),9649 AlignmentSource::Decl));9650 CombinedInfo.Pointers.push_back(PtrAddr.emitRawPointer(CGF));9651 } else {9652 CombinedInfo.Pointers.push_back(CV);9653 }9654 if (I != FirstPrivateDecls.end())9655 IsImplicit = I->getSecond();9656 }9657 // Every default map produces a single argument which is a target parameter.9658 CombinedInfo.Types.back() |=9659 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;9660 9661 // Add flag stating this is an implicit map.9662 if (IsImplicit)9663 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;9664 9665 // No user-defined mapper for default mapping.9666 CombinedInfo.Mappers.push_back(nullptr);9667 }9668};9669} // anonymous namespace9670 9671// Try to extract the base declaration from a `this->x` expression if possible.9672static ValueDecl *getDeclFromThisExpr(const Expr *E) {9673 if (!E)9674 return nullptr;9675 9676 if (const auto *OASE = dyn_cast<ArraySectionExpr>(E->IgnoreParenCasts()))9677 if (const MemberExpr *ME =9678 dyn_cast<MemberExpr>(OASE->getBase()->IgnoreParenImpCasts()))9679 return ME->getMemberDecl();9680 return nullptr;9681}9682 9683/// Emit a string constant containing the names of the values mapped to the9684/// offloading runtime library.9685static llvm::Constant *9686emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder,9687 MappableExprsHandler::MappingExprInfo &MapExprs) {9688 9689 uint32_t SrcLocStrSize;9690 if (!MapExprs.getMapDecl() && !MapExprs.getMapExpr())9691 return OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);9692 9693 SourceLocation Loc;9694 if (!MapExprs.getMapDecl() && MapExprs.getMapExpr()) {9695 if (const ValueDecl *VD = getDeclFromThisExpr(MapExprs.getMapExpr()))9696 Loc = VD->getLocation();9697 else9698 Loc = MapExprs.getMapExpr()->getExprLoc();9699 } else {9700 Loc = MapExprs.getMapDecl()->getLocation();9701 }9702 9703 std::string ExprName;9704 if (MapExprs.getMapExpr()) {9705 PrintingPolicy P(CGF.getContext().getLangOpts());9706 llvm::raw_string_ostream OS(ExprName);9707 MapExprs.getMapExpr()->printPretty(OS, nullptr, P);9708 } else {9709 ExprName = MapExprs.getMapDecl()->getNameAsString();9710 }9711 9712 std::string FileName;9713 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);9714 if (auto *DbgInfo = CGF.getDebugInfo())9715 FileName = DbgInfo->remapDIPath(PLoc.getFilename());9716 else9717 FileName = PLoc.getFilename();9718 return OMPBuilder.getOrCreateSrcLocStr(FileName, ExprName, PLoc.getLine(),9719 PLoc.getColumn(), SrcLocStrSize);9720}9721/// Emit the arrays used to pass the captures and map information to the9722/// offloading runtime library. If there is no map or capture information,9723/// return nullptr by reference.9724static void emitOffloadingArraysAndArgs(9725 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,9726 CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder,9727 bool IsNonContiguous = false, bool ForEndCall = false) {9728 CodeGenModule &CGM = CGF.CGM;9729 9730 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;9731 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(),9732 CGF.AllocaInsertPt->getIterator());9733 InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(),9734 CGF.Builder.GetInsertPoint());9735 9736 auto DeviceAddrCB = [&](unsigned int I, llvm::Value *NewDecl) {9737 if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {9738 Info.CaptureDeviceAddrMap.try_emplace(DevVD, NewDecl);9739 }9740 };9741 9742 auto CustomMapperCB = [&](unsigned int I) {9743 llvm::Function *MFunc = nullptr;9744 if (CombinedInfo.Mappers[I]) {9745 Info.HasMapper = true;9746 MFunc = CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(9747 cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));9748 }9749 return MFunc;9750 };9751 cantFail(OMPBuilder.emitOffloadingArraysAndArgs(9752 AllocaIP, CodeGenIP, Info, Info.RTArgs, CombinedInfo, CustomMapperCB,9753 IsNonContiguous, ForEndCall, DeviceAddrCB));9754}9755 9756/// Check for inner distribute directive.9757static const OMPExecutableDirective *9758getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) {9759 const auto *CS = D.getInnermostCapturedStmt();9760 const auto *Body =9761 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);9762 const Stmt *ChildStmt =9763 CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);9764 9765 if (const auto *NestedDir =9766 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {9767 OpenMPDirectiveKind DKind = NestedDir->getDirectiveKind();9768 switch (D.getDirectiveKind()) {9769 case OMPD_target:9770 // For now, treat 'target' with nested 'teams loop' as if it's9771 // distributed (target teams distribute).9772 if (isOpenMPDistributeDirective(DKind) || DKind == OMPD_teams_loop)9773 return NestedDir;9774 if (DKind == OMPD_teams) {9775 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(9776 /*IgnoreCaptured=*/true);9777 if (!Body)9778 return nullptr;9779 ChildStmt = CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);9780 if (const auto *NND =9781 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {9782 DKind = NND->getDirectiveKind();9783 if (isOpenMPDistributeDirective(DKind))9784 return NND;9785 }9786 }9787 return nullptr;9788 case OMPD_target_teams:9789 if (isOpenMPDistributeDirective(DKind))9790 return NestedDir;9791 return nullptr;9792 case OMPD_target_parallel:9793 case OMPD_target_simd:9794 case OMPD_target_parallel_for:9795 case OMPD_target_parallel_for_simd:9796 return nullptr;9797 case OMPD_target_teams_distribute:9798 case OMPD_target_teams_distribute_simd:9799 case OMPD_target_teams_distribute_parallel_for:9800 case OMPD_target_teams_distribute_parallel_for_simd:9801 case OMPD_parallel:9802 case OMPD_for:9803 case OMPD_parallel_for:9804 case OMPD_parallel_master:9805 case OMPD_parallel_sections:9806 case OMPD_for_simd:9807 case OMPD_parallel_for_simd:9808 case OMPD_cancel:9809 case OMPD_cancellation_point:9810 case OMPD_ordered:9811 case OMPD_threadprivate:9812 case OMPD_allocate:9813 case OMPD_task:9814 case OMPD_simd:9815 case OMPD_tile:9816 case OMPD_unroll:9817 case OMPD_sections:9818 case OMPD_section:9819 case OMPD_single:9820 case OMPD_master:9821 case OMPD_critical:9822 case OMPD_taskyield:9823 case OMPD_barrier:9824 case OMPD_taskwait:9825 case OMPD_taskgroup:9826 case OMPD_atomic:9827 case OMPD_flush:9828 case OMPD_depobj:9829 case OMPD_scan:9830 case OMPD_teams:9831 case OMPD_target_data:9832 case OMPD_target_exit_data:9833 case OMPD_target_enter_data:9834 case OMPD_distribute:9835 case OMPD_distribute_simd:9836 case OMPD_distribute_parallel_for:9837 case OMPD_distribute_parallel_for_simd:9838 case OMPD_teams_distribute:9839 case OMPD_teams_distribute_simd:9840 case OMPD_teams_distribute_parallel_for:9841 case OMPD_teams_distribute_parallel_for_simd:9842 case OMPD_target_update:9843 case OMPD_declare_simd:9844 case OMPD_declare_variant:9845 case OMPD_begin_declare_variant:9846 case OMPD_end_declare_variant:9847 case OMPD_declare_target:9848 case OMPD_end_declare_target:9849 case OMPD_declare_reduction:9850 case OMPD_declare_mapper:9851 case OMPD_taskloop:9852 case OMPD_taskloop_simd:9853 case OMPD_master_taskloop:9854 case OMPD_master_taskloop_simd:9855 case OMPD_parallel_master_taskloop:9856 case OMPD_parallel_master_taskloop_simd:9857 case OMPD_requires:9858 case OMPD_metadirective:9859 case OMPD_unknown:9860 default:9861 llvm_unreachable("Unexpected directive.");9862 }9863 }9864 9865 return nullptr;9866}9867 9868/// Emit the user-defined mapper function. The code generation follows the9869/// pattern in the example below.9870/// \code9871/// void .omp_mapper.<type_name>.<mapper_id>.(void *rt_mapper_handle,9872/// void *base, void *begin,9873/// int64_t size, int64_t type,9874/// void *name = nullptr) {9875/// // Allocate space for an array section first or add a base/begin for9876/// // pointer dereference.9877/// if ((size > 1 || (base != begin && maptype.IsPtrAndObj)) &&9878/// !maptype.IsDelete)9879/// __tgt_push_mapper_component(rt_mapper_handle, base, begin,9880/// size*sizeof(Ty), clearToFromMember(type));9881/// // Map members.9882/// for (unsigned i = 0; i < size; i++) {9883/// // For each component specified by this mapper:9884/// for (auto c : begin[i]->all_components) {9885/// if (c.hasMapper())9886/// (*c.Mapper())(rt_mapper_handle, c.arg_base, c.arg_begin, c.arg_size,9887/// c.arg_type, c.arg_name);9888/// else9889/// __tgt_push_mapper_component(rt_mapper_handle, c.arg_base,9890/// c.arg_begin, c.arg_size, c.arg_type,9891/// c.arg_name);9892/// }9893/// }9894/// // Delete the array section.9895/// if (size > 1 && maptype.IsDelete)9896/// __tgt_push_mapper_component(rt_mapper_handle, base, begin,9897/// size*sizeof(Ty), clearToFromMember(type));9898/// }9899/// \endcode9900void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D,9901 CodeGenFunction *CGF) {9902 if (UDMMap.count(D) > 0)9903 return;9904 ASTContext &C = CGM.getContext();9905 QualType Ty = D->getType();9906 auto *MapperVarDecl =9907 cast<VarDecl>(cast<DeclRefExpr>(D->getMapperVarRef())->getDecl());9908 CharUnits ElementSize = C.getTypeSizeInChars(Ty);9909 llvm::Type *ElemTy = CGM.getTypes().ConvertTypeForMem(Ty);9910 9911 CodeGenFunction MapperCGF(CGM);9912 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;9913 auto PrivatizeAndGenMapInfoCB =9914 [&](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP, llvm::Value *PtrPHI,9915 llvm::Value *BeginArg) -> llvm::OpenMPIRBuilder::MapInfosTy & {9916 MapperCGF.Builder.restoreIP(CodeGenIP);9917 9918 // Privatize the declared variable of mapper to be the current array9919 // element.9920 Address PtrCurrent(9921 PtrPHI, ElemTy,9922 Address(BeginArg, MapperCGF.VoidPtrTy, CGM.getPointerAlign())9923 .getAlignment()9924 .alignmentOfArrayElement(ElementSize));9925 CodeGenFunction::OMPPrivateScope Scope(MapperCGF);9926 Scope.addPrivate(MapperVarDecl, PtrCurrent);9927 (void)Scope.Privatize();9928 9929 // Get map clause information.9930 MappableExprsHandler MEHandler(*D, MapperCGF);9931 MEHandler.generateAllInfoForMapper(CombinedInfo, OMPBuilder);9932 9933 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {9934 return emitMappingInformation(MapperCGF, OMPBuilder, MapExpr);9935 };9936 if (CGM.getCodeGenOpts().getDebugInfo() !=9937 llvm::codegenoptions::NoDebugInfo) {9938 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());9939 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),9940 FillInfoMap);9941 }9942 9943 return CombinedInfo;9944 };9945 9946 auto CustomMapperCB = [&](unsigned I) {9947 llvm::Function *MapperFunc = nullptr;9948 if (CombinedInfo.Mappers[I]) {9949 // Call the corresponding mapper function.9950 MapperFunc = getOrCreateUserDefinedMapperFunc(9951 cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));9952 assert(MapperFunc && "Expect a valid mapper function is available.");9953 }9954 return MapperFunc;9955 };9956 9957 SmallString<64> TyStr;9958 llvm::raw_svector_ostream Out(TyStr);9959 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(Ty, Out);9960 std::string Name = getName({"omp_mapper", TyStr, D->getName()});9961 9962 llvm::Function *NewFn = cantFail(OMPBuilder.emitUserDefinedMapper(9963 PrivatizeAndGenMapInfoCB, ElemTy, Name, CustomMapperCB));9964 UDMMap.try_emplace(D, NewFn);9965 if (CGF)9966 FunctionUDMMap[CGF->CurFn].push_back(D);9967}9968 9969llvm::Function *CGOpenMPRuntime::getOrCreateUserDefinedMapperFunc(9970 const OMPDeclareMapperDecl *D) {9971 auto I = UDMMap.find(D);9972 if (I != UDMMap.end())9973 return I->second;9974 emitUserDefinedMapper(D);9975 return UDMMap.lookup(D);9976}9977 9978llvm::Value *CGOpenMPRuntime::emitTargetNumIterationsCall(9979 CodeGenFunction &CGF, const OMPExecutableDirective &D,9980 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,9981 const OMPLoopDirective &D)>9982 SizeEmitter) {9983 OpenMPDirectiveKind Kind = D.getDirectiveKind();9984 const OMPExecutableDirective *TD = &D;9985 // Get nested teams distribute kind directive, if any. For now, treat9986 // 'target_teams_loop' as if it's really a target_teams_distribute.9987 if ((!isOpenMPDistributeDirective(Kind) || !isOpenMPTeamsDirective(Kind)) &&9988 Kind != OMPD_target_teams_loop)9989 TD = getNestedDistributeDirective(CGM.getContext(), D);9990 if (!TD)9991 return llvm::ConstantInt::get(CGF.Int64Ty, 0);9992 9993 const auto *LD = cast<OMPLoopDirective>(TD);9994 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD))9995 return NumIterations;9996 return llvm::ConstantInt::get(CGF.Int64Ty, 0);9997}9998 9999static void10000emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,10001 const OMPExecutableDirective &D,10002 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,10003 bool RequiresOuterTask, const CapturedStmt &CS,10004 bool OffloadingMandatory, CodeGenFunction &CGF) {10005 if (OffloadingMandatory) {10006 CGF.Builder.CreateUnreachable();10007 } else {10008 if (RequiresOuterTask) {10009 CapturedVars.clear();10010 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);10011 }10012 OMPRuntime->emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn,10013 CapturedVars);10014 }10015}10016 10017static llvm::Value *emitDeviceID(10018 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,10019 CodeGenFunction &CGF) {10020 // Emit device ID if any.10021 llvm::Value *DeviceID;10022 if (Device.getPointer()) {10023 assert((Device.getInt() == OMPC_DEVICE_unknown ||10024 Device.getInt() == OMPC_DEVICE_device_num) &&10025 "Expected device_num modifier.");10026 llvm::Value *DevVal = CGF.EmitScalarExpr(Device.getPointer());10027 DeviceID =10028 CGF.Builder.CreateIntCast(DevVal, CGF.Int64Ty, /*isSigned=*/true);10029 } else {10030 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);10031 }10032 return DeviceID;10033}10034 10035static std::pair<llvm::Value *, OMPDynGroupprivateFallbackType>10036emitDynCGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF) {10037 llvm::Value *DynGP = CGF.Builder.getInt32(0);10038 auto DynGPFallback = OMPDynGroupprivateFallbackType::Abort;10039 10040 if (auto *DynGPClause = D.getSingleClause<OMPDynGroupprivateClause>()) {10041 CodeGenFunction::RunCleanupsScope DynGPScope(CGF);10042 llvm::Value *DynGPVal =10043 CGF.EmitScalarExpr(DynGPClause->getSize(), /*IgnoreResultAssign=*/true);10044 DynGP = CGF.Builder.CreateIntCast(DynGPVal, CGF.Int32Ty,10045 /*isSigned=*/false);10046 auto FallbackModifier = DynGPClause->getDynGroupprivateFallbackModifier();10047 switch (FallbackModifier) {10048 case OMPC_DYN_GROUPPRIVATE_FALLBACK_abort:10049 DynGPFallback = OMPDynGroupprivateFallbackType::Abort;10050 break;10051 case OMPC_DYN_GROUPPRIVATE_FALLBACK_null:10052 DynGPFallback = OMPDynGroupprivateFallbackType::Null;10053 break;10054 case OMPC_DYN_GROUPPRIVATE_FALLBACK_default_mem:10055 case OMPC_DYN_GROUPPRIVATE_FALLBACK_unknown:10056 // This is the default for dyn_groupprivate.10057 DynGPFallback = OMPDynGroupprivateFallbackType::DefaultMem;10058 break;10059 default:10060 llvm_unreachable("Unknown fallback modifier for OpenMP dyn_groupprivate");10061 }10062 } else if (auto *OMPXDynCGClause =10063 D.getSingleClause<OMPXDynCGroupMemClause>()) {10064 CodeGenFunction::RunCleanupsScope DynCGMemScope(CGF);10065 llvm::Value *DynCGMemVal = CGF.EmitScalarExpr(OMPXDynCGClause->getSize(),10066 /*IgnoreResultAssign=*/true);10067 DynGP = CGF.Builder.CreateIntCast(DynCGMemVal, CGF.Int32Ty,10068 /*isSigned=*/false);10069 }10070 return {DynGP, DynGPFallback};10071}10072 10073static void genMapInfoForCaptures(10074 MappableExprsHandler &MEHandler, CodeGenFunction &CGF,10075 const CapturedStmt &CS, llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,10076 llvm::OpenMPIRBuilder &OMPBuilder,10077 llvm::DenseSet<CanonicalDeclPtr<const Decl>> &MappedVarSet,10078 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {10079 10080 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;10081 auto RI = CS.getCapturedRecordDecl()->field_begin();10082 auto *CV = CapturedVars.begin();10083 for (CapturedStmt::const_capture_iterator CI = CS.capture_begin(),10084 CE = CS.capture_end();10085 CI != CE; ++CI, ++RI, ++CV) {10086 MappableExprsHandler::MapCombinedInfoTy CurInfo;10087 10088 // VLA sizes are passed to the outlined region by copy and do not have map10089 // information associated.10090 if (CI->capturesVariableArrayType()) {10091 CurInfo.Exprs.push_back(nullptr);10092 CurInfo.BasePointers.push_back(*CV);10093 CurInfo.DevicePtrDecls.push_back(nullptr);10094 CurInfo.DevicePointers.push_back(10095 MappableExprsHandler::DeviceInfoTy::None);10096 CurInfo.Pointers.push_back(*CV);10097 CurInfo.Sizes.push_back(CGF.Builder.CreateIntCast(10098 CGF.getTypeSize(RI->getType()), CGF.Int64Ty, /*isSigned=*/true));10099 // Copy to the device as an argument. No need to retrieve it.10100 CurInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |10101 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |10102 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);10103 CurInfo.Mappers.push_back(nullptr);10104 } else {10105 // If we have any information in the map clause, we use it, otherwise we10106 // just do a default mapping.10107 MEHandler.generateInfoForCaptureFromClauseInfo(10108 CI, *CV, CurInfo, OMPBuilder,10109 /*OffsetForMemberOfFlag=*/CombinedInfo.BasePointers.size());10110 10111 if (!CI->capturesThis())10112 MappedVarSet.insert(CI->getCapturedVar());10113 else10114 MappedVarSet.insert(nullptr);10115 10116 if (CurInfo.BasePointers.empty())10117 MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurInfo);10118 10119 // Generate correct mapping for variables captured by reference in10120 // lambdas.10121 if (CI->capturesVariable())10122 MEHandler.generateInfoForLambdaCaptures(CI->getCapturedVar(), *CV,10123 CurInfo, LambdaPointers);10124 }10125 // We expect to have at least an element of information for this capture.10126 assert(!CurInfo.BasePointers.empty() &&10127 "Non-existing map pointer for capture!");10128 assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() &&10129 CurInfo.BasePointers.size() == CurInfo.Sizes.size() &&10130 CurInfo.BasePointers.size() == CurInfo.Types.size() &&10131 CurInfo.BasePointers.size() == CurInfo.Mappers.size() &&10132 "Inconsistent map information sizes!");10133 10134 // We need to append the results of this capture to what we already have.10135 CombinedInfo.append(CurInfo);10136 }10137 // Adjust MEMBER_OF flags for the lambdas captures.10138 MEHandler.adjustMemberOfForLambdaCaptures(10139 OMPBuilder, LambdaPointers, CombinedInfo.BasePointers,10140 CombinedInfo.Pointers, CombinedInfo.Types);10141}10142static void10143genMapInfo(MappableExprsHandler &MEHandler, CodeGenFunction &CGF,10144 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,10145 llvm::OpenMPIRBuilder &OMPBuilder,10146 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkippedVarSet =10147 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) {10148 10149 CodeGenModule &CGM = CGF.CGM;10150 // Map any list items in a map clause that were not captures because they10151 // weren't referenced within the construct.10152 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder, SkippedVarSet);10153 10154 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {10155 return emitMappingInformation(CGF, OMPBuilder, MapExpr);10156 };10157 if (CGM.getCodeGenOpts().getDebugInfo() !=10158 llvm::codegenoptions::NoDebugInfo) {10159 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());10160 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),10161 FillInfoMap);10162 }10163}10164 10165static void genMapInfo(const OMPExecutableDirective &D, CodeGenFunction &CGF,10166 const CapturedStmt &CS,10167 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,10168 llvm::OpenMPIRBuilder &OMPBuilder,10169 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {10170 // Get mappable expression information.10171 MappableExprsHandler MEHandler(D, CGF);10172 llvm::DenseSet<CanonicalDeclPtr<const Decl>> MappedVarSet;10173 10174 genMapInfoForCaptures(MEHandler, CGF, CS, CapturedVars, OMPBuilder,10175 MappedVarSet, CombinedInfo);10176 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder, MappedVarSet);10177}10178 10179template <typename ClauseTy>10180static void10181emitClauseForBareTargetDirective(CodeGenFunction &CGF,10182 const OMPExecutableDirective &D,10183 llvm::SmallVectorImpl<llvm::Value *> &Values) {10184 const auto *C = D.getSingleClause<ClauseTy>();10185 assert(!C->varlist_empty() &&10186 "ompx_bare requires explicit num_teams and thread_limit");10187 CodeGenFunction::RunCleanupsScope Scope(CGF);10188 for (auto *E : C->varlist()) {10189 llvm::Value *V = CGF.EmitScalarExpr(E);10190 Values.push_back(10191 CGF.Builder.CreateIntCast(V, CGF.Int32Ty, /*isSigned=*/true));10192 }10193}10194 10195static void emitTargetCallKernelLaunch(10196 CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,10197 const OMPExecutableDirective &D,10198 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, bool RequiresOuterTask,10199 const CapturedStmt &CS, bool OffloadingMandatory,10200 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,10201 llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo,10202 llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray,10203 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,10204 const OMPLoopDirective &D)>10205 SizeEmitter,10206 CodeGenFunction &CGF, CodeGenModule &CGM) {10207 llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->getOMPBuilder();10208 10209 // Fill up the arrays with all the captured variables.10210 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;10211 CGOpenMPRuntime::TargetDataInfo Info;10212 genMapInfo(D, CGF, CS, CapturedVars, OMPBuilder, CombinedInfo);10213 10214 emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,10215 /*IsNonContiguous=*/true, /*ForEndCall=*/false);10216 10217 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;10218 InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray,10219 CGF.VoidPtrTy, CGM.getPointerAlign());10220 InputInfo.PointersArray =10221 Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy, CGM.getPointerAlign());10222 InputInfo.SizesArray =10223 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());10224 InputInfo.MappersArray =10225 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());10226 MapTypesArray = Info.RTArgs.MapTypesArray;10227 MapNamesArray = Info.RTArgs.MapNamesArray;10228 10229 auto &&ThenGen = [&OMPRuntime, OutlinedFn, &D, &CapturedVars,10230 RequiresOuterTask, &CS, OffloadingMandatory, Device,10231 OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,10232 SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) {10233 bool IsReverseOffloading = Device.getInt() == OMPC_DEVICE_ancestor;10234 10235 if (IsReverseOffloading) {10236 // Reverse offloading is not supported, so just execute on the host.10237 // FIXME: This fallback solution is incorrect since it ignores the10238 // OMP_TARGET_OFFLOAD environment variable. Instead it would be better to10239 // assert here and ensure SEMA emits an error.10240 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,10241 RequiresOuterTask, CS, OffloadingMandatory, CGF);10242 return;10243 }10244 10245 bool HasNoWait = D.hasClausesOfKind<OMPNowaitClause>();10246 unsigned NumTargetItems = InputInfo.NumberOfTargetItems;10247 10248 llvm::Value *BasePointersArray =10249 InputInfo.BasePointersArray.emitRawPointer(CGF);10250 llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);10251 llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);10252 llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);10253 10254 auto &&EmitTargetCallFallbackCB =10255 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,10256 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)10257 -> llvm::OpenMPIRBuilder::InsertPointTy {10258 CGF.Builder.restoreIP(IP);10259 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,10260 RequiresOuterTask, CS, OffloadingMandatory, CGF);10261 return CGF.Builder.saveIP();10262 };10263 10264 bool IsBare = D.hasClausesOfKind<OMPXBareClause>();10265 SmallVector<llvm::Value *, 3> NumTeams;10266 SmallVector<llvm::Value *, 3> NumThreads;10267 if (IsBare) {10268 emitClauseForBareTargetDirective<OMPNumTeamsClause>(CGF, D, NumTeams);10269 emitClauseForBareTargetDirective<OMPThreadLimitClause>(CGF, D,10270 NumThreads);10271 } else {10272 NumTeams.push_back(OMPRuntime->emitNumTeamsForTargetDirective(CGF, D));10273 NumThreads.push_back(10274 OMPRuntime->emitNumThreadsForTargetDirective(CGF, D));10275 }10276 10277 llvm::Value *DeviceID = emitDeviceID(Device, CGF);10278 llvm::Value *RTLoc = OMPRuntime->emitUpdateLocation(CGF, D.getBeginLoc());10279 llvm::Value *NumIterations =10280 OMPRuntime->emitTargetNumIterationsCall(CGF, D, SizeEmitter);10281 auto [DynCGroupMem, DynCGroupMemFallback] = emitDynCGroupMem(D, CGF);10282 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(10283 CGF.AllocaInsertPt->getParent(), CGF.AllocaInsertPt->getIterator());10284 10285 llvm::OpenMPIRBuilder::TargetDataRTArgs RTArgs(10286 BasePointersArray, PointersArray, SizesArray, MapTypesArray,10287 nullptr /* MapTypesArrayEnd */, MappersArray, MapNamesArray);10288 10289 llvm::OpenMPIRBuilder::TargetKernelArgs Args(10290 NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads,10291 DynCGroupMem, HasNoWait, DynCGroupMemFallback);10292 10293 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =10294 cantFail(OMPRuntime->getOMPBuilder().emitKernelLaunch(10295 CGF.Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID,10296 RTLoc, AllocaIP));10297 CGF.Builder.restoreIP(AfterIP);10298 };10299 10300 if (RequiresOuterTask)10301 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);10302 else10303 OMPRuntime->emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);10304}10305 10306static void10307emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,10308 const OMPExecutableDirective &D,10309 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,10310 bool RequiresOuterTask, const CapturedStmt &CS,10311 bool OffloadingMandatory, CodeGenFunction &CGF) {10312 10313 // Notify that the host version must be executed.10314 auto &&ElseGen =10315 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,10316 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {10317 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,10318 RequiresOuterTask, CS, OffloadingMandatory, CGF);10319 };10320 10321 if (RequiresOuterTask) {10322 CodeGenFunction::OMPTargetDataInfo InputInfo;10323 CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo);10324 } else {10325 OMPRuntime->emitInlinedDirective(CGF, D.getDirectiveKind(), ElseGen);10326 }10327}10328 10329void CGOpenMPRuntime::emitTargetCall(10330 CodeGenFunction &CGF, const OMPExecutableDirective &D,10331 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,10332 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,10333 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,10334 const OMPLoopDirective &D)>10335 SizeEmitter) {10336 if (!CGF.HaveInsertPoint())10337 return;10338 10339 const bool OffloadingMandatory = !CGM.getLangOpts().OpenMPIsTargetDevice &&10340 CGM.getLangOpts().OpenMPOffloadMandatory;10341 10342 assert((OffloadingMandatory || OutlinedFn) && "Invalid outlined function!");10343 10344 const bool RequiresOuterTask =10345 D.hasClausesOfKind<OMPDependClause>() ||10346 D.hasClausesOfKind<OMPNowaitClause>() ||10347 D.hasClausesOfKind<OMPInReductionClause>() ||10348 (CGM.getLangOpts().OpenMP >= 51 &&10349 needsTaskBasedThreadLimit(D.getDirectiveKind()) &&10350 D.hasClausesOfKind<OMPThreadLimitClause>());10351 llvm::SmallVector<llvm::Value *, 16> CapturedVars;10352 const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);10353 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF,10354 PrePostActionTy &) {10355 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);10356 };10357 emitInlinedDirective(CGF, OMPD_unknown, ArgsCodegen);10358 10359 CodeGenFunction::OMPTargetDataInfo InputInfo;10360 llvm::Value *MapTypesArray = nullptr;10361 llvm::Value *MapNamesArray = nullptr;10362 10363 auto &&TargetThenGen = [this, OutlinedFn, &D, &CapturedVars,10364 RequiresOuterTask, &CS, OffloadingMandatory, Device,10365 OutlinedFnID, &InputInfo, &MapTypesArray,10366 &MapNamesArray, SizeEmitter](CodeGenFunction &CGF,10367 PrePostActionTy &) {10368 emitTargetCallKernelLaunch(this, OutlinedFn, D, CapturedVars,10369 RequiresOuterTask, CS, OffloadingMandatory,10370 Device, OutlinedFnID, InputInfo, MapTypesArray,10371 MapNamesArray, SizeEmitter, CGF, CGM);10372 };10373 10374 auto &&TargetElseGen =10375 [this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,10376 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {10377 emitTargetCallElse(this, OutlinedFn, D, CapturedVars, RequiresOuterTask,10378 CS, OffloadingMandatory, CGF);10379 };10380 10381 // If we have a target function ID it means that we need to support10382 // offloading, otherwise, just execute on the host. We need to execute on host10383 // regardless of the conditional in the if clause if, e.g., the user do not10384 // specify target triples.10385 if (OutlinedFnID) {10386 if (IfCond) {10387 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen);10388 } else {10389 RegionCodeGenTy ThenRCG(TargetThenGen);10390 ThenRCG(CGF);10391 }10392 } else {10393 RegionCodeGenTy ElseRCG(TargetElseGen);10394 ElseRCG(CGF);10395 }10396}10397 10398void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,10399 StringRef ParentName) {10400 if (!S)10401 return;10402 10403 // Codegen OMP target directives that offload compute to the device.10404 bool RequiresDeviceCodegen =10405 isa<OMPExecutableDirective>(S) &&10406 isOpenMPTargetExecutionDirective(10407 cast<OMPExecutableDirective>(S)->getDirectiveKind());10408 10409 if (RequiresDeviceCodegen) {10410 const auto &E = *cast<OMPExecutableDirective>(S);10411 10412 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(10413 CGM, OMPBuilder, E.getBeginLoc(), ParentName);10414 10415 // Is this a target region that should not be emitted as an entry point? If10416 // so just signal we are done with this target region.10417 if (!OMPBuilder.OffloadInfoManager.hasTargetRegionEntryInfo(EntryInfo))10418 return;10419 10420 switch (E.getDirectiveKind()) {10421 case OMPD_target:10422 CodeGenFunction::EmitOMPTargetDeviceFunction(CGM, ParentName,10423 cast<OMPTargetDirective>(E));10424 break;10425 case OMPD_target_parallel:10426 CodeGenFunction::EmitOMPTargetParallelDeviceFunction(10427 CGM, ParentName, cast<OMPTargetParallelDirective>(E));10428 break;10429 case OMPD_target_teams:10430 CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(10431 CGM, ParentName, cast<OMPTargetTeamsDirective>(E));10432 break;10433 case OMPD_target_teams_distribute:10434 CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(10435 CGM, ParentName, cast<OMPTargetTeamsDistributeDirective>(E));10436 break;10437 case OMPD_target_teams_distribute_simd:10438 CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(10439 CGM, ParentName, cast<OMPTargetTeamsDistributeSimdDirective>(E));10440 break;10441 case OMPD_target_parallel_for:10442 CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(10443 CGM, ParentName, cast<OMPTargetParallelForDirective>(E));10444 break;10445 case OMPD_target_parallel_for_simd:10446 CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(10447 CGM, ParentName, cast<OMPTargetParallelForSimdDirective>(E));10448 break;10449 case OMPD_target_simd:10450 CodeGenFunction::EmitOMPTargetSimdDeviceFunction(10451 CGM, ParentName, cast<OMPTargetSimdDirective>(E));10452 break;10453 case OMPD_target_teams_distribute_parallel_for:10454 CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(10455 CGM, ParentName,10456 cast<OMPTargetTeamsDistributeParallelForDirective>(E));10457 break;10458 case OMPD_target_teams_distribute_parallel_for_simd:10459 CodeGenFunction::10460 EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(10461 CGM, ParentName,10462 cast<OMPTargetTeamsDistributeParallelForSimdDirective>(E));10463 break;10464 case OMPD_target_teams_loop:10465 CodeGenFunction::EmitOMPTargetTeamsGenericLoopDeviceFunction(10466 CGM, ParentName, cast<OMPTargetTeamsGenericLoopDirective>(E));10467 break;10468 case OMPD_target_parallel_loop:10469 CodeGenFunction::EmitOMPTargetParallelGenericLoopDeviceFunction(10470 CGM, ParentName, cast<OMPTargetParallelGenericLoopDirective>(E));10471 break;10472 case OMPD_parallel:10473 case OMPD_for:10474 case OMPD_parallel_for:10475 case OMPD_parallel_master:10476 case OMPD_parallel_sections:10477 case OMPD_for_simd:10478 case OMPD_parallel_for_simd:10479 case OMPD_cancel:10480 case OMPD_cancellation_point:10481 case OMPD_ordered:10482 case OMPD_threadprivate:10483 case OMPD_allocate:10484 case OMPD_task:10485 case OMPD_simd:10486 case OMPD_tile:10487 case OMPD_unroll:10488 case OMPD_sections:10489 case OMPD_section:10490 case OMPD_single:10491 case OMPD_master:10492 case OMPD_critical:10493 case OMPD_taskyield:10494 case OMPD_barrier:10495 case OMPD_taskwait:10496 case OMPD_taskgroup:10497 case OMPD_atomic:10498 case OMPD_flush:10499 case OMPD_depobj:10500 case OMPD_scan:10501 case OMPD_teams:10502 case OMPD_target_data:10503 case OMPD_target_exit_data:10504 case OMPD_target_enter_data:10505 case OMPD_distribute:10506 case OMPD_distribute_simd:10507 case OMPD_distribute_parallel_for:10508 case OMPD_distribute_parallel_for_simd:10509 case OMPD_teams_distribute:10510 case OMPD_teams_distribute_simd:10511 case OMPD_teams_distribute_parallel_for:10512 case OMPD_teams_distribute_parallel_for_simd:10513 case OMPD_target_update:10514 case OMPD_declare_simd:10515 case OMPD_declare_variant:10516 case OMPD_begin_declare_variant:10517 case OMPD_end_declare_variant:10518 case OMPD_declare_target:10519 case OMPD_end_declare_target:10520 case OMPD_declare_reduction:10521 case OMPD_declare_mapper:10522 case OMPD_taskloop:10523 case OMPD_taskloop_simd:10524 case OMPD_master_taskloop:10525 case OMPD_master_taskloop_simd:10526 case OMPD_parallel_master_taskloop:10527 case OMPD_parallel_master_taskloop_simd:10528 case OMPD_requires:10529 case OMPD_metadirective:10530 case OMPD_unknown:10531 default:10532 llvm_unreachable("Unknown target directive for OpenMP device codegen.");10533 }10534 return;10535 }10536 10537 if (const auto *E = dyn_cast<OMPExecutableDirective>(S)) {10538 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())10539 return;10540 10541 scanForTargetRegionsFunctions(E->getRawStmt(), ParentName);10542 return;10543 }10544 10545 // If this is a lambda function, look into its body.10546 if (const auto *L = dyn_cast<LambdaExpr>(S))10547 S = L->getBody();10548 10549 // Keep looking for target regions recursively.10550 for (const Stmt *II : S->children())10551 scanForTargetRegionsFunctions(II, ParentName);10552}10553 10554static bool isAssumedToBeNotEmitted(const ValueDecl *VD, bool IsDevice) {10555 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =10556 OMPDeclareTargetDeclAttr::getDeviceType(VD);10557 if (!DevTy)10558 return false;10559 // Do not emit device_type(nohost) functions for the host.10560 if (!IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)10561 return true;10562 // Do not emit device_type(host) functions for the device.10563 if (IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_Host)10564 return true;10565 return false;10566}10567 10568bool CGOpenMPRuntime::emitTargetFunctions(GlobalDecl GD) {10569 // If emitting code for the host, we do not process FD here. Instead we do10570 // the normal code generation.10571 if (!CGM.getLangOpts().OpenMPIsTargetDevice) {10572 if (const auto *FD = dyn_cast<FunctionDecl>(GD.getDecl()))10573 if (isAssumedToBeNotEmitted(cast<ValueDecl>(FD),10574 CGM.getLangOpts().OpenMPIsTargetDevice))10575 return true;10576 return false;10577 }10578 10579 const ValueDecl *VD = cast<ValueDecl>(GD.getDecl());10580 // Try to detect target regions in the function.10581 if (const auto *FD = dyn_cast<FunctionDecl>(VD)) {10582 StringRef Name = CGM.getMangledName(GD);10583 scanForTargetRegionsFunctions(FD->getBody(), Name);10584 if (isAssumedToBeNotEmitted(cast<ValueDecl>(FD),10585 CGM.getLangOpts().OpenMPIsTargetDevice))10586 return true;10587 }10588 10589 // Do not to emit function if it is not marked as declare target.10590 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&10591 AlreadyEmittedTargetDecls.count(VD) == 0;10592}10593 10594bool CGOpenMPRuntime::emitTargetGlobalVariable(GlobalDecl GD) {10595 if (isAssumedToBeNotEmitted(cast<ValueDecl>(GD.getDecl()),10596 CGM.getLangOpts().OpenMPIsTargetDevice))10597 return true;10598 10599 if (!CGM.getLangOpts().OpenMPIsTargetDevice)10600 return false;10601 10602 // Check if there are Ctors/Dtors in this declaration and look for target10603 // regions in it. We use the complete variant to produce the kernel name10604 // mangling.10605 QualType RDTy = cast<VarDecl>(GD.getDecl())->getType();10606 if (const auto *RD = RDTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) {10607 for (const CXXConstructorDecl *Ctor : RD->ctors()) {10608 StringRef ParentName =10609 CGM.getMangledName(GlobalDecl(Ctor, Ctor_Complete));10610 scanForTargetRegionsFunctions(Ctor->getBody(), ParentName);10611 }10612 if (const CXXDestructorDecl *Dtor = RD->getDestructor()) {10613 StringRef ParentName =10614 CGM.getMangledName(GlobalDecl(Dtor, Dtor_Complete));10615 scanForTargetRegionsFunctions(Dtor->getBody(), ParentName);10616 }10617 }10618 10619 // Do not to emit variable if it is not marked as declare target.10620 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =10621 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(10622 cast<VarDecl>(GD.getDecl()));10623 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||10624 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||10625 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&10626 HasRequiresUnifiedSharedMemory)) {10627 DeferredGlobalVariables.insert(cast<VarDecl>(GD.getDecl()));10628 return true;10629 }10630 return false;10631}10632 10633void CGOpenMPRuntime::registerTargetGlobalVariable(const VarDecl *VD,10634 llvm::Constant *Addr) {10635 if (CGM.getLangOpts().OMPTargetTriples.empty() &&10636 !CGM.getLangOpts().OpenMPIsTargetDevice)10637 return;10638 10639 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =10640 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);10641 10642 // If this is an 'extern' declaration we defer to the canonical definition and10643 // do not emit an offloading entry.10644 if (Res && *Res != OMPDeclareTargetDeclAttr::MT_Link &&10645 VD->hasExternalStorage())10646 return;10647 10648 if (!Res) {10649 if (CGM.getLangOpts().OpenMPIsTargetDevice) {10650 // Register non-target variables being emitted in device code (debug info10651 // may cause this).10652 StringRef VarName = CGM.getMangledName(VD);10653 EmittedNonTargetVariables.try_emplace(VarName, Addr);10654 }10655 return;10656 }10657 10658 auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(VD); };10659 auto LinkageForVariable = [&VD, this]() {10660 return CGM.getLLVMLinkageVarDefinition(VD);10661 };10662 10663 std::vector<llvm::GlobalVariable *> GeneratedRefs;10664 OMPBuilder.registerTargetGlobalVariable(10665 convertCaptureClause(VD), convertDeviceClause(VD),10666 VD->hasDefinition(CGM.getContext()) == VarDecl::DeclarationOnly,10667 VD->isExternallyVisible(),10668 getEntryInfoFromPresumedLoc(CGM, OMPBuilder,10669 VD->getCanonicalDecl()->getBeginLoc()),10670 CGM.getMangledName(VD), GeneratedRefs, CGM.getLangOpts().OpenMPSimd,10671 CGM.getLangOpts().OMPTargetTriples, AddrOfGlobal, LinkageForVariable,10672 CGM.getTypes().ConvertTypeForMem(10673 CGM.getContext().getPointerType(VD->getType())),10674 Addr);10675 10676 for (auto *ref : GeneratedRefs)10677 CGM.addCompilerUsedGlobal(ref);10678}10679 10680bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) {10681 if (isa<FunctionDecl>(GD.getDecl()) ||10682 isa<OMPDeclareReductionDecl>(GD.getDecl()))10683 return emitTargetFunctions(GD);10684 10685 return emitTargetGlobalVariable(GD);10686}10687 10688void CGOpenMPRuntime::emitDeferredTargetDecls() const {10689 for (const VarDecl *VD : DeferredGlobalVariables) {10690 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =10691 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);10692 if (!Res)10693 continue;10694 if ((*Res == OMPDeclareTargetDeclAttr::MT_To ||10695 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&10696 !HasRequiresUnifiedSharedMemory) {10697 CGM.EmitGlobal(VD);10698 } else {10699 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||10700 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||10701 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&10702 HasRequiresUnifiedSharedMemory)) &&10703 "Expected link clause or to clause with unified memory.");10704 (void)CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);10705 }10706 }10707}10708 10709void CGOpenMPRuntime::adjustTargetSpecificDataForLambdas(10710 CodeGenFunction &CGF, const OMPExecutableDirective &D) const {10711 assert(isOpenMPTargetExecutionDirective(D.getDirectiveKind()) &&10712 " Expected target-based directive.");10713}10714 10715void CGOpenMPRuntime::processRequiresDirective(const OMPRequiresDecl *D) {10716 for (const OMPClause *Clause : D->clauselists()) {10717 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {10718 HasRequiresUnifiedSharedMemory = true;10719 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true);10720 } else if (const auto *AC =10721 dyn_cast<OMPAtomicDefaultMemOrderClause>(Clause)) {10722 switch (AC->getAtomicDefaultMemOrderKind()) {10723 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel:10724 RequiresAtomicOrdering = llvm::AtomicOrdering::AcquireRelease;10725 break;10726 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst:10727 RequiresAtomicOrdering = llvm::AtomicOrdering::SequentiallyConsistent;10728 break;10729 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed:10730 RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;10731 break;10732 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown:10733 break;10734 }10735 }10736 }10737}10738 10739llvm::AtomicOrdering CGOpenMPRuntime::getDefaultMemoryOrdering() const {10740 return RequiresAtomicOrdering;10741}10742 10743bool CGOpenMPRuntime::hasAllocateAttributeForGlobalVar(const VarDecl *VD,10744 LangAS &AS) {10745 if (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())10746 return false;10747 const auto *A = VD->getAttr<OMPAllocateDeclAttr>();10748 switch(A->getAllocatorType()) {10749 case OMPAllocateDeclAttr::OMPNullMemAlloc:10750 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:10751 // Not supported, fallback to the default mem space.10752 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:10753 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:10754 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:10755 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:10756 case OMPAllocateDeclAttr::OMPThreadMemAlloc:10757 case OMPAllocateDeclAttr::OMPConstMemAlloc:10758 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:10759 AS = LangAS::Default;10760 return true;10761 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:10762 llvm_unreachable("Expected predefined allocator for the variables with the "10763 "static storage.");10764 }10765 return false;10766}10767 10768bool CGOpenMPRuntime::hasRequiresUnifiedSharedMemory() const {10769 return HasRequiresUnifiedSharedMemory;10770}10771 10772CGOpenMPRuntime::DisableAutoDeclareTargetRAII::DisableAutoDeclareTargetRAII(10773 CodeGenModule &CGM)10774 : CGM(CGM) {10775 if (CGM.getLangOpts().OpenMPIsTargetDevice) {10776 SavedShouldMarkAsGlobal = CGM.getOpenMPRuntime().ShouldMarkAsGlobal;10777 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = false;10778 }10779}10780 10781CGOpenMPRuntime::DisableAutoDeclareTargetRAII::~DisableAutoDeclareTargetRAII() {10782 if (CGM.getLangOpts().OpenMPIsTargetDevice)10783 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = SavedShouldMarkAsGlobal;10784}10785 10786bool CGOpenMPRuntime::markAsGlobalTarget(GlobalDecl GD) {10787 if (!CGM.getLangOpts().OpenMPIsTargetDevice || !ShouldMarkAsGlobal)10788 return true;10789 10790 const auto *D = cast<FunctionDecl>(GD.getDecl());10791 // Do not to emit function if it is marked as declare target as it was already10792 // emitted.10793 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(D)) {10794 if (D->hasBody() && AlreadyEmittedTargetDecls.count(D) == 0) {10795 if (auto *F = dyn_cast_or_null<llvm::Function>(10796 CGM.GetGlobalValue(CGM.getMangledName(GD))))10797 return !F->isDeclaration();10798 return false;10799 }10800 return true;10801 }10802 10803 return !AlreadyEmittedTargetDecls.insert(D).second;10804}10805 10806void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF,10807 const OMPExecutableDirective &D,10808 SourceLocation Loc,10809 llvm::Function *OutlinedFn,10810 ArrayRef<llvm::Value *> CapturedVars) {10811 if (!CGF.HaveInsertPoint())10812 return;10813 10814 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);10815 CodeGenFunction::RunCleanupsScope Scope(CGF);10816 10817 // Build call __kmpc_fork_teams(loc, n, microtask, var1, .., varn);10818 llvm::Value *Args[] = {10819 RTLoc,10820 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars10821 OutlinedFn};10822 llvm::SmallVector<llvm::Value *, 16> RealArgs;10823 RealArgs.append(std::begin(Args), std::end(Args));10824 RealArgs.append(CapturedVars.begin(), CapturedVars.end());10825 10826 llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(10827 CGM.getModule(), OMPRTL___kmpc_fork_teams);10828 CGF.EmitRuntimeCall(RTLFn, RealArgs);10829}10830 10831void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF,10832 const Expr *NumTeams,10833 const Expr *ThreadLimit,10834 SourceLocation Loc) {10835 if (!CGF.HaveInsertPoint())10836 return;10837 10838 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);10839 10840 llvm::Value *NumTeamsVal =10841 NumTeams10842 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams),10843 CGF.CGM.Int32Ty, /* isSigned = */ true)10844 : CGF.Builder.getInt32(0);10845 10846 llvm::Value *ThreadLimitVal =10847 ThreadLimit10848 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit),10849 CGF.CGM.Int32Ty, /* isSigned = */ true)10850 : CGF.Builder.getInt32(0);10851 10852 // Build call __kmpc_push_num_teamss(&loc, global_tid, num_teams, thread_limit)10853 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,10854 ThreadLimitVal};10855 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(10856 CGM.getModule(), OMPRTL___kmpc_push_num_teams),10857 PushNumTeamsArgs);10858}10859 10860void CGOpenMPRuntime::emitThreadLimitClause(CodeGenFunction &CGF,10861 const Expr *ThreadLimit,10862 SourceLocation Loc) {10863 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);10864 llvm::Value *ThreadLimitVal =10865 ThreadLimit10866 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit),10867 CGF.CGM.Int32Ty, /* isSigned = */ true)10868 : CGF.Builder.getInt32(0);10869 10870 // Build call __kmpc_set_thread_limit(&loc, global_tid, thread_limit)10871 llvm::Value *ThreadLimitArgs[] = {RTLoc, getThreadID(CGF, Loc),10872 ThreadLimitVal};10873 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(10874 CGM.getModule(), OMPRTL___kmpc_set_thread_limit),10875 ThreadLimitArgs);10876}10877 10878void CGOpenMPRuntime::emitTargetDataCalls(10879 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,10880 const Expr *Device, const RegionCodeGenTy &CodeGen,10881 CGOpenMPRuntime::TargetDataInfo &Info) {10882 if (!CGF.HaveInsertPoint())10883 return;10884 10885 // Action used to replace the default codegen action and turn privatization10886 // off.10887 PrePostActionTy NoPrivAction;10888 10889 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;10890 10891 llvm::Value *IfCondVal = nullptr;10892 if (IfCond)10893 IfCondVal = CGF.EvaluateExprAsBool(IfCond);10894 10895 // Emit device ID if any.10896 llvm::Value *DeviceID = nullptr;10897 if (Device) {10898 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),10899 CGF.Int64Ty, /*isSigned=*/true);10900 } else {10901 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);10902 }10903 10904 // Fill up the arrays with all the mapped variables.10905 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;10906 auto GenMapInfoCB =10907 [&](InsertPointTy CodeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {10908 CGF.Builder.restoreIP(CodeGenIP);10909 // Get map clause information.10910 MappableExprsHandler MEHandler(D, CGF);10911 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder);10912 10913 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {10914 return emitMappingInformation(CGF, OMPBuilder, MapExpr);10915 };10916 if (CGM.getCodeGenOpts().getDebugInfo() !=10917 llvm::codegenoptions::NoDebugInfo) {10918 CombinedInfo.Names.resize(CombinedInfo.Exprs.size());10919 llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),10920 FillInfoMap);10921 }10922 10923 return CombinedInfo;10924 };10925 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;10926 auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {10927 CGF.Builder.restoreIP(CodeGenIP);10928 switch (BodyGenType) {10929 case BodyGenTy::Priv:10930 if (!Info.CaptureDeviceAddrMap.empty())10931 CodeGen(CGF);10932 break;10933 case BodyGenTy::DupNoPriv:10934 if (!Info.CaptureDeviceAddrMap.empty()) {10935 CodeGen.setAction(NoPrivAction);10936 CodeGen(CGF);10937 }10938 break;10939 case BodyGenTy::NoPriv:10940 if (Info.CaptureDeviceAddrMap.empty()) {10941 CodeGen.setAction(NoPrivAction);10942 CodeGen(CGF);10943 }10944 break;10945 }10946 return InsertPointTy(CGF.Builder.GetInsertBlock(),10947 CGF.Builder.GetInsertPoint());10948 };10949 10950 auto DeviceAddrCB = [&](unsigned int I, llvm::Value *NewDecl) {10951 if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {10952 Info.CaptureDeviceAddrMap.try_emplace(DevVD, NewDecl);10953 }10954 };10955 10956 auto CustomMapperCB = [&](unsigned int I) {10957 llvm::Function *MFunc = nullptr;10958 if (CombinedInfo.Mappers[I]) {10959 Info.HasMapper = true;10960 MFunc = CGF.CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(10961 cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));10962 }10963 return MFunc;10964 };10965 10966 // Source location for the ident struct10967 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc());10968 10969 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(),10970 CGF.AllocaInsertPt->getIterator());10971 InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(),10972 CGF.Builder.GetInsertPoint());10973 llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CodeGenIP);10974 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =10975 cantFail(OMPBuilder.createTargetData(10976 OmpLoc, AllocaIP, CodeGenIP, DeviceID, IfCondVal, Info, GenMapInfoCB,10977 CustomMapperCB,10978 /*MapperFunc=*/nullptr, BodyCB, DeviceAddrCB, RTLoc));10979 CGF.Builder.restoreIP(AfterIP);10980}10981 10982void CGOpenMPRuntime::emitTargetDataStandAloneCall(10983 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,10984 const Expr *Device) {10985 if (!CGF.HaveInsertPoint())10986 return;10987 10988 assert((isa<OMPTargetEnterDataDirective>(D) ||10989 isa<OMPTargetExitDataDirective>(D) ||10990 isa<OMPTargetUpdateDirective>(D)) &&10991 "Expecting either target enter, exit data, or update directives.");10992 10993 CodeGenFunction::OMPTargetDataInfo InputInfo;10994 llvm::Value *MapTypesArray = nullptr;10995 llvm::Value *MapNamesArray = nullptr;10996 // Generate the code for the opening of the data environment.10997 auto &&ThenGen = [this, &D, Device, &InputInfo, &MapTypesArray,10998 &MapNamesArray](CodeGenFunction &CGF, PrePostActionTy &) {10999 // Emit device ID if any.11000 llvm::Value *DeviceID = nullptr;11001 if (Device) {11002 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),11003 CGF.Int64Ty, /*isSigned=*/true);11004 } else {11005 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);11006 }11007 11008 // Emit the number of elements in the offloading arrays.11009 llvm::Constant *PointerNum =11010 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);11011 11012 // Source location for the ident struct11013 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc());11014 11015 SmallVector<llvm::Value *, 13> OffloadingArgs(11016 {RTLoc, DeviceID, PointerNum,11017 InputInfo.BasePointersArray.emitRawPointer(CGF),11018 InputInfo.PointersArray.emitRawPointer(CGF),11019 InputInfo.SizesArray.emitRawPointer(CGF), MapTypesArray, MapNamesArray,11020 InputInfo.MappersArray.emitRawPointer(CGF)});11021 11022 // Select the right runtime function call for each standalone11023 // directive.11024 const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();11025 RuntimeFunction RTLFn;11026 switch (D.getDirectiveKind()) {11027 case OMPD_target_enter_data:11028 RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper11029 : OMPRTL___tgt_target_data_begin_mapper;11030 break;11031 case OMPD_target_exit_data:11032 RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper11033 : OMPRTL___tgt_target_data_end_mapper;11034 break;11035 case OMPD_target_update:11036 RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper11037 : OMPRTL___tgt_target_data_update_mapper;11038 break;11039 case OMPD_parallel:11040 case OMPD_for:11041 case OMPD_parallel_for:11042 case OMPD_parallel_master:11043 case OMPD_parallel_sections:11044 case OMPD_for_simd:11045 case OMPD_parallel_for_simd:11046 case OMPD_cancel:11047 case OMPD_cancellation_point:11048 case OMPD_ordered:11049 case OMPD_threadprivate:11050 case OMPD_allocate:11051 case OMPD_task:11052 case OMPD_simd:11053 case OMPD_tile:11054 case OMPD_unroll:11055 case OMPD_sections:11056 case OMPD_section:11057 case OMPD_single:11058 case OMPD_master:11059 case OMPD_critical:11060 case OMPD_taskyield:11061 case OMPD_barrier:11062 case OMPD_taskwait:11063 case OMPD_taskgroup:11064 case OMPD_atomic:11065 case OMPD_flush:11066 case OMPD_depobj:11067 case OMPD_scan:11068 case OMPD_teams:11069 case OMPD_target_data:11070 case OMPD_distribute:11071 case OMPD_distribute_simd:11072 case OMPD_distribute_parallel_for:11073 case OMPD_distribute_parallel_for_simd:11074 case OMPD_teams_distribute:11075 case OMPD_teams_distribute_simd:11076 case OMPD_teams_distribute_parallel_for:11077 case OMPD_teams_distribute_parallel_for_simd:11078 case OMPD_declare_simd:11079 case OMPD_declare_variant:11080 case OMPD_begin_declare_variant:11081 case OMPD_end_declare_variant:11082 case OMPD_declare_target:11083 case OMPD_end_declare_target:11084 case OMPD_declare_reduction:11085 case OMPD_declare_mapper:11086 case OMPD_taskloop:11087 case OMPD_taskloop_simd:11088 case OMPD_master_taskloop:11089 case OMPD_master_taskloop_simd:11090 case OMPD_parallel_master_taskloop:11091 case OMPD_parallel_master_taskloop_simd:11092 case OMPD_target:11093 case OMPD_target_simd:11094 case OMPD_target_teams_distribute:11095 case OMPD_target_teams_distribute_simd:11096 case OMPD_target_teams_distribute_parallel_for:11097 case OMPD_target_teams_distribute_parallel_for_simd:11098 case OMPD_target_teams:11099 case OMPD_target_parallel:11100 case OMPD_target_parallel_for:11101 case OMPD_target_parallel_for_simd:11102 case OMPD_requires:11103 case OMPD_metadirective:11104 case OMPD_unknown:11105 default:11106 llvm_unreachable("Unexpected standalone target data directive.");11107 break;11108 }11109 if (HasNowait) {11110 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.Int32Ty));11111 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.VoidPtrTy));11112 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.Int32Ty));11113 OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.VoidPtrTy));11114 }11115 CGF.EmitRuntimeCall(11116 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), RTLFn),11117 OffloadingArgs);11118 };11119 11120 auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray,11121 &MapNamesArray](CodeGenFunction &CGF,11122 PrePostActionTy &) {11123 // Fill up the arrays with all the mapped variables.11124 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;11125 CGOpenMPRuntime::TargetDataInfo Info;11126 MappableExprsHandler MEHandler(D, CGF);11127 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder);11128 emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,11129 /*IsNonContiguous=*/true, /*ForEndCall=*/false);11130 11131 bool RequiresOuterTask = D.hasClausesOfKind<OMPDependClause>() ||11132 D.hasClausesOfKind<OMPNowaitClause>();11133 11134 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;11135 InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray,11136 CGF.VoidPtrTy, CGM.getPointerAlign());11137 InputInfo.PointersArray = Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy,11138 CGM.getPointerAlign());11139 InputInfo.SizesArray =11140 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());11141 InputInfo.MappersArray =11142 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());11143 MapTypesArray = Info.RTArgs.MapTypesArray;11144 MapNamesArray = Info.RTArgs.MapNamesArray;11145 if (RequiresOuterTask)11146 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);11147 else11148 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);11149 };11150 11151 if (IfCond) {11152 emitIfClause(CGF, IfCond, TargetThenGen,11153 [](CodeGenFunction &CGF, PrePostActionTy &) {});11154 } else {11155 RegionCodeGenTy ThenRCG(TargetThenGen);11156 ThenRCG(CGF);11157 }11158}11159 11160namespace {11161 /// Kind of parameter in a function with 'declare simd' directive.11162enum ParamKindTy {11163 Linear,11164 LinearRef,11165 LinearUVal,11166 LinearVal,11167 Uniform,11168 Vector,11169};11170/// Attribute set of the parameter.11171struct ParamAttrTy {11172 ParamKindTy Kind = Vector;11173 llvm::APSInt StrideOrArg;11174 llvm::APSInt Alignment;11175 bool HasVarStride = false;11176};11177} // namespace11178 11179static unsigned evaluateCDTSize(const FunctionDecl *FD,11180 ArrayRef<ParamAttrTy> ParamAttrs) {11181 // Every vector variant of a SIMD-enabled function has a vector length (VLEN).11182 // If OpenMP clause "simdlen" is used, the VLEN is the value of the argument11183 // of that clause. The VLEN value must be power of 2.11184 // In other case the notion of the function`s "characteristic data type" (CDT)11185 // is used to compute the vector length.11186 // CDT is defined in the following order:11187 // a) For non-void function, the CDT is the return type.11188 // b) If the function has any non-uniform, non-linear parameters, then the11189 // CDT is the type of the first such parameter.11190 // c) If the CDT determined by a) or b) above is struct, union, or class11191 // type which is pass-by-value (except for the type that maps to the11192 // built-in complex data type), the characteristic data type is int.11193 // d) If none of the above three cases is applicable, the CDT is int.11194 // The VLEN is then determined based on the CDT and the size of vector11195 // register of that ISA for which current vector version is generated. The11196 // VLEN is computed using the formula below:11197 // VLEN = sizeof(vector_register) / sizeof(CDT),11198 // where vector register size specified in section 3.2.1 Registers and the11199 // Stack Frame of original AMD64 ABI document.11200 QualType RetType = FD->getReturnType();11201 if (RetType.isNull())11202 return 0;11203 ASTContext &C = FD->getASTContext();11204 QualType CDT;11205 if (!RetType.isNull() && !RetType->isVoidType()) {11206 CDT = RetType;11207 } else {11208 unsigned Offset = 0;11209 if (const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {11210 if (ParamAttrs[Offset].Kind == Vector)11211 CDT = C.getPointerType(C.getCanonicalTagType(MD->getParent()));11212 ++Offset;11213 }11214 if (CDT.isNull()) {11215 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {11216 if (ParamAttrs[I + Offset].Kind == Vector) {11217 CDT = FD->getParamDecl(I)->getType();11218 break;11219 }11220 }11221 }11222 }11223 if (CDT.isNull())11224 CDT = C.IntTy;11225 CDT = CDT->getCanonicalTypeUnqualified();11226 if (CDT->isRecordType() || CDT->isUnionType())11227 CDT = C.IntTy;11228 return C.getTypeSize(CDT);11229}11230 11231/// Mangle the parameter part of the vector function name according to11232/// their OpenMP classification. The mangling function is defined in11233/// section 4.5 of the AAVFABI(2021Q1).11234static std::string mangleVectorParameters(ArrayRef<ParamAttrTy> ParamAttrs) {11235 SmallString<256> Buffer;11236 llvm::raw_svector_ostream Out(Buffer);11237 for (const auto &ParamAttr : ParamAttrs) {11238 switch (ParamAttr.Kind) {11239 case Linear:11240 Out << 'l';11241 break;11242 case LinearRef:11243 Out << 'R';11244 break;11245 case LinearUVal:11246 Out << 'U';11247 break;11248 case LinearVal:11249 Out << 'L';11250 break;11251 case Uniform:11252 Out << 'u';11253 break;11254 case Vector:11255 Out << 'v';11256 break;11257 }11258 if (ParamAttr.HasVarStride)11259 Out << "s" << ParamAttr.StrideOrArg;11260 else if (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef ||11261 ParamAttr.Kind == LinearUVal || ParamAttr.Kind == LinearVal) {11262 // Don't print the step value if it is not present or if it is11263 // equal to 1.11264 if (ParamAttr.StrideOrArg < 0)11265 Out << 'n' << -ParamAttr.StrideOrArg;11266 else if (ParamAttr.StrideOrArg != 1)11267 Out << ParamAttr.StrideOrArg;11268 }11269 11270 if (!!ParamAttr.Alignment)11271 Out << 'a' << ParamAttr.Alignment;11272 }11273 11274 return std::string(Out.str());11275}11276 11277static void11278emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn,11279 const llvm::APSInt &VLENVal,11280 ArrayRef<ParamAttrTy> ParamAttrs,11281 OMPDeclareSimdDeclAttr::BranchStateTy State) {11282 struct ISADataTy {11283 char ISA;11284 unsigned VecRegSize;11285 };11286 ISADataTy ISAData[] = {11287 {11288 'b', 12811289 }, // SSE11290 {11291 'c', 25611292 }, // AVX11293 {11294 'd', 25611295 }, // AVX211296 {11297 'e', 51211298 }, // AVX51211299 };11300 llvm::SmallVector<char, 2> Masked;11301 switch (State) {11302 case OMPDeclareSimdDeclAttr::BS_Undefined:11303 Masked.push_back('N');11304 Masked.push_back('M');11305 break;11306 case OMPDeclareSimdDeclAttr::BS_Notinbranch:11307 Masked.push_back('N');11308 break;11309 case OMPDeclareSimdDeclAttr::BS_Inbranch:11310 Masked.push_back('M');11311 break;11312 }11313 for (char Mask : Masked) {11314 for (const ISADataTy &Data : ISAData) {11315 SmallString<256> Buffer;11316 llvm::raw_svector_ostream Out(Buffer);11317 Out << "_ZGV" << Data.ISA << Mask;11318 if (!VLENVal) {11319 unsigned NumElts = evaluateCDTSize(FD, ParamAttrs);11320 assert(NumElts && "Non-zero simdlen/cdtsize expected");11321 Out << llvm::APSInt::getUnsigned(Data.VecRegSize / NumElts);11322 } else {11323 Out << VLENVal;11324 }11325 Out << mangleVectorParameters(ParamAttrs);11326 Out << '_' << Fn->getName();11327 Fn->addFnAttr(Out.str());11328 }11329 }11330}11331 11332// This are the Functions that are needed to mangle the name of the11333// vector functions generated by the compiler, according to the rules11334// defined in the "Vector Function ABI specifications for AArch64",11335// available at11336// https://developer.arm.com/products/software-development-tools/hpc/arm-compiler-for-hpc/vector-function-abi.11337 11338/// Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1).11339static bool getAArch64MTV(QualType QT, ParamKindTy Kind) {11340 QT = QT.getCanonicalType();11341 11342 if (QT->isVoidType())11343 return false;11344 11345 if (Kind == ParamKindTy::Uniform)11346 return false;11347 11348 if (Kind == ParamKindTy::LinearUVal || Kind == ParamKindTy::LinearRef)11349 return false;11350 11351 if ((Kind == ParamKindTy::Linear || Kind == ParamKindTy::LinearVal) &&11352 !QT->isReferenceType())11353 return false;11354 11355 return true;11356}11357 11358/// Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.11359static bool getAArch64PBV(QualType QT, ASTContext &C) {11360 QT = QT.getCanonicalType();11361 unsigned Size = C.getTypeSize(QT);11362 11363 // Only scalars and complex within 16 bytes wide set PVB to true.11364 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)11365 return false;11366 11367 if (QT->isFloatingType())11368 return true;11369 11370 if (QT->isIntegerType())11371 return true;11372 11373 if (QT->isPointerType())11374 return true;11375 11376 // TODO: Add support for complex types (section 3.1.2, item 2).11377 11378 return false;11379}11380 11381/// Computes the lane size (LS) of a return type or of an input parameter,11382/// as defined by `LS(P)` in 3.2.1 of the AAVFABI.11383/// TODO: Add support for references, section 3.2.1, item 1.11384static unsigned getAArch64LS(QualType QT, ParamKindTy Kind, ASTContext &C) {11385 if (!getAArch64MTV(QT, Kind) && QT.getCanonicalType()->isPointerType()) {11386 QualType PTy = QT.getCanonicalType()->getPointeeType();11387 if (getAArch64PBV(PTy, C))11388 return C.getTypeSize(PTy);11389 }11390 if (getAArch64PBV(QT, C))11391 return C.getTypeSize(QT);11392 11393 return C.getTypeSize(C.getUIntPtrType());11394}11395 11396// Get Narrowest Data Size (NDS) and Widest Data Size (WDS) from the11397// signature of the scalar function, as defined in 3.2.2 of the11398// AAVFABI.11399static std::tuple<unsigned, unsigned, bool>11400getNDSWDS(const FunctionDecl *FD, ArrayRef<ParamAttrTy> ParamAttrs) {11401 QualType RetType = FD->getReturnType().getCanonicalType();11402 11403 ASTContext &C = FD->getASTContext();11404 11405 bool OutputBecomesInput = false;11406 11407 llvm::SmallVector<unsigned, 8> Sizes;11408 if (!RetType->isVoidType()) {11409 Sizes.push_back(getAArch64LS(RetType, ParamKindTy::Vector, C));11410 if (!getAArch64PBV(RetType, C) && getAArch64MTV(RetType, {}))11411 OutputBecomesInput = true;11412 }11413 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {11414 QualType QT = FD->getParamDecl(I)->getType().getCanonicalType();11415 Sizes.push_back(getAArch64LS(QT, ParamAttrs[I].Kind, C));11416 }11417 11418 assert(!Sizes.empty() && "Unable to determine NDS and WDS.");11419 // The LS of a function parameter / return value can only be a power11420 // of 2, starting from 8 bits, up to 128.11421 assert(llvm::all_of(Sizes,11422 [](unsigned Size) {11423 return Size == 8 || Size == 16 || Size == 32 ||11424 Size == 64 || Size == 128;11425 }) &&11426 "Invalid size");11427 11428 return std::make_tuple(*llvm::min_element(Sizes), *llvm::max_element(Sizes),11429 OutputBecomesInput);11430}11431 11432// Function used to add the attribute. The parameter `VLEN` is11433// templated to allow the use of "x" when targeting scalable functions11434// for SVE.11435template <typename T>11436static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix,11437 char ISA, StringRef ParSeq,11438 StringRef MangledName, bool OutputBecomesInput,11439 llvm::Function *Fn) {11440 SmallString<256> Buffer;11441 llvm::raw_svector_ostream Out(Buffer);11442 Out << Prefix << ISA << LMask << VLEN;11443 if (OutputBecomesInput)11444 Out << "v";11445 Out << ParSeq << "_" << MangledName;11446 Fn->addFnAttr(Out.str());11447}11448 11449// Helper function to generate the Advanced SIMD names depending on11450// the value of the NDS when simdlen is not present.11451static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask,11452 StringRef Prefix, char ISA,11453 StringRef ParSeq, StringRef MangledName,11454 bool OutputBecomesInput,11455 llvm::Function *Fn) {11456 switch (NDS) {11457 case 8:11458 addAArch64VectorName(8, Mask, Prefix, ISA, ParSeq, MangledName,11459 OutputBecomesInput, Fn);11460 addAArch64VectorName(16, Mask, Prefix, ISA, ParSeq, MangledName,11461 OutputBecomesInput, Fn);11462 break;11463 case 16:11464 addAArch64VectorName(4, Mask, Prefix, ISA, ParSeq, MangledName,11465 OutputBecomesInput, Fn);11466 addAArch64VectorName(8, Mask, Prefix, ISA, ParSeq, MangledName,11467 OutputBecomesInput, Fn);11468 break;11469 case 32:11470 addAArch64VectorName(2, Mask, Prefix, ISA, ParSeq, MangledName,11471 OutputBecomesInput, Fn);11472 addAArch64VectorName(4, Mask, Prefix, ISA, ParSeq, MangledName,11473 OutputBecomesInput, Fn);11474 break;11475 case 64:11476 case 128:11477 addAArch64VectorName(2, Mask, Prefix, ISA, ParSeq, MangledName,11478 OutputBecomesInput, Fn);11479 break;11480 default:11481 llvm_unreachable("Scalar type is too wide.");11482 }11483}11484 11485/// Emit vector function attributes for AArch64, as defined in the AAVFABI.11486static void emitAArch64DeclareSimdFunction(11487 CodeGenModule &CGM, const FunctionDecl *FD, unsigned UserVLEN,11488 ArrayRef<ParamAttrTy> ParamAttrs,11489 OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName,11490 char ISA, unsigned VecRegSize, llvm::Function *Fn, SourceLocation SLoc) {11491 11492 // Get basic data for building the vector signature.11493 const auto Data = getNDSWDS(FD, ParamAttrs);11494 const unsigned NDS = std::get<0>(Data);11495 const unsigned WDS = std::get<1>(Data);11496 const bool OutputBecomesInput = std::get<2>(Data);11497 11498 // Check the values provided via `simdlen` by the user.11499 // 1. A `simdlen(1)` doesn't produce vector signatures,11500 if (UserVLEN == 1) {11501 unsigned DiagID = CGM.getDiags().getCustomDiagID(11502 DiagnosticsEngine::Warning,11503 "The clause simdlen(1) has no effect when targeting aarch64.");11504 CGM.getDiags().Report(SLoc, DiagID);11505 return;11506 }11507 11508 // 2. Section 3.3.1, item 1: user input must be a power of 2 for11509 // Advanced SIMD output.11510 if (ISA == 'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) {11511 unsigned DiagID = CGM.getDiags().getCustomDiagID(11512 DiagnosticsEngine::Warning, "The value specified in simdlen must be a "11513 "power of 2 when targeting Advanced SIMD.");11514 CGM.getDiags().Report(SLoc, DiagID);11515 return;11516 }11517 11518 // 3. Section 3.4.1. SVE fixed lengh must obey the architectural11519 // limits.11520 if (ISA == 's' && UserVLEN != 0) {11521 if ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0)) {11522 unsigned DiagID = CGM.getDiags().getCustomDiagID(11523 DiagnosticsEngine::Warning, "The clause simdlen must fit the %0-bit "11524 "lanes in the architectural constraints "11525 "for SVE (min is 128-bit, max is "11526 "2048-bit, by steps of 128-bit)");11527 CGM.getDiags().Report(SLoc, DiagID) << WDS;11528 return;11529 }11530 }11531 11532 // Sort out parameter sequence.11533 const std::string ParSeq = mangleVectorParameters(ParamAttrs);11534 StringRef Prefix = "_ZGV";11535 // Generate simdlen from user input (if any).11536 if (UserVLEN) {11537 if (ISA == 's') {11538 // SVE generates only a masked function.11539 addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName,11540 OutputBecomesInput, Fn);11541 } else {11542 assert(ISA == 'n' && "Expected ISA either 's' or 'n'.");11543 // Advanced SIMD generates one or two functions, depending on11544 // the `[not]inbranch` clause.11545 switch (State) {11546 case OMPDeclareSimdDeclAttr::BS_Undefined:11547 addAArch64VectorName(UserVLEN, "N", Prefix, ISA, ParSeq, MangledName,11548 OutputBecomesInput, Fn);11549 addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName,11550 OutputBecomesInput, Fn);11551 break;11552 case OMPDeclareSimdDeclAttr::BS_Notinbranch:11553 addAArch64VectorName(UserVLEN, "N", Prefix, ISA, ParSeq, MangledName,11554 OutputBecomesInput, Fn);11555 break;11556 case OMPDeclareSimdDeclAttr::BS_Inbranch:11557 addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName,11558 OutputBecomesInput, Fn);11559 break;11560 }11561 }11562 } else {11563 // If no user simdlen is provided, follow the AAVFABI rules for11564 // generating the vector length.11565 if (ISA == 's') {11566 // SVE, section 3.4.1, item 1.11567 addAArch64VectorName("x", "M", Prefix, ISA, ParSeq, MangledName,11568 OutputBecomesInput, Fn);11569 } else {11570 assert(ISA == 'n' && "Expected ISA either 's' or 'n'.");11571 // Advanced SIMD, Section 3.3.1 of the AAVFABI, generates one or11572 // two vector names depending on the use of the clause11573 // `[not]inbranch`.11574 switch (State) {11575 case OMPDeclareSimdDeclAttr::BS_Undefined:11576 addAArch64AdvSIMDNDSNames(NDS, "N", Prefix, ISA, ParSeq, MangledName,11577 OutputBecomesInput, Fn);11578 addAArch64AdvSIMDNDSNames(NDS, "M", Prefix, ISA, ParSeq, MangledName,11579 OutputBecomesInput, Fn);11580 break;11581 case OMPDeclareSimdDeclAttr::BS_Notinbranch:11582 addAArch64AdvSIMDNDSNames(NDS, "N", Prefix, ISA, ParSeq, MangledName,11583 OutputBecomesInput, Fn);11584 break;11585 case OMPDeclareSimdDeclAttr::BS_Inbranch:11586 addAArch64AdvSIMDNDSNames(NDS, "M", Prefix, ISA, ParSeq, MangledName,11587 OutputBecomesInput, Fn);11588 break;11589 }11590 }11591 }11592}11593 11594void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,11595 llvm::Function *Fn) {11596 ASTContext &C = CGM.getContext();11597 FD = FD->getMostRecentDecl();11598 while (FD) {11599 // Map params to their positions in function decl.11600 llvm::DenseMap<const Decl *, unsigned> ParamPositions;11601 if (isa<CXXMethodDecl>(FD))11602 ParamPositions.try_emplace(FD, 0);11603 unsigned ParamPos = ParamPositions.size();11604 for (const ParmVarDecl *P : FD->parameters()) {11605 ParamPositions.try_emplace(P->getCanonicalDecl(), ParamPos);11606 ++ParamPos;11607 }11608 for (const auto *Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) {11609 llvm::SmallVector<ParamAttrTy, 8> ParamAttrs(ParamPositions.size());11610 // Mark uniform parameters.11611 for (const Expr *E : Attr->uniforms()) {11612 E = E->IgnoreParenImpCasts();11613 unsigned Pos;11614 if (isa<CXXThisExpr>(E)) {11615 Pos = ParamPositions[FD];11616 } else {11617 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())11618 ->getCanonicalDecl();11619 auto It = ParamPositions.find(PVD);11620 assert(It != ParamPositions.end() && "Function parameter not found");11621 Pos = It->second;11622 }11623 ParamAttrs[Pos].Kind = Uniform;11624 }11625 // Get alignment info.11626 auto *NI = Attr->alignments_begin();11627 for (const Expr *E : Attr->aligneds()) {11628 E = E->IgnoreParenImpCasts();11629 unsigned Pos;11630 QualType ParmTy;11631 if (isa<CXXThisExpr>(E)) {11632 Pos = ParamPositions[FD];11633 ParmTy = E->getType();11634 } else {11635 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())11636 ->getCanonicalDecl();11637 auto It = ParamPositions.find(PVD);11638 assert(It != ParamPositions.end() && "Function parameter not found");11639 Pos = It->second;11640 ParmTy = PVD->getType();11641 }11642 ParamAttrs[Pos].Alignment =11643 (*NI)11644 ? (*NI)->EvaluateKnownConstInt(C)11645 : llvm::APSInt::getUnsigned(11646 C.toCharUnitsFromBits(C.getOpenMPDefaultSimdAlign(ParmTy))11647 .getQuantity());11648 ++NI;11649 }11650 // Mark linear parameters.11651 auto *SI = Attr->steps_begin();11652 auto *MI = Attr->modifiers_begin();11653 for (const Expr *E : Attr->linears()) {11654 E = E->IgnoreParenImpCasts();11655 unsigned Pos;11656 bool IsReferenceType = false;11657 // Rescaling factor needed to compute the linear parameter11658 // value in the mangled name.11659 unsigned PtrRescalingFactor = 1;11660 if (isa<CXXThisExpr>(E)) {11661 Pos = ParamPositions[FD];11662 auto *P = cast<PointerType>(E->getType());11663 PtrRescalingFactor = CGM.getContext()11664 .getTypeSizeInChars(P->getPointeeType())11665 .getQuantity();11666 } else {11667 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())11668 ->getCanonicalDecl();11669 auto It = ParamPositions.find(PVD);11670 assert(It != ParamPositions.end() && "Function parameter not found");11671 Pos = It->second;11672 if (auto *P = dyn_cast<PointerType>(PVD->getType()))11673 PtrRescalingFactor = CGM.getContext()11674 .getTypeSizeInChars(P->getPointeeType())11675 .getQuantity();11676 else if (PVD->getType()->isReferenceType()) {11677 IsReferenceType = true;11678 PtrRescalingFactor =11679 CGM.getContext()11680 .getTypeSizeInChars(PVD->getType().getNonReferenceType())11681 .getQuantity();11682 }11683 }11684 ParamAttrTy &ParamAttr = ParamAttrs[Pos];11685 if (*MI == OMPC_LINEAR_ref)11686 ParamAttr.Kind = LinearRef;11687 else if (*MI == OMPC_LINEAR_uval)11688 ParamAttr.Kind = LinearUVal;11689 else if (IsReferenceType)11690 ParamAttr.Kind = LinearVal;11691 else11692 ParamAttr.Kind = Linear;11693 // Assuming a stride of 1, for `linear` without modifiers.11694 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(1);11695 if (*SI) {11696 Expr::EvalResult Result;11697 if (!(*SI)->EvaluateAsInt(Result, C, Expr::SE_AllowSideEffects)) {11698 if (const auto *DRE =11699 cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {11700 if (const auto *StridePVD =11701 dyn_cast<ParmVarDecl>(DRE->getDecl())) {11702 ParamAttr.HasVarStride = true;11703 auto It = ParamPositions.find(StridePVD->getCanonicalDecl());11704 assert(It != ParamPositions.end() &&11705 "Function parameter not found");11706 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(It->second);11707 }11708 }11709 } else {11710 ParamAttr.StrideOrArg = Result.Val.getInt();11711 }11712 }11713 // If we are using a linear clause on a pointer, we need to11714 // rescale the value of linear_step with the byte size of the11715 // pointee type.11716 if (!ParamAttr.HasVarStride &&11717 (ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef))11718 ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor;11719 ++SI;11720 ++MI;11721 }11722 llvm::APSInt VLENVal;11723 SourceLocation ExprLoc;11724 const Expr *VLENExpr = Attr->getSimdlen();11725 if (VLENExpr) {11726 VLENVal = VLENExpr->EvaluateKnownConstInt(C);11727 ExprLoc = VLENExpr->getExprLoc();11728 }11729 OMPDeclareSimdDeclAttr::BranchStateTy State = Attr->getBranchState();11730 if (CGM.getTriple().isX86()) {11731 emitX86DeclareSimdFunction(FD, Fn, VLENVal, ParamAttrs, State);11732 } else if (CGM.getTriple().getArch() == llvm::Triple::aarch64) {11733 unsigned VLEN = VLENVal.getExtValue();11734 StringRef MangledName = Fn->getName();11735 if (CGM.getTarget().hasFeature("sve"))11736 emitAArch64DeclareSimdFunction(CGM, FD, VLEN, ParamAttrs, State,11737 MangledName, 's', 128, Fn, ExprLoc);11738 else if (CGM.getTarget().hasFeature("neon"))11739 emitAArch64DeclareSimdFunction(CGM, FD, VLEN, ParamAttrs, State,11740 MangledName, 'n', 128, Fn, ExprLoc);11741 }11742 }11743 FD = FD->getPreviousDecl();11744 }11745}11746 11747namespace {11748/// Cleanup action for doacross support.11749class DoacrossCleanupTy final : public EHScopeStack::Cleanup {11750public:11751 static const int DoacrossFinArgs = 2;11752 11753private:11754 llvm::FunctionCallee RTLFn;11755 llvm::Value *Args[DoacrossFinArgs];11756 11757public:11758 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,11759 ArrayRef<llvm::Value *> CallArgs)11760 : RTLFn(RTLFn) {11761 assert(CallArgs.size() == DoacrossFinArgs);11762 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));11763 }11764 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {11765 if (!CGF.HaveInsertPoint())11766 return;11767 CGF.EmitRuntimeCall(RTLFn, Args);11768 }11769};11770} // namespace11771 11772void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,11773 const OMPLoopDirective &D,11774 ArrayRef<Expr *> NumIterations) {11775 if (!CGF.HaveInsertPoint())11776 return;11777 11778 ASTContext &C = CGM.getContext();11779 QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true);11780 RecordDecl *RD;11781 if (KmpDimTy.isNull()) {11782 // Build struct kmp_dim { // loop bounds info casted to kmp_int6411783 // kmp_int64 lo; // lower11784 // kmp_int64 up; // upper11785 // kmp_int64 st; // stride11786 // };11787 RD = C.buildImplicitRecord("kmp_dim");11788 RD->startDefinition();11789 addFieldToRecordDecl(C, RD, Int64Ty);11790 addFieldToRecordDecl(C, RD, Int64Ty);11791 addFieldToRecordDecl(C, RD, Int64Ty);11792 RD->completeDefinition();11793 KmpDimTy = C.getCanonicalTagType(RD);11794 } else {11795 RD = KmpDimTy->castAsRecordDecl();11796 }11797 llvm::APInt Size(/*numBits=*/32, NumIterations.size());11798 QualType ArrayTy = C.getConstantArrayType(KmpDimTy, Size, nullptr,11799 ArraySizeModifier::Normal, 0);11800 11801 Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims");11802 CGF.EmitNullInitialization(DimsAddr, ArrayTy);11803 enum { LowerFD = 0, UpperFD, StrideFD };11804 // Fill dims with data.11805 for (unsigned I = 0, E = NumIterations.size(); I < E; ++I) {11806 LValue DimsLVal = CGF.MakeAddrLValue(11807 CGF.Builder.CreateConstArrayGEP(DimsAddr, I), KmpDimTy);11808 // dims.upper = num_iterations;11809 LValue UpperLVal = CGF.EmitLValueForField(11810 DimsLVal, *std::next(RD->field_begin(), UpperFD));11811 llvm::Value *NumIterVal = CGF.EmitScalarConversion(11812 CGF.EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(),11813 Int64Ty, NumIterations[I]->getExprLoc());11814 CGF.EmitStoreOfScalar(NumIterVal, UpperLVal);11815 // dims.stride = 1;11816 LValue StrideLVal = CGF.EmitLValueForField(11817 DimsLVal, *std::next(RD->field_begin(), StrideFD));11818 CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1),11819 StrideLVal);11820 }11821 11822 // Build call void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid,11823 // kmp_int32 num_dims, struct kmp_dim * dims);11824 llvm::Value *Args[] = {11825 emitUpdateLocation(CGF, D.getBeginLoc()),11826 getThreadID(CGF, D.getBeginLoc()),11827 llvm::ConstantInt::getSigned(CGM.Int32Ty, NumIterations.size()),11828 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(11829 CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).emitRawPointer(CGF),11830 CGM.VoidPtrTy)};11831 11832 llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(11833 CGM.getModule(), OMPRTL___kmpc_doacross_init);11834 CGF.EmitRuntimeCall(RTLFn, Args);11835 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {11836 emitUpdateLocation(CGF, D.getEndLoc()), getThreadID(CGF, D.getEndLoc())};11837 llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction(11838 CGM.getModule(), OMPRTL___kmpc_doacross_fini);11839 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn,11840 llvm::ArrayRef(FiniArgs));11841}11842 11843template <typename T>11844static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM,11845 const T *C, llvm::Value *ULoc,11846 llvm::Value *ThreadID) {11847 QualType Int64Ty =11848 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);11849 llvm::APInt Size(/*numBits=*/32, C->getNumLoops());11850 QualType ArrayTy = CGM.getContext().getConstantArrayType(11851 Int64Ty, Size, nullptr, ArraySizeModifier::Normal, 0);11852 Address CntAddr = CGF.CreateMemTemp(ArrayTy, ".cnt.addr");11853 for (unsigned I = 0, E = C->getNumLoops(); I < E; ++I) {11854 const Expr *CounterVal = C->getLoopData(I);11855 assert(CounterVal);11856 llvm::Value *CntVal = CGF.EmitScalarConversion(11857 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty,11858 CounterVal->getExprLoc());11859 CGF.EmitStoreOfScalar(CntVal, CGF.Builder.CreateConstArrayGEP(CntAddr, I),11860 /*Volatile=*/false, Int64Ty);11861 }11862 llvm::Value *Args[] = {11863 ULoc, ThreadID,11864 CGF.Builder.CreateConstArrayGEP(CntAddr, 0).emitRawPointer(CGF)};11865 llvm::FunctionCallee RTLFn;11866 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();11867 OMPDoacrossKind<T> ODK;11868 if (ODK.isSource(C)) {11869 RTLFn = OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),11870 OMPRTL___kmpc_doacross_post);11871 } else {11872 assert(ODK.isSink(C) && "Expect sink modifier.");11873 RTLFn = OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),11874 OMPRTL___kmpc_doacross_wait);11875 }11876 CGF.EmitRuntimeCall(RTLFn, Args);11877}11878 11879void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,11880 const OMPDependClause *C) {11881 return EmitDoacrossOrdered<OMPDependClause>(11882 CGF, CGM, C, emitUpdateLocation(CGF, C->getBeginLoc()),11883 getThreadID(CGF, C->getBeginLoc()));11884}11885 11886void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,11887 const OMPDoacrossClause *C) {11888 return EmitDoacrossOrdered<OMPDoacrossClause>(11889 CGF, CGM, C, emitUpdateLocation(CGF, C->getBeginLoc()),11890 getThreadID(CGF, C->getBeginLoc()));11891}11892 11893void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc,11894 llvm::FunctionCallee Callee,11895 ArrayRef<llvm::Value *> Args) const {11896 assert(Loc.isValid() && "Outlined function call location must be valid.");11897 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);11898 11899 if (auto *Fn = dyn_cast<llvm::Function>(Callee.getCallee())) {11900 if (Fn->doesNotThrow()) {11901 CGF.EmitNounwindRuntimeCall(Fn, Args);11902 return;11903 }11904 }11905 CGF.EmitRuntimeCall(Callee, Args);11906}11907 11908void CGOpenMPRuntime::emitOutlinedFunctionCall(11909 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,11910 ArrayRef<llvm::Value *> Args) const {11911 emitCall(CGF, Loc, OutlinedFn, Args);11912}11913 11914void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) {11915 if (const auto *FD = dyn_cast<FunctionDecl>(D))11916 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(FD))11917 HasEmittedDeclareTargetRegion = true;11918}11919 11920Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF,11921 const VarDecl *NativeParam,11922 const VarDecl *TargetParam) const {11923 return CGF.GetAddrOfLocalVar(NativeParam);11924}11925 11926/// Return allocator value from expression, or return a null allocator (default11927/// when no allocator specified).11928static llvm::Value *getAllocatorVal(CodeGenFunction &CGF,11929 const Expr *Allocator) {11930 llvm::Value *AllocVal;11931 if (Allocator) {11932 AllocVal = CGF.EmitScalarExpr(Allocator);11933 // According to the standard, the original allocator type is a enum11934 // (integer). Convert to pointer type, if required.11935 AllocVal = CGF.EmitScalarConversion(AllocVal, Allocator->getType(),11936 CGF.getContext().VoidPtrTy,11937 Allocator->getExprLoc());11938 } else {11939 // If no allocator specified, it defaults to the null allocator.11940 AllocVal = llvm::Constant::getNullValue(11941 CGF.CGM.getTypes().ConvertType(CGF.getContext().VoidPtrTy));11942 }11943 return AllocVal;11944}11945 11946/// Return the alignment from an allocate directive if present.11947static llvm::Value *getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD) {11948 std::optional<CharUnits> AllocateAlignment = CGM.getOMPAllocateAlignment(VD);11949 11950 if (!AllocateAlignment)11951 return nullptr;11952 11953 return llvm::ConstantInt::get(CGM.SizeTy, AllocateAlignment->getQuantity());11954}11955 11956Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,11957 const VarDecl *VD) {11958 if (!VD)11959 return Address::invalid();11960 Address UntiedAddr = Address::invalid();11961 Address UntiedRealAddr = Address::invalid();11962 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn);11963 if (It != FunctionToUntiedTaskStackMap.end()) {11964 const UntiedLocalVarsAddressesMap &UntiedData =11965 UntiedLocalVarsStack[It->second];11966 auto I = UntiedData.find(VD);11967 if (I != UntiedData.end()) {11968 UntiedAddr = I->second.first;11969 UntiedRealAddr = I->second.second;11970 }11971 }11972 const VarDecl *CVD = VD->getCanonicalDecl();11973 if (CVD->hasAttr<OMPAllocateDeclAttr>()) {11974 // Use the default allocation.11975 if (!isAllocatableDecl(VD))11976 return UntiedAddr;11977 llvm::Value *Size;11978 CharUnits Align = CGM.getContext().getDeclAlign(CVD);11979 if (CVD->getType()->isVariablyModifiedType()) {11980 Size = CGF.getTypeSize(CVD->getType());11981 // Align the size: ((size + align - 1) / align) * align11982 Size = CGF.Builder.CreateNUWAdd(11983 Size, CGM.getSize(Align - CharUnits::fromQuantity(1)));11984 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));11985 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));11986 } else {11987 CharUnits Sz = CGM.getContext().getTypeSizeInChars(CVD->getType());11988 Size = CGM.getSize(Sz.alignTo(Align));11989 }11990 llvm::Value *ThreadID = getThreadID(CGF, CVD->getBeginLoc());11991 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();11992 const Expr *Allocator = AA->getAllocator();11993 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator);11994 llvm::Value *Alignment = getAlignmentValue(CGM, CVD);11995 SmallVector<llvm::Value *, 4> Args;11996 Args.push_back(ThreadID);11997 if (Alignment)11998 Args.push_back(Alignment);11999 Args.push_back(Size);12000 Args.push_back(AllocVal);12001 llvm::omp::RuntimeFunction FnID =12002 Alignment ? OMPRTL___kmpc_aligned_alloc : OMPRTL___kmpc_alloc;12003 llvm::Value *Addr = CGF.EmitRuntimeCall(12004 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), FnID), Args,12005 getName({CVD->getName(), ".void.addr"}));12006 llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction(12007 CGM.getModule(), OMPRTL___kmpc_free);12008 QualType Ty = CGM.getContext().getPointerType(CVD->getType());12009 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(12010 Addr, CGF.ConvertTypeForMem(Ty), getName({CVD->getName(), ".addr"}));12011 if (UntiedAddr.isValid())12012 CGF.EmitStoreOfScalar(Addr, UntiedAddr, /*Volatile=*/false, Ty);12013 12014 // Cleanup action for allocate support.12015 class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {12016 llvm::FunctionCallee RTLFn;12017 SourceLocation::UIntTy LocEncoding;12018 Address Addr;12019 const Expr *AllocExpr;12020 12021 public:12022 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,12023 SourceLocation::UIntTy LocEncoding, Address Addr,12024 const Expr *AllocExpr)12025 : RTLFn(RTLFn), LocEncoding(LocEncoding), Addr(Addr),12026 AllocExpr(AllocExpr) {}12027 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {12028 if (!CGF.HaveInsertPoint())12029 return;12030 llvm::Value *Args[3];12031 Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID(12032 CGF, SourceLocation::getFromRawEncoding(LocEncoding));12033 Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(12034 Addr.emitRawPointer(CGF), CGF.VoidPtrTy);12035 llvm::Value *AllocVal = getAllocatorVal(CGF, AllocExpr);12036 Args[2] = AllocVal;12037 CGF.EmitRuntimeCall(RTLFn, Args);12038 }12039 };12040 Address VDAddr =12041 UntiedRealAddr.isValid()12042 ? UntiedRealAddr12043 : Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align);12044 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(12045 NormalAndEHCleanup, FiniRTLFn, CVD->getLocation().getRawEncoding(),12046 VDAddr, Allocator);12047 if (UntiedRealAddr.isValid())12048 if (auto *Region =12049 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))12050 Region->emitUntiedSwitch(CGF);12051 return VDAddr;12052 }12053 return UntiedAddr;12054}12055 12056bool CGOpenMPRuntime::isLocalVarInUntiedTask(CodeGenFunction &CGF,12057 const VarDecl *VD) const {12058 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn);12059 if (It == FunctionToUntiedTaskStackMap.end())12060 return false;12061 return UntiedLocalVarsStack[It->second].count(VD) > 0;12062}12063 12064CGOpenMPRuntime::NontemporalDeclsRAII::NontemporalDeclsRAII(12065 CodeGenModule &CGM, const OMPLoopDirective &S)12066 : CGM(CGM), NeedToPush(S.hasClausesOfKind<OMPNontemporalClause>()) {12067 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");12068 if (!NeedToPush)12069 return;12070 NontemporalDeclsSet &DS =12071 CGM.getOpenMPRuntime().NontemporalDeclsStack.emplace_back();12072 for (const auto *C : S.getClausesOfKind<OMPNontemporalClause>()) {12073 for (const Stmt *Ref : C->private_refs()) {12074 const auto *SimpleRefExpr = cast<Expr>(Ref)->IgnoreParenImpCasts();12075 const ValueDecl *VD;12076 if (const auto *DRE = dyn_cast<DeclRefExpr>(SimpleRefExpr)) {12077 VD = DRE->getDecl();12078 } else {12079 const auto *ME = cast<MemberExpr>(SimpleRefExpr);12080 assert((ME->isImplicitCXXThis() ||12081 isa<CXXThisExpr>(ME->getBase()->IgnoreParenImpCasts())) &&12082 "Expected member of current class.");12083 VD = ME->getMemberDecl();12084 }12085 DS.insert(VD);12086 }12087 }12088}12089 12090CGOpenMPRuntime::NontemporalDeclsRAII::~NontemporalDeclsRAII() {12091 if (!NeedToPush)12092 return;12093 CGM.getOpenMPRuntime().NontemporalDeclsStack.pop_back();12094}12095 12096CGOpenMPRuntime::UntiedTaskLocalDeclsRAII::UntiedTaskLocalDeclsRAII(12097 CodeGenFunction &CGF,12098 const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,12099 std::pair<Address, Address>> &LocalVars)12100 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) {12101 if (!NeedToPush)12102 return;12103 CGM.getOpenMPRuntime().FunctionToUntiedTaskStackMap.try_emplace(12104 CGF.CurFn, CGM.getOpenMPRuntime().UntiedLocalVarsStack.size());12105 CGM.getOpenMPRuntime().UntiedLocalVarsStack.push_back(LocalVars);12106}12107 12108CGOpenMPRuntime::UntiedTaskLocalDeclsRAII::~UntiedTaskLocalDeclsRAII() {12109 if (!NeedToPush)12110 return;12111 CGM.getOpenMPRuntime().UntiedLocalVarsStack.pop_back();12112}12113 12114bool CGOpenMPRuntime::isNontemporalDecl(const ValueDecl *VD) const {12115 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");12116 12117 return llvm::any_of(12118 CGM.getOpenMPRuntime().NontemporalDeclsStack,12119 [VD](const NontemporalDeclsSet &Set) { return Set.contains(VD); });12120}12121 12122void CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis(12123 const OMPExecutableDirective &S,12124 llvm::DenseSet<CanonicalDeclPtr<const Decl>> &NeedToAddForLPCsAsDisabled)12125 const {12126 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToCheckForLPCs;12127 // Vars in target/task regions must be excluded completely.12128 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()) ||12129 isOpenMPTaskingDirective(S.getDirectiveKind())) {12130 SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;12131 getOpenMPCaptureRegions(CaptureRegions, S.getDirectiveKind());12132 const CapturedStmt *CS = S.getCapturedStmt(CaptureRegions.front());12133 for (const CapturedStmt::Capture &Cap : CS->captures()) {12134 if (Cap.capturesVariable() || Cap.capturesVariableByCopy())12135 NeedToCheckForLPCs.insert(Cap.getCapturedVar());12136 }12137 }12138 // Exclude vars in private clauses.12139 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {12140 for (const Expr *Ref : C->varlist()) {12141 if (!Ref->getType()->isScalarType())12142 continue;12143 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());12144 if (!DRE)12145 continue;12146 NeedToCheckForLPCs.insert(DRE->getDecl());12147 }12148 }12149 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {12150 for (const Expr *Ref : C->varlist()) {12151 if (!Ref->getType()->isScalarType())12152 continue;12153 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());12154 if (!DRE)12155 continue;12156 NeedToCheckForLPCs.insert(DRE->getDecl());12157 }12158 }12159 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {12160 for (const Expr *Ref : C->varlist()) {12161 if (!Ref->getType()->isScalarType())12162 continue;12163 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());12164 if (!DRE)12165 continue;12166 NeedToCheckForLPCs.insert(DRE->getDecl());12167 }12168 }12169 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {12170 for (const Expr *Ref : C->varlist()) {12171 if (!Ref->getType()->isScalarType())12172 continue;12173 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());12174 if (!DRE)12175 continue;12176 NeedToCheckForLPCs.insert(DRE->getDecl());12177 }12178 }12179 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {12180 for (const Expr *Ref : C->varlist()) {12181 if (!Ref->getType()->isScalarType())12182 continue;12183 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());12184 if (!DRE)12185 continue;12186 NeedToCheckForLPCs.insert(DRE->getDecl());12187 }12188 }12189 for (const Decl *VD : NeedToCheckForLPCs) {12190 for (const LastprivateConditionalData &Data :12191 llvm::reverse(CGM.getOpenMPRuntime().LastprivateConditionalStack)) {12192 if (Data.DeclToUniqueName.count(VD) > 0) {12193 if (!Data.Disabled)12194 NeedToAddForLPCsAsDisabled.insert(VD);12195 break;12196 }12197 }12198 }12199}12200 12201CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(12202 CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal)12203 : CGM(CGF.CGM),12204 Action((CGM.getLangOpts().OpenMP >= 50 &&12205 llvm::any_of(S.getClausesOfKind<OMPLastprivateClause>(),12206 [](const OMPLastprivateClause *C) {12207 return C->getKind() ==12208 OMPC_LASTPRIVATE_conditional;12209 }))12210 ? ActionToDo::PushAsLastprivateConditional12211 : ActionToDo::DoNotPush) {12212 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");12213 if (CGM.getLangOpts().OpenMP < 50 || Action == ActionToDo::DoNotPush)12214 return;12215 assert(Action == ActionToDo::PushAsLastprivateConditional &&12216 "Expected a push action.");12217 LastprivateConditionalData &Data =12218 CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();12219 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {12220 if (C->getKind() != OMPC_LASTPRIVATE_conditional)12221 continue;12222 12223 for (const Expr *Ref : C->varlist()) {12224 Data.DeclToUniqueName.insert(std::make_pair(12225 cast<DeclRefExpr>(Ref->IgnoreParenImpCasts())->getDecl(),12226 SmallString<16>(generateUniqueName(CGM, "pl_cond", Ref))));12227 }12228 }12229 Data.IVLVal = IVLVal;12230 Data.Fn = CGF.CurFn;12231}12232 12233CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(12234 CodeGenFunction &CGF, const OMPExecutableDirective &S)12235 : CGM(CGF.CGM), Action(ActionToDo::DoNotPush) {12236 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");12237 if (CGM.getLangOpts().OpenMP < 50)12238 return;12239 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToAddForLPCsAsDisabled;12240 tryToDisableInnerAnalysis(S, NeedToAddForLPCsAsDisabled);12241 if (!NeedToAddForLPCsAsDisabled.empty()) {12242 Action = ActionToDo::DisableLastprivateConditional;12243 LastprivateConditionalData &Data =12244 CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();12245 for (const Decl *VD : NeedToAddForLPCsAsDisabled)12246 Data.DeclToUniqueName.try_emplace(VD);12247 Data.Fn = CGF.CurFn;12248 Data.Disabled = true;12249 }12250}12251 12252CGOpenMPRuntime::LastprivateConditionalRAII12253CGOpenMPRuntime::LastprivateConditionalRAII::disable(12254 CodeGenFunction &CGF, const OMPExecutableDirective &S) {12255 return LastprivateConditionalRAII(CGF, S);12256}12257 12258CGOpenMPRuntime::LastprivateConditionalRAII::~LastprivateConditionalRAII() {12259 if (CGM.getLangOpts().OpenMP < 50)12260 return;12261 if (Action == ActionToDo::DisableLastprivateConditional) {12262 assert(CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&12263 "Expected list of disabled private vars.");12264 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();12265 }12266 if (Action == ActionToDo::PushAsLastprivateConditional) {12267 assert(12268 !CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&12269 "Expected list of lastprivate conditional vars.");12270 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();12271 }12272}12273 12274Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF,12275 const VarDecl *VD) {12276 ASTContext &C = CGM.getContext();12277 auto I = LastprivateConditionalToTypes.try_emplace(CGF.CurFn).first;12278 QualType NewType;12279 const FieldDecl *VDField;12280 const FieldDecl *FiredField;12281 LValue BaseLVal;12282 auto VI = I->getSecond().find(VD);12283 if (VI == I->getSecond().end()) {12284 RecordDecl *RD = C.buildImplicitRecord("lasprivate.conditional");12285 RD->startDefinition();12286 VDField = addFieldToRecordDecl(C, RD, VD->getType().getNonReferenceType());12287 FiredField = addFieldToRecordDecl(C, RD, C.CharTy);12288 RD->completeDefinition();12289 NewType = C.getCanonicalTagType(RD);12290 Address Addr = CGF.CreateMemTemp(NewType, C.getDeclAlign(VD), VD->getName());12291 BaseLVal = CGF.MakeAddrLValue(Addr, NewType, AlignmentSource::Decl);12292 I->getSecond().try_emplace(VD, NewType, VDField, FiredField, BaseLVal);12293 } else {12294 NewType = std::get<0>(VI->getSecond());12295 VDField = std::get<1>(VI->getSecond());12296 FiredField = std::get<2>(VI->getSecond());12297 BaseLVal = std::get<3>(VI->getSecond());12298 }12299 LValue FiredLVal =12300 CGF.EmitLValueForField(BaseLVal, FiredField);12301 CGF.EmitStoreOfScalar(12302 llvm::ConstantInt::getNullValue(CGF.ConvertTypeForMem(C.CharTy)),12303 FiredLVal);12304 return CGF.EmitLValueForField(BaseLVal, VDField).getAddress();12305}12306 12307namespace {12308/// Checks if the lastprivate conditional variable is referenced in LHS.12309class LastprivateConditionalRefChecker final12310 : public ConstStmtVisitor<LastprivateConditionalRefChecker, bool> {12311 ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM;12312 const Expr *FoundE = nullptr;12313 const Decl *FoundD = nullptr;12314 StringRef UniqueDeclName;12315 LValue IVLVal;12316 llvm::Function *FoundFn = nullptr;12317 SourceLocation Loc;12318 12319public:12320 bool VisitDeclRefExpr(const DeclRefExpr *E) {12321 for (const CGOpenMPRuntime::LastprivateConditionalData &D :12322 llvm::reverse(LPM)) {12323 auto It = D.DeclToUniqueName.find(E->getDecl());12324 if (It == D.DeclToUniqueName.end())12325 continue;12326 if (D.Disabled)12327 return false;12328 FoundE = E;12329 FoundD = E->getDecl()->getCanonicalDecl();12330 UniqueDeclName = It->second;12331 IVLVal = D.IVLVal;12332 FoundFn = D.Fn;12333 break;12334 }12335 return FoundE == E;12336 }12337 bool VisitMemberExpr(const MemberExpr *E) {12338 if (!CodeGenFunction::IsWrappedCXXThis(E->getBase()))12339 return false;12340 for (const CGOpenMPRuntime::LastprivateConditionalData &D :12341 llvm::reverse(LPM)) {12342 auto It = D.DeclToUniqueName.find(E->getMemberDecl());12343 if (It == D.DeclToUniqueName.end())12344 continue;12345 if (D.Disabled)12346 return false;12347 FoundE = E;12348 FoundD = E->getMemberDecl()->getCanonicalDecl();12349 UniqueDeclName = It->second;12350 IVLVal = D.IVLVal;12351 FoundFn = D.Fn;12352 break;12353 }12354 return FoundE == E;12355 }12356 bool VisitStmt(const Stmt *S) {12357 for (const Stmt *Child : S->children()) {12358 if (!Child)12359 continue;12360 if (const auto *E = dyn_cast<Expr>(Child))12361 if (!E->isGLValue())12362 continue;12363 if (Visit(Child))12364 return true;12365 }12366 return false;12367 }12368 explicit LastprivateConditionalRefChecker(12369 ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM)12370 : LPM(LPM) {}12371 std::tuple<const Expr *, const Decl *, StringRef, LValue, llvm::Function *>12372 getFoundData() const {12373 return std::make_tuple(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn);12374 }12375};12376} // namespace12377 12378void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF,12379 LValue IVLVal,12380 StringRef UniqueDeclName,12381 LValue LVal,12382 SourceLocation Loc) {12383 // Last updated loop counter for the lastprivate conditional var.12384 // int<xx> last_iv = 0;12385 llvm::Type *LLIVTy = CGF.ConvertTypeForMem(IVLVal.getType());12386 llvm::Constant *LastIV = OMPBuilder.getOrCreateInternalVariable(12387 LLIVTy, getName({UniqueDeclName, "iv"}));12388 cast<llvm::GlobalVariable>(LastIV)->setAlignment(12389 IVLVal.getAlignment().getAsAlign());12390 LValue LastIVLVal =12391 CGF.MakeNaturalAlignRawAddrLValue(LastIV, IVLVal.getType());12392 12393 // Last value of the lastprivate conditional.12394 // decltype(priv_a) last_a;12395 llvm::GlobalVariable *Last = OMPBuilder.getOrCreateInternalVariable(12396 CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName);12397 cast<llvm::GlobalVariable>(Last)->setAlignment(12398 LVal.getAlignment().getAsAlign());12399 LValue LastLVal =12400 CGF.MakeRawAddrLValue(Last, LVal.getType(), LVal.getAlignment());12401 12402 // Global loop counter. Required to handle inner parallel-for regions.12403 // iv12404 llvm::Value *IVVal = CGF.EmitLoadOfScalar(IVLVal, Loc);12405 12406 // #pragma omp critical(a)12407 // if (last_iv <= iv) {12408 // last_iv = iv;12409 // last_a = priv_a;12410 // }12411 auto &&CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,12412 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {12413 Action.Enter(CGF);12414 llvm::Value *LastIVVal = CGF.EmitLoadOfScalar(LastIVLVal, Loc);12415 // (last_iv <= iv) ? Check if the variable is updated and store new12416 // value in global var.12417 llvm::Value *CmpRes;12418 if (IVLVal.getType()->isSignedIntegerType()) {12419 CmpRes = CGF.Builder.CreateICmpSLE(LastIVVal, IVVal);12420 } else {12421 assert(IVLVal.getType()->isUnsignedIntegerType() &&12422 "Loop iteration variable must be integer.");12423 CmpRes = CGF.Builder.CreateICmpULE(LastIVVal, IVVal);12424 }12425 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lp_cond_then");12426 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("lp_cond_exit");12427 CGF.Builder.CreateCondBr(CmpRes, ThenBB, ExitBB);12428 // {12429 CGF.EmitBlock(ThenBB);12430 12431 // last_iv = iv;12432 CGF.EmitStoreOfScalar(IVVal, LastIVLVal);12433 12434 // last_a = priv_a;12435 switch (CGF.getEvaluationKind(LVal.getType())) {12436 case TEK_Scalar: {12437 llvm::Value *PrivVal = CGF.EmitLoadOfScalar(LVal, Loc);12438 CGF.EmitStoreOfScalar(PrivVal, LastLVal);12439 break;12440 }12441 case TEK_Complex: {12442 CodeGenFunction::ComplexPairTy PrivVal = CGF.EmitLoadOfComplex(LVal, Loc);12443 CGF.EmitStoreOfComplex(PrivVal, LastLVal, /*isInit=*/false);12444 break;12445 }12446 case TEK_Aggregate:12447 llvm_unreachable(12448 "Aggregates are not supported in lastprivate conditional.");12449 }12450 // }12451 CGF.EmitBranch(ExitBB);12452 // There is no need to emit line number for unconditional branch.12453 (void)ApplyDebugLocation::CreateEmpty(CGF);12454 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);12455 };12456 12457 if (CGM.getLangOpts().OpenMPSimd) {12458 // Do not emit as a critical region as no parallel region could be emitted.12459 RegionCodeGenTy ThenRCG(CodeGen);12460 ThenRCG(CGF);12461 } else {12462 emitCriticalRegion(CGF, UniqueDeclName, CodeGen, Loc);12463 }12464}12465 12466void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF,12467 const Expr *LHS) {12468 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())12469 return;12470 LastprivateConditionalRefChecker Checker(LastprivateConditionalStack);12471 if (!Checker.Visit(LHS))12472 return;12473 const Expr *FoundE;12474 const Decl *FoundD;12475 StringRef UniqueDeclName;12476 LValue IVLVal;12477 llvm::Function *FoundFn;12478 std::tie(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn) =12479 Checker.getFoundData();12480 if (FoundFn != CGF.CurFn) {12481 // Special codegen for inner parallel regions.12482 // ((struct.lastprivate.conditional*)&priv_a)->Fired = 1;12483 auto It = LastprivateConditionalToTypes[FoundFn].find(FoundD);12484 assert(It != LastprivateConditionalToTypes[FoundFn].end() &&12485 "Lastprivate conditional is not found in outer region.");12486 QualType StructTy = std::get<0>(It->getSecond());12487 const FieldDecl* FiredDecl = std::get<2>(It->getSecond());12488 LValue PrivLVal = CGF.EmitLValue(FoundE);12489 Address StructAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(12490 PrivLVal.getAddress(),12491 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(StructTy)),12492 CGF.ConvertTypeForMem(StructTy));12493 LValue BaseLVal =12494 CGF.MakeAddrLValue(StructAddr, StructTy, AlignmentSource::Decl);12495 LValue FiredLVal = CGF.EmitLValueForField(BaseLVal, FiredDecl);12496 CGF.EmitAtomicStore(RValue::get(llvm::ConstantInt::get(12497 CGF.ConvertTypeForMem(FiredDecl->getType()), 1)),12498 FiredLVal, llvm::AtomicOrdering::Unordered,12499 /*IsVolatile=*/true, /*isInit=*/false);12500 return;12501 }12502 12503 // Private address of the lastprivate conditional in the current context.12504 // priv_a12505 LValue LVal = CGF.EmitLValue(FoundE);12506 emitLastprivateConditionalUpdate(CGF, IVLVal, UniqueDeclName, LVal,12507 FoundE->getExprLoc());12508}12509 12510void CGOpenMPRuntime::checkAndEmitSharedLastprivateConditional(12511 CodeGenFunction &CGF, const OMPExecutableDirective &D,12512 const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls) {12513 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())12514 return;12515 auto Range = llvm::reverse(LastprivateConditionalStack);12516 auto It = llvm::find_if(12517 Range, [](const LastprivateConditionalData &D) { return !D.Disabled; });12518 if (It == Range.end() || It->Fn != CGF.CurFn)12519 return;12520 auto LPCI = LastprivateConditionalToTypes.find(It->Fn);12521 assert(LPCI != LastprivateConditionalToTypes.end() &&12522 "Lastprivates must be registered already.");12523 SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;12524 getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());12525 const CapturedStmt *CS = D.getCapturedStmt(CaptureRegions.back());12526 for (const auto &Pair : It->DeclToUniqueName) {12527 const auto *VD = cast<VarDecl>(Pair.first->getCanonicalDecl());12528 if (!CS->capturesVariable(VD) || IgnoredDecls.contains(VD))12529 continue;12530 auto I = LPCI->getSecond().find(Pair.first);12531 assert(I != LPCI->getSecond().end() &&12532 "Lastprivate must be rehistered already.");12533 // bool Cmp = priv_a.Fired != 0;12534 LValue BaseLVal = std::get<3>(I->getSecond());12535 LValue FiredLVal =12536 CGF.EmitLValueForField(BaseLVal, std::get<2>(I->getSecond()));12537 llvm::Value *Res = CGF.EmitLoadOfScalar(FiredLVal, D.getBeginLoc());12538 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Res);12539 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lpc.then");12540 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("lpc.done");12541 // if (Cmp) {12542 CGF.Builder.CreateCondBr(Cmp, ThenBB, DoneBB);12543 CGF.EmitBlock(ThenBB);12544 Address Addr = CGF.GetAddrOfLocalVar(VD);12545 LValue LVal;12546 if (VD->getType()->isReferenceType())12547 LVal = CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),12548 AlignmentSource::Decl);12549 else12550 LVal = CGF.MakeAddrLValue(Addr, VD->getType().getNonReferenceType(),12551 AlignmentSource::Decl);12552 emitLastprivateConditionalUpdate(CGF, It->IVLVal, Pair.second, LVal,12553 D.getBeginLoc());12554 auto AL = ApplyDebugLocation::CreateArtificial(CGF);12555 CGF.EmitBlock(DoneBB, /*IsFinal=*/true);12556 // }12557 }12558}12559 12560void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate(12561 CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD,12562 SourceLocation Loc) {12563 if (CGF.getLangOpts().OpenMP < 50)12564 return;12565 auto It = LastprivateConditionalStack.back().DeclToUniqueName.find(VD);12566 assert(It != LastprivateConditionalStack.back().DeclToUniqueName.end() &&12567 "Unknown lastprivate conditional variable.");12568 StringRef UniqueName = It->second;12569 llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(UniqueName);12570 // The variable was not updated in the region - exit.12571 if (!GV)12572 return;12573 LValue LPLVal = CGF.MakeRawAddrLValue(12574 GV, PrivLVal.getType().getNonReferenceType(), PrivLVal.getAlignment());12575 llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc);12576 CGF.EmitStoreOfScalar(Res, PrivLVal);12577}12578 12579llvm::Function *CGOpenMPSIMDRuntime::emitParallelOutlinedFunction(12580 CodeGenFunction &CGF, const OMPExecutableDirective &D,12581 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,12582 const RegionCodeGenTy &CodeGen) {12583 llvm_unreachable("Not supported in SIMD-only mode");12584}12585 12586llvm::Function *CGOpenMPSIMDRuntime::emitTeamsOutlinedFunction(12587 CodeGenFunction &CGF, const OMPExecutableDirective &D,12588 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,12589 const RegionCodeGenTy &CodeGen) {12590 llvm_unreachable("Not supported in SIMD-only mode");12591}12592 12593llvm::Function *CGOpenMPSIMDRuntime::emitTaskOutlinedFunction(12594 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,12595 const VarDecl *PartIDVar, const VarDecl *TaskTVar,12596 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,12597 bool Tied, unsigned &NumberOfParts) {12598 llvm_unreachable("Not supported in SIMD-only mode");12599}12600 12601void CGOpenMPSIMDRuntime::emitParallelCall(12602 CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn,12603 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,12604 llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier,12605 OpenMPSeverityClauseKind Severity, const Expr *Message) {12606 llvm_unreachable("Not supported in SIMD-only mode");12607}12608 12609void CGOpenMPSIMDRuntime::emitCriticalRegion(12610 CodeGenFunction &CGF, StringRef CriticalName,12611 const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc,12612 const Expr *Hint) {12613 llvm_unreachable("Not supported in SIMD-only mode");12614}12615 12616void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF,12617 const RegionCodeGenTy &MasterOpGen,12618 SourceLocation Loc) {12619 llvm_unreachable("Not supported in SIMD-only mode");12620}12621 12622void CGOpenMPSIMDRuntime::emitMaskedRegion(CodeGenFunction &CGF,12623 const RegionCodeGenTy &MasterOpGen,12624 SourceLocation Loc,12625 const Expr *Filter) {12626 llvm_unreachable("Not supported in SIMD-only mode");12627}12628 12629void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF,12630 SourceLocation Loc) {12631 llvm_unreachable("Not supported in SIMD-only mode");12632}12633 12634void CGOpenMPSIMDRuntime::emitTaskgroupRegion(12635 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen,12636 SourceLocation Loc) {12637 llvm_unreachable("Not supported in SIMD-only mode");12638}12639 12640void CGOpenMPSIMDRuntime::emitSingleRegion(12641 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen,12642 SourceLocation Loc, ArrayRef<const Expr *> CopyprivateVars,12643 ArrayRef<const Expr *> DestExprs, ArrayRef<const Expr *> SrcExprs,12644 ArrayRef<const Expr *> AssignmentOps) {12645 llvm_unreachable("Not supported in SIMD-only mode");12646}12647 12648void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF,12649 const RegionCodeGenTy &OrderedOpGen,12650 SourceLocation Loc,12651 bool IsThreads) {12652 llvm_unreachable("Not supported in SIMD-only mode");12653}12654 12655void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF,12656 SourceLocation Loc,12657 OpenMPDirectiveKind Kind,12658 bool EmitChecks,12659 bool ForceSimpleCall) {12660 llvm_unreachable("Not supported in SIMD-only mode");12661}12662 12663void CGOpenMPSIMDRuntime::emitForDispatchInit(12664 CodeGenFunction &CGF, SourceLocation Loc,12665 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,12666 bool Ordered, const DispatchRTInput &DispatchValues) {12667 llvm_unreachable("Not supported in SIMD-only mode");12668}12669 12670void CGOpenMPSIMDRuntime::emitForDispatchDeinit(CodeGenFunction &CGF,12671 SourceLocation Loc) {12672 llvm_unreachable("Not supported in SIMD-only mode");12673}12674 12675void CGOpenMPSIMDRuntime::emitForStaticInit(12676 CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind,12677 const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) {12678 llvm_unreachable("Not supported in SIMD-only mode");12679}12680 12681void CGOpenMPSIMDRuntime::emitDistributeStaticInit(12682 CodeGenFunction &CGF, SourceLocation Loc,12683 OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) {12684 llvm_unreachable("Not supported in SIMD-only mode");12685}12686 12687void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,12688 SourceLocation Loc,12689 unsigned IVSize,12690 bool IVSigned) {12691 llvm_unreachable("Not supported in SIMD-only mode");12692}12693 12694void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF,12695 SourceLocation Loc,12696 OpenMPDirectiveKind DKind) {12697 llvm_unreachable("Not supported in SIMD-only mode");12698}12699 12700llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF,12701 SourceLocation Loc,12702 unsigned IVSize, bool IVSigned,12703 Address IL, Address LB,12704 Address UB, Address ST) {12705 llvm_unreachable("Not supported in SIMD-only mode");12706}12707 12708void CGOpenMPSIMDRuntime::emitNumThreadsClause(12709 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,12710 OpenMPNumThreadsClauseModifier Modifier, OpenMPSeverityClauseKind Severity,12711 SourceLocation SeverityLoc, const Expr *Message,12712 SourceLocation MessageLoc) {12713 llvm_unreachable("Not supported in SIMD-only mode");12714}12715 12716void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF,12717 ProcBindKind ProcBind,12718 SourceLocation Loc) {12719 llvm_unreachable("Not supported in SIMD-only mode");12720}12721 12722Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,12723 const VarDecl *VD,12724 Address VDAddr,12725 SourceLocation Loc) {12726 llvm_unreachable("Not supported in SIMD-only mode");12727}12728 12729llvm::Function *CGOpenMPSIMDRuntime::emitThreadPrivateVarDefinition(12730 const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit,12731 CodeGenFunction *CGF) {12732 llvm_unreachable("Not supported in SIMD-only mode");12733}12734 12735Address CGOpenMPSIMDRuntime::getAddrOfArtificialThreadPrivate(12736 CodeGenFunction &CGF, QualType VarType, StringRef Name) {12737 llvm_unreachable("Not supported in SIMD-only mode");12738}12739 12740void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF,12741 ArrayRef<const Expr *> Vars,12742 SourceLocation Loc,12743 llvm::AtomicOrdering AO) {12744 llvm_unreachable("Not supported in SIMD-only mode");12745}12746 12747void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,12748 const OMPExecutableDirective &D,12749 llvm::Function *TaskFunction,12750 QualType SharedsTy, Address Shareds,12751 const Expr *IfCond,12752 const OMPTaskDataTy &Data) {12753 llvm_unreachable("Not supported in SIMD-only mode");12754}12755 12756void CGOpenMPSIMDRuntime::emitTaskLoopCall(12757 CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D,12758 llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,12759 const Expr *IfCond, const OMPTaskDataTy &Data) {12760 llvm_unreachable("Not supported in SIMD-only mode");12761}12762 12763void CGOpenMPSIMDRuntime::emitReduction(12764 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates,12765 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,12766 ArrayRef<const Expr *> ReductionOps, ReductionOptionsTy Options) {12767 assert(Options.SimpleReduction && "Only simple reduction is expected.");12768 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs,12769 ReductionOps, Options);12770}12771 12772llvm::Value *CGOpenMPSIMDRuntime::emitTaskReductionInit(12773 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,12774 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {12775 llvm_unreachable("Not supported in SIMD-only mode");12776}12777 12778void CGOpenMPSIMDRuntime::emitTaskReductionFini(CodeGenFunction &CGF,12779 SourceLocation Loc,12780 bool IsWorksharingReduction) {12781 llvm_unreachable("Not supported in SIMD-only mode");12782}12783 12784void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,12785 SourceLocation Loc,12786 ReductionCodeGen &RCG,12787 unsigned N) {12788 llvm_unreachable("Not supported in SIMD-only mode");12789}12790 12791Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF,12792 SourceLocation Loc,12793 llvm::Value *ReductionsPtr,12794 LValue SharedLVal) {12795 llvm_unreachable("Not supported in SIMD-only mode");12796}12797 12798void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF,12799 SourceLocation Loc,12800 const OMPTaskDataTy &Data) {12801 llvm_unreachable("Not supported in SIMD-only mode");12802}12803 12804void CGOpenMPSIMDRuntime::emitCancellationPointCall(12805 CodeGenFunction &CGF, SourceLocation Loc,12806 OpenMPDirectiveKind CancelRegion) {12807 llvm_unreachable("Not supported in SIMD-only mode");12808}12809 12810void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF,12811 SourceLocation Loc, const Expr *IfCond,12812 OpenMPDirectiveKind CancelRegion) {12813 llvm_unreachable("Not supported in SIMD-only mode");12814}12815 12816void CGOpenMPSIMDRuntime::emitTargetOutlinedFunction(12817 const OMPExecutableDirective &D, StringRef ParentName,12818 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,12819 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {12820 llvm_unreachable("Not supported in SIMD-only mode");12821}12822 12823void CGOpenMPSIMDRuntime::emitTargetCall(12824 CodeGenFunction &CGF, const OMPExecutableDirective &D,12825 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,12826 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,12827 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,12828 const OMPLoopDirective &D)>12829 SizeEmitter) {12830 llvm_unreachable("Not supported in SIMD-only mode");12831}12832 12833bool CGOpenMPSIMDRuntime::emitTargetFunctions(GlobalDecl GD) {12834 llvm_unreachable("Not supported in SIMD-only mode");12835}12836 12837bool CGOpenMPSIMDRuntime::emitTargetGlobalVariable(GlobalDecl GD) {12838 llvm_unreachable("Not supported in SIMD-only mode");12839}12840 12841bool CGOpenMPSIMDRuntime::emitTargetGlobal(GlobalDecl GD) {12842 return false;12843}12844 12845void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF,12846 const OMPExecutableDirective &D,12847 SourceLocation Loc,12848 llvm::Function *OutlinedFn,12849 ArrayRef<llvm::Value *> CapturedVars) {12850 llvm_unreachable("Not supported in SIMD-only mode");12851}12852 12853void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF,12854 const Expr *NumTeams,12855 const Expr *ThreadLimit,12856 SourceLocation Loc) {12857 llvm_unreachable("Not supported in SIMD-only mode");12858}12859 12860void CGOpenMPSIMDRuntime::emitTargetDataCalls(12861 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,12862 const Expr *Device, const RegionCodeGenTy &CodeGen,12863 CGOpenMPRuntime::TargetDataInfo &Info) {12864 llvm_unreachable("Not supported in SIMD-only mode");12865}12866 12867void CGOpenMPSIMDRuntime::emitTargetDataStandAloneCall(12868 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,12869 const Expr *Device) {12870 llvm_unreachable("Not supported in SIMD-only mode");12871}12872 12873void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF,12874 const OMPLoopDirective &D,12875 ArrayRef<Expr *> NumIterations) {12876 llvm_unreachable("Not supported in SIMD-only mode");12877}12878 12879void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,12880 const OMPDependClause *C) {12881 llvm_unreachable("Not supported in SIMD-only mode");12882}12883 12884void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,12885 const OMPDoacrossClause *C) {12886 llvm_unreachable("Not supported in SIMD-only mode");12887}12888 12889const VarDecl *12890CGOpenMPSIMDRuntime::translateParameter(const FieldDecl *FD,12891 const VarDecl *NativeParam) const {12892 llvm_unreachable("Not supported in SIMD-only mode");12893}12894 12895Address12896CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF,12897 const VarDecl *NativeParam,12898 const VarDecl *TargetParam) const {12899 llvm_unreachable("Not supported in SIMD-only mode");12900}12901