2370 lines · c
1//===----- CGOpenMPRuntime.h - Interface to OpenMP Runtimes -----*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This provides a class for OpenMP runtime code generation.10//11//===----------------------------------------------------------------------===//12 13#ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H14#define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H15 16#include "CGValue.h"17#include "clang/AST/DeclOpenMP.h"18#include "clang/AST/GlobalDecl.h"19#include "clang/AST/Type.h"20#include "clang/Basic/OpenMPKinds.h"21#include "clang/Basic/SourceLocation.h"22#include "llvm/ADT/DenseMap.h"23#include "llvm/ADT/PointerIntPair.h"24#include "llvm/ADT/SmallPtrSet.h"25#include "llvm/ADT/StringMap.h"26#include "llvm/ADT/StringSet.h"27#include "llvm/Frontend/OpenMP/OMPConstants.h"28#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"29#include "llvm/IR/Function.h"30#include "llvm/IR/ValueHandle.h"31#include "llvm/Support/AtomicOrdering.h"32 33namespace llvm {34class ArrayType;35class Constant;36class FunctionType;37class GlobalVariable;38class Type;39class Value;40class OpenMPIRBuilder;41} // namespace llvm42 43namespace clang {44class Expr;45class OMPDependClause;46class OMPExecutableDirective;47class OMPLoopDirective;48class VarDecl;49class OMPDeclareReductionDecl;50 51namespace CodeGen {52class Address;53class CodeGenFunction;54class CodeGenModule;55 56/// A basic class for pre|post-action for advanced codegen sequence for OpenMP57/// region.58class PrePostActionTy {59public:60 explicit PrePostActionTy() {}61 virtual void Enter(CodeGenFunction &CGF) {}62 virtual void Exit(CodeGenFunction &CGF) {}63 virtual ~PrePostActionTy() {}64};65 66/// Class provides a way to call simple version of codegen for OpenMP region, or67/// an advanced with possible pre|post-actions in codegen.68class RegionCodeGenTy final {69 intptr_t CodeGen;70 typedef void (*CodeGenTy)(intptr_t, CodeGenFunction &, PrePostActionTy &);71 CodeGenTy Callback;72 mutable PrePostActionTy *PrePostAction;73 RegionCodeGenTy() = delete;74 template <typename Callable>75 static void CallbackFn(intptr_t CodeGen, CodeGenFunction &CGF,76 PrePostActionTy &Action) {77 return (*reinterpret_cast<Callable *>(CodeGen))(CGF, Action);78 }79 80public:81 template <typename Callable>82 RegionCodeGenTy(83 Callable &&CodeGen,84 std::enable_if_t<!std::is_same<std::remove_reference_t<Callable>,85 RegionCodeGenTy>::value> * = nullptr)86 : CodeGen(reinterpret_cast<intptr_t>(&CodeGen)),87 Callback(CallbackFn<std::remove_reference_t<Callable>>),88 PrePostAction(nullptr) {}89 void setAction(PrePostActionTy &Action) const { PrePostAction = &Action; }90 void operator()(CodeGenFunction &CGF) const;91};92 93struct OMPTaskDataTy final {94 SmallVector<const Expr *, 4> PrivateVars;95 SmallVector<const Expr *, 4> PrivateCopies;96 SmallVector<const Expr *, 4> FirstprivateVars;97 SmallVector<const Expr *, 4> FirstprivateCopies;98 SmallVector<const Expr *, 4> FirstprivateInits;99 SmallVector<const Expr *, 4> LastprivateVars;100 SmallVector<const Expr *, 4> LastprivateCopies;101 SmallVector<const Expr *, 4> ReductionVars;102 SmallVector<const Expr *, 4> ReductionOrigs;103 SmallVector<const Expr *, 4> ReductionCopies;104 SmallVector<const Expr *, 4> ReductionOps;105 SmallVector<CanonicalDeclPtr<const VarDecl>, 4> PrivateLocals;106 struct DependData {107 OpenMPDependClauseKind DepKind = OMPC_DEPEND_unknown;108 const Expr *IteratorExpr = nullptr;109 SmallVector<const Expr *, 4> DepExprs;110 explicit DependData() = default;111 DependData(OpenMPDependClauseKind DepKind, const Expr *IteratorExpr)112 : DepKind(DepKind), IteratorExpr(IteratorExpr) {}113 };114 SmallVector<DependData, 4> Dependences;115 llvm::PointerIntPair<llvm::Value *, 1, bool> Final;116 llvm::PointerIntPair<llvm::Value *, 1, bool> Schedule;117 llvm::PointerIntPair<llvm::Value *, 1, bool> Priority;118 llvm::Value *Reductions = nullptr;119 unsigned NumberOfParts = 0;120 bool Tied = true;121 bool Nogroup = false;122 bool IsReductionWithTaskMod = false;123 bool IsWorksharingReduction = false;124 bool HasNowaitClause = false;125 bool HasModifier = false;126};127 128/// Class intended to support codegen of all kind of the reduction clauses.129class ReductionCodeGen {130private:131 /// Data required for codegen of reduction clauses.132 struct ReductionData {133 /// Reference to the item shared between tasks to reduce into.134 const Expr *Shared = nullptr;135 /// Reference to the original item.136 const Expr *Ref = nullptr;137 /// Helper expression for generation of private copy.138 const Expr *Private = nullptr;139 /// Helper expression for generation reduction operation.140 const Expr *ReductionOp = nullptr;141 ReductionData(const Expr *Shared, const Expr *Ref, const Expr *Private,142 const Expr *ReductionOp)143 : Shared(Shared), Ref(Ref), Private(Private), ReductionOp(ReductionOp) {144 }145 };146 /// List of reduction-based clauses.147 SmallVector<ReductionData, 4> ClausesData;148 149 /// List of addresses of shared variables/expressions.150 SmallVector<std::pair<LValue, LValue>, 4> SharedAddresses;151 /// List of addresses of original variables/expressions.152 SmallVector<std::pair<LValue, LValue>, 4> OrigAddresses;153 /// Sizes of the reduction items in chars.154 SmallVector<std::pair<llvm::Value *, llvm::Value *>, 4> Sizes;155 /// Base declarations for the reduction items.156 SmallVector<const VarDecl *, 4> BaseDecls;157 158 /// Emits lvalue for shared expression.159 LValue emitSharedLValue(CodeGenFunction &CGF, const Expr *E);160 /// Emits upper bound for shared expression (if array section).161 LValue emitSharedLValueUB(CodeGenFunction &CGF, const Expr *E);162 /// Performs aggregate initialization.163 /// \param N Number of reduction item in the common list.164 /// \param PrivateAddr Address of the corresponding private item.165 /// \param SharedAddr Address of the original shared variable.166 /// \param DRD Declare reduction construct used for reduction item.167 void emitAggregateInitialization(CodeGenFunction &CGF, unsigned N,168 Address PrivateAddr, Address SharedAddr,169 const OMPDeclareReductionDecl *DRD);170 171public:172 ReductionCodeGen(ArrayRef<const Expr *> Shareds, ArrayRef<const Expr *> Origs,173 ArrayRef<const Expr *> Privates,174 ArrayRef<const Expr *> ReductionOps);175 /// Emits lvalue for the shared and original reduction item.176 /// \param N Number of the reduction item.177 void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N);178 /// Emits the code for the variable-modified type, if required.179 /// \param N Number of the reduction item.180 void emitAggregateType(CodeGenFunction &CGF, unsigned N);181 /// Emits the code for the variable-modified type, if required.182 /// \param N Number of the reduction item.183 /// \param Size Size of the type in chars.184 void emitAggregateType(CodeGenFunction &CGF, unsigned N, llvm::Value *Size);185 /// Performs initialization of the private copy for the reduction item.186 /// \param N Number of the reduction item.187 /// \param PrivateAddr Address of the corresponding private item.188 /// \param DefaultInit Default initialization sequence that should be189 /// performed if no reduction specific initialization is found.190 /// \param SharedAddr Address of the original shared variable.191 void192 emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr,193 Address SharedAddr,194 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit);195 /// Returns true if the private copy requires cleanups.196 bool needCleanups(unsigned N);197 /// Emits cleanup code for the reduction item.198 /// \param N Number of the reduction item.199 /// \param PrivateAddr Address of the corresponding private item.200 void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr);201 /// Adjusts \p PrivatedAddr for using instead of the original variable202 /// address in normal operations.203 /// \param N Number of the reduction item.204 /// \param PrivateAddr Address of the corresponding private item.205 Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,206 Address PrivateAddr);207 /// Returns LValue for the reduction item.208 LValue getSharedLValue(unsigned N) const { return SharedAddresses[N].first; }209 /// Returns LValue for the original reduction item.210 LValue getOrigLValue(unsigned N) const { return OrigAddresses[N].first; }211 /// Returns the size of the reduction item (in chars and total number of212 /// elements in the item), or nullptr, if the size is a constant.213 std::pair<llvm::Value *, llvm::Value *> getSizes(unsigned N) const {214 return Sizes[N];215 }216 /// Returns the base declaration of the reduction item.217 const VarDecl *getBaseDecl(unsigned N) const { return BaseDecls[N]; }218 /// Returns the base declaration of the reduction item.219 const Expr *getRefExpr(unsigned N) const { return ClausesData[N].Ref; }220 /// Returns true if the initialization of the reduction item uses initializer221 /// from declare reduction construct.222 bool usesReductionInitializer(unsigned N) const;223 /// Return the type of the private item.224 QualType getPrivateType(unsigned N) const {225 return cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl())226 ->getType();227 }228};229 230class CGOpenMPRuntime {231public:232 /// Allows to disable automatic handling of functions used in target regions233 /// as those marked as `omp declare target`.234 class DisableAutoDeclareTargetRAII {235 CodeGenModule &CGM;236 bool SavedShouldMarkAsGlobal = false;237 238 public:239 DisableAutoDeclareTargetRAII(CodeGenModule &CGM);240 ~DisableAutoDeclareTargetRAII();241 };242 243 /// Manages list of nontemporal decls for the specified directive.244 class NontemporalDeclsRAII {245 CodeGenModule &CGM;246 const bool NeedToPush;247 248 public:249 NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S);250 ~NontemporalDeclsRAII();251 };252 253 /// Manages list of nontemporal decls for the specified directive.254 class UntiedTaskLocalDeclsRAII {255 CodeGenModule &CGM;256 const bool NeedToPush;257 258 public:259 UntiedTaskLocalDeclsRAII(260 CodeGenFunction &CGF,261 const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,262 std::pair<Address, Address>> &LocalVars);263 ~UntiedTaskLocalDeclsRAII();264 };265 266 /// Maps the expression for the lastprivate variable to the global copy used267 /// to store new value because original variables are not mapped in inner268 /// parallel regions. Only private copies are captured but we need also to269 /// store private copy in shared address.270 /// Also, stores the expression for the private loop counter and it271 /// threaprivate name.272 struct LastprivateConditionalData {273 llvm::MapVector<CanonicalDeclPtr<const Decl>, SmallString<16>>274 DeclToUniqueName;275 LValue IVLVal;276 llvm::Function *Fn = nullptr;277 bool Disabled = false;278 };279 /// Manages list of lastprivate conditional decls for the specified directive.280 class LastprivateConditionalRAII {281 enum class ActionToDo {282 DoNotPush,283 PushAsLastprivateConditional,284 DisableLastprivateConditional,285 };286 CodeGenModule &CGM;287 ActionToDo Action = ActionToDo::DoNotPush;288 289 /// Check and try to disable analysis of inner regions for changes in290 /// lastprivate conditional.291 void tryToDisableInnerAnalysis(const OMPExecutableDirective &S,292 llvm::DenseSet<CanonicalDeclPtr<const Decl>>293 &NeedToAddForLPCsAsDisabled) const;294 295 LastprivateConditionalRAII(CodeGenFunction &CGF,296 const OMPExecutableDirective &S);297 298 public:299 explicit LastprivateConditionalRAII(CodeGenFunction &CGF,300 const OMPExecutableDirective &S,301 LValue IVLVal);302 static LastprivateConditionalRAII disable(CodeGenFunction &CGF,303 const OMPExecutableDirective &S);304 ~LastprivateConditionalRAII();305 };306 307 llvm::OpenMPIRBuilder &getOMPBuilder() { return OMPBuilder; }308 309protected:310 CodeGenModule &CGM;311 312 /// An OpenMP-IR-Builder instance.313 llvm::OpenMPIRBuilder OMPBuilder;314 315 /// Helper to determine the min/max number of threads/teams for \p D.316 void computeMinAndMaxThreadsAndTeams(317 const OMPExecutableDirective &D, CodeGenFunction &CGF,318 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs);319 320 /// Helper to emit outlined function for 'target' directive.321 /// \param D Directive to emit.322 /// \param ParentName Name of the function that encloses the target region.323 /// \param OutlinedFn Outlined function value to be defined by this call.324 /// \param OutlinedFnID Outlined function ID value to be defined by this call.325 /// \param IsOffloadEntry True if the outlined function is an offload entry.326 /// \param CodeGen Lambda codegen specific to an accelerator device.327 /// An outlined function may not be an entry if, e.g. the if clause always328 /// evaluates to false.329 virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D,330 StringRef ParentName,331 llvm::Function *&OutlinedFn,332 llvm::Constant *&OutlinedFnID,333 bool IsOffloadEntry,334 const RegionCodeGenTy &CodeGen);335 336 /// Returns pointer to ident_t type.337 llvm::Type *getIdentTyPointerTy();338 339 /// Gets thread id value for the current thread.340 ///341 llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc);342 343 /// Get the function name of an outlined region.344 std::string getOutlinedHelperName(StringRef Name) const;345 std::string getOutlinedHelperName(CodeGenFunction &CGF) const;346 347 /// Get the function name of a reduction function.348 std::string getReductionFuncName(StringRef Name) const;349 350 /// Emits \p Callee function call with arguments \p Args with location \p Loc.351 void emitCall(CodeGenFunction &CGF, SourceLocation Loc,352 llvm::FunctionCallee Callee,353 ArrayRef<llvm::Value *> Args = {}) const;354 355 /// Emits address of the word in a memory where current thread id is356 /// stored.357 virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc);358 359 void setLocThreadIdInsertPt(CodeGenFunction &CGF,360 bool AtCurrentPoint = false);361 void clearLocThreadIdInsertPt(CodeGenFunction &CGF);362 363 /// Check if the default location must be constant.364 /// Default is false to support OMPT/OMPD.365 virtual bool isDefaultLocationConstant() const { return false; }366 367 /// Returns additional flags that can be stored in reserved_2 field of the368 /// default location.369 virtual unsigned getDefaultLocationReserved2Flags() const { return 0; }370 371 /// Returns default flags for the barriers depending on the directive, for372 /// which this barier is going to be emitted.373 static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind);374 375 /// Get the LLVM type for the critical name.376 llvm::ArrayType *getKmpCriticalNameTy() const {return KmpCriticalNameTy;}377 378 /// Returns corresponding lock object for the specified critical region379 /// name. If the lock object does not exist it is created, otherwise the380 /// reference to the existing copy is returned.381 /// \param CriticalName Name of the critical region.382 ///383 llvm::Value *getCriticalRegionLock(StringRef CriticalName);384 385protected:386 /// Map for SourceLocation and OpenMP runtime library debug locations.387 typedef llvm::DenseMap<SourceLocation, llvm::Value *> OpenMPDebugLocMapTy;388 OpenMPDebugLocMapTy OpenMPDebugLocMap;389 /// Stores debug location and ThreadID for the function.390 struct DebugLocThreadIdTy {391 llvm::Value *DebugLoc;392 llvm::Value *ThreadID;393 /// Insert point for the service instructions.394 llvm::AssertingVH<llvm::Instruction> ServiceInsertPt = nullptr;395 };396 /// Map of local debug location, ThreadId and functions.397 typedef llvm::DenseMap<llvm::Function *, DebugLocThreadIdTy>398 OpenMPLocThreadIDMapTy;399 OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap;400 /// Map of UDRs and corresponding combiner/initializer.401 typedef llvm::DenseMap<const OMPDeclareReductionDecl *,402 std::pair<llvm::Function *, llvm::Function *>>403 UDRMapTy;404 UDRMapTy UDRMap;405 /// Map of functions and locally defined UDRs.406 typedef llvm::DenseMap<llvm::Function *,407 SmallVector<const OMPDeclareReductionDecl *, 4>>408 FunctionUDRMapTy;409 FunctionUDRMapTy FunctionUDRMap;410 /// Map from the user-defined mapper declaration to its corresponding411 /// functions.412 llvm::DenseMap<const OMPDeclareMapperDecl *, llvm::Function *> UDMMap;413 /// Map of functions and their local user-defined mappers.414 using FunctionUDMMapTy =415 llvm::DenseMap<llvm::Function *,416 SmallVector<const OMPDeclareMapperDecl *, 4>>;417 FunctionUDMMapTy FunctionUDMMap;418 /// Maps local variables marked as lastprivate conditional to their internal419 /// types.420 llvm::DenseMap<llvm::Function *,421 llvm::DenseMap<CanonicalDeclPtr<const Decl>,422 std::tuple<QualType, const FieldDecl *,423 const FieldDecl *, LValue>>>424 LastprivateConditionalToTypes;425 /// Maps function to the position of the untied task locals stack.426 llvm::DenseMap<llvm::Function *, unsigned> FunctionToUntiedTaskStackMap;427 /// Type kmp_critical_name, originally defined as typedef kmp_int32428 /// kmp_critical_name[8];429 llvm::ArrayType *KmpCriticalNameTy;430 /// An ordered map of auto-generated variables to their unique names.431 /// It stores variables with the following names: 1) ".gomp_critical_user_" +432 /// <critical_section_name> + ".var" for "omp critical" directives; 2)433 /// <mangled_name_for_global_var> + ".cache." for cache for threadprivate434 /// variables.435 llvm::StringMap<llvm::AssertingVH<llvm::GlobalVariable>,436 llvm::BumpPtrAllocator> InternalVars;437 /// Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);438 llvm::Type *KmpRoutineEntryPtrTy = nullptr;439 QualType KmpRoutineEntryPtrQTy;440 /// Type typedef struct kmp_task {441 /// void * shareds; /**< pointer to block of pointers to442 /// shared vars */443 /// kmp_routine_entry_t routine; /**< pointer to routine to call for444 /// executing task */445 /// kmp_int32 part_id; /**< part id for the task */446 /// kmp_routine_entry_t destructors; /* pointer to function to invoke447 /// deconstructors of firstprivate C++ objects */448 /// } kmp_task_t;449 QualType KmpTaskTQTy;450 /// Saved kmp_task_t for task directive.451 QualType SavedKmpTaskTQTy;452 /// Saved kmp_task_t for taskloop-based directive.453 QualType SavedKmpTaskloopTQTy;454 /// Type typedef struct kmp_depend_info {455 /// kmp_intptr_t base_addr;456 /// size_t len;457 /// struct {458 /// bool in:1;459 /// bool out:1;460 /// } flags;461 /// } kmp_depend_info_t;462 QualType KmpDependInfoTy;463 /// Type typedef struct kmp_task_affinity_info {464 /// kmp_intptr_t base_addr;465 /// size_t len;466 /// struct {467 /// bool flag1 : 1;468 /// bool flag2 : 1;469 /// kmp_int32 reserved : 30;470 /// } flags;471 /// } kmp_task_affinity_info_t;472 QualType KmpTaskAffinityInfoTy;473 /// struct kmp_dim { // loop bounds info casted to kmp_int64474 /// kmp_int64 lo; // lower475 /// kmp_int64 up; // upper476 /// kmp_int64 st; // stride477 /// };478 QualType KmpDimTy;479 480 bool ShouldMarkAsGlobal = true;481 /// List of the emitted declarations.482 llvm::DenseSet<CanonicalDeclPtr<const Decl>> AlreadyEmittedTargetDecls;483 /// List of the global variables with their addresses that should not be484 /// emitted for the target.485 llvm::StringMap<llvm::WeakTrackingVH> EmittedNonTargetVariables;486 487 /// List of variables that can become declare target implicitly and, thus,488 /// must be emitted.489 llvm::SmallDenseSet<const VarDecl *> DeferredGlobalVariables;490 491 using NontemporalDeclsSet = llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>>;492 /// Stack for list of declarations in current context marked as nontemporal.493 /// The set is the union of all current stack elements.494 llvm::SmallVector<NontemporalDeclsSet, 4> NontemporalDeclsStack;495 496 using UntiedLocalVarsAddressesMap =497 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,498 std::pair<Address, Address>>;499 llvm::SmallVector<UntiedLocalVarsAddressesMap, 4> UntiedLocalVarsStack;500 501 /// Stack for list of addresses of declarations in current context marked as502 /// lastprivate conditional. The set is the union of all current stack503 /// elements.504 llvm::SmallVector<LastprivateConditionalData, 4> LastprivateConditionalStack;505 506 /// Flag for keeping track of weather a requires unified_shared_memory507 /// directive is present.508 bool HasRequiresUnifiedSharedMemory = false;509 510 /// Atomic ordering from the omp requires directive.511 llvm::AtomicOrdering RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;512 513 /// Flag for keeping track of weather a target region has been emitted.514 bool HasEmittedTargetRegion = false;515 516 /// Flag for keeping track of weather a device routine has been emitted.517 /// Device routines are specific to the518 bool HasEmittedDeclareTargetRegion = false;519 520 /// Start scanning from statement \a S and emit all target regions521 /// found along the way.522 /// \param S Starting statement.523 /// \param ParentName Name of the function declaration that is being scanned.524 void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName);525 526 /// Build type kmp_routine_entry_t (if not built yet).527 void emitKmpRoutineEntryT(QualType KmpInt32Ty);528 529 /// If the specified mangled name is not in the module, create and530 /// return threadprivate cache object. This object is a pointer's worth of531 /// storage that's reserved for use by the OpenMP runtime.532 /// \param VD Threadprivate variable.533 /// \return Cache variable for the specified threadprivate.534 llvm::Constant *getOrCreateThreadPrivateCache(const VarDecl *VD);535 536 /// Set of threadprivate variables with the generated initializer.537 llvm::StringSet<> ThreadPrivateWithDefinition;538 539 /// Set of declare target variables with the generated initializer.540 llvm::StringSet<> DeclareTargetWithDefinition;541 542 /// Emits initialization code for the threadprivate variables.543 /// \param VDAddr Address of the global variable \a VD.544 /// \param Ctor Pointer to a global init function for \a VD.545 /// \param CopyCtor Pointer to a global copy function for \a VD.546 /// \param Dtor Pointer to a global destructor function for \a VD.547 /// \param Loc Location of threadprivate declaration.548 void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr,549 llvm::Value *Ctor, llvm::Value *CopyCtor,550 llvm::Value *Dtor, SourceLocation Loc);551 552 struct TaskResultTy {553 llvm::Value *NewTask = nullptr;554 llvm::Function *TaskEntry = nullptr;555 llvm::Value *NewTaskNewTaskTTy = nullptr;556 LValue TDBase;557 const RecordDecl *KmpTaskTQTyRD = nullptr;558 llvm::Value *TaskDupFn = nullptr;559 };560 /// Emit task region for the task directive. The task region is emitted in561 /// several steps:562 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32563 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,564 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the565 /// function:566 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {567 /// TaskFunction(gtid, tt->part_id, tt->shareds);568 /// return 0;569 /// }570 /// 2. Copy a list of shared variables to field shareds of the resulting571 /// structure kmp_task_t returned by the previous call (if any).572 /// 3. Copy a pointer to destructions function to field destructions of the573 /// resulting structure kmp_task_t.574 /// \param D Current task directive.575 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32576 /// /*part_id*/, captured_struct */*__context*/);577 /// \param SharedsTy A type which contains references the shared variables.578 /// \param Shareds Context with the list of shared variables from the \p579 /// TaskFunction.580 /// \param Data Additional data for task generation like tiednsee, final581 /// state, list of privates etc.582 TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,583 const OMPExecutableDirective &D,584 llvm::Function *TaskFunction, QualType SharedsTy,585 Address Shareds, const OMPTaskDataTy &Data);586 587 /// Emit update for lastprivate conditional data.588 void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal,589 StringRef UniqueDeclName, LValue LVal,590 SourceLocation Loc);591 592 /// Returns the number of the elements and the address of the depobj593 /// dependency array.594 /// \return Number of elements in depobj array and the pointer to the array of595 /// dependencies.596 std::pair<llvm::Value *, LValue> getDepobjElements(CodeGenFunction &CGF,597 LValue DepobjLVal,598 SourceLocation Loc);599 600 SmallVector<llvm::Value *, 4>601 emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,602 const OMPTaskDataTy::DependData &Data);603 604 void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,605 LValue PosLVal, const OMPTaskDataTy::DependData &Data,606 Address DependenciesArray);607 608public:609 explicit CGOpenMPRuntime(CodeGenModule &CGM);610 virtual ~CGOpenMPRuntime() {}611 virtual void clear();612 613 /// Emits object of ident_t type with info for source location.614 /// \param Flags Flags for OpenMP location.615 /// \param EmitLoc emit source location with debug-info is off.616 ///617 llvm::Value *emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc,618 unsigned Flags = 0, bool EmitLoc = false);619 620 /// Emit the number of teams for a target directive. Inspect the num_teams621 /// clause associated with a teams construct combined or closely nested622 /// with the target directive.623 ///624 /// Emit a team of size one for directives such as 'target parallel' that625 /// have no associated teams construct.626 ///627 /// Otherwise, return nullptr.628 const Expr *getNumTeamsExprForTargetDirective(CodeGenFunction &CGF,629 const OMPExecutableDirective &D,630 int32_t &MinTeamsVal,631 int32_t &MaxTeamsVal);632 llvm::Value *emitNumTeamsForTargetDirective(CodeGenFunction &CGF,633 const OMPExecutableDirective &D);634 635 /// Check for a number of threads upper bound constant value (stored in \p636 /// UpperBound), or expression (returned). If the value is conditional (via an637 /// if-clause), store the condition in \p CondExpr. Similarly, a potential638 /// thread limit expression is stored in \p ThreadLimitExpr. If \p639 /// UpperBoundOnly is true, no expression evaluation is perfomed.640 const Expr *getNumThreadsExprForTargetDirective(641 CodeGenFunction &CGF, const OMPExecutableDirective &D,642 int32_t &UpperBound, bool UpperBoundOnly,643 llvm::Value **CondExpr = nullptr, const Expr **ThreadLimitExpr = nullptr);644 645 /// Emit an expression that denotes the number of threads a target region646 /// shall use. Will generate "i32 0" to allow the runtime to choose.647 llvm::Value *648 emitNumThreadsForTargetDirective(CodeGenFunction &CGF,649 const OMPExecutableDirective &D);650 651 /// Return the trip count of loops associated with constructs / 'target teams652 /// distribute' and 'teams distribute parallel for'. \param SizeEmitter Emits653 /// the int64 value for the number of iterations of the associated loop.654 llvm::Value *emitTargetNumIterationsCall(655 CodeGenFunction &CGF, const OMPExecutableDirective &D,656 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,657 const OMPLoopDirective &D)>658 SizeEmitter);659 660 /// Returns true if the current target is a GPU.661 virtual bool isGPU() const { return false; }662 663 /// Check if the variable length declaration is delayed:664 virtual bool isDelayedVariableLengthDecl(CodeGenFunction &CGF,665 const VarDecl *VD) const {666 return false;667 };668 669 /// Get call to __kmpc_alloc_shared670 virtual std::pair<llvm::Value *, llvm::Value *>671 getKmpcAllocShared(CodeGenFunction &CGF, const VarDecl *VD) {672 llvm_unreachable("not implemented");673 }674 675 /// Get call to __kmpc_free_shared676 virtual void getKmpcFreeShared(677 CodeGenFunction &CGF,678 const std::pair<llvm::Value *, llvm::Value *> &AddrSizePair) {679 llvm_unreachable("not implemented");680 }681 682 /// Emits code for OpenMP 'if' clause using specified \a CodeGen683 /// function. Here is the logic:684 /// if (Cond) {685 /// ThenGen();686 /// } else {687 /// ElseGen();688 /// }689 void emitIfClause(CodeGenFunction &CGF, const Expr *Cond,690 const RegionCodeGenTy &ThenGen,691 const RegionCodeGenTy &ElseGen);692 693 /// Checks if the \p Body is the \a CompoundStmt and returns its child694 /// statement iff there is only one that is not evaluatable at the compile695 /// time.696 static const Stmt *getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body);697 698 /// Get the platform-specific name separator.699 std::string getName(ArrayRef<StringRef> Parts) const;700 701 /// Emit code for the specified user defined reduction construct.702 virtual void emitUserDefinedReduction(CodeGenFunction *CGF,703 const OMPDeclareReductionDecl *D);704 /// Get combiner/initializer for the specified user-defined reduction, if any.705 virtual std::pair<llvm::Function *, llvm::Function *>706 getUserDefinedReduction(const OMPDeclareReductionDecl *D);707 708 /// Emit the function for the user defined mapper construct.709 void emitUserDefinedMapper(const OMPDeclareMapperDecl *D,710 CodeGenFunction *CGF = nullptr);711 /// Get the function for the specified user-defined mapper. If it does not712 /// exist, create one.713 llvm::Function *714 getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D);715 716 /// Emits outlined function for the specified OpenMP parallel directive717 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,718 /// kmp_int32 BoundID, struct context_vars*).719 /// \param CGF Reference to current CodeGenFunction.720 /// \param D OpenMP directive.721 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.722 /// \param InnermostKind Kind of innermost directive (for simple directives it723 /// is a directive itself, for combined - its innermost directive).724 /// \param CodeGen Code generation sequence for the \a D directive.725 virtual llvm::Function *emitParallelOutlinedFunction(726 CodeGenFunction &CGF, const OMPExecutableDirective &D,727 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,728 const RegionCodeGenTy &CodeGen);729 730 /// Emits outlined function for the specified OpenMP teams directive731 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,732 /// kmp_int32 BoundID, struct context_vars*).733 /// \param CGF Reference to current CodeGenFunction.734 /// \param D OpenMP directive.735 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.736 /// \param InnermostKind Kind of innermost directive (for simple directives it737 /// is a directive itself, for combined - its innermost directive).738 /// \param CodeGen Code generation sequence for the \a D directive.739 virtual llvm::Function *emitTeamsOutlinedFunction(740 CodeGenFunction &CGF, const OMPExecutableDirective &D,741 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,742 const RegionCodeGenTy &CodeGen);743 744 /// Emits outlined function for the OpenMP task directive \a D. This745 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*746 /// TaskT).747 /// \param D OpenMP directive.748 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.749 /// \param PartIDVar Variable for partition id in the current OpenMP untied750 /// task region.751 /// \param TaskTVar Variable for task_t argument.752 /// \param InnermostKind Kind of innermost directive (for simple directives it753 /// is a directive itself, for combined - its innermost directive).754 /// \param CodeGen Code generation sequence for the \a D directive.755 /// \param Tied true if task is generated for tied task, false otherwise.756 /// \param NumberOfParts Number of parts in untied task. Ignored for tied757 /// tasks.758 ///759 virtual llvm::Function *emitTaskOutlinedFunction(760 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,761 const VarDecl *PartIDVar, const VarDecl *TaskTVar,762 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,763 bool Tied, unsigned &NumberOfParts);764 765 /// Cleans up references to the objects in finished function.766 ///767 virtual void functionFinished(CodeGenFunction &CGF);768 769 /// Emits code for parallel or serial call of the \a OutlinedFn with770 /// variables captured in a record which address is stored in \a771 /// CapturedStruct.772 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of773 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).774 /// \param CapturedVars A pointer to the record with the references to775 /// variables used in \a OutlinedFn function.776 /// \param IfCond Condition in the associated 'if' clause, if it was777 /// specified, nullptr otherwise.778 /// \param NumThreads The value corresponding to the num_threads clause, if779 /// any, or nullptr.780 /// \param NumThreadsModifier The modifier of the num_threads clause, if781 /// any, ignored otherwise.782 /// \param Severity The severity corresponding to the num_threads clause, if783 /// any, ignored otherwise.784 /// \param Message The message string corresponding to the num_threads clause,785 /// if any, or nullptr.786 ///787 virtual void788 emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,789 llvm::Function *OutlinedFn,790 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,791 llvm::Value *NumThreads,792 OpenMPNumThreadsClauseModifier NumThreadsModifier =793 OMPC_NUMTHREADS_unknown,794 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,795 const Expr *Message = nullptr);796 797 /// Emits a critical region.798 /// \param CriticalName Name of the critical region.799 /// \param CriticalOpGen Generator for the statement associated with the given800 /// critical region.801 /// \param Hint Value of the 'hint' clause (optional).802 virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,803 const RegionCodeGenTy &CriticalOpGen,804 SourceLocation Loc,805 const Expr *Hint = nullptr);806 807 /// Emits a master region.808 /// \param MasterOpGen Generator for the statement associated with the given809 /// master region.810 virtual void emitMasterRegion(CodeGenFunction &CGF,811 const RegionCodeGenTy &MasterOpGen,812 SourceLocation Loc);813 814 /// Emits a masked region.815 /// \param MaskedOpGen Generator for the statement associated with the given816 /// masked region.817 virtual void emitMaskedRegion(CodeGenFunction &CGF,818 const RegionCodeGenTy &MaskedOpGen,819 SourceLocation Loc,820 const Expr *Filter = nullptr);821 822 /// Emits code for a taskyield directive.823 virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc);824 825 /// Emit __kmpc_error call for error directive826 /// extern void __kmpc_error(ident_t *loc, int severity, const char *message);827 virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME,828 bool IsFatal);829 830 /// Emit a taskgroup region.831 /// \param TaskgroupOpGen Generator for the statement associated with the832 /// given taskgroup region.833 virtual void emitTaskgroupRegion(CodeGenFunction &CGF,834 const RegionCodeGenTy &TaskgroupOpGen,835 SourceLocation Loc);836 837 /// Emits a single region.838 /// \param SingleOpGen Generator for the statement associated with the given839 /// single region.840 virtual void emitSingleRegion(CodeGenFunction &CGF,841 const RegionCodeGenTy &SingleOpGen,842 SourceLocation Loc,843 ArrayRef<const Expr *> CopyprivateVars,844 ArrayRef<const Expr *> DestExprs,845 ArrayRef<const Expr *> SrcExprs,846 ArrayRef<const Expr *> AssignmentOps);847 848 /// Emit an ordered region.849 /// \param OrderedOpGen Generator for the statement associated with the given850 /// ordered region.851 virtual void emitOrderedRegion(CodeGenFunction &CGF,852 const RegionCodeGenTy &OrderedOpGen,853 SourceLocation Loc, bool IsThreads);854 855 /// Emit an implicit/explicit barrier for OpenMP threads.856 /// \param Kind Directive for which this implicit barrier call must be857 /// generated. Must be OMPD_barrier for explicit barrier generation.858 /// \param EmitChecks true if need to emit checks for cancellation barriers.859 /// \param ForceSimpleCall true simple barrier call must be emitted, false if860 /// runtime class decides which one to emit (simple or with cancellation861 /// checks).862 ///863 virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,864 OpenMPDirectiveKind Kind,865 bool EmitChecks = true,866 bool ForceSimpleCall = false);867 868 /// Check if the specified \a ScheduleKind is static non-chunked.869 /// This kind of worksharing directive is emitted without outer loop.870 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause.871 /// \param Chunked True if chunk is specified in the clause.872 ///873 virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,874 bool Chunked) const;875 876 /// Check if the specified \a ScheduleKind is static non-chunked.877 /// This kind of distribute directive is emitted without outer loop.878 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.879 /// \param Chunked True if chunk is specified in the clause.880 ///881 virtual bool isStaticNonchunked(OpenMPDistScheduleClauseKind ScheduleKind,882 bool Chunked) const;883 884 /// Check if the specified \a ScheduleKind is static chunked.885 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause.886 /// \param Chunked True if chunk is specified in the clause.887 ///888 virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,889 bool Chunked) const;890 891 /// Check if the specified \a ScheduleKind is static non-chunked.892 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.893 /// \param Chunked True if chunk is specified in the clause.894 ///895 virtual bool isStaticChunked(OpenMPDistScheduleClauseKind ScheduleKind,896 bool Chunked) const;897 898 /// Check if the specified \a ScheduleKind is dynamic.899 /// This kind of worksharing directive is emitted without outer loop.900 /// \param ScheduleKind Schedule Kind specified in the 'schedule' clause.901 ///902 virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const;903 904 /// struct with the values to be passed to the dispatch runtime function905 struct DispatchRTInput {906 /// Loop lower bound907 llvm::Value *LB = nullptr;908 /// Loop upper bound909 llvm::Value *UB = nullptr;910 /// Chunk size specified using 'schedule' clause (nullptr if chunk911 /// was not specified)912 llvm::Value *Chunk = nullptr;913 DispatchRTInput() = default;914 DispatchRTInput(llvm::Value *LB, llvm::Value *UB, llvm::Value *Chunk)915 : LB(LB), UB(UB), Chunk(Chunk) {}916 };917 918 /// Call the appropriate runtime routine to initialize it before start919 /// of loop.920 921 /// This is used for non static scheduled types and when the ordered922 /// clause is present on the loop construct.923 /// Depending on the loop schedule, it is necessary to call some runtime924 /// routine before start of the OpenMP loop to get the loop upper / lower925 /// bounds \a LB and \a UB and stride \a ST.926 ///927 /// \param CGF Reference to current CodeGenFunction.928 /// \param Loc Clang source location.929 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.930 /// \param IVSize Size of the iteration variable in bits.931 /// \param IVSigned Sign of the iteration variable.932 /// \param Ordered true if loop is ordered, false otherwise.933 /// \param DispatchValues struct containing llvm values for lower bound, upper934 /// bound, and chunk expression.935 /// For the default (nullptr) value, the chunk 1 will be used.936 ///937 virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,938 const OpenMPScheduleTy &ScheduleKind,939 unsigned IVSize, bool IVSigned, bool Ordered,940 const DispatchRTInput &DispatchValues);941 942 /// This is used for non static scheduled types and when the ordered943 /// clause is present on the loop construct.944 ///945 /// \param CGF Reference to current CodeGenFunction.946 /// \param Loc Clang source location.947 ///948 virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc);949 950 /// Struct with the values to be passed to the static runtime function951 struct StaticRTInput {952 /// Size of the iteration variable in bits.953 unsigned IVSize = 0;954 /// Sign of the iteration variable.955 bool IVSigned = false;956 /// true if loop is ordered, false otherwise.957 bool Ordered = false;958 /// Address of the output variable in which the flag of the last iteration959 /// is returned.960 Address IL = Address::invalid();961 /// Address of the output variable in which the lower iteration number is962 /// returned.963 Address LB = Address::invalid();964 /// Address of the output variable in which the upper iteration number is965 /// returned.966 Address UB = Address::invalid();967 /// Address of the output variable in which the stride value is returned968 /// necessary to generated the static_chunked scheduled loop.969 Address ST = Address::invalid();970 /// Value of the chunk for the static_chunked scheduled loop. For the971 /// default (nullptr) value, the chunk 1 will be used.972 llvm::Value *Chunk = nullptr;973 StaticRTInput(unsigned IVSize, bool IVSigned, bool Ordered, Address IL,974 Address LB, Address UB, Address ST,975 llvm::Value *Chunk = nullptr)976 : IVSize(IVSize), IVSigned(IVSigned), Ordered(Ordered), IL(IL), LB(LB),977 UB(UB), ST(ST), Chunk(Chunk) {}978 };979 /// Call the appropriate runtime routine to initialize it before start980 /// of loop.981 ///982 /// This is used only in case of static schedule, when the user did not983 /// specify a ordered clause on the loop construct.984 /// Depending on the loop schedule, it is necessary to call some runtime985 /// routine before start of the OpenMP loop to get the loop upper / lower986 /// bounds LB and UB and stride ST.987 ///988 /// \param CGF Reference to current CodeGenFunction.989 /// \param Loc Clang source location.990 /// \param DKind Kind of the directive.991 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.992 /// \param Values Input arguments for the construct.993 ///994 virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,995 OpenMPDirectiveKind DKind,996 const OpenMPScheduleTy &ScheduleKind,997 const StaticRTInput &Values);998 999 ///1000 /// \param CGF Reference to current CodeGenFunction.1001 /// \param Loc Clang source location.1002 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.1003 /// \param Values Input arguments for the construct.1004 ///1005 virtual void emitDistributeStaticInit(CodeGenFunction &CGF,1006 SourceLocation Loc,1007 OpenMPDistScheduleClauseKind SchedKind,1008 const StaticRTInput &Values);1009 1010 /// Call the appropriate runtime routine to notify that we finished1011 /// iteration of the ordered loop with the dynamic scheduling.1012 ///1013 /// \param CGF Reference to current CodeGenFunction.1014 /// \param Loc Clang source location.1015 /// \param IVSize Size of the iteration variable in bits.1016 /// \param IVSigned Sign of the iteration variable.1017 ///1018 virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF,1019 SourceLocation Loc, unsigned IVSize,1020 bool IVSigned);1021 1022 /// Call the appropriate runtime routine to notify that we finished1023 /// all the work with current loop.1024 ///1025 /// \param CGF Reference to current CodeGenFunction.1026 /// \param Loc Clang source location.1027 /// \param DKind Kind of the directive for which the static finish is emitted.1028 ///1029 virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,1030 OpenMPDirectiveKind DKind);1031 1032 /// Call __kmpc_dispatch_next(1033 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,1034 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,1035 /// kmp_int[32|64] *p_stride);1036 /// \param IVSize Size of the iteration variable in bits.1037 /// \param IVSigned Sign of the iteration variable.1038 /// \param IL Address of the output variable in which the flag of the1039 /// last iteration is returned.1040 /// \param LB Address of the output variable in which the lower iteration1041 /// number is returned.1042 /// \param UB Address of the output variable in which the upper iteration1043 /// number is returned.1044 /// \param ST Address of the output variable in which the stride value is1045 /// returned.1046 virtual llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,1047 unsigned IVSize, bool IVSigned,1048 Address IL, Address LB,1049 Address UB, Address ST);1050 1051 virtual llvm::Value *emitMessageClause(CodeGenFunction &CGF,1052 const Expr *Message,1053 SourceLocation Loc);1054 1055 virtual llvm::Value *emitSeverityClause(OpenMPSeverityClauseKind Severity,1056 SourceLocation Loc);1057 1058 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int321059 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'1060 /// clause.1061 /// If the modifier 'strict' is given:1062 /// Emits call to void __kmpc_push_num_threads_strict(ident_t *loc, kmp_int321063 /// global_tid, kmp_int32 num_threads, int severity, const char *message) to1064 /// generate code for 'num_threads' clause with 'strict' modifier.1065 /// \param NumThreads An integer value of threads.1066 virtual void emitNumThreadsClause(1067 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,1068 OpenMPNumThreadsClauseModifier Modifier = OMPC_NUMTHREADS_unknown,1069 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,1070 SourceLocation SeverityLoc = SourceLocation(),1071 const Expr *Message = nullptr,1072 SourceLocation MessageLoc = SourceLocation());1073 1074 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int321075 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.1076 virtual void emitProcBindClause(CodeGenFunction &CGF,1077 llvm::omp::ProcBindKind ProcBind,1078 SourceLocation Loc);1079 1080 /// Returns address of the threadprivate variable for the current1081 /// thread.1082 /// \param VD Threadprivate variable.1083 /// \param VDAddr Address of the global variable \a VD.1084 /// \param Loc Location of the reference to threadprivate var.1085 /// \return Address of the threadprivate variable for the current thread.1086 virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF,1087 const VarDecl *VD, Address VDAddr,1088 SourceLocation Loc);1089 1090 /// Returns the address of the variable marked as declare target with link1091 /// clause OR as declare target with to clause and unified memory.1092 virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD);1093 1094 /// Emit a code for initialization of threadprivate variable. It emits1095 /// a call to runtime library which adds initial value to the newly created1096 /// threadprivate variable (if it is not constant) and registers destructor1097 /// for the variable (if any).1098 /// \param VD Threadprivate variable.1099 /// \param VDAddr Address of the global variable \a VD.1100 /// \param Loc Location of threadprivate declaration.1101 /// \param PerformInit true if initialization expression is not constant.1102 virtual llvm::Function *1103 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,1104 SourceLocation Loc, bool PerformInit,1105 CodeGenFunction *CGF = nullptr);1106 1107 /// Emit code for handling declare target functions in the runtime.1108 /// \param FD Declare target function.1109 /// \param Addr Address of the global \a FD.1110 /// \param PerformInit true if initialization expression is not constant.1111 virtual void emitDeclareTargetFunction(const FunctionDecl *FD,1112 llvm::GlobalValue *GV);1113 1114 /// Creates artificial threadprivate variable with name \p Name and type \p1115 /// VarType.1116 /// \param VarType Type of the artificial threadprivate variable.1117 /// \param Name Name of the artificial threadprivate variable.1118 virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,1119 QualType VarType,1120 StringRef Name);1121 1122 /// Emit flush of the variables specified in 'omp flush' directive.1123 /// \param Vars List of variables to flush.1124 virtual void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,1125 SourceLocation Loc, llvm::AtomicOrdering AO);1126 1127 /// Emit task region for the task directive. The task region is1128 /// emitted in several steps:1129 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int321130 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,1131 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the1132 /// function:1133 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {1134 /// TaskFunction(gtid, tt->part_id, tt->shareds);1135 /// return 0;1136 /// }1137 /// 2. Copy a list of shared variables to field shareds of the resulting1138 /// structure kmp_task_t returned by the previous call (if any).1139 /// 3. Copy a pointer to destructions function to field destructions of the1140 /// resulting structure kmp_task_t.1141 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,1142 /// kmp_task_t *new_task), where new_task is a resulting structure from1143 /// previous items.1144 /// \param D Current task directive.1145 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i321146 /// /*part_id*/, captured_struct */*__context*/);1147 /// \param SharedsTy A type which contains references the shared variables.1148 /// \param Shareds Context with the list of shared variables from the \p1149 /// TaskFunction.1150 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr1151 /// otherwise.1152 /// \param Data Additional data for task generation like tiednsee, final1153 /// state, list of privates etc.1154 virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,1155 const OMPExecutableDirective &D,1156 llvm::Function *TaskFunction, QualType SharedsTy,1157 Address Shareds, const Expr *IfCond,1158 const OMPTaskDataTy &Data);1159 1160 /// Emit task region for the taskloop directive. The taskloop region is1161 /// emitted in several steps:1162 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int321163 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,1164 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the1165 /// function:1166 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {1167 /// TaskFunction(gtid, tt->part_id, tt->shareds);1168 /// return 0;1169 /// }1170 /// 2. Copy a list of shared variables to field shareds of the resulting1171 /// structure kmp_task_t returned by the previous call (if any).1172 /// 3. Copy a pointer to destructions function to field destructions of the1173 /// resulting structure kmp_task_t.1174 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t1175 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int1176 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task1177 /// is a resulting structure from1178 /// previous items.1179 /// \param D Current task directive.1180 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i321181 /// /*part_id*/, captured_struct */*__context*/);1182 /// \param SharedsTy A type which contains references the shared variables.1183 /// \param Shareds Context with the list of shared variables from the \p1184 /// TaskFunction.1185 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr1186 /// otherwise.1187 /// \param Data Additional data for task generation like tiednsee, final1188 /// state, list of privates etc.1189 virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,1190 const OMPLoopDirective &D,1191 llvm::Function *TaskFunction,1192 QualType SharedsTy, Address Shareds,1193 const Expr *IfCond, const OMPTaskDataTy &Data);1194 1195 /// Emit code for the directive that does not require outlining.1196 ///1197 /// \param InnermostKind Kind of innermost directive (for simple directives it1198 /// is a directive itself, for combined - its innermost directive).1199 /// \param CodeGen Code generation sequence for the \a D directive.1200 /// \param HasCancel true if region has inner cancel directive, false1201 /// otherwise.1202 virtual void emitInlinedDirective(CodeGenFunction &CGF,1203 OpenMPDirectiveKind InnermostKind,1204 const RegionCodeGenTy &CodeGen,1205 bool HasCancel = false);1206 1207 /// Emits reduction function.1208 /// \param ReducerName Name of the function calling the reduction.1209 /// \param ArgsElemType Array type containing pointers to reduction variables.1210 /// \param Privates List of private copies for original reduction arguments.1211 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.1212 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.1213 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'1214 /// or 'operator binop(LHS, RHS)'.1215 llvm::Function *emitReductionFunction(1216 StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType,1217 ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,1218 ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps);1219 1220 /// Emits single reduction combiner1221 void emitSingleReductionCombiner(CodeGenFunction &CGF,1222 const Expr *ReductionOp,1223 const Expr *PrivateRef,1224 const DeclRefExpr *LHS,1225 const DeclRefExpr *RHS);1226 1227 struct ReductionOptionsTy {1228 bool WithNowait;1229 bool SimpleReduction;1230 llvm::SmallVector<bool, 8> IsPrivateVarReduction;1231 OpenMPDirectiveKind ReductionKind;1232 };1233 1234 /// Emits code for private variable reduction1235 /// \param Privates List of private copies for original reduction arguments.1236 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.1237 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.1238 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'1239 /// or 'operator binop(LHS, RHS)'.1240 void emitPrivateReduction(CodeGenFunction &CGF, SourceLocation Loc,1241 const Expr *Privates, const Expr *LHSExprs,1242 const Expr *RHSExprs, const Expr *ReductionOps);1243 1244 /// Emit a code for reduction clause. Next code should be emitted for1245 /// reduction:1246 /// \code1247 ///1248 /// static kmp_critical_name lock = { 0 };1249 ///1250 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {1251 /// ...1252 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);1253 /// ...1254 /// }1255 ///1256 /// ...1257 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};1258 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),1259 /// RedList, reduce_func, &<lock>)) {1260 /// case 1:1261 /// ...1262 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);1263 /// ...1264 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);1265 /// break;1266 /// case 2:1267 /// ...1268 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));1269 /// ...1270 /// break;1271 /// default:;1272 /// }1273 /// \endcode1274 ///1275 /// \param Privates List of private copies for original reduction arguments.1276 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.1277 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.1278 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'1279 /// or 'operator binop(LHS, RHS)'.1280 /// \param Options List of options for reduction codegen:1281 /// WithNowait true if parent directive has also nowait clause, false1282 /// otherwise.1283 /// SimpleReduction Emit reduction operation only. Used for omp simd1284 /// directive on the host.1285 /// ReductionKind The kind of reduction to perform.1286 virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,1287 ArrayRef<const Expr *> Privates,1288 ArrayRef<const Expr *> LHSExprs,1289 ArrayRef<const Expr *> RHSExprs,1290 ArrayRef<const Expr *> ReductionOps,1291 ReductionOptionsTy Options);1292 1293 /// Emit a code for initialization of task reduction clause. Next code1294 /// should be emitted for reduction:1295 /// \code1296 ///1297 /// _taskred_item_t red_data[n];1298 /// ...1299 /// red_data[i].shar = &shareds[i];1300 /// red_data[i].orig = &origs[i];1301 /// red_data[i].size = sizeof(origs[i]);1302 /// red_data[i].f_init = (void*)RedInit<i>;1303 /// red_data[i].f_fini = (void*)RedDest<i>;1304 /// red_data[i].f_comb = (void*)RedOp<i>;1305 /// red_data[i].flags = <Flag_i>;1306 /// ...1307 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);1308 /// \endcode1309 /// For reduction clause with task modifier it emits the next call:1310 /// \code1311 ///1312 /// _taskred_item_t red_data[n];1313 /// ...1314 /// red_data[i].shar = &shareds[i];1315 /// red_data[i].orig = &origs[i];1316 /// red_data[i].size = sizeof(origs[i]);1317 /// red_data[i].f_init = (void*)RedInit<i>;1318 /// red_data[i].f_fini = (void*)RedDest<i>;1319 /// red_data[i].f_comb = (void*)RedOp<i>;1320 /// red_data[i].flags = <Flag_i>;1321 /// ...1322 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,1323 /// red_data);1324 /// \endcode1325 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.1326 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.1327 /// \param Data Additional data for task generation like tiedness, final1328 /// state, list of privates, reductions etc.1329 virtual llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF,1330 SourceLocation Loc,1331 ArrayRef<const Expr *> LHSExprs,1332 ArrayRef<const Expr *> RHSExprs,1333 const OMPTaskDataTy &Data);1334 1335 /// Emits the following code for reduction clause with task modifier:1336 /// \code1337 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);1338 /// \endcode1339 virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,1340 bool IsWorksharingReduction);1341 1342 /// Required to resolve existing problems in the runtime. Emits threadprivate1343 /// variables to store the size of the VLAs/array sections for1344 /// initializer/combiner/finalizer functions.1345 /// \param RCG Allows to reuse an existing data for the reductions.1346 /// \param N Reduction item for which fixups must be emitted.1347 virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,1348 ReductionCodeGen &RCG, unsigned N);1349 1350 /// Get the address of `void *` type of the privatue copy of the reduction1351 /// item specified by the \p SharedLVal.1352 /// \param ReductionsPtr Pointer to the reduction data returned by the1353 /// emitTaskReductionInit function.1354 /// \param SharedLVal Address of the original reduction item.1355 virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,1356 llvm::Value *ReductionsPtr,1357 LValue SharedLVal);1358 1359 /// Emit code for 'taskwait' directive.1360 virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,1361 const OMPTaskDataTy &Data);1362 1363 /// Emit code for 'cancellation point' construct.1364 /// \param CancelRegion Region kind for which the cancellation point must be1365 /// emitted.1366 ///1367 virtual void emitCancellationPointCall(CodeGenFunction &CGF,1368 SourceLocation Loc,1369 OpenMPDirectiveKind CancelRegion);1370 1371 /// Emit code for 'cancel' construct.1372 /// \param IfCond Condition in the associated 'if' clause, if it was1373 /// specified, nullptr otherwise.1374 /// \param CancelRegion Region kind for which the cancel must be emitted.1375 ///1376 virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,1377 const Expr *IfCond,1378 OpenMPDirectiveKind CancelRegion);1379 1380 /// Emit outilined function for 'target' directive.1381 /// \param D Directive to emit.1382 /// \param ParentName Name of the function that encloses the target region.1383 /// \param OutlinedFn Outlined function value to be defined by this call.1384 /// \param OutlinedFnID Outlined function ID value to be defined by this call.1385 /// \param IsOffloadEntry True if the outlined function is an offload entry.1386 /// \param CodeGen Code generation sequence for the \a D directive.1387 /// An outlined function may not be an entry if, e.g. the if clause always1388 /// evaluates to false.1389 virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D,1390 StringRef ParentName,1391 llvm::Function *&OutlinedFn,1392 llvm::Constant *&OutlinedFnID,1393 bool IsOffloadEntry,1394 const RegionCodeGenTy &CodeGen);1395 1396 /// Emit the target offloading code associated with \a D. The emitted1397 /// code attempts offloading the execution to the device, an the event of1398 /// a failure it executes the host version outlined in \a OutlinedFn.1399 /// \param D Directive to emit.1400 /// \param OutlinedFn Host version of the code to be offloaded.1401 /// \param OutlinedFnID ID of host version of the code to be offloaded.1402 /// \param IfCond Expression evaluated in if clause associated with the target1403 /// directive, or null if no if clause is used.1404 /// \param Device Expression evaluated in device clause associated with the1405 /// target directive, or null if no device clause is used and device modifier.1406 /// \param SizeEmitter Callback to emit number of iterations for loop-based1407 /// directives.1408 virtual void emitTargetCall(1409 CodeGenFunction &CGF, const OMPExecutableDirective &D,1410 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,1411 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,1412 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,1413 const OMPLoopDirective &D)>1414 SizeEmitter);1415 1416 /// Emit the target regions enclosed in \a GD function definition or1417 /// the function itself in case it is a valid device function. Returns true if1418 /// \a GD was dealt with successfully.1419 /// \param GD Function to scan.1420 virtual bool emitTargetFunctions(GlobalDecl GD);1421 1422 /// Emit the global variable if it is a valid device global variable.1423 /// Returns true if \a GD was dealt with successfully.1424 /// \param GD Variable declaration to emit.1425 virtual bool emitTargetGlobalVariable(GlobalDecl GD);1426 1427 /// Checks if the provided global decl \a GD is a declare target variable and1428 /// registers it when emitting code for the host.1429 virtual void registerTargetGlobalVariable(const VarDecl *VD,1430 llvm::Constant *Addr);1431 1432 /// Emit the global \a GD if it is meaningful for the target. Returns1433 /// if it was emitted successfully.1434 /// \param GD Global to scan.1435 virtual bool emitTargetGlobal(GlobalDecl GD);1436 1437 /// Creates all the offload entries in the current compilation unit1438 /// along with the associated metadata.1439 void createOffloadEntriesAndInfoMetadata();1440 1441 /// Emits code for teams call of the \a OutlinedFn with1442 /// variables captured in a record which address is stored in \a1443 /// CapturedStruct.1444 /// \param OutlinedFn Outlined function to be run by team masters. Type of1445 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).1446 /// \param CapturedVars A pointer to the record with the references to1447 /// variables used in \a OutlinedFn function.1448 ///1449 virtual void emitTeamsCall(CodeGenFunction &CGF,1450 const OMPExecutableDirective &D,1451 SourceLocation Loc, llvm::Function *OutlinedFn,1452 ArrayRef<llvm::Value *> CapturedVars);1453 1454 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int321455 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code1456 /// for num_teams clause.1457 /// \param NumTeams An integer expression of teams.1458 /// \param ThreadLimit An integer expression of threads.1459 virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,1460 const Expr *ThreadLimit, SourceLocation Loc);1461 1462 /// Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int321463 /// global_tid, kmp_int32 thread_limit) to generate code for1464 /// thread_limit clause on target directive1465 /// \param ThreadLimit An integer expression of threads.1466 virtual void emitThreadLimitClause(CodeGenFunction &CGF,1467 const Expr *ThreadLimit,1468 SourceLocation Loc);1469 1470 /// Struct that keeps all the relevant information that should be kept1471 /// throughout a 'target data' region.1472 class TargetDataInfo : public llvm::OpenMPIRBuilder::TargetDataInfo {1473 public:1474 explicit TargetDataInfo() : llvm::OpenMPIRBuilder::TargetDataInfo() {}1475 explicit TargetDataInfo(bool RequiresDevicePointerInfo,1476 bool SeparateBeginEndCalls)1477 : llvm::OpenMPIRBuilder::TargetDataInfo(RequiresDevicePointerInfo,1478 SeparateBeginEndCalls) {}1479 /// Map between the a declaration of a capture and the corresponding new1480 /// llvm address where the runtime returns the device pointers.1481 llvm::DenseMap<const ValueDecl *, llvm::Value *> CaptureDeviceAddrMap;1482 };1483 1484 /// Emit the target data mapping code associated with \a D.1485 /// \param D Directive to emit.1486 /// \param IfCond Expression evaluated in if clause associated with the1487 /// target directive, or null if no device clause is used.1488 /// \param Device Expression evaluated in device clause associated with the1489 /// target directive, or null if no device clause is used.1490 /// \param Info A record used to store information that needs to be preserved1491 /// until the region is closed.1492 virtual void emitTargetDataCalls(CodeGenFunction &CGF,1493 const OMPExecutableDirective &D,1494 const Expr *IfCond, const Expr *Device,1495 const RegionCodeGenTy &CodeGen,1496 CGOpenMPRuntime::TargetDataInfo &Info);1497 1498 /// Emit the data mapping/movement code associated with the directive1499 /// \a D that should be of the form 'target [{enter|exit} data | update]'.1500 /// \param D Directive to emit.1501 /// \param IfCond Expression evaluated in if clause associated with the target1502 /// directive, or null if no if clause is used.1503 /// \param Device Expression evaluated in device clause associated with the1504 /// target directive, or null if no device clause is used.1505 virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF,1506 const OMPExecutableDirective &D,1507 const Expr *IfCond,1508 const Expr *Device);1509 1510 /// Marks function \a Fn with properly mangled versions of vector functions.1511 /// \param FD Function marked as 'declare simd'.1512 /// \param Fn LLVM function that must be marked with 'declare simd'1513 /// attributes.1514 virtual void emitDeclareSimdFunction(const FunctionDecl *FD,1515 llvm::Function *Fn);1516 1517 /// Emit initialization for doacross loop nesting support.1518 /// \param D Loop-based construct used in doacross nesting construct.1519 virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,1520 ArrayRef<Expr *> NumIterations);1521 1522 /// Emit code for doacross ordered directive with 'depend' clause.1523 /// \param C 'depend' clause with 'sink|source' dependency kind.1524 virtual void emitDoacrossOrdered(CodeGenFunction &CGF,1525 const OMPDependClause *C);1526 1527 /// Emit code for doacross ordered directive with 'doacross' clause.1528 /// \param C 'doacross' clause with 'sink|source' dependence type.1529 virtual void emitDoacrossOrdered(CodeGenFunction &CGF,1530 const OMPDoacrossClause *C);1531 1532 /// Translates the native parameter of outlined function if this is required1533 /// for target.1534 /// \param FD Field decl from captured record for the parameter.1535 /// \param NativeParam Parameter itself.1536 virtual const VarDecl *translateParameter(const FieldDecl *FD,1537 const VarDecl *NativeParam) const {1538 return NativeParam;1539 }1540 1541 /// Gets the address of the native argument basing on the address of the1542 /// target-specific parameter.1543 /// \param NativeParam Parameter itself.1544 /// \param TargetParam Corresponding target-specific parameter.1545 virtual Address getParameterAddress(CodeGenFunction &CGF,1546 const VarDecl *NativeParam,1547 const VarDecl *TargetParam) const;1548 1549 /// Choose default schedule type and chunk value for the1550 /// dist_schedule clause.1551 virtual void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,1552 const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,1553 llvm::Value *&Chunk) const {}1554 1555 /// Choose default schedule type and chunk value for the1556 /// schedule clause.1557 virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF,1558 const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,1559 const Expr *&ChunkExpr) const;1560 1561 /// Emits call of the outlined function with the provided arguments,1562 /// translating these arguments to correct target-specific arguments.1563 virtual void1564 emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc,1565 llvm::FunctionCallee OutlinedFn,1566 ArrayRef<llvm::Value *> Args = {}) const;1567 1568 /// Emits OpenMP-specific function prolog.1569 /// Required for device constructs.1570 virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D);1571 1572 /// Gets the OpenMP-specific address of the local variable.1573 virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF,1574 const VarDecl *VD);1575 1576 /// Marks the declaration as already emitted for the device code and returns1577 /// true, if it was marked already, and false, otherwise.1578 bool markAsGlobalTarget(GlobalDecl GD);1579 1580 /// Emit deferred declare target variables marked for deferred emission.1581 void emitDeferredTargetDecls() const;1582 1583 /// Adjust some parameters for the target-based directives, like addresses of1584 /// the variables captured by reference in lambdas.1585 virtual void1586 adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF,1587 const OMPExecutableDirective &D) const;1588 1589 /// Perform check on requires decl to ensure that target architecture1590 /// supports unified addressing1591 virtual void processRequiresDirective(const OMPRequiresDecl *D);1592 1593 /// Gets default memory ordering as specified in requires directive.1594 llvm::AtomicOrdering getDefaultMemoryOrdering() const;1595 1596 /// Checks if the variable has associated OMPAllocateDeclAttr attribute with1597 /// the predefined allocator and translates it into the corresponding address1598 /// space.1599 virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS);1600 1601 /// Return whether the unified_shared_memory has been specified.1602 bool hasRequiresUnifiedSharedMemory() const;1603 1604 /// Checks if the \p VD variable is marked as nontemporal declaration in1605 /// current context.1606 bool isNontemporalDecl(const ValueDecl *VD) const;1607 1608 /// Create specialized alloca to handle lastprivate conditionals.1609 Address emitLastprivateConditionalInit(CodeGenFunction &CGF,1610 const VarDecl *VD);1611 1612 /// Checks if the provided \p LVal is lastprivate conditional and emits the1613 /// code to update the value of the original variable.1614 /// \code1615 /// lastprivate(conditional: a)1616 /// ...1617 /// <type> a;1618 /// lp_a = ...;1619 /// #pragma omp critical(a)1620 /// if (last_iv_a <= iv) {1621 /// last_iv_a = iv;1622 /// global_a = lp_a;1623 /// }1624 /// \endcode1625 virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF,1626 const Expr *LHS);1627 1628 /// Checks if the lastprivate conditional was updated in inner region and1629 /// writes the value.1630 /// \code1631 /// lastprivate(conditional: a)1632 /// ...1633 /// <type> a;bool Fired = false;1634 /// #pragma omp ... shared(a)1635 /// {1636 /// lp_a = ...;1637 /// Fired = true;1638 /// }1639 /// if (Fired) {1640 /// #pragma omp critical(a)1641 /// if (last_iv_a <= iv) {1642 /// last_iv_a = iv;1643 /// global_a = lp_a;1644 /// }1645 /// Fired = false;1646 /// }1647 /// \endcode1648 virtual void checkAndEmitSharedLastprivateConditional(1649 CodeGenFunction &CGF, const OMPExecutableDirective &D,1650 const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls);1651 1652 /// Gets the address of the global copy used for lastprivate conditional1653 /// update, if any.1654 /// \param PrivLVal LValue for the private copy.1655 /// \param VD Original lastprivate declaration.1656 virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF,1657 LValue PrivLVal,1658 const VarDecl *VD,1659 SourceLocation Loc);1660 1661 /// Emits list of dependecies based on the provided data (array of1662 /// dependence/expression pairs).1663 /// \returns Pointer to the first element of the array casted to VoidPtr type.1664 std::pair<llvm::Value *, Address>1665 emitDependClause(CodeGenFunction &CGF,1666 ArrayRef<OMPTaskDataTy::DependData> Dependencies,1667 SourceLocation Loc);1668 1669 /// Emits list of dependecies based on the provided data (array of1670 /// dependence/expression pairs) for depobj construct. In this case, the1671 /// variable is allocated in dynamically. \returns Pointer to the first1672 /// element of the array casted to VoidPtr type.1673 Address emitDepobjDependClause(CodeGenFunction &CGF,1674 const OMPTaskDataTy::DependData &Dependencies,1675 SourceLocation Loc);1676 1677 /// Emits the code to destroy the dependency object provided in depobj1678 /// directive.1679 void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,1680 SourceLocation Loc);1681 1682 /// Updates the dependency kind in the specified depobj object.1683 /// \param DepobjLVal LValue for the main depobj object.1684 /// \param NewDepKind New dependency kind.1685 void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,1686 OpenMPDependClauseKind NewDepKind, SourceLocation Loc);1687 1688 /// Initializes user defined allocators specified in the uses_allocators1689 /// clauses.1690 void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator,1691 const Expr *AllocatorTraits);1692 1693 /// Destroys user defined allocators specified in the uses_allocators clause.1694 void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator);1695 1696 /// Returns true if the variable is a local variable in untied task.1697 bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const;1698};1699 1700/// Class supports emissionof SIMD-only code.1701class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {1702public:1703 explicit CGOpenMPSIMDRuntime(CodeGenModule &CGM) : CGOpenMPRuntime(CGM) {}1704 ~CGOpenMPSIMDRuntime() override {}1705 1706 /// Emits outlined function for the specified OpenMP parallel directive1707 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,1708 /// kmp_int32 BoundID, struct context_vars*).1709 /// \param CGF Reference to current CodeGenFunction.1710 /// \param D OpenMP directive.1711 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.1712 /// \param InnermostKind Kind of innermost directive (for simple directives it1713 /// is a directive itself, for combined - its innermost directive).1714 /// \param CodeGen Code generation sequence for the \a D directive.1715 llvm::Function *emitParallelOutlinedFunction(1716 CodeGenFunction &CGF, const OMPExecutableDirective &D,1717 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,1718 const RegionCodeGenTy &CodeGen) override;1719 1720 /// Emits outlined function for the specified OpenMP teams directive1721 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,1722 /// kmp_int32 BoundID, struct context_vars*).1723 /// \param CGF Reference to current CodeGenFunction.1724 /// \param D OpenMP directive.1725 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.1726 /// \param InnermostKind Kind of innermost directive (for simple directives it1727 /// is a directive itself, for combined - its innermost directive).1728 /// \param CodeGen Code generation sequence for the \a D directive.1729 llvm::Function *emitTeamsOutlinedFunction(1730 CodeGenFunction &CGF, const OMPExecutableDirective &D,1731 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,1732 const RegionCodeGenTy &CodeGen) override;1733 1734 /// Emits outlined function for the OpenMP task directive \a D. This1735 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*1736 /// TaskT).1737 /// \param D OpenMP directive.1738 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.1739 /// \param PartIDVar Variable for partition id in the current OpenMP untied1740 /// task region.1741 /// \param TaskTVar Variable for task_t argument.1742 /// \param InnermostKind Kind of innermost directive (for simple directives it1743 /// is a directive itself, for combined - its innermost directive).1744 /// \param CodeGen Code generation sequence for the \a D directive.1745 /// \param Tied true if task is generated for tied task, false otherwise.1746 /// \param NumberOfParts Number of parts in untied task. Ignored for tied1747 /// tasks.1748 ///1749 llvm::Function *emitTaskOutlinedFunction(1750 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,1751 const VarDecl *PartIDVar, const VarDecl *TaskTVar,1752 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,1753 bool Tied, unsigned &NumberOfParts) override;1754 1755 /// Emits code for parallel or serial call of the \a OutlinedFn with1756 /// variables captured in a record which address is stored in \a1757 /// CapturedStruct.1758 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of1759 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).1760 /// \param CapturedVars A pointer to the record with the references to1761 /// variables used in \a OutlinedFn function.1762 /// \param IfCond Condition in the associated 'if' clause, if it was1763 /// specified, nullptr otherwise.1764 /// \param NumThreads The value corresponding to the num_threads clause, if1765 /// any, or nullptr.1766 /// \param NumThreadsModifier The modifier of the num_threads clause, if1767 /// any, ignored otherwise.1768 /// \param Severity The severity corresponding to the num_threads clause, if1769 /// any, ignored otherwise.1770 /// \param Message The message string corresponding to the num_threads clause,1771 /// if any, or nullptr.1772 ///1773 void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,1774 llvm::Function *OutlinedFn,1775 ArrayRef<llvm::Value *> CapturedVars,1776 const Expr *IfCond, llvm::Value *NumThreads,1777 OpenMPNumThreadsClauseModifier NumThreadsModifier =1778 OMPC_NUMTHREADS_unknown,1779 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,1780 const Expr *Message = nullptr) override;1781 1782 /// Emits a critical region.1783 /// \param CriticalName Name of the critical region.1784 /// \param CriticalOpGen Generator for the statement associated with the given1785 /// critical region.1786 /// \param Hint Value of the 'hint' clause (optional).1787 void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,1788 const RegionCodeGenTy &CriticalOpGen,1789 SourceLocation Loc,1790 const Expr *Hint = nullptr) override;1791 1792 /// Emits a master region.1793 /// \param MasterOpGen Generator for the statement associated with the given1794 /// master region.1795 void emitMasterRegion(CodeGenFunction &CGF,1796 const RegionCodeGenTy &MasterOpGen,1797 SourceLocation Loc) override;1798 1799 /// Emits a masked region.1800 /// \param MaskedOpGen Generator for the statement associated with the given1801 /// masked region.1802 void emitMaskedRegion(CodeGenFunction &CGF,1803 const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc,1804 const Expr *Filter = nullptr) override;1805 1806 /// Emits a masked region.1807 /// \param MaskedOpGen Generator for the statement associated with the given1808 /// masked region.1809 1810 /// Emits code for a taskyield directive.1811 void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override;1812 1813 /// Emit a taskgroup region.1814 /// \param TaskgroupOpGen Generator for the statement associated with the1815 /// given taskgroup region.1816 void emitTaskgroupRegion(CodeGenFunction &CGF,1817 const RegionCodeGenTy &TaskgroupOpGen,1818 SourceLocation Loc) override;1819 1820 /// Emits a single region.1821 /// \param SingleOpGen Generator for the statement associated with the given1822 /// single region.1823 void emitSingleRegion(CodeGenFunction &CGF,1824 const RegionCodeGenTy &SingleOpGen, SourceLocation Loc,1825 ArrayRef<const Expr *> CopyprivateVars,1826 ArrayRef<const Expr *> DestExprs,1827 ArrayRef<const Expr *> SrcExprs,1828 ArrayRef<const Expr *> AssignmentOps) override;1829 1830 /// Emit an ordered region.1831 /// \param OrderedOpGen Generator for the statement associated with the given1832 /// ordered region.1833 void emitOrderedRegion(CodeGenFunction &CGF,1834 const RegionCodeGenTy &OrderedOpGen,1835 SourceLocation Loc, bool IsThreads) override;1836 1837 /// Emit an implicit/explicit barrier for OpenMP threads.1838 /// \param Kind Directive for which this implicit barrier call must be1839 /// generated. Must be OMPD_barrier for explicit barrier generation.1840 /// \param EmitChecks true if need to emit checks for cancellation barriers.1841 /// \param ForceSimpleCall true simple barrier call must be emitted, false if1842 /// runtime class decides which one to emit (simple or with cancellation1843 /// checks).1844 ///1845 void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,1846 OpenMPDirectiveKind Kind, bool EmitChecks = true,1847 bool ForceSimpleCall = false) override;1848 1849 /// This is used for non static scheduled types and when the ordered1850 /// clause is present on the loop construct.1851 /// Depending on the loop schedule, it is necessary to call some runtime1852 /// routine before start of the OpenMP loop to get the loop upper / lower1853 /// bounds \a LB and \a UB and stride \a ST.1854 ///1855 /// \param CGF Reference to current CodeGenFunction.1856 /// \param Loc Clang source location.1857 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.1858 /// \param IVSize Size of the iteration variable in bits.1859 /// \param IVSigned Sign of the iteration variable.1860 /// \param Ordered true if loop is ordered, false otherwise.1861 /// \param DispatchValues struct containing llvm values for lower bound, upper1862 /// bound, and chunk expression.1863 /// For the default (nullptr) value, the chunk 1 will be used.1864 ///1865 void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,1866 const OpenMPScheduleTy &ScheduleKind,1867 unsigned IVSize, bool IVSigned, bool Ordered,1868 const DispatchRTInput &DispatchValues) override;1869 1870 /// This is used for non static scheduled types and when the ordered1871 /// clause is present on the loop construct.1872 ///1873 /// \param CGF Reference to current CodeGenFunction.1874 /// \param Loc Clang source location.1875 ///1876 void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc) override;1877 1878 /// Call the appropriate runtime routine to initialize it before start1879 /// of loop.1880 ///1881 /// This is used only in case of static schedule, when the user did not1882 /// specify a ordered clause on the loop construct.1883 /// Depending on the loop schedule, it is necessary to call some runtime1884 /// routine before start of the OpenMP loop to get the loop upper / lower1885 /// bounds LB and UB and stride ST.1886 ///1887 /// \param CGF Reference to current CodeGenFunction.1888 /// \param Loc Clang source location.1889 /// \param DKind Kind of the directive.1890 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.1891 /// \param Values Input arguments for the construct.1892 ///1893 void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,1894 OpenMPDirectiveKind DKind,1895 const OpenMPScheduleTy &ScheduleKind,1896 const StaticRTInput &Values) override;1897 1898 ///1899 /// \param CGF Reference to current CodeGenFunction.1900 /// \param Loc Clang source location.1901 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.1902 /// \param Values Input arguments for the construct.1903 ///1904 void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc,1905 OpenMPDistScheduleClauseKind SchedKind,1906 const StaticRTInput &Values) override;1907 1908 /// Call the appropriate runtime routine to notify that we finished1909 /// iteration of the ordered loop with the dynamic scheduling.1910 ///1911 /// \param CGF Reference to current CodeGenFunction.1912 /// \param Loc Clang source location.1913 /// \param IVSize Size of the iteration variable in bits.1914 /// \param IVSigned Sign of the iteration variable.1915 ///1916 void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc,1917 unsigned IVSize, bool IVSigned) override;1918 1919 /// Call the appropriate runtime routine to notify that we finished1920 /// all the work with current loop.1921 ///1922 /// \param CGF Reference to current CodeGenFunction.1923 /// \param Loc Clang source location.1924 /// \param DKind Kind of the directive for which the static finish is emitted.1925 ///1926 void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,1927 OpenMPDirectiveKind DKind) override;1928 1929 /// Call __kmpc_dispatch_next(1930 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,1931 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,1932 /// kmp_int[32|64] *p_stride);1933 /// \param IVSize Size of the iteration variable in bits.1934 /// \param IVSigned Sign of the iteration variable.1935 /// \param IL Address of the output variable in which the flag of the1936 /// last iteration is returned.1937 /// \param LB Address of the output variable in which the lower iteration1938 /// number is returned.1939 /// \param UB Address of the output variable in which the upper iteration1940 /// number is returned.1941 /// \param ST Address of the output variable in which the stride value is1942 /// returned.1943 llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,1944 unsigned IVSize, bool IVSigned, Address IL,1945 Address LB, Address UB, Address ST) override;1946 1947 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int321948 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'1949 /// clause.1950 /// If the modifier 'strict' is given:1951 /// Emits call to void __kmpc_push_num_threads_strict(ident_t *loc, kmp_int321952 /// global_tid, kmp_int32 num_threads, int severity, const char *message) to1953 /// generate code for 'num_threads' clause with 'strict' modifier.1954 /// \param NumThreads An integer value of threads.1955 void emitNumThreadsClause(1956 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,1957 OpenMPNumThreadsClauseModifier Modifier = OMPC_NUMTHREADS_unknown,1958 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,1959 SourceLocation SeverityLoc = SourceLocation(),1960 const Expr *Message = nullptr,1961 SourceLocation MessageLoc = SourceLocation()) override;1962 1963 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int321964 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.1965 void emitProcBindClause(CodeGenFunction &CGF,1966 llvm::omp::ProcBindKind ProcBind,1967 SourceLocation Loc) override;1968 1969 /// Returns address of the threadprivate variable for the current1970 /// thread.1971 /// \param VD Threadprivate variable.1972 /// \param VDAddr Address of the global variable \a VD.1973 /// \param Loc Location of the reference to threadprivate var.1974 /// \return Address of the threadprivate variable for the current thread.1975 Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD,1976 Address VDAddr, SourceLocation Loc) override;1977 1978 /// Emit a code for initialization of threadprivate variable. It emits1979 /// a call to runtime library which adds initial value to the newly created1980 /// threadprivate variable (if it is not constant) and registers destructor1981 /// for the variable (if any).1982 /// \param VD Threadprivate variable.1983 /// \param VDAddr Address of the global variable \a VD.1984 /// \param Loc Location of threadprivate declaration.1985 /// \param PerformInit true if initialization expression is not constant.1986 llvm::Function *1987 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,1988 SourceLocation Loc, bool PerformInit,1989 CodeGenFunction *CGF = nullptr) override;1990 1991 /// Creates artificial threadprivate variable with name \p Name and type \p1992 /// VarType.1993 /// \param VarType Type of the artificial threadprivate variable.1994 /// \param Name Name of the artificial threadprivate variable.1995 Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,1996 QualType VarType,1997 StringRef Name) override;1998 1999 /// Emit flush of the variables specified in 'omp flush' directive.2000 /// \param Vars List of variables to flush.2001 void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,2002 SourceLocation Loc, llvm::AtomicOrdering AO) override;2003 2004 /// Emit task region for the task directive. The task region is2005 /// emitted in several steps:2006 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int322007 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,2008 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the2009 /// function:2010 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {2011 /// TaskFunction(gtid, tt->part_id, tt->shareds);2012 /// return 0;2013 /// }2014 /// 2. Copy a list of shared variables to field shareds of the resulting2015 /// structure kmp_task_t returned by the previous call (if any).2016 /// 3. Copy a pointer to destructions function to field destructions of the2017 /// resulting structure kmp_task_t.2018 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,2019 /// kmp_task_t *new_task), where new_task is a resulting structure from2020 /// previous items.2021 /// \param D Current task directive.2022 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i322023 /// /*part_id*/, captured_struct */*__context*/);2024 /// \param SharedsTy A type which contains references the shared variables.2025 /// \param Shareds Context with the list of shared variables from the \p2026 /// TaskFunction.2027 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr2028 /// otherwise.2029 /// \param Data Additional data for task generation like tiednsee, final2030 /// state, list of privates etc.2031 void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,2032 const OMPExecutableDirective &D,2033 llvm::Function *TaskFunction, QualType SharedsTy,2034 Address Shareds, const Expr *IfCond,2035 const OMPTaskDataTy &Data) override;2036 2037 /// Emit task region for the taskloop directive. The taskloop region is2038 /// emitted in several steps:2039 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int322040 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,2041 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the2042 /// function:2043 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {2044 /// TaskFunction(gtid, tt->part_id, tt->shareds);2045 /// return 0;2046 /// }2047 /// 2. Copy a list of shared variables to field shareds of the resulting2048 /// structure kmp_task_t returned by the previous call (if any).2049 /// 3. Copy a pointer to destructions function to field destructions of the2050 /// resulting structure kmp_task_t.2051 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t2052 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int2053 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task2054 /// is a resulting structure from2055 /// previous items.2056 /// \param D Current task directive.2057 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i322058 /// /*part_id*/, captured_struct */*__context*/);2059 /// \param SharedsTy A type which contains references the shared variables.2060 /// \param Shareds Context with the list of shared variables from the \p2061 /// TaskFunction.2062 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr2063 /// otherwise.2064 /// \param Data Additional data for task generation like tiednsee, final2065 /// state, list of privates etc.2066 void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,2067 const OMPLoopDirective &D, llvm::Function *TaskFunction,2068 QualType SharedsTy, Address Shareds, const Expr *IfCond,2069 const OMPTaskDataTy &Data) override;2070 2071 /// Emit a code for reduction clause. Next code should be emitted for2072 /// reduction:2073 /// \code2074 ///2075 /// static kmp_critical_name lock = { 0 };2076 ///2077 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {2078 /// ...2079 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);2080 /// ...2081 /// }2082 ///2083 /// ...2084 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};2085 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),2086 /// RedList, reduce_func, &<lock>)) {2087 /// case 1:2088 /// ...2089 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);2090 /// ...2091 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);2092 /// break;2093 /// case 2:2094 /// ...2095 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));2096 /// ...2097 /// break;2098 /// default:;2099 /// }2100 /// \endcode2101 ///2102 /// \param Privates List of private copies for original reduction arguments.2103 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.2104 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.2105 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'2106 /// or 'operator binop(LHS, RHS)'.2107 /// \param Options List of options for reduction codegen:2108 /// WithNowait true if parent directive has also nowait clause, false2109 /// otherwise.2110 /// SimpleReduction Emit reduction operation only. Used for omp simd2111 /// directive on the host.2112 /// ReductionKind The kind of reduction to perform.2113 void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,2114 ArrayRef<const Expr *> Privates,2115 ArrayRef<const Expr *> LHSExprs,2116 ArrayRef<const Expr *> RHSExprs,2117 ArrayRef<const Expr *> ReductionOps,2118 ReductionOptionsTy Options) override;2119 2120 /// Emit a code for initialization of task reduction clause. Next code2121 /// should be emitted for reduction:2122 /// \code2123 ///2124 /// _taskred_item_t red_data[n];2125 /// ...2126 /// red_data[i].shar = &shareds[i];2127 /// red_data[i].orig = &origs[i];2128 /// red_data[i].size = sizeof(origs[i]);2129 /// red_data[i].f_init = (void*)RedInit<i>;2130 /// red_data[i].f_fini = (void*)RedDest<i>;2131 /// red_data[i].f_comb = (void*)RedOp<i>;2132 /// red_data[i].flags = <Flag_i>;2133 /// ...2134 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);2135 /// \endcode2136 /// For reduction clause with task modifier it emits the next call:2137 /// \code2138 ///2139 /// _taskred_item_t red_data[n];2140 /// ...2141 /// red_data[i].shar = &shareds[i];2142 /// red_data[i].orig = &origs[i];2143 /// red_data[i].size = sizeof(origs[i]);2144 /// red_data[i].f_init = (void*)RedInit<i>;2145 /// red_data[i].f_fini = (void*)RedDest<i>;2146 /// red_data[i].f_comb = (void*)RedOp<i>;2147 /// red_data[i].flags = <Flag_i>;2148 /// ...2149 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,2150 /// red_data);2151 /// \endcode2152 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.2153 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.2154 /// \param Data Additional data for task generation like tiedness, final2155 /// state, list of privates, reductions etc.2156 llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc,2157 ArrayRef<const Expr *> LHSExprs,2158 ArrayRef<const Expr *> RHSExprs,2159 const OMPTaskDataTy &Data) override;2160 2161 /// Emits the following code for reduction clause with task modifier:2162 /// \code2163 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);2164 /// \endcode2165 void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,2166 bool IsWorksharingReduction) override;2167 2168 /// Required to resolve existing problems in the runtime. Emits threadprivate2169 /// variables to store the size of the VLAs/array sections for2170 /// initializer/combiner/finalizer functions + emits threadprivate variable to2171 /// store the pointer to the original reduction item for the custom2172 /// initializer defined by declare reduction construct.2173 /// \param RCG Allows to reuse an existing data for the reductions.2174 /// \param N Reduction item for which fixups must be emitted.2175 void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,2176 ReductionCodeGen &RCG, unsigned N) override;2177 2178 /// Get the address of `void *` type of the privatue copy of the reduction2179 /// item specified by the \p SharedLVal.2180 /// \param ReductionsPtr Pointer to the reduction data returned by the2181 /// emitTaskReductionInit function.2182 /// \param SharedLVal Address of the original reduction item.2183 Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,2184 llvm::Value *ReductionsPtr,2185 LValue SharedLVal) override;2186 2187 /// Emit code for 'taskwait' directive.2188 void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,2189 const OMPTaskDataTy &Data) override;2190 2191 /// Emit code for 'cancellation point' construct.2192 /// \param CancelRegion Region kind for which the cancellation point must be2193 /// emitted.2194 ///2195 void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc,2196 OpenMPDirectiveKind CancelRegion) override;2197 2198 /// Emit code for 'cancel' construct.2199 /// \param IfCond Condition in the associated 'if' clause, if it was2200 /// specified, nullptr otherwise.2201 /// \param CancelRegion Region kind for which the cancel must be emitted.2202 ///2203 void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,2204 const Expr *IfCond,2205 OpenMPDirectiveKind CancelRegion) override;2206 2207 /// Emit outilined function for 'target' directive.2208 /// \param D Directive to emit.2209 /// \param ParentName Name of the function that encloses the target region.2210 /// \param OutlinedFn Outlined function value to be defined by this call.2211 /// \param OutlinedFnID Outlined function ID value to be defined by this call.2212 /// \param IsOffloadEntry True if the outlined function is an offload entry.2213 /// \param CodeGen Code generation sequence for the \a D directive.2214 /// An outlined function may not be an entry if, e.g. the if clause always2215 /// evaluates to false.2216 void emitTargetOutlinedFunction(const OMPExecutableDirective &D,2217 StringRef ParentName,2218 llvm::Function *&OutlinedFn,2219 llvm::Constant *&OutlinedFnID,2220 bool IsOffloadEntry,2221 const RegionCodeGenTy &CodeGen) override;2222 2223 /// Emit the target offloading code associated with \a D. The emitted2224 /// code attempts offloading the execution to the device, an the event of2225 /// a failure it executes the host version outlined in \a OutlinedFn.2226 /// \param D Directive to emit.2227 /// \param OutlinedFn Host version of the code to be offloaded.2228 /// \param OutlinedFnID ID of host version of the code to be offloaded.2229 /// \param IfCond Expression evaluated in if clause associated with the target2230 /// directive, or null if no if clause is used.2231 /// \param Device Expression evaluated in device clause associated with the2232 /// target directive, or null if no device clause is used and device modifier.2233 void emitTargetCall(2234 CodeGenFunction &CGF, const OMPExecutableDirective &D,2235 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,2236 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,2237 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,2238 const OMPLoopDirective &D)>2239 SizeEmitter) override;2240 2241 /// Emit the target regions enclosed in \a GD function definition or2242 /// the function itself in case it is a valid device function. Returns true if2243 /// \a GD was dealt with successfully.2244 /// \param GD Function to scan.2245 bool emitTargetFunctions(GlobalDecl GD) override;2246 2247 /// Emit the global variable if it is a valid device global variable.2248 /// Returns true if \a GD was dealt with successfully.2249 /// \param GD Variable declaration to emit.2250 bool emitTargetGlobalVariable(GlobalDecl GD) override;2251 2252 /// Emit the global \a GD if it is meaningful for the target. Returns2253 /// if it was emitted successfully.2254 /// \param GD Global to scan.2255 bool emitTargetGlobal(GlobalDecl GD) override;2256 2257 /// Emits code for teams call of the \a OutlinedFn with2258 /// variables captured in a record which address is stored in \a2259 /// CapturedStruct.2260 /// \param OutlinedFn Outlined function to be run by team masters. Type of2261 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).2262 /// \param CapturedVars A pointer to the record with the references to2263 /// variables used in \a OutlinedFn function.2264 ///2265 void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,2266 SourceLocation Loc, llvm::Function *OutlinedFn,2267 ArrayRef<llvm::Value *> CapturedVars) override;2268 2269 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int322270 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code2271 /// for num_teams clause.2272 /// \param NumTeams An integer expression of teams.2273 /// \param ThreadLimit An integer expression of threads.2274 void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,2275 const Expr *ThreadLimit, SourceLocation Loc) override;2276 2277 /// Emit the target data mapping code associated with \a D.2278 /// \param D Directive to emit.2279 /// \param IfCond Expression evaluated in if clause associated with the2280 /// target directive, or null if no device clause is used.2281 /// \param Device Expression evaluated in device clause associated with the2282 /// target directive, or null if no device clause is used.2283 /// \param Info A record used to store information that needs to be preserved2284 /// until the region is closed.2285 void emitTargetDataCalls(CodeGenFunction &CGF,2286 const OMPExecutableDirective &D, const Expr *IfCond,2287 const Expr *Device, const RegionCodeGenTy &CodeGen,2288 CGOpenMPRuntime::TargetDataInfo &Info) override;2289 2290 /// Emit the data mapping/movement code associated with the directive2291 /// \a D that should be of the form 'target [{enter|exit} data | update]'.2292 /// \param D Directive to emit.2293 /// \param IfCond Expression evaluated in if clause associated with the target2294 /// directive, or null if no if clause is used.2295 /// \param Device Expression evaluated in device clause associated with the2296 /// target directive, or null if no device clause is used.2297 void emitTargetDataStandAloneCall(CodeGenFunction &CGF,2298 const OMPExecutableDirective &D,2299 const Expr *IfCond,2300 const Expr *Device) override;2301 2302 /// Emit initialization for doacross loop nesting support.2303 /// \param D Loop-based construct used in doacross nesting construct.2304 void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,2305 ArrayRef<Expr *> NumIterations) override;2306 2307 /// Emit code for doacross ordered directive with 'depend' clause.2308 /// \param C 'depend' clause with 'sink|source' dependency kind.2309 void emitDoacrossOrdered(CodeGenFunction &CGF,2310 const OMPDependClause *C) override;2311 2312 /// Emit code for doacross ordered directive with 'doacross' clause.2313 /// \param C 'doacross' clause with 'sink|source' dependence type.2314 void emitDoacrossOrdered(CodeGenFunction &CGF,2315 const OMPDoacrossClause *C) override;2316 2317 /// Translates the native parameter of outlined function if this is required2318 /// for target.2319 /// \param FD Field decl from captured record for the parameter.2320 /// \param NativeParam Parameter itself.2321 const VarDecl *translateParameter(const FieldDecl *FD,2322 const VarDecl *NativeParam) const override;2323 2324 /// Gets the address of the native argument basing on the address of the2325 /// target-specific parameter.2326 /// \param NativeParam Parameter itself.2327 /// \param TargetParam Corresponding target-specific parameter.2328 Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,2329 const VarDecl *TargetParam) const override;2330 2331 /// Gets the OpenMP-specific address of the local variable.2332 Address getAddressOfLocalVariable(CodeGenFunction &CGF,2333 const VarDecl *VD) override {2334 return Address::invalid();2335 }2336};2337 2338} // namespace CodeGen2339// Utility for openmp doacross clause kind2340namespace {2341template <typename T> class OMPDoacrossKind {2342public:2343 bool isSink(const T *) { return false; }2344 bool isSource(const T *) { return false; }2345};2346template <> class OMPDoacrossKind<OMPDependClause> {2347public:2348 bool isSink(const OMPDependClause *C) {2349 return C->getDependencyKind() == OMPC_DEPEND_sink;2350 }2351 bool isSource(const OMPDependClause *C) {2352 return C->getDependencyKind() == OMPC_DEPEND_source;2353 }2354};2355template <> class OMPDoacrossKind<OMPDoacrossClause> {2356public:2357 bool isSource(const OMPDoacrossClause *C) {2358 return C->getDependenceType() == OMPC_DOACROSS_source ||2359 C->getDependenceType() == OMPC_DOACROSS_source_omp_cur_iteration;2360 }2361 bool isSink(const OMPDoacrossClause *C) {2362 return C->getDependenceType() == OMPC_DOACROSS_sink ||2363 C->getDependenceType() == OMPC_DOACROSS_sink_omp_cur_iteration;2364 }2365};2366} // namespace2367} // namespace clang2368 2369#endif2370