424 lines · c
1//===------ CGOpenMPRuntimeGPU.h - Interface to OpenMP GPU Runtimes ------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This provides a generalized class for OpenMP runtime code generation10// specialized by GPU targets NVPTX and AMDGCN.11//12//===----------------------------------------------------------------------===//13 14#ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEGPU_H15#define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEGPU_H16 17#include "CGOpenMPRuntime.h"18#include "CodeGenFunction.h"19#include "clang/AST/StmtOpenMP.h"20 21namespace clang {22namespace CodeGen {23 24class CGOpenMPRuntimeGPU : public CGOpenMPRuntime {25public:26 /// Defines the execution mode.27 enum ExecutionMode {28 /// SPMD execution mode (all threads are worker threads).29 EM_SPMD,30 /// Non-SPMD execution mode (1 master thread, others are workers).31 EM_NonSPMD,32 /// Unknown execution mode (orphaned directive).33 EM_Unknown,34 };35 36 /// Target codegen is specialized based on two data-sharing modes: CUDA, in37 /// which the local variables are actually global threadlocal, and Generic, in38 /// which the local variables are placed in global memory if they may escape39 /// their declaration context.40 enum DataSharingMode {41 /// CUDA data sharing mode.42 DS_CUDA,43 /// Generic data-sharing mode.44 DS_Generic,45 };46 47private:48 /// Parallel outlined function work for workers to execute.49 llvm::SmallVector<llvm::Function *, 16> Work;50 51 struct EntryFunctionState {52 SourceLocation Loc;53 };54 55 ExecutionMode getExecutionMode() const;56 57 DataSharingMode getDataSharingMode() const;58 59 /// Get barrier to synchronize all threads in a block.60 void syncCTAThreads(CodeGenFunction &CGF);61 62 /// Helper for target directive initialization.63 void emitKernelInit(const OMPExecutableDirective &D, CodeGenFunction &CGF,64 EntryFunctionState &EST, bool IsSPMD);65 66 /// Helper for target directive finalization.67 void emitKernelDeinit(CodeGenFunction &CGF, EntryFunctionState &EST,68 bool IsSPMD);69 70 /// Helper for generic variables globalization prolog.71 void emitGenericVarsProlog(CodeGenFunction &CGF, SourceLocation Loc);72 73 /// Helper for generic variables globalization epilog.74 void emitGenericVarsEpilog(CodeGenFunction &CGF);75 76 //77 // Base class overrides.78 //79 80 /// Emit outlined function specialized for the Fork-Join81 /// programming model for applicable target directives on the NVPTX device.82 /// \param D Directive to emit.83 /// \param ParentName Name of the function that encloses the target region.84 /// \param OutlinedFn Outlined function value to be defined by this call.85 /// \param OutlinedFnID Outlined function ID value to be defined by this call.86 /// \param IsOffloadEntry True if the outlined function is an offload entry.87 /// An outlined function may not be an entry if, e.g. the if clause always88 /// evaluates to false.89 void emitNonSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName,90 llvm::Function *&OutlinedFn,91 llvm::Constant *&OutlinedFnID, bool IsOffloadEntry,92 const RegionCodeGenTy &CodeGen);93 94 /// Emit outlined function specialized for the Single Program95 /// Multiple Data programming model for applicable target directives on the96 /// NVPTX device.97 /// \param D Directive to emit.98 /// \param ParentName Name of the function that encloses the target region.99 /// \param OutlinedFn Outlined function value to be defined by this call.100 /// \param OutlinedFnID Outlined function ID value to be defined by this call.101 /// \param IsOffloadEntry True if the outlined function is an offload entry.102 /// \param CodeGen Object containing the target statements.103 /// An outlined function may not be an entry if, e.g. the if clause always104 /// evaluates to false.105 void emitSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName,106 llvm::Function *&OutlinedFn,107 llvm::Constant *&OutlinedFnID, bool IsOffloadEntry,108 const RegionCodeGenTy &CodeGen);109 110 /// Emit outlined function for 'target' directive on the NVPTX111 /// device.112 /// \param D Directive to emit.113 /// \param ParentName Name of the function that encloses the target region.114 /// \param OutlinedFn Outlined function value to be defined by this call.115 /// \param OutlinedFnID Outlined function ID value to be defined by this call.116 /// \param IsOffloadEntry True if the outlined function is an offload entry.117 /// An outlined function may not be an entry if, e.g. the if clause always118 /// evaluates to false.119 void emitTargetOutlinedFunction(const OMPExecutableDirective &D,120 StringRef ParentName,121 llvm::Function *&OutlinedFn,122 llvm::Constant *&OutlinedFnID,123 bool IsOffloadEntry,124 const RegionCodeGenTy &CodeGen) override;125 126protected:127 /// Check if the default location must be constant.128 /// Constant for NVPTX for better optimization.129 bool isDefaultLocationConstant() const override { return true; }130 131public:132 explicit CGOpenMPRuntimeGPU(CodeGenModule &CGM);133 134 bool isGPU() const override { return true; };135 136 /// Declare generalized virtual functions which need to be defined137 /// by all specializations of OpenMPGPURuntime Targets like AMDGCN138 /// and NVPTX.139 140 /// Check if the variable length declaration is delayed:141 bool isDelayedVariableLengthDecl(CodeGenFunction &CGF,142 const VarDecl *VD) const override;143 144 /// Get call to __kmpc_alloc_shared145 std::pair<llvm::Value *, llvm::Value *>146 getKmpcAllocShared(CodeGenFunction &CGF, const VarDecl *VD) override;147 148 /// Get call to __kmpc_free_shared149 void getKmpcFreeShared(150 CodeGenFunction &CGF,151 const std::pair<llvm::Value *, llvm::Value *> &AddrSizePair) override;152 153 /// Get the id of the current thread on the GPU.154 llvm::Value *getGPUThreadID(CodeGenFunction &CGF);155 156 /// Get the maximum number of threads in a block of the GPU.157 llvm::Value *getGPUNumThreads(CodeGenFunction &CGF);158 159 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32160 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.161 void emitProcBindClause(CodeGenFunction &CGF,162 llvm::omp::ProcBindKind ProcBind,163 SourceLocation Loc) override;164 165 // Currently unsupported on the device.166 using CGOpenMPRuntime::emitMessageClause;167 llvm::Value *emitMessageClause(CodeGenFunction &CGF, const Expr *Message,168 SourceLocation Loc) override;169 170 // Currently unsupported on the device.171 using CGOpenMPRuntime::emitSeverityClause;172 llvm::Value *emitSeverityClause(OpenMPSeverityClauseKind Severity,173 SourceLocation Loc) override;174 175 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32176 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'177 /// clause.178 void emitNumThreadsClause(179 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,180 OpenMPNumThreadsClauseModifier Modifier = OMPC_NUMTHREADS_unknown,181 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,182 SourceLocation SeverityLoc = SourceLocation(),183 const Expr *Message = nullptr,184 SourceLocation MessageLoc = SourceLocation()) override;185 186 /// This function ought to emit, in the general case, a call to187 // the openmp runtime kmpc_push_num_teams. In NVPTX backend it is not needed188 // as these numbers are obtained through the PTX grid and block configuration.189 /// \param NumTeams An integer expression of teams.190 /// \param ThreadLimit An integer expression of threads.191 void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,192 const Expr *ThreadLimit, SourceLocation Loc) override;193 194 /// Emits inlined function for the specified OpenMP parallel195 // directive.196 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,197 /// kmp_int32 BoundID, struct context_vars*).198 /// \param CGF Reference to current CodeGenFunction.199 /// \param D OpenMP directive.200 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.201 /// \param InnermostKind Kind of innermost directive (for simple directives it202 /// is a directive itself, for combined - its innermost directive).203 /// \param CodeGen Code generation sequence for the \a D directive.204 llvm::Function *emitParallelOutlinedFunction(205 CodeGenFunction &CGF, const OMPExecutableDirective &D,206 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,207 const RegionCodeGenTy &CodeGen) override;208 209 /// Emits inlined function for the specified OpenMP teams210 // directive.211 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,212 /// kmp_int32 BoundID, struct context_vars*).213 /// \param CGF Reference to current CodeGenFunction.214 /// \param D OpenMP directive.215 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.216 /// \param InnermostKind Kind of innermost directive (for simple directives it217 /// is a directive itself, for combined - its innermost directive).218 /// \param CodeGen Code generation sequence for the \a D directive.219 llvm::Function *emitTeamsOutlinedFunction(220 CodeGenFunction &CGF, const OMPExecutableDirective &D,221 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,222 const RegionCodeGenTy &CodeGen) override;223 224 /// Emits code for teams call of the \a OutlinedFn with225 /// variables captured in a record which address is stored in \a226 /// CapturedStruct.227 /// \param OutlinedFn Outlined function to be run by team masters. Type of228 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).229 /// \param CapturedVars A pointer to the record with the references to230 /// variables used in \a OutlinedFn function.231 ///232 void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,233 SourceLocation Loc, llvm::Function *OutlinedFn,234 ArrayRef<llvm::Value *> CapturedVars) override;235 236 /// Emits code for parallel or serial call of the \a OutlinedFn with237 /// variables captured in a record which address is stored in \a238 /// CapturedStruct.239 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of240 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).241 /// \param CapturedVars A pointer to the record with the references to242 /// variables used in \a OutlinedFn function.243 /// \param IfCond Condition in the associated 'if' clause, if it was244 /// specified, nullptr otherwise.245 /// \param NumThreads The value corresponding to the num_threads clause, if246 /// any, or nullptr.247 /// \param NumThreadsModifier The modifier of the num_threads clause, if248 /// any, ignored otherwise. Currently unused on the device.249 /// \param Severity The severity corresponding to the num_threads clause, if250 /// any, ignored otherwise. Currently unused on the device.251 /// \param Message The message string corresponding to the num_threads clause,252 /// if any, or nullptr. Currently unused on the device.253 void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,254 llvm::Function *OutlinedFn,255 ArrayRef<llvm::Value *> CapturedVars,256 const Expr *IfCond, llvm::Value *NumThreads,257 OpenMPNumThreadsClauseModifier NumThreadsModifier =258 OMPC_NUMTHREADS_unknown,259 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,260 const Expr *Message = nullptr) override;261 262 /// Emit an implicit/explicit barrier for OpenMP threads.263 /// \param Kind Directive for which this implicit barrier call must be264 /// generated. Must be OMPD_barrier for explicit barrier generation.265 /// \param EmitChecks true if need to emit checks for cancellation barriers.266 /// \param ForceSimpleCall true simple barrier call must be emitted, false if267 /// runtime class decides which one to emit (simple or with cancellation268 /// checks).269 ///270 void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,271 OpenMPDirectiveKind Kind, bool EmitChecks = true,272 bool ForceSimpleCall = false) override;273 274 /// Emits a critical region.275 /// \param CriticalName Name of the critical region.276 /// \param CriticalOpGen Generator for the statement associated with the given277 /// critical region.278 /// \param Hint Value of the 'hint' clause (optional).279 void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,280 const RegionCodeGenTy &CriticalOpGen,281 SourceLocation Loc,282 const Expr *Hint = nullptr) override;283 284 /// Emit a code for reduction clause.285 ///286 /// \param Privates List of private copies for original reduction arguments.287 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.288 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.289 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'290 /// or 'operator binop(LHS, RHS)'.291 /// \param Options List of options for reduction codegen:292 /// WithNowait true if parent directive has also nowait clause, false293 /// otherwise.294 /// SimpleReduction Emit reduction operation only. Used for omp simd295 /// directive on the host.296 /// ReductionKind The kind of reduction to perform.297 void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,298 ArrayRef<const Expr *> Privates,299 ArrayRef<const Expr *> LHSExprs,300 ArrayRef<const Expr *> RHSExprs,301 ArrayRef<const Expr *> ReductionOps,302 ReductionOptionsTy Options) override;303 304 /// Translates the native parameter of outlined function if this is required305 /// for target.306 /// \param FD Field decl from captured record for the parameter.307 /// \param NativeParam Parameter itself.308 const VarDecl *translateParameter(const FieldDecl *FD,309 const VarDecl *NativeParam) const override;310 311 /// Gets the address of the native argument basing on the address of the312 /// target-specific parameter.313 /// \param NativeParam Parameter itself.314 /// \param TargetParam Corresponding target-specific parameter.315 Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,316 const VarDecl *TargetParam) const override;317 318 /// Emits call of the outlined function with the provided arguments,319 /// translating these arguments to correct target-specific arguments.320 void321 emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc,322 llvm::FunctionCallee OutlinedFn,323 ArrayRef<llvm::Value *> Args = {}) const override;324 325 /// Emits OpenMP-specific function prolog.326 /// Required for device constructs.327 void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) override;328 329 /// Gets the OpenMP-specific address of the local variable.330 Address getAddressOfLocalVariable(CodeGenFunction &CGF,331 const VarDecl *VD) override;332 333 /// Cleans up references to the objects in finished function.334 ///335 void functionFinished(CodeGenFunction &CGF) override;336 337 /// Choose a default value for the dist_schedule clause.338 void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,339 const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,340 llvm::Value *&Chunk) const override;341 342 /// Choose a default value for the schedule clause.343 void getDefaultScheduleAndChunk(CodeGenFunction &CGF,344 const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,345 const Expr *&ChunkExpr) const override;346 347 /// Adjust some parameters for the target-based directives, like addresses of348 /// the variables captured by reference in lambdas.349 void adjustTargetSpecificDataForLambdas(350 CodeGenFunction &CGF, const OMPExecutableDirective &D) const override;351 352 /// Perform check on requires decl to ensure that target architecture353 /// supports unified addressing354 void processRequiresDirective(const OMPRequiresDecl *D) override;355 356 /// Checks if the variable has associated OMPAllocateDeclAttr attribute with357 /// the predefined allocator and translates it into the corresponding address358 /// space.359 bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS) override;360 361private:362 /// Track the execution mode when codegening directives within a target363 /// region. The appropriate mode (SPMD/NON-SPMD) is set on entry to the364 /// target region and used by containing directives such as 'parallel'365 /// to emit optimized code.366 ExecutionMode CurrentExecutionMode = EM_Unknown;367 368 /// Track the data sharing mode when codegening directives within a target369 /// region.370 DataSharingMode CurrentDataSharingMode = DataSharingMode::DS_Generic;371 372 /// true if currently emitting code for target/teams/distribute region, false373 /// - otherwise.374 bool IsInTTDRegion = false;375 376 /// Map between an outlined function and its wrapper.377 llvm::DenseMap<llvm::Function *, llvm::Function *> WrapperFunctionsMap;378 379 /// Emit function which wraps the outline parallel region380 /// and controls the parameters which are passed to this function.381 /// The wrapper ensures that the outlined function is called382 /// with the correct arguments when data is shared.383 llvm::Function *createParallelDataSharingWrapper(384 llvm::Function *OutlinedParallelFn, const OMPExecutableDirective &D);385 386 /// The data for the single globalized variable.387 struct MappedVarData {388 /// Corresponding field in the global record.389 llvm::Value *GlobalizedVal = nullptr;390 /// Corresponding address.391 Address PrivateAddr = Address::invalid();392 };393 /// The map of local variables to their addresses in the global memory.394 using DeclToAddrMapTy = llvm::MapVector<const Decl *, MappedVarData>;395 /// Set of the parameters passed by value escaping OpenMP context.396 using EscapedParamsTy = llvm::SmallPtrSet<const Decl *, 4>;397 struct FunctionData {398 DeclToAddrMapTy LocalVarData;399 EscapedParamsTy EscapedParameters;400 llvm::SmallVector<const ValueDecl*, 4> EscapedVariableLengthDecls;401 llvm::SmallVector<const ValueDecl *, 4> DelayedVariableLengthDecls;402 llvm::SmallVector<std::pair<llvm::Value *, llvm::Value *>, 4>403 EscapedVariableLengthDeclsAddrs;404 std::unique_ptr<CodeGenFunction::OMPMapVars> MappedParams;405 };406 /// Maps the function to the list of the globalized variables with their407 /// addresses.408 llvm::SmallDenseMap<llvm::Function *, FunctionData> FunctionGlobalizedDecls;409 /// List of the records with the list of fields for the reductions across the410 /// teams. Used to build the intermediate buffer for the fast teams411 /// reductions.412 /// All the records are gathered into a union `union.type` is created.413 llvm::SmallVector<const RecordDecl *, 4> TeamsReductions;414 /// Pair of the Non-SPMD team and all reductions variables in this team415 /// region.416 std::pair<const Decl *, llvm::SmallVector<const ValueDecl *, 4>>417 TeamAndReductions;418};419 420} // CodeGen namespace.421} // clang namespace.422 423#endif // LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEGPU_H424