1299 lines · cpp
1//===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the IRBuilder class, which is used as a convenient way10// to create LLVM instructions with a consistent and simplified interface.11//12//===----------------------------------------------------------------------===//13 14#include "llvm/IR/IRBuilder.h"15#include "llvm/ADT/ArrayRef.h"16#include "llvm/IR/Constant.h"17#include "llvm/IR/Constants.h"18#include "llvm/IR/DerivedTypes.h"19#include "llvm/IR/Function.h"20#include "llvm/IR/GlobalValue.h"21#include "llvm/IR/GlobalVariable.h"22#include "llvm/IR/IntrinsicInst.h"23#include "llvm/IR/Intrinsics.h"24#include "llvm/IR/LLVMContext.h"25#include "llvm/IR/Module.h"26#include "llvm/IR/NoFolder.h"27#include "llvm/IR/Operator.h"28#include "llvm/IR/ProfDataUtils.h"29#include "llvm/IR/Statepoint.h"30#include "llvm/IR/Type.h"31#include "llvm/IR/Value.h"32#include "llvm/Support/Casting.h"33#include <cassert>34#include <cstdint>35#include <optional>36#include <vector>37 38using namespace llvm;39 40/// CreateGlobalString - Make a new global variable with an initializer that41/// has array of i8 type filled in with the nul terminated string value42/// specified. If Name is specified, it is the name of the global variable43/// created.44GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,45 const Twine &Name,46 unsigned AddressSpace,47 Module *M, bool AddNull) {48 Constant *StrConstant = ConstantDataArray::getString(Context, Str, AddNull);49 if (!M)50 M = BB->getParent()->getParent();51 auto *GV = new GlobalVariable(52 *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,53 StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);54 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);55 GV->setAlignment(M->getDataLayout().getPrefTypeAlign(getInt8Ty()));56 return GV;57}58 59Type *IRBuilderBase::getCurrentFunctionReturnType() const {60 assert(BB && BB->getParent() && "No current function!");61 return BB->getParent()->getReturnType();62}63 64DebugLoc IRBuilderBase::getCurrentDebugLocation() const { return StoredDL; }65void IRBuilderBase::SetInstDebugLocation(Instruction *I) const {66 // We prefer to set our current debug location if any has been set, but if67 // our debug location is empty and I has a valid location, we shouldn't68 // overwrite it.69 I->setDebugLoc(StoredDL.orElse(I->getDebugLoc()));70}71 72Value *IRBuilderBase::CreateAggregateCast(Value *V, Type *DestTy) {73 Type *SrcTy = V->getType();74 if (SrcTy == DestTy)75 return V;76 77 if (SrcTy->isAggregateType()) {78 unsigned NumElements;79 if (SrcTy->isStructTy()) {80 assert(DestTy->isStructTy() && "Expected StructType");81 assert(SrcTy->getStructNumElements() == DestTy->getStructNumElements() &&82 "Expected StructTypes with equal number of elements");83 NumElements = SrcTy->getStructNumElements();84 } else {85 assert(SrcTy->isArrayTy() && DestTy->isArrayTy() && "Expected ArrayType");86 assert(SrcTy->getArrayNumElements() == DestTy->getArrayNumElements() &&87 "Expected ArrayTypes with equal number of elements");88 NumElements = SrcTy->getArrayNumElements();89 }90 91 Value *Result = PoisonValue::get(DestTy);92 for (unsigned I = 0; I < NumElements; ++I) {93 Type *ElementTy = SrcTy->isStructTy() ? DestTy->getStructElementType(I)94 : DestTy->getArrayElementType();95 Value *Element =96 CreateAggregateCast(CreateExtractValue(V, ArrayRef(I)), ElementTy);97 98 Result = CreateInsertValue(Result, Element, ArrayRef(I));99 }100 return Result;101 }102 103 return CreateBitOrPointerCast(V, DestTy);104}105 106CallInst *107IRBuilderBase::createCallHelper(Function *Callee, ArrayRef<Value *> Ops,108 const Twine &Name, FMFSource FMFSource,109 ArrayRef<OperandBundleDef> OpBundles) {110 CallInst *CI = CreateCall(Callee, Ops, OpBundles, Name);111 if (isa<FPMathOperator>(CI))112 CI->setFastMathFlags(FMFSource.get(FMF));113 return CI;114}115 116static Value *CreateVScaleMultiple(IRBuilderBase &B, Type *Ty, uint64_t Scale) {117 Value *VScale = B.CreateVScale(Ty);118 if (Scale == 1)119 return VScale;120 121 return B.CreateNUWMul(VScale, ConstantInt::get(Ty, Scale));122}123 124Value *IRBuilderBase::CreateElementCount(Type *Ty, ElementCount EC) {125 if (EC.isFixed() || EC.isZero())126 return ConstantInt::get(Ty, EC.getKnownMinValue());127 128 return CreateVScaleMultiple(*this, Ty, EC.getKnownMinValue());129}130 131Value *IRBuilderBase::CreateTypeSize(Type *Ty, TypeSize Size) {132 if (Size.isFixed() || Size.isZero())133 return ConstantInt::get(Ty, Size.getKnownMinValue());134 135 return CreateVScaleMultiple(*this, Ty, Size.getKnownMinValue());136}137 138Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) {139 Type *STy = DstType->getScalarType();140 if (isa<ScalableVectorType>(DstType)) {141 Type *StepVecType = DstType;142 // TODO: We expect this special case (element type < 8 bits) to be143 // temporary - once the intrinsic properly supports < 8 bits this code144 // can be removed.145 if (STy->getScalarSizeInBits() < 8)146 StepVecType =147 VectorType::get(getInt8Ty(), cast<ScalableVectorType>(DstType));148 Value *Res = CreateIntrinsic(Intrinsic::stepvector, {StepVecType}, {},149 nullptr, Name);150 if (StepVecType != DstType)151 Res = CreateTrunc(Res, DstType);152 return Res;153 }154 155 unsigned NumEls = cast<FixedVectorType>(DstType)->getNumElements();156 157 // Create a vector of consecutive numbers from zero to VF.158 SmallVector<Constant *, 8> Indices;159 for (unsigned i = 0; i < NumEls; ++i)160 Indices.push_back(ConstantInt::get(STy, i));161 162 // Add the consecutive indices to the vector value.163 return ConstantVector::get(Indices);164}165 166CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,167 MaybeAlign Align, bool isVolatile,168 const AAMDNodes &AAInfo) {169 Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};170 Type *Tys[] = {Ptr->getType(), Size->getType()};171 172 CallInst *CI = CreateIntrinsic(Intrinsic::memset, Tys, Ops);173 174 if (Align)175 cast<MemSetInst>(CI)->setDestAlignment(*Align);176 CI->setAAMetadata(AAInfo);177 return CI;178}179 180CallInst *IRBuilderBase::CreateMemSetInline(Value *Dst, MaybeAlign DstAlign,181 Value *Val, Value *Size,182 bool IsVolatile,183 const AAMDNodes &AAInfo) {184 Value *Ops[] = {Dst, Val, Size, getInt1(IsVolatile)};185 Type *Tys[] = {Dst->getType(), Size->getType()};186 187 CallInst *CI = CreateIntrinsic(Intrinsic::memset_inline, Tys, Ops);188 189 if (DstAlign)190 cast<MemSetInst>(CI)->setDestAlignment(*DstAlign);191 CI->setAAMetadata(AAInfo);192 return CI;193}194 195CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(196 Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,197 const AAMDNodes &AAInfo) {198 199 Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};200 Type *Tys[] = {Ptr->getType(), Size->getType()};201 202 CallInst *CI =203 CreateIntrinsic(Intrinsic::memset_element_unordered_atomic, Tys, Ops);204 205 cast<AnyMemSetInst>(CI)->setDestAlignment(Alignment);206 CI->setAAMetadata(AAInfo);207 return CI;208}209 210CallInst *IRBuilderBase::CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst,211 MaybeAlign DstAlign, Value *Src,212 MaybeAlign SrcAlign, Value *Size,213 bool isVolatile,214 const AAMDNodes &AAInfo) {215 assert((IntrID == Intrinsic::memcpy || IntrID == Intrinsic::memcpy_inline ||216 IntrID == Intrinsic::memmove) &&217 "Unexpected intrinsic ID");218 Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};219 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};220 221 CallInst *CI = CreateIntrinsic(IntrID, Tys, Ops);222 223 auto* MCI = cast<MemTransferInst>(CI);224 if (DstAlign)225 MCI->setDestAlignment(*DstAlign);226 if (SrcAlign)227 MCI->setSourceAlignment(*SrcAlign);228 MCI->setAAMetadata(AAInfo);229 return CI;230}231 232CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(233 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,234 uint32_t ElementSize, const AAMDNodes &AAInfo) {235 assert(DstAlign >= ElementSize &&236 "Pointer alignment must be at least element size");237 assert(SrcAlign >= ElementSize &&238 "Pointer alignment must be at least element size");239 Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};240 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};241 242 CallInst *CI =243 CreateIntrinsic(Intrinsic::memcpy_element_unordered_atomic, Tys, Ops);244 245 // Set the alignment of the pointer args.246 auto *AMCI = cast<AnyMemCpyInst>(CI);247 AMCI->setDestAlignment(DstAlign);248 AMCI->setSourceAlignment(SrcAlign);249 AMCI->setAAMetadata(AAInfo);250 return CI;251}252 253/// isConstantOne - Return true only if val is constant int 1254static bool isConstantOne(const Value *Val) {255 assert(Val && "isConstantOne does not work with nullptr Val");256 const ConstantInt *CVal = dyn_cast<ConstantInt>(Val);257 return CVal && CVal->isOne();258}259 260CallInst *IRBuilderBase::CreateMalloc(Type *IntPtrTy, Type *AllocTy,261 Value *AllocSize, Value *ArraySize,262 ArrayRef<OperandBundleDef> OpB,263 Function *MallocF, const Twine &Name) {264 // malloc(type) becomes:265 // i8* malloc(typeSize)266 // malloc(type, arraySize) becomes:267 // i8* malloc(typeSize*arraySize)268 if (!ArraySize)269 ArraySize = ConstantInt::get(IntPtrTy, 1);270 else if (ArraySize->getType() != IntPtrTy)271 ArraySize = CreateIntCast(ArraySize, IntPtrTy, false);272 273 if (!isConstantOne(ArraySize)) {274 if (isConstantOne(AllocSize)) {275 AllocSize = ArraySize; // Operand * 1 = Operand276 } else {277 // Multiply type size by the array size...278 AllocSize = CreateMul(ArraySize, AllocSize, "mallocsize");279 }280 }281 282 assert(AllocSize->getType() == IntPtrTy && "malloc arg is wrong size");283 // Create the call to Malloc.284 Module *M = BB->getParent()->getParent();285 Type *BPTy = PointerType::getUnqual(Context);286 FunctionCallee MallocFunc = MallocF;287 if (!MallocFunc)288 // prototype malloc as "void *malloc(size_t)"289 MallocFunc = M->getOrInsertFunction("malloc", BPTy, IntPtrTy);290 CallInst *MCall = CreateCall(MallocFunc, AllocSize, OpB, Name);291 292 MCall->setTailCall();293 if (Function *F = dyn_cast<Function>(MallocFunc.getCallee())) {294 MCall->setCallingConv(F->getCallingConv());295 F->setReturnDoesNotAlias();296 }297 298 assert(!MCall->getType()->isVoidTy() && "Malloc has void return type");299 300 return MCall;301}302 303CallInst *IRBuilderBase::CreateMalloc(Type *IntPtrTy, Type *AllocTy,304 Value *AllocSize, Value *ArraySize,305 Function *MallocF, const Twine &Name) {306 307 return CreateMalloc(IntPtrTy, AllocTy, AllocSize, ArraySize, {}, MallocF,308 Name);309}310 311/// CreateFree - Generate the IR for a call to the builtin free function.312CallInst *IRBuilderBase::CreateFree(Value *Source,313 ArrayRef<OperandBundleDef> Bundles) {314 assert(Source->getType()->isPointerTy() &&315 "Can not free something of nonpointer type!");316 317 Module *M = BB->getParent()->getParent();318 319 Type *VoidTy = Type::getVoidTy(M->getContext());320 Type *VoidPtrTy = PointerType::getUnqual(M->getContext());321 // prototype free as "void free(void*)"322 FunctionCallee FreeFunc = M->getOrInsertFunction("free", VoidTy, VoidPtrTy);323 CallInst *Result = CreateCall(FreeFunc, Source, Bundles, "");324 Result->setTailCall();325 if (Function *F = dyn_cast<Function>(FreeFunc.getCallee()))326 Result->setCallingConv(F->getCallingConv());327 328 return Result;329}330 331CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(332 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,333 uint32_t ElementSize, const AAMDNodes &AAInfo) {334 assert(DstAlign >= ElementSize &&335 "Pointer alignment must be at least element size");336 assert(SrcAlign >= ElementSize &&337 "Pointer alignment must be at least element size");338 Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};339 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};340 341 CallInst *CI =342 CreateIntrinsic(Intrinsic::memmove_element_unordered_atomic, Tys, Ops);343 344 // Set the alignment of the pointer args.345 CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));346 CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));347 CI->setAAMetadata(AAInfo);348 return CI;349}350 351CallInst *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) {352 Value *Ops[] = {Src};353 Type *Tys[] = { Src->getType() };354 return CreateIntrinsic(ID, Tys, Ops);355}356 357CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {358 Value *Ops[] = {Acc, Src};359 return CreateIntrinsic(Intrinsic::vector_reduce_fadd, {Src->getType()}, Ops);360}361 362CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {363 Value *Ops[] = {Acc, Src};364 return CreateIntrinsic(Intrinsic::vector_reduce_fmul, {Src->getType()}, Ops);365}366 367CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {368 return getReductionIntrinsic(Intrinsic::vector_reduce_add, Src);369}370 371CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {372 return getReductionIntrinsic(Intrinsic::vector_reduce_mul, Src);373}374 375CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {376 return getReductionIntrinsic(Intrinsic::vector_reduce_and, Src);377}378 379CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {380 return getReductionIntrinsic(Intrinsic::vector_reduce_or, Src);381}382 383CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {384 return getReductionIntrinsic(Intrinsic::vector_reduce_xor, Src);385}386 387CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {388 auto ID =389 IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;390 return getReductionIntrinsic(ID, Src);391}392 393CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {394 auto ID =395 IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;396 return getReductionIntrinsic(ID, Src);397}398 399CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src) {400 return getReductionIntrinsic(Intrinsic::vector_reduce_fmax, Src);401}402 403CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src) {404 return getReductionIntrinsic(Intrinsic::vector_reduce_fmin, Src);405}406 407CallInst *IRBuilderBase::CreateFPMaximumReduce(Value *Src) {408 return getReductionIntrinsic(Intrinsic::vector_reduce_fmaximum, Src);409}410 411CallInst *IRBuilderBase::CreateFPMinimumReduce(Value *Src) {412 return getReductionIntrinsic(Intrinsic::vector_reduce_fminimum, Src);413}414 415CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr) {416 assert(isa<PointerType>(Ptr->getType()) &&417 "lifetime.start only applies to pointers.");418 return CreateIntrinsic(Intrinsic::lifetime_start, {Ptr->getType()}, {Ptr});419}420 421CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr) {422 assert(isa<PointerType>(Ptr->getType()) &&423 "lifetime.end only applies to pointers.");424 return CreateIntrinsic(Intrinsic::lifetime_end, {Ptr->getType()}, {Ptr});425}426 427CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {428 429 assert(isa<PointerType>(Ptr->getType()) &&430 "invariant.start only applies to pointers.");431 if (!Size)432 Size = getInt64(-1);433 else434 assert(Size->getType() == getInt64Ty() &&435 "invariant.start requires the size to be an i64");436 437 Value *Ops[] = {Size, Ptr};438 // Fill in the single overloaded type: memory object type.439 Type *ObjectPtr[1] = {Ptr->getType()};440 return CreateIntrinsic(Intrinsic::invariant_start, ObjectPtr, Ops);441}442 443static MaybeAlign getAlign(Value *Ptr) {444 if (auto *V = dyn_cast<GlobalVariable>(Ptr))445 return V->getAlign();446 if (auto *A = dyn_cast<GlobalAlias>(Ptr))447 return getAlign(A->getAliaseeObject());448 return {};449}450 451CallInst *IRBuilderBase::CreateThreadLocalAddress(Value *Ptr) {452 assert(isa<GlobalValue>(Ptr) && cast<GlobalValue>(Ptr)->isThreadLocal() &&453 "threadlocal_address only applies to thread local variables.");454 CallInst *CI = CreateIntrinsic(llvm::Intrinsic::threadlocal_address,455 {Ptr->getType()}, {Ptr});456 if (MaybeAlign A = getAlign(Ptr)) {457 CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), *A));458 CI->addRetAttr(Attribute::getWithAlignment(CI->getContext(), *A));459 }460 return CI;461}462 463CallInst *464IRBuilderBase::CreateAssumption(Value *Cond,465 ArrayRef<OperandBundleDef> OpBundles) {466 assert(Cond->getType() == getInt1Ty() &&467 "an assumption condition must be of type i1");468 469 Value *Ops[] = { Cond };470 Module *M = BB->getParent()->getParent();471 Function *FnAssume = Intrinsic::getOrInsertDeclaration(M, Intrinsic::assume);472 return CreateCall(FnAssume, Ops, OpBundles);473}474 475Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {476 return CreateIntrinsic(Intrinsic::experimental_noalias_scope_decl, {},477 {Scope});478}479 480/// Create a call to a Masked Load intrinsic.481/// \p Ty - vector type to load482/// \p Ptr - base pointer for the load483/// \p Alignment - alignment of the source location484/// \p Mask - vector of booleans which indicates what vector lanes should485/// be accessed in memory486/// \p PassThru - pass-through value that is used to fill the masked-off lanes487/// of the result488/// \p Name - name of the result variable489CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,490 Value *Mask, Value *PassThru,491 const Twine &Name) {492 auto *PtrTy = cast<PointerType>(Ptr->getType());493 assert(Ty->isVectorTy() && "Type should be vector");494 assert(Mask && "Mask should not be all-ones (null)");495 if (!PassThru)496 PassThru = PoisonValue::get(Ty);497 Type *OverloadedTypes[] = { Ty, PtrTy };498 Value *Ops[] = {Ptr, Mask, PassThru};499 CallInst *CI =500 CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, OverloadedTypes, Name);501 CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), Alignment));502 return CI;503}504 505/// Create a call to a Masked Store intrinsic.506/// \p Val - data to be stored,507/// \p Ptr - base pointer for the store508/// \p Alignment - alignment of the destination location509/// \p Mask - vector of booleans which indicates what vector lanes should510/// be accessed in memory511CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,512 Align Alignment, Value *Mask) {513 auto *PtrTy = cast<PointerType>(Ptr->getType());514 Type *DataTy = Val->getType();515 assert(DataTy->isVectorTy() && "Val should be a vector");516 assert(Mask && "Mask should not be all-ones (null)");517 Type *OverloadedTypes[] = { DataTy, PtrTy };518 Value *Ops[] = {Val, Ptr, Mask};519 CallInst *CI =520 CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);521 CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), Alignment));522 return CI;523}524 525/// Create a call to a Masked intrinsic, with given intrinsic Id,526/// an array of operands - Ops, and an array of overloaded types -527/// OverloadedTypes.528CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,529 ArrayRef<Value *> Ops,530 ArrayRef<Type *> OverloadedTypes,531 const Twine &Name) {532 return CreateIntrinsic(Id, OverloadedTypes, Ops, {}, Name);533}534 535/// Create a call to a Masked Gather intrinsic.536/// \p Ty - vector type to gather537/// \p Ptrs - vector of pointers for loading538/// \p Align - alignment for one element539/// \p Mask - vector of booleans which indicates what vector lanes should540/// be accessed in memory541/// \p PassThru - pass-through value that is used to fill the masked-off lanes542/// of the result543/// \p Name - name of the result variable544CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,545 Align Alignment, Value *Mask,546 Value *PassThru,547 const Twine &Name) {548 auto *VecTy = cast<VectorType>(Ty);549 ElementCount NumElts = VecTy->getElementCount();550 auto *PtrsTy = cast<VectorType>(Ptrs->getType());551 assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch");552 553 if (!Mask)554 Mask = getAllOnesMask(NumElts);555 556 if (!PassThru)557 PassThru = PoisonValue::get(Ty);558 559 Type *OverloadedTypes[] = {Ty, PtrsTy};560 Value *Ops[] = {Ptrs, Mask, PassThru};561 562 // We specify only one type when we create this intrinsic. Types of other563 // arguments are derived from this type.564 CallInst *CI = CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops,565 OverloadedTypes, Name);566 CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), Alignment));567 return CI;568}569 570/// Create a call to a Masked Scatter intrinsic.571/// \p Data - data to be stored,572/// \p Ptrs - the vector of pointers, where the \p Data elements should be573/// stored574/// \p Align - alignment for one element575/// \p Mask - vector of booleans which indicates what vector lanes should576/// be accessed in memory577CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,578 Align Alignment, Value *Mask) {579 auto *PtrsTy = cast<VectorType>(Ptrs->getType());580 auto *DataTy = cast<VectorType>(Data->getType());581 ElementCount NumElts = PtrsTy->getElementCount();582 583 if (!Mask)584 Mask = getAllOnesMask(NumElts);585 586 Type *OverloadedTypes[] = {DataTy, PtrsTy};587 Value *Ops[] = {Data, Ptrs, Mask};588 589 // We specify only one type when we create this intrinsic. Types of other590 // arguments are derived from this type.591 CallInst *CI =592 CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);593 CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), Alignment));594 return CI;595}596 597/// Create a call to Masked Expand Load intrinsic598/// \p Ty - vector type to load599/// \p Ptr - base pointer for the load600/// \p Align - alignment of \p Ptr601/// \p Mask - vector of booleans which indicates what vector lanes should602/// be accessed in memory603/// \p PassThru - pass-through value that is used to fill the masked-off lanes604/// of the result605/// \p Name - name of the result variable606CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,607 MaybeAlign Align, Value *Mask,608 Value *PassThru,609 const Twine &Name) {610 assert(Ty->isVectorTy() && "Type should be vector");611 assert(Mask && "Mask should not be all-ones (null)");612 if (!PassThru)613 PassThru = PoisonValue::get(Ty);614 Type *OverloadedTypes[] = {Ty};615 Value *Ops[] = {Ptr, Mask, PassThru};616 CallInst *CI = CreateMaskedIntrinsic(Intrinsic::masked_expandload, Ops,617 OverloadedTypes, Name);618 if (Align)619 CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), *Align));620 return CI;621}622 623/// Create a call to Masked Compress Store intrinsic624/// \p Val - data to be stored,625/// \p Ptr - base pointer for the store626/// \p Align - alignment of \p Ptr627/// \p Mask - vector of booleans which indicates what vector lanes should628/// be accessed in memory629CallInst *IRBuilderBase::CreateMaskedCompressStore(Value *Val, Value *Ptr,630 MaybeAlign Align,631 Value *Mask) {632 Type *DataTy = Val->getType();633 assert(DataTy->isVectorTy() && "Val should be a vector");634 assert(Mask && "Mask should not be all-ones (null)");635 Type *OverloadedTypes[] = {DataTy};636 Value *Ops[] = {Val, Ptr, Mask};637 CallInst *CI = CreateMaskedIntrinsic(Intrinsic::masked_compressstore, Ops,638 OverloadedTypes);639 if (Align)640 CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), *Align));641 return CI;642}643 644template <typename T0>645static std::vector<Value *>646getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,647 Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {648 std::vector<Value *> Args;649 Args.push_back(B.getInt64(ID));650 Args.push_back(B.getInt32(NumPatchBytes));651 Args.push_back(ActualCallee);652 Args.push_back(B.getInt32(CallArgs.size()));653 Args.push_back(B.getInt32(Flags));654 llvm::append_range(Args, CallArgs);655 // GC Transition and Deopt args are now always handled via operand bundle.656 // They will be removed from the signature of gc.statepoint shortly.657 Args.push_back(B.getInt32(0));658 Args.push_back(B.getInt32(0));659 // GC args are now encoded in the gc-live operand bundle660 return Args;661}662 663template<typename T1, typename T2, typename T3>664static std::vector<OperandBundleDef>665getStatepointBundles(std::optional<ArrayRef<T1>> TransitionArgs,666 std::optional<ArrayRef<T2>> DeoptArgs,667 ArrayRef<T3> GCArgs) {668 std::vector<OperandBundleDef> Rval;669 if (DeoptArgs)670 Rval.emplace_back("deopt", SmallVector<Value *, 16>(*DeoptArgs));671 if (TransitionArgs)672 Rval.emplace_back("gc-transition",673 SmallVector<Value *, 16>(*TransitionArgs));674 if (GCArgs.size())675 Rval.emplace_back("gc-live", SmallVector<Value *, 16>(GCArgs));676 return Rval;677}678 679template <typename T0, typename T1, typename T2, typename T3>680static CallInst *CreateGCStatepointCallCommon(681 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,682 FunctionCallee ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,683 std::optional<ArrayRef<T1>> TransitionArgs,684 std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,685 const Twine &Name) {686 Module *M = Builder->GetInsertBlock()->getParent()->getParent();687 // Fill in the one generic type'd argument (the function is also vararg)688 Function *FnStatepoint = Intrinsic::getOrInsertDeclaration(689 M, Intrinsic::experimental_gc_statepoint,690 {ActualCallee.getCallee()->getType()});691 692 std::vector<Value *> Args = getStatepointArgs(693 *Builder, ID, NumPatchBytes, ActualCallee.getCallee(), Flags, CallArgs);694 695 CallInst *CI = Builder->CreateCall(696 FnStatepoint, Args,697 getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);698 CI->addParamAttr(2,699 Attribute::get(Builder->getContext(), Attribute::ElementType,700 ActualCallee.getFunctionType()));701 return CI;702}703 704CallInst *IRBuilderBase::CreateGCStatepointCall(705 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,706 ArrayRef<Value *> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,707 ArrayRef<Value *> GCArgs, const Twine &Name) {708 return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(709 this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),710 CallArgs, std::nullopt /* No Transition Args */, DeoptArgs, GCArgs, Name);711}712 713CallInst *IRBuilderBase::CreateGCStatepointCall(714 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,715 uint32_t Flags, ArrayRef<Value *> CallArgs,716 std::optional<ArrayRef<Use>> TransitionArgs,717 std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,718 const Twine &Name) {719 return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(720 this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,721 DeoptArgs, GCArgs, Name);722}723 724CallInst *IRBuilderBase::CreateGCStatepointCall(725 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,726 ArrayRef<Use> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,727 ArrayRef<Value *> GCArgs, const Twine &Name) {728 return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(729 this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),730 CallArgs, std::nullopt, DeoptArgs, GCArgs, Name);731}732 733template <typename T0, typename T1, typename T2, typename T3>734static InvokeInst *CreateGCStatepointInvokeCommon(735 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,736 FunctionCallee ActualInvokee, BasicBlock *NormalDest,737 BasicBlock *UnwindDest, uint32_t Flags, ArrayRef<T0> InvokeArgs,738 std::optional<ArrayRef<T1>> TransitionArgs,739 std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,740 const Twine &Name) {741 Module *M = Builder->GetInsertBlock()->getParent()->getParent();742 // Fill in the one generic type'd argument (the function is also vararg)743 Function *FnStatepoint = Intrinsic::getOrInsertDeclaration(744 M, Intrinsic::experimental_gc_statepoint,745 {ActualInvokee.getCallee()->getType()});746 747 std::vector<Value *> Args =748 getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee.getCallee(),749 Flags, InvokeArgs);750 751 InvokeInst *II = Builder->CreateInvoke(752 FnStatepoint, NormalDest, UnwindDest, Args,753 getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);754 II->addParamAttr(2,755 Attribute::get(Builder->getContext(), Attribute::ElementType,756 ActualInvokee.getFunctionType()));757 return II;758}759 760InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(761 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,762 BasicBlock *NormalDest, BasicBlock *UnwindDest,763 ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Value *>> DeoptArgs,764 ArrayRef<Value *> GCArgs, const Twine &Name) {765 return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(766 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,767 uint32_t(StatepointFlags::None), InvokeArgs,768 std::nullopt /* No Transition Args*/, DeoptArgs, GCArgs, Name);769}770 771InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(772 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,773 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,774 ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Use>> TransitionArgs,775 std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,776 const Twine &Name) {777 return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(778 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,779 InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);780}781 782InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(783 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,784 BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,785 std::optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs,786 const Twine &Name) {787 return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(788 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,789 uint32_t(StatepointFlags::None), InvokeArgs, std::nullopt, DeoptArgs,790 GCArgs, Name);791}792 793CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,794 Type *ResultType, const Twine &Name) {795 Intrinsic::ID ID = Intrinsic::experimental_gc_result;796 Type *Types[] = {ResultType};797 798 Value *Args[] = {Statepoint};799 return CreateIntrinsic(ID, Types, Args, {}, Name);800}801 802CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,803 int BaseOffset, int DerivedOffset,804 Type *ResultType, const Twine &Name) {805 Type *Types[] = {ResultType};806 807 Value *Args[] = {Statepoint, getInt32(BaseOffset), getInt32(DerivedOffset)};808 return CreateIntrinsic(Intrinsic::experimental_gc_relocate, Types, Args, {},809 Name);810}811 812CallInst *IRBuilderBase::CreateGCGetPointerBase(Value *DerivedPtr,813 const Twine &Name) {814 Type *PtrTy = DerivedPtr->getType();815 return CreateIntrinsic(Intrinsic::experimental_gc_get_pointer_base,816 {PtrTy, PtrTy}, {DerivedPtr}, {}, Name);817}818 819CallInst *IRBuilderBase::CreateGCGetPointerOffset(Value *DerivedPtr,820 const Twine &Name) {821 Type *PtrTy = DerivedPtr->getType();822 return CreateIntrinsic(Intrinsic::experimental_gc_get_pointer_offset, {PtrTy},823 {DerivedPtr}, {}, Name);824}825 826CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,827 FMFSource FMFSource,828 const Twine &Name) {829 Module *M = BB->getModule();830 Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, {V->getType()});831 return createCallHelper(Fn, {V}, Name, FMFSource);832}833 834Value *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,835 Value *RHS, FMFSource FMFSource,836 const Twine &Name) {837 Module *M = BB->getModule();838 Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, {LHS->getType()});839 if (Value *V = Folder.FoldBinaryIntrinsic(ID, LHS, RHS, Fn->getReturnType(),840 /*FMFSource=*/nullptr))841 return V;842 return createCallHelper(Fn, {LHS, RHS}, Name, FMFSource);843}844 845CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,846 ArrayRef<Type *> Types,847 ArrayRef<Value *> Args,848 FMFSource FMFSource,849 const Twine &Name) {850 Module *M = BB->getModule();851 Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, Types);852 return createCallHelper(Fn, Args, Name, FMFSource);853}854 855CallInst *IRBuilderBase::CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,856 ArrayRef<Value *> Args,857 FMFSource FMFSource,858 const Twine &Name) {859 Module *M = BB->getModule();860 861 SmallVector<Intrinsic::IITDescriptor> Table;862 Intrinsic::getIntrinsicInfoTableEntries(ID, Table);863 ArrayRef<Intrinsic::IITDescriptor> TableRef(Table);864 865 SmallVector<Type *> ArgTys;866 ArgTys.reserve(Args.size());867 for (auto &I : Args)868 ArgTys.push_back(I->getType());869 FunctionType *FTy = FunctionType::get(RetTy, ArgTys, false);870 SmallVector<Type *> OverloadTys;871 Intrinsic::MatchIntrinsicTypesResult Res =872 matchIntrinsicSignature(FTy, TableRef, OverloadTys);873 (void)Res;874 assert(Res == Intrinsic::MatchIntrinsicTypes_Match && TableRef.empty() &&875 "Wrong types for intrinsic!");876 // TODO: Handle varargs intrinsics.877 878 Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, OverloadTys);879 return createCallHelper(Fn, Args, Name, FMFSource);880}881 882CallInst *IRBuilderBase::CreateConstrainedFPBinOp(883 Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource,884 const Twine &Name, MDNode *FPMathTag, std::optional<RoundingMode> Rounding,885 std::optional<fp::ExceptionBehavior> Except) {886 Value *RoundingV = getConstrainedFPRounding(Rounding);887 Value *ExceptV = getConstrainedFPExcept(Except);888 889 FastMathFlags UseFMF = FMFSource.get(FMF);890 891 CallInst *C = CreateIntrinsic(ID, {L->getType()},892 {L, R, RoundingV, ExceptV}, nullptr, Name);893 setConstrainedFPCallAttr(C);894 setFPAttrs(C, FPMathTag, UseFMF);895 return C;896}897 898CallInst *IRBuilderBase::CreateConstrainedFPIntrinsic(899 Intrinsic::ID ID, ArrayRef<Type *> Types, ArrayRef<Value *> Args,900 FMFSource FMFSource, const Twine &Name, MDNode *FPMathTag,901 std::optional<RoundingMode> Rounding,902 std::optional<fp::ExceptionBehavior> Except) {903 Value *RoundingV = getConstrainedFPRounding(Rounding);904 Value *ExceptV = getConstrainedFPExcept(Except);905 906 FastMathFlags UseFMF = FMFSource.get(FMF);907 908 llvm::SmallVector<Value *, 5> ExtArgs(Args);909 ExtArgs.push_back(RoundingV);910 ExtArgs.push_back(ExceptV);911 912 CallInst *C = CreateIntrinsic(ID, Types, ExtArgs, nullptr, Name);913 setConstrainedFPCallAttr(C);914 setFPAttrs(C, FPMathTag, UseFMF);915 return C;916}917 918CallInst *IRBuilderBase::CreateConstrainedFPUnroundedBinOp(919 Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource,920 const Twine &Name, MDNode *FPMathTag,921 std::optional<fp::ExceptionBehavior> Except) {922 Value *ExceptV = getConstrainedFPExcept(Except);923 924 FastMathFlags UseFMF = FMFSource.get(FMF);925 926 CallInst *C =927 CreateIntrinsic(ID, {L->getType()}, {L, R, ExceptV}, nullptr, Name);928 setConstrainedFPCallAttr(C);929 setFPAttrs(C, FPMathTag, UseFMF);930 return C;931}932 933Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,934 const Twine &Name, MDNode *FPMathTag) {935 if (Instruction::isBinaryOp(Opc)) {936 assert(Ops.size() == 2 && "Invalid number of operands!");937 return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc),938 Ops[0], Ops[1], Name, FPMathTag);939 }940 if (Instruction::isUnaryOp(Opc)) {941 assert(Ops.size() == 1 && "Invalid number of operands!");942 return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc),943 Ops[0], Name, FPMathTag);944 }945 llvm_unreachable("Unexpected opcode!");946}947 948CallInst *IRBuilderBase::CreateConstrainedFPCast(949 Intrinsic::ID ID, Value *V, Type *DestTy, FMFSource FMFSource,950 const Twine &Name, MDNode *FPMathTag, std::optional<RoundingMode> Rounding,951 std::optional<fp::ExceptionBehavior> Except) {952 Value *ExceptV = getConstrainedFPExcept(Except);953 954 FastMathFlags UseFMF = FMFSource.get(FMF);955 956 CallInst *C;957 if (Intrinsic::hasConstrainedFPRoundingModeOperand(ID)) {958 Value *RoundingV = getConstrainedFPRounding(Rounding);959 C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV},960 nullptr, Name);961 } else962 C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr,963 Name);964 965 setConstrainedFPCallAttr(C);966 967 if (isa<FPMathOperator>(C))968 setFPAttrs(C, FPMathTag, UseFMF);969 return C;970}971 972Value *IRBuilderBase::CreateFCmpHelper(CmpInst::Predicate P, Value *LHS,973 Value *RHS, const Twine &Name,974 MDNode *FPMathTag, FMFSource FMFSource,975 bool IsSignaling) {976 if (IsFPConstrained) {977 auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps978 : Intrinsic::experimental_constrained_fcmp;979 return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name);980 }981 982 if (auto *V = Folder.FoldCmp(P, LHS, RHS))983 return V;984 return Insert(985 setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMFSource.get(FMF)),986 Name);987}988 989CallInst *IRBuilderBase::CreateConstrainedFPCmp(990 Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,991 const Twine &Name, std::optional<fp::ExceptionBehavior> Except) {992 Value *PredicateV = getConstrainedFPPredicate(P);993 Value *ExceptV = getConstrainedFPExcept(Except);994 995 CallInst *C = CreateIntrinsic(ID, {L->getType()},996 {L, R, PredicateV, ExceptV}, nullptr, Name);997 setConstrainedFPCallAttr(C);998 return C;999}1000 1001CallInst *IRBuilderBase::CreateConstrainedFPCall(1002 Function *Callee, ArrayRef<Value *> Args, const Twine &Name,1003 std::optional<RoundingMode> Rounding,1004 std::optional<fp::ExceptionBehavior> Except) {1005 llvm::SmallVector<Value *, 6> UseArgs(Args);1006 1007 if (Intrinsic::hasConstrainedFPRoundingModeOperand(Callee->getIntrinsicID()))1008 UseArgs.push_back(getConstrainedFPRounding(Rounding));1009 UseArgs.push_back(getConstrainedFPExcept(Except));1010 1011 CallInst *C = CreateCall(Callee, UseArgs, Name);1012 setConstrainedFPCallAttr(C);1013 return C;1014}1015 1016Value *IRBuilderBase::CreateSelectWithUnknownProfile(Value *C, Value *True,1017 Value *False,1018 StringRef PassName,1019 const Twine &Name) {1020 Value *Ret = CreateSelectFMF(C, True, False, {}, Name);1021 if (auto *SI = dyn_cast<SelectInst>(Ret)) {1022 setExplicitlyUnknownBranchWeightsIfProfiled(*SI, PassName);1023 }1024 return Ret;1025}1026 1027Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False,1028 const Twine &Name, Instruction *MDFrom) {1029 return CreateSelectFMF(C, True, False, {}, Name, MDFrom);1030}1031 1032Value *IRBuilderBase::CreateSelectFMF(Value *C, Value *True, Value *False,1033 FMFSource FMFSource, const Twine &Name,1034 Instruction *MDFrom) {1035 if (auto *V = Folder.FoldSelect(C, True, False))1036 return V;1037 1038 SelectInst *Sel = SelectInst::Create(C, True, False);1039 if (MDFrom) {1040 MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof);1041 MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable);1042 Sel = addBranchMetadata(Sel, Prof, Unpred);1043 }1044 if (isa<FPMathOperator>(Sel))1045 setFPAttrs(Sel, /*MDNode=*/nullptr, FMFSource.get(FMF));1046 return Insert(Sel, Name);1047}1048 1049Value *IRBuilderBase::CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS,1050 const Twine &Name) {1051 assert(LHS->getType() == RHS->getType() &&1052 "Pointer subtraction operand types must match!");1053 Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context));1054 Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context));1055 Value *Difference = CreateSub(LHS_int, RHS_int);1056 return CreateExactSDiv(Difference, ConstantExpr::getSizeOf(ElemTy),1057 Name);1058}1059 1060Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) {1061 assert(isa<PointerType>(Ptr->getType()) &&1062 "launder.invariant.group only applies to pointers.");1063 auto *PtrType = Ptr->getType();1064 Module *M = BB->getParent()->getParent();1065 Function *FnLaunderInvariantGroup = Intrinsic::getOrInsertDeclaration(1066 M, Intrinsic::launder_invariant_group, {PtrType});1067 1068 assert(FnLaunderInvariantGroup->getReturnType() == PtrType &&1069 FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==1070 PtrType &&1071 "LaunderInvariantGroup should take and return the same type");1072 1073 return CreateCall(FnLaunderInvariantGroup, {Ptr});1074}1075 1076Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {1077 assert(isa<PointerType>(Ptr->getType()) &&1078 "strip.invariant.group only applies to pointers.");1079 1080 auto *PtrType = Ptr->getType();1081 Module *M = BB->getParent()->getParent();1082 Function *FnStripInvariantGroup = Intrinsic::getOrInsertDeclaration(1083 M, Intrinsic::strip_invariant_group, {PtrType});1084 1085 assert(FnStripInvariantGroup->getReturnType() == PtrType &&1086 FnStripInvariantGroup->getFunctionType()->getParamType(0) ==1087 PtrType &&1088 "StripInvariantGroup should take and return the same type");1089 1090 return CreateCall(FnStripInvariantGroup, {Ptr});1091}1092 1093Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) {1094 auto *Ty = cast<VectorType>(V->getType());1095 if (isa<ScalableVectorType>(Ty)) {1096 Module *M = BB->getParent()->getParent();1097 Function *F =1098 Intrinsic::getOrInsertDeclaration(M, Intrinsic::vector_reverse, Ty);1099 return Insert(CallInst::Create(F, V), Name);1100 }1101 // Keep the original behaviour for fixed vector1102 SmallVector<int, 8> ShuffleMask;1103 int NumElts = Ty->getElementCount().getKnownMinValue();1104 for (int i = 0; i < NumElts; ++i)1105 ShuffleMask.push_back(NumElts - i - 1);1106 return CreateShuffleVector(V, ShuffleMask, Name);1107}1108 1109Value *IRBuilderBase::CreateVectorSplice(Value *V1, Value *V2, int64_t Imm,1110 const Twine &Name) {1111 assert(isa<VectorType>(V1->getType()) && "Unexpected type");1112 assert(V1->getType() == V2->getType() &&1113 "Splice expects matching operand types!");1114 1115 if (auto *VTy = dyn_cast<ScalableVectorType>(V1->getType())) {1116 Module *M = BB->getParent()->getParent();1117 Function *F =1118 Intrinsic::getOrInsertDeclaration(M, Intrinsic::vector_splice, VTy);1119 1120 Value *Ops[] = {V1, V2, getInt32(Imm)};1121 return Insert(CallInst::Create(F, Ops), Name);1122 }1123 1124 unsigned NumElts = cast<FixedVectorType>(V1->getType())->getNumElements();1125 assert(((-Imm <= NumElts) || (Imm < NumElts)) &&1126 "Invalid immediate for vector splice!");1127 1128 // Keep the original behaviour for fixed vector1129 unsigned Idx = (NumElts + Imm) % NumElts;1130 SmallVector<int, 8> Mask;1131 for (unsigned I = 0; I < NumElts; ++I)1132 Mask.push_back(Idx + I);1133 1134 return CreateShuffleVector(V1, V2, Mask);1135}1136 1137Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,1138 const Twine &Name) {1139 auto EC = ElementCount::getFixed(NumElts);1140 return CreateVectorSplat(EC, V, Name);1141}1142 1143Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,1144 const Twine &Name) {1145 assert(EC.isNonZero() && "Cannot splat to an empty vector!");1146 1147 // First insert it into a poison vector so we can shuffle it.1148 Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC));1149 V = CreateInsertElement(Poison, V, getInt64(0), Name + ".splatinsert");1150 1151 // Shuffle the value across the desired number of elements.1152 SmallVector<int, 16> Zeros;1153 Zeros.resize(EC.getKnownMinValue());1154 return CreateShuffleVector(V, Zeros, Name + ".splat");1155}1156 1157Value *IRBuilderBase::CreateVectorInterleave(ArrayRef<Value *> Ops,1158 const Twine &Name) {1159 assert(Ops.size() >= 2 && Ops.size() <= 8 &&1160 "Unexpected number of operands to interleave");1161 1162 // Make sure all operands are the same type.1163 assert(isa<VectorType>(Ops[0]->getType()) && "Unexpected type");1164 1165#ifndef NDEBUG1166 for (unsigned I = 1; I < Ops.size(); I++) {1167 assert(Ops[I]->getType() == Ops[0]->getType() &&1168 "Vector interleave expects matching operand types!");1169 }1170#endif1171 1172 unsigned IID = Intrinsic::getInterleaveIntrinsicID(Ops.size());1173 auto *SubvecTy = cast<VectorType>(Ops[0]->getType());1174 Type *DestTy = VectorType::get(SubvecTy->getElementType(),1175 SubvecTy->getElementCount() * Ops.size());1176 return CreateIntrinsic(IID, {DestTy}, Ops, {}, Name);1177}1178 1179Value *IRBuilderBase::CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base,1180 unsigned Dimension,1181 unsigned LastIndex,1182 MDNode *DbgInfo) {1183 auto *BaseType = Base->getType();1184 assert(isa<PointerType>(BaseType) &&1185 "Invalid Base ptr type for preserve.array.access.index.");1186 1187 Value *LastIndexV = getInt32(LastIndex);1188 Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);1189 SmallVector<Value *, 4> IdxList(Dimension, Zero);1190 IdxList.push_back(LastIndexV);1191 1192 Type *ResultType = GetElementPtrInst::getGEPReturnType(Base, IdxList);1193 1194 Value *DimV = getInt32(Dimension);1195 CallInst *Fn =1196 CreateIntrinsic(Intrinsic::preserve_array_access_index,1197 {ResultType, BaseType}, {Base, DimV, LastIndexV});1198 Fn->addParamAttr(1199 0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));1200 if (DbgInfo)1201 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);1202 1203 return Fn;1204}1205 1206Value *IRBuilderBase::CreatePreserveUnionAccessIndex(1207 Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {1208 assert(isa<PointerType>(Base->getType()) &&1209 "Invalid Base ptr type for preserve.union.access.index.");1210 auto *BaseType = Base->getType();1211 1212 Value *DIIndex = getInt32(FieldIndex);1213 CallInst *Fn = CreateIntrinsic(Intrinsic::preserve_union_access_index,1214 {BaseType, BaseType}, {Base, DIIndex});1215 if (DbgInfo)1216 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);1217 1218 return Fn;1219}1220 1221Value *IRBuilderBase::CreatePreserveStructAccessIndex(1222 Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,1223 MDNode *DbgInfo) {1224 auto *BaseType = Base->getType();1225 assert(isa<PointerType>(BaseType) &&1226 "Invalid Base ptr type for preserve.struct.access.index.");1227 1228 Value *GEPIndex = getInt32(Index);1229 Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);1230 Type *ResultType =1231 GetElementPtrInst::getGEPReturnType(Base, {Zero, GEPIndex});1232 1233 Value *DIIndex = getInt32(FieldIndex);1234 CallInst *Fn =1235 CreateIntrinsic(Intrinsic::preserve_struct_access_index,1236 {ResultType, BaseType}, {Base, GEPIndex, DIIndex});1237 Fn->addParamAttr(1238 0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));1239 if (DbgInfo)1240 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);1241 1242 return Fn;1243}1244 1245Value *IRBuilderBase::createIsFPClass(Value *FPNum, unsigned Test) {1246 ConstantInt *TestV = getInt32(Test);1247 return CreateIntrinsic(Intrinsic::is_fpclass, {FPNum->getType()},1248 {FPNum, TestV});1249}1250 1251CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,1252 Value *PtrValue,1253 Value *AlignValue,1254 Value *OffsetValue) {1255 SmallVector<Value *, 4> Vals({PtrValue, AlignValue});1256 if (OffsetValue)1257 Vals.push_back(OffsetValue);1258 OperandBundleDefT<Value *> AlignOpB("align", Vals);1259 return CreateAssumption(ConstantInt::getTrue(getContext()), {AlignOpB});1260}1261 1262CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,1263 Value *PtrValue,1264 unsigned Alignment,1265 Value *OffsetValue) {1266 assert(isa<PointerType>(PtrValue->getType()) &&1267 "trying to create an alignment assumption on a non-pointer?");1268 assert(Alignment != 0 && "Invalid Alignment");1269 auto *PtrTy = cast<PointerType>(PtrValue->getType());1270 Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace());1271 Value *AlignValue = ConstantInt::get(IntPtrTy, Alignment);1272 return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);1273}1274 1275CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,1276 Value *PtrValue,1277 Value *Alignment,1278 Value *OffsetValue) {1279 assert(isa<PointerType>(PtrValue->getType()) &&1280 "trying to create an alignment assumption on a non-pointer?");1281 return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue);1282}1283 1284CallInst *IRBuilderBase::CreateDereferenceableAssumption(Value *PtrValue,1285 Value *SizeValue) {1286 assert(isa<PointerType>(PtrValue->getType()) &&1287 "trying to create an deferenceable assumption on a non-pointer?");1288 SmallVector<Value *, 4> Vals({PtrValue, SizeValue});1289 OperandBundleDefT<Value *> DereferenceableOpB("dereferenceable", Vals);1290 return CreateAssumption(ConstantInt::getTrue(getContext()),1291 {DereferenceableOpB});1292}1293 1294IRBuilderDefaultInserter::~IRBuilderDefaultInserter() = default;1295IRBuilderCallbackInserter::~IRBuilderCallbackInserter() = default;1296IRBuilderFolder::~IRBuilderFolder() = default;1297void ConstantFolder::anchor() {}1298void NoFolder::anchor() {}1299