674 lines · cpp
1//===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This pass implements IR lowering for the llvm.memcpy, llvm.memmove,10// llvm.memset, llvm.load.relative and llvm.objc.* intrinsics.11//12//===----------------------------------------------------------------------===//13 14#include "llvm/CodeGen/PreISelIntrinsicLowering.h"15#include "llvm/Analysis/ObjCARCInstKind.h"16#include "llvm/Analysis/ObjCARCUtil.h"17#include "llvm/Analysis/TargetLibraryInfo.h"18#include "llvm/Analysis/TargetTransformInfo.h"19#include "llvm/CodeGen/ExpandVectorPredication.h"20#include "llvm/CodeGen/Passes.h"21#include "llvm/CodeGen/TargetLowering.h"22#include "llvm/CodeGen/TargetPassConfig.h"23#include "llvm/IR/Function.h"24#include "llvm/IR/IRBuilder.h"25#include "llvm/IR/Instructions.h"26#include "llvm/IR/IntrinsicInst.h"27#include "llvm/IR/Module.h"28#include "llvm/IR/RuntimeLibcalls.h"29#include "llvm/IR/Type.h"30#include "llvm/IR/Use.h"31#include "llvm/InitializePasses.h"32#include "llvm/Pass.h"33#include "llvm/Support/Casting.h"34#include "llvm/Target/TargetMachine.h"35#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"36#include "llvm/Transforms/Utils/BuildLibCalls.h"37#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"38#include "llvm/Transforms/Utils/LowerVectorIntrinsics.h"39 40using namespace llvm;41 42/// Threshold to leave statically sized memory intrinsic calls. Calls of known43/// size larger than this will be expanded by the pass. Calls of unknown or44/// lower size will be left for expansion in codegen.45static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt(46 "mem-intrinsic-expand-size",47 cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(-1),48 cl::Hidden);49 50namespace {51 52struct PreISelIntrinsicLowering {53 const TargetMachine *TM;54 const function_ref<TargetTransformInfo &(Function &)> LookupTTI;55 const function_ref<TargetLibraryInfo &(Function &)> LookupTLI;56 57 /// If this is true, assume it's preferably to leave memory intrinsic calls58 /// for replacement with a library call later. Otherwise this depends on59 /// TargetLoweringInfo availability of the corresponding function.60 const bool UseMemIntrinsicLibFunc;61 62 explicit PreISelIntrinsicLowering(63 const TargetMachine *TM_,64 function_ref<TargetTransformInfo &(Function &)> LookupTTI_,65 function_ref<TargetLibraryInfo &(Function &)> LookupTLI_,66 bool UseMemIntrinsicLibFunc_ = true)67 : TM(TM_), LookupTTI(LookupTTI_), LookupTLI(LookupTLI_),68 UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {}69 70 static bool shouldExpandMemIntrinsicWithSize(Value *Size,71 const TargetTransformInfo &TTI);72 bool73 expandMemIntrinsicUses(Function &F,74 DenseMap<Constant *, GlobalVariable *> &CMap) const;75 bool lowerIntrinsics(Module &M) const;76};77 78} // namespace79 80template <class T> static bool forEachCall(Function &Intrin, T Callback) {81 // Lowering all intrinsics in a function will delete multiple uses, so we82 // can't use an early-inc-range. In case some remain, we don't want to look83 // at them again. Unfortunately, Value::UseList is private, so we can't use a84 // simple Use**. If LastUse is null, the next use to consider is85 // Intrin.use_begin(), otherwise it's LastUse->getNext().86 Use *LastUse = nullptr;87 bool Changed = false;88 while (!Intrin.use_empty() && (!LastUse || LastUse->getNext())) {89 Use *U = LastUse ? LastUse->getNext() : &*Intrin.use_begin();90 bool Removed = false;91 // An intrinsic cannot have its address taken, so it cannot be an argument92 // operand. It might be used as operand in debug metadata, though.93 if (auto CI = dyn_cast<CallInst>(U->getUser()))94 Changed |= Removed = Callback(CI);95 if (!Removed)96 LastUse = U;97 }98 return Changed;99}100 101static bool lowerLoadRelative(Function &F) {102 if (F.use_empty())103 return false;104 105 bool Changed = false;106 Type *Int32Ty = Type::getInt32Ty(F.getContext());107 108 for (Use &U : llvm::make_early_inc_range(F.uses())) {109 auto CI = dyn_cast<CallInst>(U.getUser());110 if (!CI || CI->getCalledOperand() != &F)111 continue;112 113 IRBuilder<> B(CI);114 Value *OffsetPtr =115 B.CreatePtrAdd(CI->getArgOperand(0), CI->getArgOperand(1));116 Value *OffsetI32 = B.CreateAlignedLoad(Int32Ty, OffsetPtr, Align(4));117 118 Value *ResultPtr = B.CreatePtrAdd(CI->getArgOperand(0), OffsetI32);119 120 CI->replaceAllUsesWith(ResultPtr);121 CI->eraseFromParent();122 Changed = true;123 }124 125 return Changed;126}127 128// ObjCARC has knowledge about whether an obj-c runtime function needs to be129// always tail-called or never tail-called.130static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) {131 objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(&F);132 if (objcarc::IsAlwaysTail(Kind))133 return CallInst::TCK_Tail;134 else if (objcarc::IsNeverTail(Kind))135 return CallInst::TCK_NoTail;136 return CallInst::TCK_None;137}138 139static bool lowerObjCCall(Function &F, RTLIB::LibcallImpl NewFn,140 bool setNonLazyBind = false) {141 assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) &&142 "Pre-ISel intrinsics do lower into regular function calls");143 if (F.use_empty())144 return false;145 146 // FIXME: When RuntimeLibcalls is an analysis, check if the function is really147 // supported, and go through RTLIB::Libcall.148 StringRef NewFnName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(NewFn);149 150 // If we haven't already looked up this function, check to see if the151 // program already contains a function with this name.152 Module *M = F.getParent();153 FunctionCallee FCache =154 M->getOrInsertFunction(NewFnName, F.getFunctionType());155 156 if (Function *Fn = dyn_cast<Function>(FCache.getCallee())) {157 Fn->setLinkage(F.getLinkage());158 if (setNonLazyBind && !Fn->isWeakForLinker()) {159 // If we have Native ARC, set nonlazybind attribute for these APIs for160 // performance.161 Fn->addFnAttr(Attribute::NonLazyBind);162 }163 }164 165 CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F);166 167 for (Use &U : llvm::make_early_inc_range(F.uses())) {168 auto *CB = cast<CallBase>(U.getUser());169 170 if (CB->getCalledFunction() != &F) {171 assert(objcarc::getAttachedARCFunction(CB) == &F &&172 "use expected to be the argument of operand bundle "173 "\"clang.arc.attachedcall\"");174 U.set(FCache.getCallee());175 continue;176 }177 178 auto *CI = cast<CallInst>(CB);179 assert(CI->getCalledFunction() && "Cannot lower an indirect call!");180 181 IRBuilder<> Builder(CI->getParent(), CI->getIterator());182 SmallVector<Value *, 8> Args(CI->args());183 SmallVector<llvm::OperandBundleDef, 1> BundleList;184 CI->getOperandBundlesAsDefs(BundleList);185 CallInst *NewCI = Builder.CreateCall(FCache, Args, BundleList);186 NewCI->setName(CI->getName());187 188 // Try to set the most appropriate TailCallKind based on both the current189 // attributes and the ones that we could get from ObjCARC's special190 // knowledge of the runtime functions.191 //192 // std::max respects both requirements of notail and tail here:193 // * notail on either the call or from ObjCARC becomes notail194 // * tail on either side is stronger than none, but not notail195 CallInst::TailCallKind TCK = CI->getTailCallKind();196 NewCI->setTailCallKind(std::max(TCK, OverridingTCK));197 198 // Transfer the 'returned' attribute from the intrinsic to the call site.199 // By applying this only to intrinsic call sites, we avoid applying it to200 // non-ARC explicit calls to things like objc_retain which have not been201 // auto-upgraded to use the intrinsics.202 unsigned Index;203 if (F.getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) &&204 Index)205 NewCI->addParamAttr(Index - AttributeList::FirstArgIndex,206 Attribute::Returned);207 208 if (!CI->use_empty())209 CI->replaceAllUsesWith(NewCI);210 CI->eraseFromParent();211 }212 213 return true;214}215 216// TODO: Should refine based on estimated number of accesses (e.g. does it217// require splitting based on alignment)218bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(219 Value *Size, const TargetTransformInfo &TTI) {220 ConstantInt *CI = dyn_cast<ConstantInt>(Size);221 if (!CI)222 return true;223 uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences()224 ? MemIntrinsicExpandSizeThresholdOpt225 : TTI.getMaxMemIntrinsicInlineSizeThreshold();226 uint64_t SizeVal = CI->getZExtValue();227 228 // Treat a threshold of 0 as a special case to force expansion of all229 // intrinsics, including size 0.230 return SizeVal > Threshold || Threshold == 0;231}232 233static bool canEmitLibcall(const TargetMachine *TM, Function *F,234 RTLIB::Libcall LC) {235 // TODO: Should this consider the address space of the memcpy?236 if (!TM)237 return true;238 const TargetLowering *TLI = TM->getSubtargetImpl(*F)->getTargetLowering();239 return TLI->getLibcallName(LC) != nullptr;240}241 242static bool canEmitMemcpy(const TargetMachine *TM, Function *F) {243 // TODO: Should this consider the address space of the memcpy?244 if (!TM)245 return true;246 const TargetLowering *TLI = TM->getSubtargetImpl(*F)->getTargetLowering();247 return TLI->getMemcpyImpl() != RTLIB::Unsupported;248}249 250// Return a value appropriate for use with the memset_pattern16 libcall, if251// possible and if we know how. (Adapted from equivalent helper in252// LoopIdiomRecognize).253static Constant *getMemSetPattern16Value(MemSetPatternInst *Inst,254 const TargetLibraryInfo &TLI) {255 // TODO: This could check for UndefValue because it can be merged into any256 // other valid pattern.257 258 // Don't emit libcalls if a non-default address space is being used.259 if (Inst->getRawDest()->getType()->getPointerAddressSpace() != 0)260 return nullptr;261 262 Value *V = Inst->getValue();263 Type *VTy = V->getType();264 const DataLayout &DL = Inst->getDataLayout();265 Module *M = Inst->getModule();266 267 if (!isLibFuncEmittable(M, &TLI, LibFunc_memset_pattern16))268 return nullptr;269 270 // If the value isn't a constant, we can't promote it to being in a constant271 // array. We could theoretically do a store to an alloca or something, but272 // that doesn't seem worthwhile.273 Constant *C = dyn_cast<Constant>(V);274 if (!C || isa<ConstantExpr>(C))275 return nullptr;276 277 // Only handle simple values that are a power of two bytes in size.278 uint64_t Size = DL.getTypeSizeInBits(VTy);279 if (!DL.typeSizeEqualsStoreSize(VTy) || !isPowerOf2_64(Size))280 return nullptr;281 282 // Don't care enough about darwin/ppc to implement this.283 if (DL.isBigEndian())284 return nullptr;285 286 // Convert to size in bytes.287 Size /= 8;288 289 // TODO: If CI is larger than 16-bytes, we can try slicing it in half to see290 // if the top and bottom are the same (e.g. for vectors and large integers).291 if (Size > 16)292 return nullptr;293 294 // If the constant is exactly 16 bytes, just use it.295 if (Size == 16)296 return C;297 298 // Otherwise, we'll use an array of the constants.299 uint64_t ArraySize = 16 / Size;300 ArrayType *AT = ArrayType::get(V->getType(), ArraySize);301 return ConstantArray::get(AT, std::vector<Constant *>(ArraySize, C));302}303 304// TODO: Handle atomic memcpy and memcpy.inline305// TODO: Pass ScalarEvolution306bool PreISelIntrinsicLowering::expandMemIntrinsicUses(307 Function &F, DenseMap<Constant *, GlobalVariable *> &CMap) const {308 Intrinsic::ID ID = F.getIntrinsicID();309 bool Changed = false;310 311 for (User *U : llvm::make_early_inc_range(F.users())) {312 Instruction *Inst = cast<Instruction>(U);313 314 switch (ID) {315 case Intrinsic::memcpy: {316 auto *Memcpy = cast<MemCpyInst>(Inst);317 Function *ParentFunc = Memcpy->getFunction();318 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);319 if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) {320 if (UseMemIntrinsicLibFunc && canEmitMemcpy(TM, ParentFunc))321 break;322 323 // TODO: For optsize, emit the loop into a separate function324 expandMemCpyAsLoop(Memcpy, TTI);325 Changed = true;326 Memcpy->eraseFromParent();327 }328 329 break;330 }331 case Intrinsic::memcpy_inline: {332 // Only expand llvm.memcpy.inline with non-constant length in this333 // codepath, leaving the current SelectionDAG expansion for constant334 // length memcpy intrinsics undisturbed.335 auto *Memcpy = cast<MemCpyInst>(Inst);336 if (isa<ConstantInt>(Memcpy->getLength()))337 break;338 339 Function *ParentFunc = Memcpy->getFunction();340 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);341 expandMemCpyAsLoop(Memcpy, TTI);342 Changed = true;343 Memcpy->eraseFromParent();344 break;345 }346 case Intrinsic::memmove: {347 auto *Memmove = cast<MemMoveInst>(Inst);348 Function *ParentFunc = Memmove->getFunction();349 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);350 if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) {351 if (UseMemIntrinsicLibFunc &&352 canEmitLibcall(TM, ParentFunc, RTLIB::MEMMOVE))353 break;354 355 if (expandMemMoveAsLoop(Memmove, TTI)) {356 Changed = true;357 Memmove->eraseFromParent();358 }359 }360 361 break;362 }363 case Intrinsic::memset: {364 auto *Memset = cast<MemSetInst>(Inst);365 Function *ParentFunc = Memset->getFunction();366 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);367 if (shouldExpandMemIntrinsicWithSize(Memset->getLength(), TTI)) {368 if (UseMemIntrinsicLibFunc &&369 canEmitLibcall(TM, ParentFunc, RTLIB::MEMSET))370 break;371 372 expandMemSetAsLoop(Memset);373 Changed = true;374 Memset->eraseFromParent();375 }376 377 break;378 }379 case Intrinsic::memset_inline: {380 // Only expand llvm.memset.inline with non-constant length in this381 // codepath, leaving the current SelectionDAG expansion for constant382 // length memset intrinsics undisturbed.383 auto *Memset = cast<MemSetInst>(Inst);384 if (isa<ConstantInt>(Memset->getLength()))385 break;386 387 expandMemSetAsLoop(Memset);388 Changed = true;389 Memset->eraseFromParent();390 break;391 }392 case Intrinsic::experimental_memset_pattern: {393 auto *Memset = cast<MemSetPatternInst>(Inst);394 const TargetLibraryInfo &TLI = LookupTLI(*Memset->getFunction());395 Constant *PatternValue = getMemSetPattern16Value(Memset, TLI);396 if (!PatternValue) {397 // If it isn't possible to emit a memset_pattern16 libcall, expand to398 // a loop instead.399 expandMemSetPatternAsLoop(Memset);400 Changed = true;401 Memset->eraseFromParent();402 break;403 }404 // FIXME: There is currently no profitability calculation for emitting405 // the libcall vs expanding the memset.pattern directly.406 IRBuilder<> Builder(Inst);407 Module *M = Memset->getModule();408 const DataLayout &DL = Memset->getDataLayout();409 410 Type *DestPtrTy = Memset->getRawDest()->getType();411 Type *SizeTTy = TLI.getSizeTType(*M);412 StringRef FuncName = "memset_pattern16";413 FunctionCallee MSP = getOrInsertLibFunc(M, TLI, LibFunc_memset_pattern16,414 Builder.getVoidTy(), DestPtrTy,415 Builder.getPtrTy(), SizeTTy);416 inferNonMandatoryLibFuncAttrs(M, FuncName, TLI);417 418 // Otherwise we should form a memset_pattern16. PatternValue is known419 // to be an constant array of 16-bytes. Put the value into a mergable420 // global.421 assert(Memset->getRawDest()->getType()->getPointerAddressSpace() == 0 &&422 "Should have skipped if non-zero AS");423 GlobalVariable *GV;424 auto It = CMap.find(PatternValue);425 if (It != CMap.end()) {426 GV = It->second;427 } else {428 GV = new GlobalVariable(429 *M, PatternValue->getType(), /*isConstant=*/true,430 GlobalValue::PrivateLinkage, PatternValue, ".memset_pattern");431 GV->setUnnamedAddr(432 GlobalValue::UnnamedAddr::Global); // Ok to merge these.433 // TODO: Consider relaxing alignment requirement.434 GV->setAlignment(Align(16));435 CMap[PatternValue] = GV;436 }437 Value *PatternPtr = GV;438 Value *NumBytes = Builder.CreateMul(439 TLI.getAsSizeT(DL.getTypeAllocSize(Memset->getValue()->getType()),440 *M),441 Builder.CreateZExtOrTrunc(Memset->getLength(), SizeTTy));442 CallInst *MemsetPattern16Call =443 Builder.CreateCall(MSP, {Memset->getRawDest(), PatternPtr, NumBytes});444 MemsetPattern16Call->setAAMetadata(Memset->getAAMetadata());445 // Preserve any call site attributes on the destination pointer446 // argument (e.g. alignment).447 AttrBuilder ArgAttrs(Memset->getContext(),448 Memset->getAttributes().getParamAttrs(0));449 MemsetPattern16Call->setAttributes(450 MemsetPattern16Call->getAttributes().addParamAttributes(451 Memset->getContext(), 0, ArgAttrs));452 Changed = true;453 Memset->eraseFromParent();454 break;455 }456 default:457 llvm_unreachable("unhandled intrinsic");458 }459 }460 461 return Changed;462}463 464bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {465 // Map unique constants to globals.466 DenseMap<Constant *, GlobalVariable *> CMap;467 bool Changed = false;468 for (Function &F : M) {469 switch (F.getIntrinsicID()) {470 default:471 break;472 case Intrinsic::memcpy:473 case Intrinsic::memcpy_inline:474 case Intrinsic::memmove:475 case Intrinsic::memset:476 case Intrinsic::memset_inline:477 case Intrinsic::experimental_memset_pattern:478 Changed |= expandMemIntrinsicUses(F, CMap);479 break;480 case Intrinsic::load_relative:481 Changed |= lowerLoadRelative(F);482 break;483 case Intrinsic::is_constant:484 case Intrinsic::objectsize:485 Changed |= forEachCall(F, [&](CallInst *CI) {486 Function *Parent = CI->getParent()->getParent();487 TargetLibraryInfo &TLI = LookupTLI(*Parent);488 // Intrinsics in unreachable code are not lowered.489 bool Changed = lowerConstantIntrinsics(*Parent, TLI, /*DT=*/nullptr);490 return Changed;491 });492 break;493#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \494 case Intrinsic::VPID:495#include "llvm/IR/VPIntrinsics.def"496 forEachCall(F, [&](CallInst *CI) {497 Function *Parent = CI->getParent()->getParent();498 const TargetTransformInfo &TTI = LookupTTI(*Parent);499 auto *VPI = cast<VPIntrinsic>(CI);500 VPExpansionDetails ED = expandVectorPredicationIntrinsic(*VPI, TTI);501 // Expansion of VP intrinsics may change the IR but not actually502 // replace the intrinsic, so update Changed for the pass503 // and compute Removed for forEachCall.504 Changed |= ED != VPExpansionDetails::IntrinsicUnchanged;505 bool Removed = ED == VPExpansionDetails::IntrinsicReplaced;506 return Removed;507 });508 break;509 case Intrinsic::objc_autorelease:510 Changed |= lowerObjCCall(F, RTLIB::impl_objc_autorelease);511 break;512 case Intrinsic::objc_autoreleasePoolPop:513 Changed |= lowerObjCCall(F, RTLIB::impl_objc_autoreleasePoolPop);514 break;515 case Intrinsic::objc_autoreleasePoolPush:516 Changed |= lowerObjCCall(F, RTLIB::impl_objc_autoreleasePoolPush);517 break;518 case Intrinsic::objc_autoreleaseReturnValue:519 Changed |= lowerObjCCall(F, RTLIB::impl_objc_autoreleaseReturnValue);520 break;521 case Intrinsic::objc_copyWeak:522 Changed |= lowerObjCCall(F, RTLIB::impl_objc_copyWeak);523 break;524 case Intrinsic::objc_destroyWeak:525 Changed |= lowerObjCCall(F, RTLIB::impl_objc_destroyWeak);526 break;527 case Intrinsic::objc_initWeak:528 Changed |= lowerObjCCall(F, RTLIB::impl_objc_initWeak);529 break;530 case Intrinsic::objc_loadWeak:531 Changed |= lowerObjCCall(F, RTLIB::impl_objc_loadWeak);532 break;533 case Intrinsic::objc_loadWeakRetained:534 Changed |= lowerObjCCall(F, RTLIB::impl_objc_loadWeakRetained);535 break;536 case Intrinsic::objc_moveWeak:537 Changed |= lowerObjCCall(F, RTLIB::impl_objc_moveWeak);538 break;539 case Intrinsic::objc_release:540 Changed |= lowerObjCCall(F, RTLIB::impl_objc_release, true);541 break;542 case Intrinsic::objc_retain:543 Changed |= lowerObjCCall(F, RTLIB::impl_objc_retain, true);544 break;545 case Intrinsic::objc_retainAutorelease:546 Changed |= lowerObjCCall(F, RTLIB::impl_objc_retainAutorelease);547 break;548 case Intrinsic::objc_retainAutoreleaseReturnValue:549 Changed |=550 lowerObjCCall(F, RTLIB::impl_objc_retainAutoreleaseReturnValue);551 break;552 case Intrinsic::objc_retainAutoreleasedReturnValue:553 Changed |=554 lowerObjCCall(F, RTLIB::impl_objc_retainAutoreleasedReturnValue);555 break;556 case Intrinsic::objc_claimAutoreleasedReturnValue:557 Changed |=558 lowerObjCCall(F, RTLIB::impl_objc_claimAutoreleasedReturnValue);559 break;560 case Intrinsic::objc_retainBlock:561 Changed |= lowerObjCCall(F, RTLIB::impl_objc_retainBlock);562 break;563 case Intrinsic::objc_storeStrong:564 Changed |= lowerObjCCall(F, RTLIB::impl_objc_storeStrong);565 break;566 case Intrinsic::objc_storeWeak:567 Changed |= lowerObjCCall(F, RTLIB::impl_objc_storeWeak);568 break;569 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:570 Changed |=571 lowerObjCCall(F, RTLIB::impl_objc_unsafeClaimAutoreleasedReturnValue);572 break;573 case Intrinsic::objc_retainedObject:574 Changed |= lowerObjCCall(F, RTLIB::impl_objc_retainedObject);575 break;576 case Intrinsic::objc_unretainedObject:577 Changed |= lowerObjCCall(F, RTLIB::impl_objc_unretainedObject);578 break;579 case Intrinsic::objc_unretainedPointer:580 Changed |= lowerObjCCall(F, RTLIB::impl_objc_unretainedPointer);581 break;582 case Intrinsic::objc_retain_autorelease:583 Changed |= lowerObjCCall(F, RTLIB::impl_objc_retain_autorelease);584 break;585 case Intrinsic::objc_sync_enter:586 Changed |= lowerObjCCall(F, RTLIB::impl_objc_sync_enter);587 break;588 case Intrinsic::objc_sync_exit:589 Changed |= lowerObjCCall(F, RTLIB::impl_objc_sync_exit);590 break;591 case Intrinsic::exp:592 case Intrinsic::exp2:593 case Intrinsic::log:594 Changed |= forEachCall(F, [&](CallInst *CI) {595 Type *Ty = CI->getArgOperand(0)->getType();596 if (!TM || !isa<ScalableVectorType>(Ty))597 return false;598 const TargetLowering *TL = TM->getSubtargetImpl(F)->getTargetLowering();599 unsigned Op = TL->IntrinsicIDToISD(F.getIntrinsicID());600 assert(Op != ISD::DELETED_NODE && "unsupported intrinsic");601 if (!TL->isOperationExpand(Op, EVT::getEVT(Ty)))602 return false;603 return lowerUnaryVectorIntrinsicAsLoop(M, CI);604 });605 break;606 }607 }608 return Changed;609}610 611namespace {612 613class PreISelIntrinsicLoweringLegacyPass : public ModulePass {614public:615 static char ID;616 617 PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}618 619 void getAnalysisUsage(AnalysisUsage &AU) const override {620 AU.addRequired<TargetTransformInfoWrapperPass>();621 AU.addRequired<TargetLibraryInfoWrapperPass>();622 AU.addRequired<TargetPassConfig>();623 }624 625 bool runOnModule(Module &M) override {626 auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {627 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);628 };629 auto LookupTLI = [this](Function &F) -> TargetLibraryInfo & {630 return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);631 };632 633 const auto *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();634 PreISelIntrinsicLowering Lowering(TM, LookupTTI, LookupTLI);635 return Lowering.lowerIntrinsics(M);636 }637};638 639} // end anonymous namespace640 641char PreISelIntrinsicLoweringLegacyPass::ID;642 643INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass,644 "pre-isel-intrinsic-lowering",645 "Pre-ISel Intrinsic Lowering", false, false)646INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)647INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)648INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)649INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,650 "pre-isel-intrinsic-lowering",651 "Pre-ISel Intrinsic Lowering", false, false)652 653ModulePass *llvm::createPreISelIntrinsicLoweringPass() {654 return new PreISelIntrinsicLoweringLegacyPass();655}656 657PreservedAnalyses PreISelIntrinsicLoweringPass::run(Module &M,658 ModuleAnalysisManager &AM) {659 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();660 661 auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {662 return FAM.getResult<TargetIRAnalysis>(F);663 };664 auto LookupTLI = [&FAM](Function &F) -> TargetLibraryInfo & {665 return FAM.getResult<TargetLibraryAnalysis>(F);666 };667 668 PreISelIntrinsicLowering Lowering(TM, LookupTTI, LookupTLI);669 if (!Lowering.lowerIntrinsics(M))670 return PreservedAnalyses::all();671 else672 return PreservedAnalyses::none();673}674