1747 lines · cpp
1//===- AMDGPULibCalls.cpp -------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file10/// This file does AMD library function optimizations.11//12//===----------------------------------------------------------------------===//13 14#include "AMDGPU.h"15#include "AMDGPULibFunc.h"16#include "llvm/Analysis/AssumptionCache.h"17#include "llvm/Analysis/TargetLibraryInfo.h"18#include "llvm/Analysis/ValueTracking.h"19#include "llvm/IR/AttributeMask.h"20#include "llvm/IR/Dominators.h"21#include "llvm/IR/IRBuilder.h"22#include "llvm/IR/MDBuilder.h"23#include "llvm/IR/PatternMatch.h"24#include <cmath>25 26#define DEBUG_TYPE "amdgpu-simplifylib"27 28using namespace llvm;29using namespace llvm::PatternMatch;30 31static cl::opt<bool> EnablePreLink("amdgpu-prelink",32 cl::desc("Enable pre-link mode optimizations"),33 cl::init(false),34 cl::Hidden);35 36static cl::list<std::string> UseNative("amdgpu-use-native",37 cl::desc("Comma separated list of functions to replace with native, or all"),38 cl::CommaSeparated, cl::ValueOptional,39 cl::Hidden);40 41#define MATH_PI numbers::pi42#define MATH_E numbers::e43#define MATH_SQRT2 numbers::sqrt244#define MATH_SQRT1_2 numbers::inv_sqrt245 46namespace llvm {47 48class AMDGPULibCalls {49private:50 const TargetLibraryInfo *TLInfo = nullptr;51 AssumptionCache *AC = nullptr;52 DominatorTree *DT = nullptr;53 54 using FuncInfo = llvm::AMDGPULibFunc;55 56 // -fuse-native.57 bool AllNative = false;58 59 bool useNativeFunc(const StringRef F) const;60 61 // Return a pointer (pointer expr) to the function if function definition with62 // "FuncName" exists. It may create a new function prototype in pre-link mode.63 FunctionCallee getFunction(Module *M, const FuncInfo &fInfo);64 65 bool parseFunctionName(const StringRef &FMangledName, FuncInfo &FInfo);66 67 bool TDOFold(CallInst *CI, const FuncInfo &FInfo);68 69 /* Specialized optimizations */70 71 // pow/powr/pown72 bool fold_pow(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);73 74 // rootn75 bool fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);76 77 // -fuse-native for sincos78 bool sincosUseNative(CallInst *aCI, const FuncInfo &FInfo);79 80 // evaluate calls if calls' arguments are constants.81 bool evaluateScalarMathFunc(const FuncInfo &FInfo, double &Res0, double &Res1,82 Constant *copr0, Constant *copr1);83 bool evaluateCall(CallInst *aCI, const FuncInfo &FInfo);84 85 /// Insert a value to sincos function \p Fsincos. Returns (value of sin, value86 /// of cos, sincos call).87 std::tuple<Value *, Value *, Value *> insertSinCos(Value *Arg,88 FastMathFlags FMF,89 IRBuilder<> &B,90 FunctionCallee Fsincos);91 92 // sin/cos93 bool fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);94 95 // __read_pipe/__write_pipe96 bool fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,97 const FuncInfo &FInfo);98 99 // Get a scalar native builtin single argument FP function100 FunctionCallee getNativeFunction(Module *M, const FuncInfo &FInfo);101 102 /// Substitute a call to a known libcall with an intrinsic call. If \p103 /// AllowMinSize is true, allow the replacement in a minsize function.104 bool shouldReplaceLibcallWithIntrinsic(const CallInst *CI,105 bool AllowMinSizeF32 = false,106 bool AllowF64 = false,107 bool AllowStrictFP = false);108 void replaceLibCallWithSimpleIntrinsic(IRBuilder<> &B, CallInst *CI,109 Intrinsic::ID IntrID);110 111 bool tryReplaceLibcallWithSimpleIntrinsic(IRBuilder<> &B, CallInst *CI,112 Intrinsic::ID IntrID,113 bool AllowMinSizeF32 = false,114 bool AllowF64 = false,115 bool AllowStrictFP = false);116 117protected:118 bool isUnsafeFiniteOnlyMath(const FPMathOperator *FPOp) const;119 120 bool canIncreasePrecisionOfConstantFold(const FPMathOperator *FPOp) const;121 122 static void replaceCall(Instruction *I, Value *With) {123 I->replaceAllUsesWith(With);124 I->eraseFromParent();125 }126 127 static void replaceCall(FPMathOperator *I, Value *With) {128 replaceCall(cast<Instruction>(I), With);129 }130 131public:132 AMDGPULibCalls() = default;133 134 bool fold(CallInst *CI);135 136 void initFunction(Function &F, FunctionAnalysisManager &FAM);137 void initNativeFuncs();138 139 // Replace a normal math function call with that native version140 bool useNative(CallInst *CI);141};142 143} // end namespace llvm144 145template <typename IRB>146static CallInst *CreateCallEx(IRB &B, FunctionCallee Callee, Value *Arg,147 const Twine &Name = "") {148 CallInst *R = B.CreateCall(Callee, Arg, Name);149 if (Function *F = dyn_cast<Function>(Callee.getCallee()))150 R->setCallingConv(F->getCallingConv());151 return R;152}153 154template <typename IRB>155static CallInst *CreateCallEx2(IRB &B, FunctionCallee Callee, Value *Arg1,156 Value *Arg2, const Twine &Name = "") {157 CallInst *R = B.CreateCall(Callee, {Arg1, Arg2}, Name);158 if (Function *F = dyn_cast<Function>(Callee.getCallee()))159 R->setCallingConv(F->getCallingConv());160 return R;161}162 163static FunctionType *getPownType(FunctionType *FT) {164 Type *PowNExpTy = Type::getInt32Ty(FT->getContext());165 if (VectorType *VecTy = dyn_cast<VectorType>(FT->getReturnType()))166 PowNExpTy = VectorType::get(PowNExpTy, VecTy->getElementCount());167 168 return FunctionType::get(FT->getReturnType(),169 {FT->getParamType(0), PowNExpTy}, false);170}171 172// Data structures for table-driven optimizations.173// FuncTbl works for both f32 and f64 functions with 1 input argument174 175struct TableEntry {176 double result;177 double input;178};179 180/* a list of {result, input} */181static const TableEntry tbl_acos[] = {182 {MATH_PI / 2.0, 0.0},183 {MATH_PI / 2.0, -0.0},184 {0.0, 1.0},185 {MATH_PI, -1.0}186};187static const TableEntry tbl_acosh[] = {188 {0.0, 1.0}189};190static const TableEntry tbl_acospi[] = {191 {0.5, 0.0},192 {0.5, -0.0},193 {0.0, 1.0},194 {1.0, -1.0}195};196static const TableEntry tbl_asin[] = {197 {0.0, 0.0},198 {-0.0, -0.0},199 {MATH_PI / 2.0, 1.0},200 {-MATH_PI / 2.0, -1.0}201};202static const TableEntry tbl_asinh[] = {203 {0.0, 0.0},204 {-0.0, -0.0}205};206static const TableEntry tbl_asinpi[] = {207 {0.0, 0.0},208 {-0.0, -0.0},209 {0.5, 1.0},210 {-0.5, -1.0}211};212static const TableEntry tbl_atan[] = {213 {0.0, 0.0},214 {-0.0, -0.0},215 {MATH_PI / 4.0, 1.0},216 {-MATH_PI / 4.0, -1.0}217};218static const TableEntry tbl_atanh[] = {219 {0.0, 0.0},220 {-0.0, -0.0}221};222static const TableEntry tbl_atanpi[] = {223 {0.0, 0.0},224 {-0.0, -0.0},225 {0.25, 1.0},226 {-0.25, -1.0}227};228static const TableEntry tbl_cbrt[] = {229 {0.0, 0.0},230 {-0.0, -0.0},231 {1.0, 1.0},232 {-1.0, -1.0},233};234static const TableEntry tbl_cos[] = {235 {1.0, 0.0},236 {1.0, -0.0}237};238static const TableEntry tbl_cosh[] = {239 {1.0, 0.0},240 {1.0, -0.0}241};242static const TableEntry tbl_cospi[] = {243 {1.0, 0.0},244 {1.0, -0.0}245};246static const TableEntry tbl_erfc[] = {247 {1.0, 0.0},248 {1.0, -0.0}249};250static const TableEntry tbl_erf[] = {251 {0.0, 0.0},252 {-0.0, -0.0}253};254static const TableEntry tbl_exp[] = {255 {1.0, 0.0},256 {1.0, -0.0},257 {MATH_E, 1.0}258};259static const TableEntry tbl_exp2[] = {260 {1.0, 0.0},261 {1.0, -0.0},262 {2.0, 1.0}263};264static const TableEntry tbl_exp10[] = {265 {1.0, 0.0},266 {1.0, -0.0},267 {10.0, 1.0}268};269static const TableEntry tbl_expm1[] = {270 {0.0, 0.0},271 {-0.0, -0.0}272};273static const TableEntry tbl_log[] = {274 {0.0, 1.0},275 {1.0, MATH_E}276};277static const TableEntry tbl_log2[] = {278 {0.0, 1.0},279 {1.0, 2.0}280};281static const TableEntry tbl_log10[] = {282 {0.0, 1.0},283 {1.0, 10.0}284};285static const TableEntry tbl_rsqrt[] = {286 {1.0, 1.0},287 {MATH_SQRT1_2, 2.0}288};289static const TableEntry tbl_sin[] = {290 {0.0, 0.0},291 {-0.0, -0.0}292};293static const TableEntry tbl_sinh[] = {294 {0.0, 0.0},295 {-0.0, -0.0}296};297static const TableEntry tbl_sinpi[] = {298 {0.0, 0.0},299 {-0.0, -0.0}300};301static const TableEntry tbl_sqrt[] = {302 {0.0, 0.0},303 {1.0, 1.0},304 {MATH_SQRT2, 2.0}305};306static const TableEntry tbl_tan[] = {307 {0.0, 0.0},308 {-0.0, -0.0}309};310static const TableEntry tbl_tanh[] = {311 {0.0, 0.0},312 {-0.0, -0.0}313};314static const TableEntry tbl_tanpi[] = {315 {0.0, 0.0},316 {-0.0, -0.0}317};318static const TableEntry tbl_tgamma[] = {319 {1.0, 1.0},320 {1.0, 2.0},321 {2.0, 3.0},322 {6.0, 4.0}323};324 325static bool HasNative(AMDGPULibFunc::EFuncId id) {326 switch(id) {327 case AMDGPULibFunc::EI_DIVIDE:328 case AMDGPULibFunc::EI_COS:329 case AMDGPULibFunc::EI_EXP:330 case AMDGPULibFunc::EI_EXP2:331 case AMDGPULibFunc::EI_EXP10:332 case AMDGPULibFunc::EI_LOG:333 case AMDGPULibFunc::EI_LOG2:334 case AMDGPULibFunc::EI_LOG10:335 case AMDGPULibFunc::EI_POWR:336 case AMDGPULibFunc::EI_RECIP:337 case AMDGPULibFunc::EI_RSQRT:338 case AMDGPULibFunc::EI_SIN:339 case AMDGPULibFunc::EI_SINCOS:340 case AMDGPULibFunc::EI_SQRT:341 case AMDGPULibFunc::EI_TAN:342 return true;343 default:;344 }345 return false;346}347 348using TableRef = ArrayRef<TableEntry>;349 350static TableRef getOptTable(AMDGPULibFunc::EFuncId id) {351 switch(id) {352 case AMDGPULibFunc::EI_ACOS: return TableRef(tbl_acos);353 case AMDGPULibFunc::EI_ACOSH: return TableRef(tbl_acosh);354 case AMDGPULibFunc::EI_ACOSPI: return TableRef(tbl_acospi);355 case AMDGPULibFunc::EI_ASIN: return TableRef(tbl_asin);356 case AMDGPULibFunc::EI_ASINH: return TableRef(tbl_asinh);357 case AMDGPULibFunc::EI_ASINPI: return TableRef(tbl_asinpi);358 case AMDGPULibFunc::EI_ATAN: return TableRef(tbl_atan);359 case AMDGPULibFunc::EI_ATANH: return TableRef(tbl_atanh);360 case AMDGPULibFunc::EI_ATANPI: return TableRef(tbl_atanpi);361 case AMDGPULibFunc::EI_CBRT: return TableRef(tbl_cbrt);362 case AMDGPULibFunc::EI_NCOS:363 case AMDGPULibFunc::EI_COS: return TableRef(tbl_cos);364 case AMDGPULibFunc::EI_COSH: return TableRef(tbl_cosh);365 case AMDGPULibFunc::EI_COSPI: return TableRef(tbl_cospi);366 case AMDGPULibFunc::EI_ERFC: return TableRef(tbl_erfc);367 case AMDGPULibFunc::EI_ERF: return TableRef(tbl_erf);368 case AMDGPULibFunc::EI_EXP: return TableRef(tbl_exp);369 case AMDGPULibFunc::EI_NEXP2:370 case AMDGPULibFunc::EI_EXP2: return TableRef(tbl_exp2);371 case AMDGPULibFunc::EI_EXP10: return TableRef(tbl_exp10);372 case AMDGPULibFunc::EI_EXPM1: return TableRef(tbl_expm1);373 case AMDGPULibFunc::EI_LOG: return TableRef(tbl_log);374 case AMDGPULibFunc::EI_NLOG2:375 case AMDGPULibFunc::EI_LOG2: return TableRef(tbl_log2);376 case AMDGPULibFunc::EI_LOG10: return TableRef(tbl_log10);377 case AMDGPULibFunc::EI_NRSQRT:378 case AMDGPULibFunc::EI_RSQRT: return TableRef(tbl_rsqrt);379 case AMDGPULibFunc::EI_NSIN:380 case AMDGPULibFunc::EI_SIN: return TableRef(tbl_sin);381 case AMDGPULibFunc::EI_SINH: return TableRef(tbl_sinh);382 case AMDGPULibFunc::EI_SINPI: return TableRef(tbl_sinpi);383 case AMDGPULibFunc::EI_NSQRT:384 case AMDGPULibFunc::EI_SQRT: return TableRef(tbl_sqrt);385 case AMDGPULibFunc::EI_TAN: return TableRef(tbl_tan);386 case AMDGPULibFunc::EI_TANH: return TableRef(tbl_tanh);387 case AMDGPULibFunc::EI_TANPI: return TableRef(tbl_tanpi);388 case AMDGPULibFunc::EI_TGAMMA: return TableRef(tbl_tgamma);389 default:;390 }391 return TableRef();392}393 394static inline int getVecSize(const AMDGPULibFunc& FInfo) {395 return FInfo.getLeads()[0].VectorSize;396}397 398static inline AMDGPULibFunc::EType getArgType(const AMDGPULibFunc& FInfo) {399 return (AMDGPULibFunc::EType)FInfo.getLeads()[0].ArgType;400}401 402FunctionCallee AMDGPULibCalls::getFunction(Module *M, const FuncInfo &fInfo) {403 // If we are doing PreLinkOpt, the function is external. So it is safe to404 // use getOrInsertFunction() at this stage.405 406 return EnablePreLink ? AMDGPULibFunc::getOrInsertFunction(M, fInfo)407 : AMDGPULibFunc::getFunction(M, fInfo);408}409 410bool AMDGPULibCalls::parseFunctionName(const StringRef &FMangledName,411 FuncInfo &FInfo) {412 return AMDGPULibFunc::parse(FMangledName, FInfo);413}414 415bool AMDGPULibCalls::isUnsafeFiniteOnlyMath(const FPMathOperator *FPOp) const {416 return FPOp->hasApproxFunc() && FPOp->hasNoNaNs() && FPOp->hasNoInfs();417}418 419bool AMDGPULibCalls::canIncreasePrecisionOfConstantFold(420 const FPMathOperator *FPOp) const {421 // TODO: Refine to approxFunc or contract422 return FPOp->isFast();423}424 425void AMDGPULibCalls::initFunction(Function &F, FunctionAnalysisManager &FAM) {426 AC = &FAM.getResult<AssumptionAnalysis>(F);427 TLInfo = &FAM.getResult<TargetLibraryAnalysis>(F);428 DT = FAM.getCachedResult<DominatorTreeAnalysis>(F);429}430 431bool AMDGPULibCalls::useNativeFunc(const StringRef F) const {432 return AllNative || llvm::is_contained(UseNative, F);433}434 435void AMDGPULibCalls::initNativeFuncs() {436 AllNative = useNativeFunc("all") ||437 (UseNative.getNumOccurrences() && UseNative.size() == 1 &&438 UseNative.begin()->empty());439}440 441bool AMDGPULibCalls::sincosUseNative(CallInst *aCI, const FuncInfo &FInfo) {442 bool native_sin = useNativeFunc("sin");443 bool native_cos = useNativeFunc("cos");444 445 if (native_sin && native_cos) {446 Module *M = aCI->getModule();447 Value *opr0 = aCI->getArgOperand(0);448 449 AMDGPULibFunc nf;450 nf.getLeads()[0].ArgType = FInfo.getLeads()[0].ArgType;451 nf.getLeads()[0].VectorSize = FInfo.getLeads()[0].VectorSize;452 453 nf.setPrefix(AMDGPULibFunc::NATIVE);454 nf.setId(AMDGPULibFunc::EI_SIN);455 FunctionCallee sinExpr = getFunction(M, nf);456 457 nf.setPrefix(AMDGPULibFunc::NATIVE);458 nf.setId(AMDGPULibFunc::EI_COS);459 FunctionCallee cosExpr = getFunction(M, nf);460 if (sinExpr && cosExpr) {461 Value *sinval =462 CallInst::Create(sinExpr, opr0, "splitsin", aCI->getIterator());463 Value *cosval =464 CallInst::Create(cosExpr, opr0, "splitcos", aCI->getIterator());465 new StoreInst(cosval, aCI->getArgOperand(1), aCI->getIterator());466 467 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI468 << " with native version of sin/cos");469 470 replaceCall(aCI, sinval);471 return true;472 }473 }474 return false;475}476 477bool AMDGPULibCalls::useNative(CallInst *aCI) {478 Function *Callee = aCI->getCalledFunction();479 if (!Callee || aCI->isNoBuiltin())480 return false;481 482 FuncInfo FInfo;483 if (!parseFunctionName(Callee->getName(), FInfo) || !FInfo.isMangled() ||484 FInfo.getPrefix() != AMDGPULibFunc::NOPFX ||485 getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()) ||486 !(AllNative || useNativeFunc(FInfo.getName()))) {487 return false;488 }489 490 if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS)491 return sincosUseNative(aCI, FInfo);492 493 FInfo.setPrefix(AMDGPULibFunc::NATIVE);494 FunctionCallee F = getFunction(aCI->getModule(), FInfo);495 if (!F)496 return false;497 498 aCI->setCalledFunction(F);499 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI500 << " with native version");501 return true;502}503 504// Clang emits call of __read_pipe_2 or __read_pipe_4 for OpenCL read_pipe505// builtin, with appended type size and alignment arguments, where 2 or 4506// indicates the original number of arguments. The library has optimized version507// of __read_pipe_2/__read_pipe_4 when the type size and alignment has the same508// power of 2 value. This function transforms __read_pipe_2 to __read_pipe_2_N509// for such cases where N is the size in bytes of the type (N = 1, 2, 4, 8, ...,510// 128). The same for __read_pipe_4, write_pipe_2, and write_pipe_4.511bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,512 const FuncInfo &FInfo) {513 auto *Callee = CI->getCalledFunction();514 if (!Callee->isDeclaration())515 return false;516 517 assert(Callee->hasName() && "Invalid read_pipe/write_pipe function");518 auto *M = Callee->getParent();519 std::string Name = std::string(Callee->getName());520 auto NumArg = CI->arg_size();521 if (NumArg != 4 && NumArg != 6)522 return false;523 ConstantInt *PacketSize =524 dyn_cast<ConstantInt>(CI->getArgOperand(NumArg - 2));525 ConstantInt *PacketAlign =526 dyn_cast<ConstantInt>(CI->getArgOperand(NumArg - 1));527 if (!PacketSize || !PacketAlign)528 return false;529 530 unsigned Size = PacketSize->getZExtValue();531 Align Alignment = PacketAlign->getAlignValue();532 if (Alignment != Size)533 return false;534 535 unsigned PtrArgLoc = CI->arg_size() - 3;536 Value *PtrArg = CI->getArgOperand(PtrArgLoc);537 Type *PtrTy = PtrArg->getType();538 539 SmallVector<llvm::Type *, 6> ArgTys;540 for (unsigned I = 0; I != PtrArgLoc; ++I)541 ArgTys.push_back(CI->getArgOperand(I)->getType());542 ArgTys.push_back(PtrTy);543 544 Name = Name + "_" + std::to_string(Size);545 auto *FTy = FunctionType::get(Callee->getReturnType(),546 ArrayRef<Type *>(ArgTys), false);547 AMDGPULibFunc NewLibFunc(Name, FTy);548 FunctionCallee F = AMDGPULibFunc::getOrInsertFunction(M, NewLibFunc);549 if (!F)550 return false;551 552 SmallVector<Value *, 6> Args;553 for (unsigned I = 0; I != PtrArgLoc; ++I)554 Args.push_back(CI->getArgOperand(I));555 Args.push_back(PtrArg);556 557 auto *NCI = B.CreateCall(F, Args);558 NCI->setAttributes(CI->getAttributes());559 CI->replaceAllUsesWith(NCI);560 CI->dropAllReferences();561 CI->eraseFromParent();562 563 return true;564}565 566static bool isKnownIntegral(const Value *V, const DataLayout &DL,567 FastMathFlags FMF) {568 if (isa<PoisonValue>(V))569 return true;570 if (isa<UndefValue>(V))571 return false;572 573 if (const ConstantFP *CF = dyn_cast<ConstantFP>(V))574 return CF->getValueAPF().isInteger();575 576 auto *VFVTy = dyn_cast<FixedVectorType>(V->getType());577 const Constant *CV = dyn_cast<Constant>(V);578 if (VFVTy && CV) {579 unsigned NumElts = VFVTy->getNumElements();580 for (unsigned i = 0; i != NumElts; ++i) {581 Constant *Elt = CV->getAggregateElement(i);582 if (!Elt)583 return false;584 if (isa<PoisonValue>(Elt))585 continue;586 587 const ConstantFP *CFP = dyn_cast<ConstantFP>(Elt);588 if (!CFP || !CFP->getValue().isInteger())589 return false;590 }591 592 return true;593 }594 595 const Instruction *I = dyn_cast<Instruction>(V);596 if (!I)597 return false;598 599 switch (I->getOpcode()) {600 case Instruction::SIToFP:601 case Instruction::UIToFP:602 // TODO: Could check nofpclass(inf) on incoming argument603 if (FMF.noInfs())604 return true;605 606 // Need to check int size cannot produce infinity, which computeKnownFPClass607 // knows how to do already.608 return isKnownNeverInfinity(I, SimplifyQuery(DL));609 case Instruction::Call: {610 const CallInst *CI = cast<CallInst>(I);611 switch (CI->getIntrinsicID()) {612 case Intrinsic::trunc:613 case Intrinsic::floor:614 case Intrinsic::ceil:615 case Intrinsic::rint:616 case Intrinsic::nearbyint:617 case Intrinsic::round:618 case Intrinsic::roundeven:619 return (FMF.noInfs() && FMF.noNaNs()) ||620 isKnownNeverInfOrNaN(I, SimplifyQuery(DL));621 default:622 break;623 }624 625 break;626 }627 default:628 break;629 }630 631 return false;632}633 634// This function returns false if no change; return true otherwise.635bool AMDGPULibCalls::fold(CallInst *CI) {636 Function *Callee = CI->getCalledFunction();637 // Ignore indirect calls.638 if (!Callee || Callee->isIntrinsic() || CI->isNoBuiltin())639 return false;640 641 FuncInfo FInfo;642 if (!parseFunctionName(Callee->getName(), FInfo))643 return false;644 645 // Further check the number of arguments to see if they match.646 // TODO: Check calling convention matches too647 if (!FInfo.isCompatibleSignature(*Callee->getParent(), CI->getFunctionType()))648 return false;649 650 LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << '\n');651 652 if (TDOFold(CI, FInfo))653 return true;654 655 IRBuilder<> B(CI);656 if (CI->isStrictFP())657 B.setIsFPConstrained(true);658 659 if (FPMathOperator *FPOp = dyn_cast<FPMathOperator>(CI)) {660 // Under unsafe-math, evaluate calls if possible.661 // According to Brian Sumner, we can do this for all f32 function calls662 // using host's double function calls.663 if (canIncreasePrecisionOfConstantFold(FPOp) && evaluateCall(CI, FInfo))664 return true;665 666 // Copy fast flags from the original call.667 FastMathFlags FMF = FPOp->getFastMathFlags();668 B.setFastMathFlags(FMF);669 670 // Specialized optimizations for each function call.671 //672 // TODO: Handle native functions673 switch (FInfo.getId()) {674 case AMDGPULibFunc::EI_EXP:675 if (FMF.none())676 return false;677 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::exp,678 FMF.approxFunc());679 case AMDGPULibFunc::EI_EXP2:680 if (FMF.none())681 return false;682 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::exp2,683 FMF.approxFunc());684 case AMDGPULibFunc::EI_LOG:685 if (FMF.none())686 return false;687 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::log,688 FMF.approxFunc());689 case AMDGPULibFunc::EI_LOG2:690 if (FMF.none())691 return false;692 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::log2,693 FMF.approxFunc());694 case AMDGPULibFunc::EI_LOG10:695 if (FMF.none())696 return false;697 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::log10,698 FMF.approxFunc());699 case AMDGPULibFunc::EI_FMIN:700 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::minnum,701 true, true);702 case AMDGPULibFunc::EI_FMAX:703 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::maxnum,704 true, true);705 case AMDGPULibFunc::EI_FMA:706 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::fma, true,707 true);708 case AMDGPULibFunc::EI_MAD:709 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::fmuladd,710 true, true);711 case AMDGPULibFunc::EI_FABS:712 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::fabs, true,713 true, true);714 case AMDGPULibFunc::EI_COPYSIGN:715 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::copysign,716 true, true, true);717 case AMDGPULibFunc::EI_FLOOR:718 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::floor, true,719 true);720 case AMDGPULibFunc::EI_CEIL:721 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::ceil, true,722 true);723 case AMDGPULibFunc::EI_TRUNC:724 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::trunc, true,725 true);726 case AMDGPULibFunc::EI_RINT:727 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::rint, true,728 true);729 case AMDGPULibFunc::EI_ROUND:730 return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::round, true,731 true);732 case AMDGPULibFunc::EI_LDEXP: {733 if (!shouldReplaceLibcallWithIntrinsic(CI, true, true))734 return false;735 736 Value *Arg1 = CI->getArgOperand(1);737 if (VectorType *VecTy = dyn_cast<VectorType>(CI->getType());738 VecTy && !isa<VectorType>(Arg1->getType())) {739 Value *SplatArg1 = B.CreateVectorSplat(VecTy->getElementCount(), Arg1);740 CI->setArgOperand(1, SplatArg1);741 }742 743 CI->setCalledFunction(Intrinsic::getOrInsertDeclaration(744 CI->getModule(), Intrinsic::ldexp,745 {CI->getType(), CI->getArgOperand(1)->getType()}));746 return true;747 }748 case AMDGPULibFunc::EI_POW: {749 Module *M = Callee->getParent();750 AMDGPULibFunc PowrInfo(AMDGPULibFunc::EI_POWR, FInfo);751 FunctionCallee PowrFunc = getFunction(M, PowrInfo);752 CallInst *Call = cast<CallInst>(FPOp);753 754 // pow(x, y) -> powr(x, y) for x >= -0.0755 // TODO: Account for flags on current call756 if (PowrFunc &&757 cannotBeOrderedLessThanZero(758 FPOp->getOperand(0),759 SimplifyQuery(M->getDataLayout(), TLInfo, DT, AC, Call))) {760 Call->setCalledFunction(PowrFunc);761 return fold_pow(FPOp, B, PowrInfo) || true;762 }763 764 // pow(x, y) -> pown(x, y) for known integral y765 if (isKnownIntegral(FPOp->getOperand(1), M->getDataLayout(),766 FPOp->getFastMathFlags())) {767 FunctionType *PownType = getPownType(CI->getFunctionType());768 AMDGPULibFunc PownInfo(AMDGPULibFunc::EI_POWN, PownType, true);769 FunctionCallee PownFunc = getFunction(M, PownInfo);770 if (PownFunc) {771 // TODO: If the incoming integral value is an sitofp/uitofp, it won't772 // fold out without a known range. We can probably take the source773 // value directly.774 Value *CastedArg =775 B.CreateFPToSI(FPOp->getOperand(1), PownType->getParamType(1));776 // Have to drop any nofpclass attributes on the original call site.777 Call->removeParamAttrs(778 1, AttributeFuncs::typeIncompatible(CastedArg->getType(),779 Call->getParamAttributes(1)));780 Call->setCalledFunction(PownFunc);781 Call->setArgOperand(1, CastedArg);782 return fold_pow(FPOp, B, PownInfo) || true;783 }784 }785 786 return fold_pow(FPOp, B, FInfo);787 }788 case AMDGPULibFunc::EI_POWR:789 case AMDGPULibFunc::EI_POWN:790 return fold_pow(FPOp, B, FInfo);791 case AMDGPULibFunc::EI_ROOTN:792 return fold_rootn(FPOp, B, FInfo);793 case AMDGPULibFunc::EI_SQRT:794 // TODO: Allow with strictfp + constrained intrinsic795 return tryReplaceLibcallWithSimpleIntrinsic(796 B, CI, Intrinsic::sqrt, true, true, /*AllowStrictFP=*/false);797 case AMDGPULibFunc::EI_COS:798 case AMDGPULibFunc::EI_SIN:799 return fold_sincos(FPOp, B, FInfo);800 default:801 break;802 }803 } else {804 // Specialized optimizations for each function call805 switch (FInfo.getId()) {806 case AMDGPULibFunc::EI_READ_PIPE_2:807 case AMDGPULibFunc::EI_READ_PIPE_4:808 case AMDGPULibFunc::EI_WRITE_PIPE_2:809 case AMDGPULibFunc::EI_WRITE_PIPE_4:810 return fold_read_write_pipe(CI, B, FInfo);811 default:812 break;813 }814 }815 816 return false;817}818 819bool AMDGPULibCalls::TDOFold(CallInst *CI, const FuncInfo &FInfo) {820 // Table-Driven optimization821 const TableRef tr = getOptTable(FInfo.getId());822 if (tr.empty())823 return false;824 825 int const sz = (int)tr.size();826 Value *opr0 = CI->getArgOperand(0);827 828 if (getVecSize(FInfo) > 1) {829 if (ConstantDataVector *CV = dyn_cast<ConstantDataVector>(opr0)) {830 SmallVector<double, 0> DVal;831 for (int eltNo = 0; eltNo < getVecSize(FInfo); ++eltNo) {832 ConstantFP *eltval = dyn_cast<ConstantFP>(833 CV->getElementAsConstant((unsigned)eltNo));834 assert(eltval && "Non-FP arguments in math function!");835 bool found = false;836 for (int i=0; i < sz; ++i) {837 if (eltval->isExactlyValue(tr[i].input)) {838 DVal.push_back(tr[i].result);839 found = true;840 break;841 }842 }843 if (!found) {844 // This vector constants not handled yet.845 return false;846 }847 }848 LLVMContext &context = CI->getContext();849 Constant *nval;850 if (getArgType(FInfo) == AMDGPULibFunc::F32) {851 SmallVector<float, 0> FVal;852 for (double D : DVal)853 FVal.push_back((float)D);854 ArrayRef<float> tmp(FVal);855 nval = ConstantDataVector::get(context, tmp);856 } else { // F64857 ArrayRef<double> tmp(DVal);858 nval = ConstantDataVector::get(context, tmp);859 }860 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");861 replaceCall(CI, nval);862 return true;863 }864 } else {865 // Scalar version866 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) {867 for (int i = 0; i < sz; ++i) {868 if (CF->isExactlyValue(tr[i].input)) {869 Value *nval = ConstantFP::get(CF->getType(), tr[i].result);870 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");871 replaceCall(CI, nval);872 return true;873 }874 }875 }876 }877 878 return false;879}880 881namespace llvm {882static double log2(double V) {883#if _XOPEN_SOURCE >= 600 || defined(_ISOC99_SOURCE) || _POSIX_C_SOURCE >= 200112L884 return ::log2(V);885#else886 return log(V) / numbers::ln2;887#endif888}889} // namespace llvm890 891bool AMDGPULibCalls::fold_pow(FPMathOperator *FPOp, IRBuilder<> &B,892 const FuncInfo &FInfo) {893 assert((FInfo.getId() == AMDGPULibFunc::EI_POW ||894 FInfo.getId() == AMDGPULibFunc::EI_POWR ||895 FInfo.getId() == AMDGPULibFunc::EI_POWN) &&896 "fold_pow: encounter a wrong function call");897 898 Module *M = B.GetInsertBlock()->getModule();899 Type *eltType = FPOp->getType()->getScalarType();900 Value *opr0 = FPOp->getOperand(0);901 Value *opr1 = FPOp->getOperand(1);902 903 const APFloat *CF = nullptr;904 const APInt *CINT = nullptr;905 if (!match(opr1, m_APFloatAllowPoison(CF)))906 match(opr1, m_APIntAllowPoison(CINT));907 908 // 0x1111111 means that we don't do anything for this call.909 int ci_opr1 = (CINT ? (int)CINT->getSExtValue() : 0x1111111);910 911 if ((CF && CF->isZero()) || (CINT && ci_opr1 == 0)) {912 // pow/powr/pown(x, 0) == 1913 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1\n");914 Constant *cnval = ConstantFP::get(eltType, 1.0);915 if (getVecSize(FInfo) > 1) {916 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);917 }918 replaceCall(FPOp, cnval);919 return true;920 }921 if ((CF && CF->isExactlyValue(1.0)) || (CINT && ci_opr1 == 1)) {922 // pow/powr/pown(x, 1.0) = x923 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << "\n");924 replaceCall(FPOp, opr0);925 return true;926 }927 if ((CF && CF->isExactlyValue(2.0)) || (CINT && ci_opr1 == 2)) {928 // pow/powr/pown(x, 2.0) = x*x929 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << " * "930 << *opr0 << "\n");931 Value *nval = B.CreateFMul(opr0, opr0, "__pow2");932 replaceCall(FPOp, nval);933 return true;934 }935 if ((CF && CF->isExactlyValue(-1.0)) || (CINT && ci_opr1 == -1)) {936 // pow/powr/pown(x, -1.0) = 1.0/x937 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1 / " << *opr0 << "\n");938 Constant *cnval = ConstantFP::get(eltType, 1.0);939 if (getVecSize(FInfo) > 1) {940 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);941 }942 Value *nval = B.CreateFDiv(cnval, opr0, "__powrecip");943 replaceCall(FPOp, nval);944 return true;945 }946 947 if (CF && (CF->isExactlyValue(0.5) || CF->isExactlyValue(-0.5))) {948 // pow[r](x, [-]0.5) = sqrt(x)949 bool issqrt = CF->isExactlyValue(0.5);950 if (FunctionCallee FPExpr =951 getFunction(M, AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT952 : AMDGPULibFunc::EI_RSQRT,953 FInfo))) {954 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << FInfo.getName()955 << '(' << *opr0 << ")\n");956 Value *nval = CreateCallEx(B,FPExpr, opr0, issqrt ? "__pow2sqrt"957 : "__pow2rsqrt");958 replaceCall(FPOp, nval);959 return true;960 }961 }962 963 if (!isUnsafeFiniteOnlyMath(FPOp))964 return false;965 966 // Unsafe Math optimization967 968 // Remember that ci_opr1 is set if opr1 is integral969 if (CF) {970 double dval = (getArgType(FInfo) == AMDGPULibFunc::F32)971 ? (double)CF->convertToFloat()972 : CF->convertToDouble();973 int ival = (int)dval;974 if ((double)ival == dval) {975 ci_opr1 = ival;976 } else977 ci_opr1 = 0x11111111;978 }979 980 // pow/powr/pown(x, c) = [1/](x*x*..x); where981 // trunc(c) == c && the number of x == c && |c| <= 12982 unsigned abs_opr1 = (ci_opr1 < 0) ? -ci_opr1 : ci_opr1;983 if (abs_opr1 <= 12) {984 Constant *cnval;985 Value *nval;986 if (abs_opr1 == 0) {987 cnval = ConstantFP::get(eltType, 1.0);988 if (getVecSize(FInfo) > 1) {989 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);990 }991 nval = cnval;992 } else {993 Value *valx2 = nullptr;994 nval = nullptr;995 while (abs_opr1 > 0) {996 valx2 = valx2 ? B.CreateFMul(valx2, valx2, "__powx2") : opr0;997 if (abs_opr1 & 1) {998 nval = nval ? B.CreateFMul(nval, valx2, "__powprod") : valx2;999 }1000 abs_opr1 >>= 1;1001 }1002 }1003 1004 if (ci_opr1 < 0) {1005 cnval = ConstantFP::get(eltType, 1.0);1006 if (getVecSize(FInfo) > 1) {1007 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);1008 }1009 nval = B.CreateFDiv(cnval, nval, "__1powprod");1010 }1011 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> "1012 << ((ci_opr1 < 0) ? "1/prod(" : "prod(") << *opr01013 << ")\n");1014 replaceCall(FPOp, nval);1015 return true;1016 }1017 1018 // If we should use the generic intrinsic instead of emitting a libcall1019 const bool ShouldUseIntrinsic = eltType->isFloatTy() || eltType->isHalfTy();1020 1021 // powr ---> exp2(y * log2(x))1022 // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31))1023 FunctionCallee ExpExpr;1024 if (ShouldUseIntrinsic)1025 ExpExpr = Intrinsic::getOrInsertDeclaration(M, Intrinsic::exp2,1026 {FPOp->getType()});1027 else {1028 ExpExpr = getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo));1029 if (!ExpExpr)1030 return false;1031 }1032 1033 bool needlog = false;1034 bool needabs = false;1035 bool needcopysign = false;1036 Constant *cnval = nullptr;1037 if (getVecSize(FInfo) == 1) {1038 CF = nullptr;1039 match(opr0, m_APFloatAllowPoison(CF));1040 1041 if (CF) {1042 double V = (getArgType(FInfo) == AMDGPULibFunc::F32)1043 ? (double)CF->convertToFloat()1044 : CF->convertToDouble();1045 1046 V = log2(std::abs(V));1047 cnval = ConstantFP::get(eltType, V);1048 needcopysign = (FInfo.getId() != AMDGPULibFunc::EI_POWR) &&1049 CF->isNegative();1050 } else {1051 needlog = true;1052 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR;1053 }1054 } else {1055 ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr0);1056 1057 if (!CDV) {1058 needlog = true;1059 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR;1060 } else {1061 assert ((int)CDV->getNumElements() == getVecSize(FInfo) &&1062 "Wrong vector size detected");1063 1064 SmallVector<double, 0> DVal;1065 for (int i=0; i < getVecSize(FInfo); ++i) {1066 double V = CDV->getElementAsAPFloat(i).convertToDouble();1067 if (V < 0.0) needcopysign = true;1068 V = log2(std::abs(V));1069 DVal.push_back(V);1070 }1071 if (getArgType(FInfo) == AMDGPULibFunc::F32) {1072 SmallVector<float, 0> FVal;1073 for (double D : DVal)1074 FVal.push_back((float)D);1075 ArrayRef<float> tmp(FVal);1076 cnval = ConstantDataVector::get(M->getContext(), tmp);1077 } else {1078 ArrayRef<double> tmp(DVal);1079 cnval = ConstantDataVector::get(M->getContext(), tmp);1080 }1081 }1082 }1083 1084 if (needcopysign && (FInfo.getId() == AMDGPULibFunc::EI_POW)) {1085 // We cannot handle corner cases for a general pow() function, give up1086 // unless y is a constant integral value. Then proceed as if it were pown.1087 if (!isKnownIntegral(opr1, M->getDataLayout(), FPOp->getFastMathFlags()))1088 return false;1089 }1090 1091 Value *nval;1092 if (needabs) {1093 nval = B.CreateUnaryIntrinsic(Intrinsic::fabs, opr0, nullptr, "__fabs");1094 } else {1095 nval = cnval ? cnval : opr0;1096 }1097 if (needlog) {1098 FunctionCallee LogExpr;1099 if (ShouldUseIntrinsic) {1100 LogExpr = Intrinsic::getOrInsertDeclaration(M, Intrinsic::log2,1101 {FPOp->getType()});1102 } else {1103 LogExpr = getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo));1104 if (!LogExpr)1105 return false;1106 }1107 1108 nval = CreateCallEx(B,LogExpr, nval, "__log2");1109 }1110 1111 if (FInfo.getId() == AMDGPULibFunc::EI_POWN) {1112 // convert int(32) to fp(f32 or f64)1113 opr1 = B.CreateSIToFP(opr1, nval->getType(), "pownI2F");1114 }1115 nval = B.CreateFMul(opr1, nval, "__ylogx");1116 nval = CreateCallEx(B,ExpExpr, nval, "__exp2");1117 1118 if (needcopysign) {1119 Type* nTyS = B.getIntNTy(eltType->getPrimitiveSizeInBits());1120 Type *nTy = FPOp->getType()->getWithNewType(nTyS);1121 unsigned size = nTy->getScalarSizeInBits();1122 Value *opr_n = FPOp->getOperand(1);1123 if (opr_n->getType()->getScalarType()->isIntegerTy())1124 opr_n = B.CreateZExtOrTrunc(opr_n, nTy, "__ytou");1125 else1126 opr_n = B.CreateFPToSI(opr1, nTy, "__ytou");1127 1128 Value *sign = B.CreateShl(opr_n, size-1, "__yeven");1129 sign = B.CreateAnd(B.CreateBitCast(opr0, nTy), sign, "__pow_sign");1130 nval = B.CreateOr(B.CreateBitCast(nval, nTy), sign);1131 nval = B.CreateBitCast(nval, opr0->getType());1132 }1133 1134 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> "1135 << "exp2(" << *opr1 << " * log2(" << *opr0 << "))\n");1136 replaceCall(FPOp, nval);1137 1138 return true;1139}1140 1141bool AMDGPULibCalls::fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B,1142 const FuncInfo &FInfo) {1143 Value *opr0 = FPOp->getOperand(0);1144 Value *opr1 = FPOp->getOperand(1);1145 1146 const APInt *CINT = nullptr;1147 if (!match(opr1, m_APIntAllowPoison(CINT)))1148 return false;1149 1150 Function *Parent = B.GetInsertBlock()->getParent();1151 1152 int ci_opr1 = (int)CINT->getSExtValue();1153 if (ci_opr1 == 1 && !Parent->hasFnAttribute(Attribute::StrictFP)) {1154 // rootn(x, 1) = x1155 //1156 // TODO: Insert constrained canonicalize for strictfp case.1157 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << '\n');1158 replaceCall(FPOp, opr0);1159 return true;1160 }1161 1162 Module *M = B.GetInsertBlock()->getModule();1163 1164 CallInst *CI = cast<CallInst>(FPOp);1165 if (ci_opr1 == 2 &&1166 shouldReplaceLibcallWithIntrinsic(CI,1167 /*AllowMinSizeF32=*/true,1168 /*AllowF64=*/true)) {1169 // rootn(x, 2) = sqrt(x)1170 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> sqrt(" << *opr0 << ")\n");1171 1172 CallInst *NewCall = B.CreateUnaryIntrinsic(Intrinsic::sqrt, opr0, CI);1173 NewCall->takeName(CI);1174 1175 // OpenCL rootn has a looser ulp of 2 requirement than sqrt, so add some1176 // metadata.1177 MDBuilder MDHelper(M->getContext());1178 MDNode *FPMD = MDHelper.createFPMath(std::max(FPOp->getFPAccuracy(), 2.0f));1179 NewCall->setMetadata(LLVMContext::MD_fpmath, FPMD);1180 1181 replaceCall(CI, NewCall);1182 return true;1183 }1184 1185 if (ci_opr1 == 3) { // rootn(x, 3) = cbrt(x)1186 if (FunctionCallee FPExpr =1187 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_CBRT, FInfo))) {1188 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> cbrt(" << *opr01189 << ")\n");1190 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2cbrt");1191 replaceCall(FPOp, nval);1192 return true;1193 }1194 } else if (ci_opr1 == -1) { // rootn(x, -1) = 1.0/x1195 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1.0 / " << *opr0 << "\n");1196 Value *nval = B.CreateFDiv(ConstantFP::get(opr0->getType(), 1.0),1197 opr0,1198 "__rootn2div");1199 replaceCall(FPOp, nval);1200 return true;1201 }1202 1203 if (ci_opr1 == -2 &&1204 shouldReplaceLibcallWithIntrinsic(CI,1205 /*AllowMinSizeF32=*/true,1206 /*AllowF64=*/true)) {1207 // rootn(x, -2) = rsqrt(x)1208 1209 // The original rootn had looser ulp requirements than the resultant sqrt1210 // and fdiv.1211 MDBuilder MDHelper(M->getContext());1212 MDNode *FPMD = MDHelper.createFPMath(std::max(FPOp->getFPAccuracy(), 2.0f));1213 1214 // TODO: Could handle strictfp but need to fix strict sqrt emission1215 FastMathFlags FMF = FPOp->getFastMathFlags();1216 FMF.setAllowContract(true);1217 1218 CallInst *Sqrt = B.CreateUnaryIntrinsic(Intrinsic::sqrt, opr0, CI);1219 Instruction *RSqrt = cast<Instruction>(1220 B.CreateFDiv(ConstantFP::get(opr0->getType(), 1.0), Sqrt));1221 Sqrt->setFastMathFlags(FMF);1222 RSqrt->setFastMathFlags(FMF);1223 RSqrt->setMetadata(LLVMContext::MD_fpmath, FPMD);1224 1225 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> rsqrt(" << *opr01226 << ")\n");1227 replaceCall(CI, RSqrt);1228 return true;1229 }1230 1231 return false;1232}1233 1234// Get a scalar native builtin single argument FP function1235FunctionCallee AMDGPULibCalls::getNativeFunction(Module *M,1236 const FuncInfo &FInfo) {1237 if (getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()))1238 return nullptr;1239 FuncInfo nf = FInfo;1240 nf.setPrefix(AMDGPULibFunc::NATIVE);1241 return getFunction(M, nf);1242}1243 1244// Some library calls are just wrappers around llvm intrinsics, but compiled1245// conservatively. Preserve the flags from the original call site by1246// substituting them with direct calls with all the flags.1247bool AMDGPULibCalls::shouldReplaceLibcallWithIntrinsic(const CallInst *CI,1248 bool AllowMinSizeF32,1249 bool AllowF64,1250 bool AllowStrictFP) {1251 Type *FltTy = CI->getType()->getScalarType();1252 const bool IsF32 = FltTy->isFloatTy();1253 1254 // f64 intrinsics aren't implemented for most operations.1255 if (!IsF32 && !FltTy->isHalfTy() && (!AllowF64 || !FltTy->isDoubleTy()))1256 return false;1257 1258 // We're implicitly inlining by replacing the libcall with the intrinsic, so1259 // don't do it for noinline call sites.1260 if (CI->isNoInline())1261 return false;1262 1263 const Function *ParentF = CI->getFunction();1264 // TODO: Handle strictfp1265 if (!AllowStrictFP && ParentF->hasFnAttribute(Attribute::StrictFP))1266 return false;1267 1268 if (IsF32 && !AllowMinSizeF32 && ParentF->hasMinSize())1269 return false;1270 return true;1271}1272 1273void AMDGPULibCalls::replaceLibCallWithSimpleIntrinsic(IRBuilder<> &B,1274 CallInst *CI,1275 Intrinsic::ID IntrID) {1276 if (CI->arg_size() == 2) {1277 Value *Arg0 = CI->getArgOperand(0);1278 Value *Arg1 = CI->getArgOperand(1);1279 VectorType *Arg0VecTy = dyn_cast<VectorType>(Arg0->getType());1280 VectorType *Arg1VecTy = dyn_cast<VectorType>(Arg1->getType());1281 if (Arg0VecTy && !Arg1VecTy) {1282 Value *SplatRHS = B.CreateVectorSplat(Arg0VecTy->getElementCount(), Arg1);1283 CI->setArgOperand(1, SplatRHS);1284 } else if (!Arg0VecTy && Arg1VecTy) {1285 Value *SplatLHS = B.CreateVectorSplat(Arg1VecTy->getElementCount(), Arg0);1286 CI->setArgOperand(0, SplatLHS);1287 }1288 }1289 1290 CI->setCalledFunction(Intrinsic::getOrInsertDeclaration(1291 CI->getModule(), IntrID, {CI->getType()}));1292}1293 1294bool AMDGPULibCalls::tryReplaceLibcallWithSimpleIntrinsic(1295 IRBuilder<> &B, CallInst *CI, Intrinsic::ID IntrID, bool AllowMinSizeF32,1296 bool AllowF64, bool AllowStrictFP) {1297 if (!shouldReplaceLibcallWithIntrinsic(CI, AllowMinSizeF32, AllowF64,1298 AllowStrictFP))1299 return false;1300 replaceLibCallWithSimpleIntrinsic(B, CI, IntrID);1301 return true;1302}1303 1304std::tuple<Value *, Value *, Value *>1305AMDGPULibCalls::insertSinCos(Value *Arg, FastMathFlags FMF, IRBuilder<> &B,1306 FunctionCallee Fsincos) {1307 DebugLoc DL = B.getCurrentDebugLocation();1308 Function *F = B.GetInsertBlock()->getParent();1309 B.SetInsertPointPastAllocas(F);1310 1311 AllocaInst *Alloc = B.CreateAlloca(Arg->getType(), nullptr, "__sincos_");1312 1313 if (Instruction *ArgInst = dyn_cast<Instruction>(Arg)) {1314 // If the argument is an instruction, it must dominate all uses so put our1315 // sincos call there. Otherwise, right after the allocas works well enough1316 // if it's an argument or constant.1317 1318 B.SetInsertPoint(ArgInst->getParent(), ++ArgInst->getIterator());1319 1320 // SetInsertPoint unwelcomely always tries to set the debug loc.1321 B.SetCurrentDebugLocation(DL);1322 }1323 1324 Type *CosPtrTy = Fsincos.getFunctionType()->getParamType(1);1325 1326 // The allocaInst allocates the memory in private address space. This need1327 // to be addrspacecasted to point to the address space of cos pointer type.1328 // In OpenCL 2.0 this is generic, while in 1.2 that is private.1329 Value *CastAlloc = B.CreateAddrSpaceCast(Alloc, CosPtrTy);1330 1331 CallInst *SinCos = CreateCallEx2(B, Fsincos, Arg, CastAlloc);1332 1333 // TODO: Is it worth trying to preserve the location for the cos calls for the1334 // load?1335 1336 LoadInst *LoadCos = B.CreateLoad(Alloc->getAllocatedType(), Alloc);1337 return {SinCos, LoadCos, SinCos};1338}1339 1340// fold sin, cos -> sincos.1341bool AMDGPULibCalls::fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B,1342 const FuncInfo &fInfo) {1343 assert(fInfo.getId() == AMDGPULibFunc::EI_SIN ||1344 fInfo.getId() == AMDGPULibFunc::EI_COS);1345 1346 if ((getArgType(fInfo) != AMDGPULibFunc::F32 &&1347 getArgType(fInfo) != AMDGPULibFunc::F64) ||1348 fInfo.getPrefix() != AMDGPULibFunc::NOPFX)1349 return false;1350 1351 bool const isSin = fInfo.getId() == AMDGPULibFunc::EI_SIN;1352 1353 Value *CArgVal = FPOp->getOperand(0);1354 1355 // TODO: Constant fold the call1356 if (isa<ConstantData>(CArgVal))1357 return false;1358 1359 CallInst *CI = cast<CallInst>(FPOp);1360 1361 Function *F = B.GetInsertBlock()->getParent();1362 Module *M = F->getParent();1363 1364 // Merge the sin and cos. For OpenCL 2.0, there may only be a generic pointer1365 // implementation. Prefer the private form if available.1366 AMDGPULibFunc SinCosLibFuncPrivate(AMDGPULibFunc::EI_SINCOS, fInfo);1367 SinCosLibFuncPrivate.getLeads()[0].PtrKind =1368 AMDGPULibFunc::getEPtrKindFromAddrSpace(AMDGPUAS::PRIVATE_ADDRESS);1369 1370 AMDGPULibFunc SinCosLibFuncGeneric(AMDGPULibFunc::EI_SINCOS, fInfo);1371 SinCosLibFuncGeneric.getLeads()[0].PtrKind =1372 AMDGPULibFunc::getEPtrKindFromAddrSpace(AMDGPUAS::FLAT_ADDRESS);1373 1374 FunctionCallee FSinCosPrivate = getFunction(M, SinCosLibFuncPrivate);1375 FunctionCallee FSinCosGeneric = getFunction(M, SinCosLibFuncGeneric);1376 FunctionCallee FSinCos = FSinCosPrivate ? FSinCosPrivate : FSinCosGeneric;1377 if (!FSinCos)1378 return false;1379 1380 SmallVector<CallInst *> SinCalls;1381 SmallVector<CallInst *> CosCalls;1382 SmallVector<CallInst *> SinCosCalls;1383 FuncInfo PartnerInfo(isSin ? AMDGPULibFunc::EI_COS : AMDGPULibFunc::EI_SIN,1384 fInfo);1385 const std::string PairName = PartnerInfo.mangle();1386 1387 StringRef SinName = isSin ? CI->getCalledFunction()->getName() : PairName;1388 StringRef CosName = isSin ? PairName : CI->getCalledFunction()->getName();1389 const std::string SinCosPrivateName = SinCosLibFuncPrivate.mangle();1390 const std::string SinCosGenericName = SinCosLibFuncGeneric.mangle();1391 1392 // Intersect the two sets of flags.1393 FastMathFlags FMF = FPOp->getFastMathFlags();1394 MDNode *FPMath = CI->getMetadata(LLVMContext::MD_fpmath);1395 1396 SmallVector<DILocation *> MergeDbgLocs = {CI->getDebugLoc()};1397 1398 for (User* U : CArgVal->users()) {1399 CallInst *XI = dyn_cast<CallInst>(U);1400 if (!XI || XI->getFunction() != F || XI->isNoBuiltin())1401 continue;1402 1403 Function *UCallee = XI->getCalledFunction();1404 if (!UCallee)1405 continue;1406 1407 bool Handled = true;1408 1409 if (UCallee->getName() == SinName)1410 SinCalls.push_back(XI);1411 else if (UCallee->getName() == CosName)1412 CosCalls.push_back(XI);1413 else if (UCallee->getName() == SinCosPrivateName ||1414 UCallee->getName() == SinCosGenericName)1415 SinCosCalls.push_back(XI);1416 else1417 Handled = false;1418 1419 if (Handled) {1420 MergeDbgLocs.push_back(XI->getDebugLoc());1421 auto *OtherOp = cast<FPMathOperator>(XI);1422 FMF &= OtherOp->getFastMathFlags();1423 FPMath = MDNode::getMostGenericFPMath(1424 FPMath, XI->getMetadata(LLVMContext::MD_fpmath));1425 }1426 }1427 1428 if (SinCalls.empty() || CosCalls.empty())1429 return false;1430 1431 B.setFastMathFlags(FMF);1432 B.setDefaultFPMathTag(FPMath);1433 DILocation *DbgLoc = DILocation::getMergedLocations(MergeDbgLocs);1434 B.SetCurrentDebugLocation(DbgLoc);1435 1436 auto [Sin, Cos, SinCos] = insertSinCos(CArgVal, FMF, B, FSinCos);1437 1438 auto replaceTrigInsts = [](ArrayRef<CallInst *> Calls, Value *Res) {1439 for (CallInst *C : Calls)1440 C->replaceAllUsesWith(Res);1441 1442 // Leave the other dead instructions to avoid clobbering iterators.1443 };1444 1445 replaceTrigInsts(SinCalls, Sin);1446 replaceTrigInsts(CosCalls, Cos);1447 replaceTrigInsts(SinCosCalls, SinCos);1448 1449 // It's safe to delete the original now.1450 CI->eraseFromParent();1451 return true;1452}1453 1454bool AMDGPULibCalls::evaluateScalarMathFunc(const FuncInfo &FInfo, double &Res0,1455 double &Res1, Constant *copr0,1456 Constant *copr1) {1457 // By default, opr0/opr1/opr3 holds values of float/double type.1458 // If they are not float/double, each function has to its1459 // operand separately.1460 double opr0 = 0.0, opr1 = 0.0;1461 ConstantFP *fpopr0 = dyn_cast_or_null<ConstantFP>(copr0);1462 ConstantFP *fpopr1 = dyn_cast_or_null<ConstantFP>(copr1);1463 if (fpopr0) {1464 opr0 = (getArgType(FInfo) == AMDGPULibFunc::F64)1465 ? fpopr0->getValueAPF().convertToDouble()1466 : (double)fpopr0->getValueAPF().convertToFloat();1467 }1468 1469 if (fpopr1) {1470 opr1 = (getArgType(FInfo) == AMDGPULibFunc::F64)1471 ? fpopr1->getValueAPF().convertToDouble()1472 : (double)fpopr1->getValueAPF().convertToFloat();1473 }1474 1475 switch (FInfo.getId()) {1476 default : return false;1477 1478 case AMDGPULibFunc::EI_ACOS:1479 Res0 = acos(opr0);1480 return true;1481 1482 case AMDGPULibFunc::EI_ACOSH:1483 // acosh(x) == log(x + sqrt(x*x - 1))1484 Res0 = log(opr0 + sqrt(opr0*opr0 - 1.0));1485 return true;1486 1487 case AMDGPULibFunc::EI_ACOSPI:1488 Res0 = acos(opr0) / MATH_PI;1489 return true;1490 1491 case AMDGPULibFunc::EI_ASIN:1492 Res0 = asin(opr0);1493 return true;1494 1495 case AMDGPULibFunc::EI_ASINH:1496 // asinh(x) == log(x + sqrt(x*x + 1))1497 Res0 = log(opr0 + sqrt(opr0*opr0 + 1.0));1498 return true;1499 1500 case AMDGPULibFunc::EI_ASINPI:1501 Res0 = asin(opr0) / MATH_PI;1502 return true;1503 1504 case AMDGPULibFunc::EI_ATAN:1505 Res0 = atan(opr0);1506 return true;1507 1508 case AMDGPULibFunc::EI_ATANH:1509 // atanh(x) == (log(x+1) - log(x-1))/2;1510 Res0 = (log(opr0 + 1.0) - log(opr0 - 1.0))/2.0;1511 return true;1512 1513 case AMDGPULibFunc::EI_ATANPI:1514 Res0 = atan(opr0) / MATH_PI;1515 return true;1516 1517 case AMDGPULibFunc::EI_CBRT:1518 Res0 = (opr0 < 0.0) ? -pow(-opr0, 1.0/3.0) : pow(opr0, 1.0/3.0);1519 return true;1520 1521 case AMDGPULibFunc::EI_COS:1522 Res0 = cos(opr0);1523 return true;1524 1525 case AMDGPULibFunc::EI_COSH:1526 Res0 = cosh(opr0);1527 return true;1528 1529 case AMDGPULibFunc::EI_COSPI:1530 Res0 = cos(MATH_PI * opr0);1531 return true;1532 1533 case AMDGPULibFunc::EI_EXP:1534 Res0 = exp(opr0);1535 return true;1536 1537 case AMDGPULibFunc::EI_EXP2:1538 Res0 = pow(2.0, opr0);1539 return true;1540 1541 case AMDGPULibFunc::EI_EXP10:1542 Res0 = pow(10.0, opr0);1543 return true;1544 1545 case AMDGPULibFunc::EI_LOG:1546 Res0 = log(opr0);1547 return true;1548 1549 case AMDGPULibFunc::EI_LOG2:1550 Res0 = log(opr0) / log(2.0);1551 return true;1552 1553 case AMDGPULibFunc::EI_LOG10:1554 Res0 = log(opr0) / log(10.0);1555 return true;1556 1557 case AMDGPULibFunc::EI_RSQRT:1558 Res0 = 1.0 / sqrt(opr0);1559 return true;1560 1561 case AMDGPULibFunc::EI_SIN:1562 Res0 = sin(opr0);1563 return true;1564 1565 case AMDGPULibFunc::EI_SINH:1566 Res0 = sinh(opr0);1567 return true;1568 1569 case AMDGPULibFunc::EI_SINPI:1570 Res0 = sin(MATH_PI * opr0);1571 return true;1572 1573 case AMDGPULibFunc::EI_TAN:1574 Res0 = tan(opr0);1575 return true;1576 1577 case AMDGPULibFunc::EI_TANH:1578 Res0 = tanh(opr0);1579 return true;1580 1581 case AMDGPULibFunc::EI_TANPI:1582 Res0 = tan(MATH_PI * opr0);1583 return true;1584 1585 // two-arg functions1586 case AMDGPULibFunc::EI_POW:1587 case AMDGPULibFunc::EI_POWR:1588 Res0 = pow(opr0, opr1);1589 return true;1590 1591 case AMDGPULibFunc::EI_POWN: {1592 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) {1593 double val = (double)iopr1->getSExtValue();1594 Res0 = pow(opr0, val);1595 return true;1596 }1597 return false;1598 }1599 1600 case AMDGPULibFunc::EI_ROOTN: {1601 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) {1602 double val = (double)iopr1->getSExtValue();1603 Res0 = pow(opr0, 1.0 / val);1604 return true;1605 }1606 return false;1607 }1608 1609 // with ptr arg1610 case AMDGPULibFunc::EI_SINCOS:1611 Res0 = sin(opr0);1612 Res1 = cos(opr0);1613 return true;1614 }1615 1616 return false;1617}1618 1619bool AMDGPULibCalls::evaluateCall(CallInst *aCI, const FuncInfo &FInfo) {1620 int numArgs = (int)aCI->arg_size();1621 if (numArgs > 3)1622 return false;1623 1624 Constant *copr0 = nullptr;1625 Constant *copr1 = nullptr;1626 if (numArgs > 0) {1627 if ((copr0 = dyn_cast<Constant>(aCI->getArgOperand(0))) == nullptr)1628 return false;1629 }1630 1631 if (numArgs > 1) {1632 if ((copr1 = dyn_cast<Constant>(aCI->getArgOperand(1))) == nullptr) {1633 if (FInfo.getId() != AMDGPULibFunc::EI_SINCOS)1634 return false;1635 }1636 }1637 1638 // At this point, all arguments to aCI are constants.1639 1640 // max vector size is 16, and sincos will generate two results.1641 double DVal0[16], DVal1[16];1642 int FuncVecSize = getVecSize(FInfo);1643 bool hasTwoResults = (FInfo.getId() == AMDGPULibFunc::EI_SINCOS);1644 if (FuncVecSize == 1) {1645 if (!evaluateScalarMathFunc(FInfo, DVal0[0], DVal1[0], copr0, copr1)) {1646 return false;1647 }1648 } else {1649 ConstantDataVector *CDV0 = dyn_cast_or_null<ConstantDataVector>(copr0);1650 ConstantDataVector *CDV1 = dyn_cast_or_null<ConstantDataVector>(copr1);1651 for (int i = 0; i < FuncVecSize; ++i) {1652 Constant *celt0 = CDV0 ? CDV0->getElementAsConstant(i) : nullptr;1653 Constant *celt1 = CDV1 ? CDV1->getElementAsConstant(i) : nullptr;1654 if (!evaluateScalarMathFunc(FInfo, DVal0[i], DVal1[i], celt0, celt1)) {1655 return false;1656 }1657 }1658 }1659 1660 LLVMContext &context = aCI->getContext();1661 Constant *nval0, *nval1;1662 if (FuncVecSize == 1) {1663 nval0 = ConstantFP::get(aCI->getType(), DVal0[0]);1664 if (hasTwoResults)1665 nval1 = ConstantFP::get(aCI->getType(), DVal1[0]);1666 } else {1667 if (getArgType(FInfo) == AMDGPULibFunc::F32) {1668 SmallVector <float, 0> FVal0, FVal1;1669 for (int i = 0; i < FuncVecSize; ++i)1670 FVal0.push_back((float)DVal0[i]);1671 ArrayRef<float> tmp0(FVal0);1672 nval0 = ConstantDataVector::get(context, tmp0);1673 if (hasTwoResults) {1674 for (int i = 0; i < FuncVecSize; ++i)1675 FVal1.push_back((float)DVal1[i]);1676 ArrayRef<float> tmp1(FVal1);1677 nval1 = ConstantDataVector::get(context, tmp1);1678 }1679 } else {1680 ArrayRef<double> tmp0(DVal0);1681 nval0 = ConstantDataVector::get(context, tmp0);1682 if (hasTwoResults) {1683 ArrayRef<double> tmp1(DVal1);1684 nval1 = ConstantDataVector::get(context, tmp1);1685 }1686 }1687 }1688 1689 if (hasTwoResults) {1690 // sincos1691 assert(FInfo.getId() == AMDGPULibFunc::EI_SINCOS &&1692 "math function with ptr arg not supported yet");1693 new StoreInst(nval1, aCI->getArgOperand(1), aCI->getIterator());1694 }1695 1696 replaceCall(aCI, nval0);1697 return true;1698}1699 1700PreservedAnalyses AMDGPUSimplifyLibCallsPass::run(Function &F,1701 FunctionAnalysisManager &AM) {1702 AMDGPULibCalls Simplifier;1703 Simplifier.initNativeFuncs();1704 Simplifier.initFunction(F, AM);1705 1706 bool Changed = false;1707 1708 LLVM_DEBUG(dbgs() << "AMDIC: process function ";1709 F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';);1710 1711 for (auto &BB : F) {1712 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) {1713 // Ignore non-calls.1714 CallInst *CI = dyn_cast<CallInst>(I);1715 ++I;1716 1717 if (CI) {1718 if (Simplifier.fold(CI))1719 Changed = true;1720 }1721 }1722 }1723 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();1724}1725 1726PreservedAnalyses AMDGPUUseNativeCallsPass::run(Function &F,1727 FunctionAnalysisManager &AM) {1728 if (UseNative.empty())1729 return PreservedAnalyses::all();1730 1731 AMDGPULibCalls Simplifier;1732 Simplifier.initNativeFuncs();1733 Simplifier.initFunction(F, AM);1734 1735 bool Changed = false;1736 for (auto &BB : F) {1737 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) {1738 // Ignore non-calls.1739 CallInst *CI = dyn_cast<CallInst>(I);1740 ++I;1741 if (CI && Simplifier.useNative(CI))1742 Changed = true;1743 }1744 }1745 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();1746}1747