183 lines · cpp
1//===-- AMDGPUUniformIntrinsicCombine.cpp ---------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file10/// This pass simplifies certain intrinsic calls when the arguments are uniform.11/// It's true that this pass has transforms that can lead to a situation where12/// some instruction whose operand was previously recognized as statically13/// uniform is later on no longer recognized as statically uniform. However, the14/// semantics of how programs execute don't (and must not, for this precise15/// reason) care about static uniformity, they only ever care about dynamic16/// uniformity. And every instruction that's downstream and cares about dynamic17/// uniformity must be convergent (and isel will introduce v_readfirstlane for18/// them if their operands can't be proven statically uniform).19//===----------------------------------------------------------------------===//20 21#include "AMDGPU.h"22#include "GCNSubtarget.h"23#include "llvm/Analysis/DomTreeUpdater.h"24#include "llvm/Analysis/LoopInfo.h"25#include "llvm/Analysis/ScalarEvolution.h"26#include "llvm/Analysis/TargetLibraryInfo.h"27#include "llvm/Analysis/UniformityAnalysis.h"28#include "llvm/CodeGen/TargetPassConfig.h"29#include "llvm/IR/IRBuilder.h"30#include "llvm/IR/InstIterator.h"31#include "llvm/IR/InstVisitor.h"32#include "llvm/IR/IntrinsicsAMDGPU.h"33#include "llvm/IR/PatternMatch.h"34#include "llvm/InitializePasses.h"35#include "llvm/Target/TargetMachine.h"36#include "llvm/Transforms/Utils/BasicBlockUtils.h"37 38#define DEBUG_TYPE "amdgpu-uniform-intrinsic-combine"39 40using namespace llvm;41using namespace llvm::AMDGPU;42using namespace llvm::PatternMatch;43 44/// Wrapper for querying uniformity info that first checks locally tracked45/// instructions.46static bool47isDivergentUseWithNew(const Use &U, const UniformityInfo &UI,48 const ValueMap<const Value *, bool> &Tracker) {49 Value *V = U.get();50 if (auto It = Tracker.find(V); It != Tracker.end())51 return !It->second; // divergent if marked false52 return UI.isDivergentUse(U);53}54 55/// Optimizes uniform intrinsics calls if their operand can be proven uniform.56static bool optimizeUniformIntrinsic(IntrinsicInst &II,57 const UniformityInfo &UI,58 ValueMap<const Value *, bool> &Tracker) {59 llvm::Intrinsic::ID IID = II.getIntrinsicID();60 /// We deliberately do not simplify readfirstlane with a uniform argument, so61 /// that frontends can use it to force a copy to SGPR and thereby prevent the62 /// backend from generating unwanted waterfall loops.63 switch (IID) {64 case Intrinsic::amdgcn_permlane64:65 case Intrinsic::amdgcn_readlane: {66 Value *Src = II.getArgOperand(0);67 if (isDivergentUseWithNew(II.getOperandUse(0), UI, Tracker))68 return false;69 LLVM_DEBUG(dbgs() << "Replacing " << II << " with " << *Src << '\n');70 II.replaceAllUsesWith(Src);71 II.eraseFromParent();72 return true;73 }74 case Intrinsic::amdgcn_ballot: {75 Value *Src = II.getArgOperand(0);76 if (isDivergentUseWithNew(II.getOperandUse(0), UI, Tracker))77 return false;78 LLVM_DEBUG(dbgs() << "Found uniform ballot intrinsic: " << II << '\n');79 80 bool Changed = false;81 for (User *U : make_early_inc_range(II.users())) {82 if (auto *ICmp = dyn_cast<ICmpInst>(U)) {83 Value *Op0 = ICmp->getOperand(0);84 Value *Op1 = ICmp->getOperand(1);85 ICmpInst::Predicate Pred = ICmp->getPredicate();86 Value *OtherOp = Op0 == &II ? Op1 : Op0;87 88 if (Pred == ICmpInst::ICMP_EQ && match(OtherOp, m_Zero())) {89 // Case: (icmp eq %ballot, 0) -> xor %ballot_arg, 190 Instruction *NotOp =91 BinaryOperator::CreateNot(Src, "", ICmp->getIterator());92 Tracker[NotOp] = true; // NOT preserves uniformity93 LLVM_DEBUG(dbgs() << "Replacing ICMP_EQ: " << *NotOp << '\n');94 ICmp->replaceAllUsesWith(NotOp);95 Changed = true;96 } else if (Pred == ICmpInst::ICMP_NE && match(OtherOp, m_Zero())) {97 // Case: (icmp ne %ballot, 0) -> %ballot_arg98 LLVM_DEBUG(dbgs() << "Replacing ICMP_NE with ballot argument: "99 << *Src << '\n');100 ICmp->replaceAllUsesWith(Src);101 Changed = true;102 }103 }104 }105 // Erase the intrinsic if it has no remaining uses.106 if (II.use_empty())107 II.eraseFromParent();108 return Changed;109 }110 default:111 return false;112 }113 return false;114}115 116/// Iterates over intrinsic calls in the Function to optimize.117static bool runUniformIntrinsicCombine(Function &F, const UniformityInfo &UI) {118 bool IsChanged = false;119 ValueMap<const Value *, bool> Tracker;120 121 for (Instruction &I : make_early_inc_range(instructions(F))) {122 auto *II = dyn_cast<IntrinsicInst>(&I);123 if (!II)124 continue;125 IsChanged |= optimizeUniformIntrinsic(*II, UI, Tracker);126 }127 return IsChanged;128}129 130PreservedAnalyses131AMDGPUUniformIntrinsicCombinePass::run(Function &F,132 FunctionAnalysisManager &AM) {133 const auto &UI = AM.getResult<UniformityInfoAnalysis>(F);134 if (!runUniformIntrinsicCombine(F, UI))135 return PreservedAnalyses::all();136 137 PreservedAnalyses PA;138 PA.preserve<UniformityInfoAnalysis>();139 return PA;140}141 142namespace {143class AMDGPUUniformIntrinsicCombineLegacy : public FunctionPass {144public:145 static char ID;146 AMDGPUUniformIntrinsicCombineLegacy() : FunctionPass(ID) {147 initializeAMDGPUUniformIntrinsicCombineLegacyPass(148 *PassRegistry::getPassRegistry());149 }150 151private:152 bool runOnFunction(Function &F) override;153 void getAnalysisUsage(AnalysisUsage &AU) const override {154 AU.setPreservesCFG();155 AU.addRequired<UniformityInfoWrapperPass>();156 AU.addRequired<TargetPassConfig>();157 }158};159} // namespace160 161char AMDGPUUniformIntrinsicCombineLegacy::ID = 0;162char &llvm::AMDGPUUniformIntrinsicCombineLegacyPassID =163 AMDGPUUniformIntrinsicCombineLegacy::ID;164 165bool AMDGPUUniformIntrinsicCombineLegacy::runOnFunction(Function &F) {166 if (skipFunction(F))167 return false;168 const UniformityInfo &UI =169 getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();170 return runUniformIntrinsicCombine(F, UI);171}172 173INITIALIZE_PASS_BEGIN(AMDGPUUniformIntrinsicCombineLegacy, DEBUG_TYPE,174 "AMDGPU Uniform Intrinsic Combine", false, false)175INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)176INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)177INITIALIZE_PASS_END(AMDGPUUniformIntrinsicCombineLegacy, DEBUG_TYPE,178 "AMDGPU Uniform Intrinsic Combine", false, false)179 180FunctionPass *llvm::createAMDGPUUniformIntrinsicCombineLegacyPass() {181 return new AMDGPUUniformIntrinsicCombineLegacy();182}183