7702 lines · cpp
1//===- InstructionSimplify.cpp - Fold instruction operands ----------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements routines for folding instructions into simpler forms10// that do not require creating new instructions. This does constant folding11// ("add i32 1, 1" -> "2") but can also handle non-constant operands, either12// returning a constant ("and i32 %x, 0" -> "0") or an already existing value13// ("and i32 %x, %x" -> "%x"). All operands are assumed to have already been14// simplified: This is usually true and assuming it simplifies the logic (if15// they have not been simplified then results are correct but maybe suboptimal).16//17//===----------------------------------------------------------------------===//18 19#include "llvm/Analysis/InstructionSimplify.h"20 21#include "llvm/ADT/STLExtras.h"22#include "llvm/ADT/SetVector.h"23#include "llvm/ADT/Statistic.h"24#include "llvm/Analysis/AliasAnalysis.h"25#include "llvm/Analysis/AssumptionCache.h"26#include "llvm/Analysis/CaptureTracking.h"27#include "llvm/Analysis/CmpInstAnalysis.h"28#include "llvm/Analysis/ConstantFolding.h"29#include "llvm/Analysis/FloatingPointPredicateUtils.h"30#include "llvm/Analysis/InstSimplifyFolder.h"31#include "llvm/Analysis/Loads.h"32#include "llvm/Analysis/LoopAnalysisManager.h"33#include "llvm/Analysis/MemoryBuiltins.h"34#include "llvm/Analysis/OverflowInstAnalysis.h"35#include "llvm/Analysis/TargetLibraryInfo.h"36#include "llvm/Analysis/ValueTracking.h"37#include "llvm/Analysis/VectorUtils.h"38#include "llvm/IR/ConstantFPRange.h"39#include "llvm/IR/ConstantRange.h"40#include "llvm/IR/DataLayout.h"41#include "llvm/IR/Dominators.h"42#include "llvm/IR/InstrTypes.h"43#include "llvm/IR/Instructions.h"44#include "llvm/IR/IntrinsicsAArch64.h"45#include "llvm/IR/Operator.h"46#include "llvm/IR/PatternMatch.h"47#include "llvm/IR/Statepoint.h"48#include "llvm/Support/KnownBits.h"49#include "llvm/Support/KnownFPClass.h"50#include <algorithm>51#include <optional>52using namespace llvm;53using namespace llvm::PatternMatch;54 55#define DEBUG_TYPE "instsimplify"56 57enum { RecursionLimit = 3 };58 59STATISTIC(NumExpand, "Number of expansions");60STATISTIC(NumReassoc, "Number of reassociations");61 62static Value *simplifyAndInst(Value *, Value *, const SimplifyQuery &,63 unsigned);64static Value *simplifyUnOp(unsigned, Value *, const SimplifyQuery &, unsigned);65static Value *simplifyFPUnOp(unsigned, Value *, const FastMathFlags &,66 const SimplifyQuery &, unsigned);67static Value *simplifyBinOp(unsigned, Value *, Value *, const SimplifyQuery &,68 unsigned);69static Value *simplifyBinOp(unsigned, Value *, Value *, const FastMathFlags &,70 const SimplifyQuery &, unsigned);71static Value *simplifyCmpInst(CmpPredicate, Value *, Value *,72 const SimplifyQuery &, unsigned);73static Value *simplifyICmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS,74 const SimplifyQuery &Q, unsigned MaxRecurse);75static Value *simplifyOrInst(Value *, Value *, const SimplifyQuery &, unsigned);76static Value *simplifyXorInst(Value *, Value *, const SimplifyQuery &,77 unsigned);78static Value *simplifyCastInst(unsigned, Value *, Type *, const SimplifyQuery &,79 unsigned);80static Value *simplifyGEPInst(Type *, Value *, ArrayRef<Value *>,81 GEPNoWrapFlags, const SimplifyQuery &, unsigned);82static Value *simplifySelectInst(Value *, Value *, Value *,83 const SimplifyQuery &, unsigned);84static Value *simplifyInstructionWithOperands(Instruction *I,85 ArrayRef<Value *> NewOps,86 const SimplifyQuery &SQ,87 unsigned MaxRecurse);88 89/// For a boolean type or a vector of boolean type, return false or a vector90/// with every element false.91static Constant *getFalse(Type *Ty) { return ConstantInt::getFalse(Ty); }92 93/// For a boolean type or a vector of boolean type, return true or a vector94/// with every element true.95static Constant *getTrue(Type *Ty) { return ConstantInt::getTrue(Ty); }96 97/// isSameCompare - Is V equivalent to the comparison "LHS Pred RHS"?98static bool isSameCompare(Value *V, CmpPredicate Pred, Value *LHS, Value *RHS) {99 CmpInst *Cmp = dyn_cast<CmpInst>(V);100 if (!Cmp)101 return false;102 CmpInst::Predicate CPred = Cmp->getPredicate();103 Value *CLHS = Cmp->getOperand(0), *CRHS = Cmp->getOperand(1);104 if (CPred == Pred && CLHS == LHS && CRHS == RHS)105 return true;106 return CPred == CmpInst::getSwappedPredicate(Pred) && CLHS == RHS &&107 CRHS == LHS;108}109 110/// Simplify comparison with true or false branch of select:111/// %sel = select i1 %cond, i32 %tv, i32 %fv112/// %cmp = icmp sle i32 %sel, %rhs113/// Compose new comparison by substituting %sel with either %tv or %fv114/// and see if it simplifies.115static Value *simplifyCmpSelCase(CmpPredicate Pred, Value *LHS, Value *RHS,116 Value *Cond, const SimplifyQuery &Q,117 unsigned MaxRecurse, Constant *TrueOrFalse) {118 Value *SimplifiedCmp = simplifyCmpInst(Pred, LHS, RHS, Q, MaxRecurse);119 if (SimplifiedCmp == Cond) {120 // %cmp simplified to the select condition (%cond).121 return TrueOrFalse;122 } else if (!SimplifiedCmp && isSameCompare(Cond, Pred, LHS, RHS)) {123 // It didn't simplify. However, if composed comparison is equivalent124 // to the select condition (%cond) then we can replace it.125 return TrueOrFalse;126 }127 return SimplifiedCmp;128}129 130/// Simplify comparison with true branch of select131static Value *simplifyCmpSelTrueCase(CmpPredicate Pred, Value *LHS, Value *RHS,132 Value *Cond, const SimplifyQuery &Q,133 unsigned MaxRecurse) {134 return simplifyCmpSelCase(Pred, LHS, RHS, Cond, Q, MaxRecurse,135 getTrue(Cond->getType()));136}137 138/// Simplify comparison with false branch of select139static Value *simplifyCmpSelFalseCase(CmpPredicate Pred, Value *LHS, Value *RHS,140 Value *Cond, const SimplifyQuery &Q,141 unsigned MaxRecurse) {142 return simplifyCmpSelCase(Pred, LHS, RHS, Cond, Q, MaxRecurse,143 getFalse(Cond->getType()));144}145 146/// We know comparison with both branches of select can be simplified, but they147/// are not equal. This routine handles some logical simplifications.148static Value *handleOtherCmpSelSimplifications(Value *TCmp, Value *FCmp,149 Value *Cond,150 const SimplifyQuery &Q,151 unsigned MaxRecurse) {152 // If the false value simplified to false, then the result of the compare153 // is equal to "Cond && TCmp". This also catches the case when the false154 // value simplified to false and the true value to true, returning "Cond".155 // Folding select to and/or isn't poison-safe in general; impliesPoison156 // checks whether folding it does not convert a well-defined value into157 // poison.158 if (match(FCmp, m_Zero()) && impliesPoison(TCmp, Cond))159 if (Value *V = simplifyAndInst(Cond, TCmp, Q, MaxRecurse))160 return V;161 // If the true value simplified to true, then the result of the compare162 // is equal to "Cond || FCmp".163 if (match(TCmp, m_One()) && impliesPoison(FCmp, Cond))164 if (Value *V = simplifyOrInst(Cond, FCmp, Q, MaxRecurse))165 return V;166 // Finally, if the false value simplified to true and the true value to167 // false, then the result of the compare is equal to "!Cond".168 if (match(FCmp, m_One()) && match(TCmp, m_Zero()))169 if (Value *V = simplifyXorInst(170 Cond, Constant::getAllOnesValue(Cond->getType()), Q, MaxRecurse))171 return V;172 return nullptr;173}174 175/// Does the given value dominate the specified phi node?176static bool valueDominatesPHI(Value *V, PHINode *P, const DominatorTree *DT) {177 Instruction *I = dyn_cast<Instruction>(V);178 if (!I)179 // Arguments and constants dominate all instructions.180 return true;181 182 // If we have a DominatorTree then do a precise test.183 if (DT)184 return DT->dominates(I, P);185 186 // Otherwise, if the instruction is in the entry block and is not an invoke,187 // then it obviously dominates all phi nodes.188 if (I->getParent()->isEntryBlock() && !isa<InvokeInst>(I) &&189 !isa<CallBrInst>(I))190 return true;191 192 return false;193}194 195/// Try to simplify a binary operator of form "V op OtherOp" where V is196/// "(B0 opex B1)" by distributing 'op' across 'opex' as197/// "(B0 op OtherOp) opex (B1 op OtherOp)".198static Value *expandBinOp(Instruction::BinaryOps Opcode, Value *V,199 Value *OtherOp, Instruction::BinaryOps OpcodeToExpand,200 const SimplifyQuery &Q, unsigned MaxRecurse) {201 auto *B = dyn_cast<BinaryOperator>(V);202 if (!B || B->getOpcode() != OpcodeToExpand)203 return nullptr;204 Value *B0 = B->getOperand(0), *B1 = B->getOperand(1);205 Value *L =206 simplifyBinOp(Opcode, B0, OtherOp, Q.getWithoutUndef(), MaxRecurse);207 if (!L)208 return nullptr;209 Value *R =210 simplifyBinOp(Opcode, B1, OtherOp, Q.getWithoutUndef(), MaxRecurse);211 if (!R)212 return nullptr;213 214 // Does the expanded pair of binops simplify to the existing binop?215 if ((L == B0 && R == B1) ||216 (Instruction::isCommutative(OpcodeToExpand) && L == B1 && R == B0)) {217 ++NumExpand;218 return B;219 }220 221 // Otherwise, return "L op' R" if it simplifies.222 Value *S = simplifyBinOp(OpcodeToExpand, L, R, Q, MaxRecurse);223 if (!S)224 return nullptr;225 226 ++NumExpand;227 return S;228}229 230/// Try to simplify binops of form "A op (B op' C)" or the commuted variant by231/// distributing op over op'.232static Value *expandCommutativeBinOp(Instruction::BinaryOps Opcode, Value *L,233 Value *R,234 Instruction::BinaryOps OpcodeToExpand,235 const SimplifyQuery &Q,236 unsigned MaxRecurse) {237 // Recursion is always used, so bail out at once if we already hit the limit.238 if (!MaxRecurse--)239 return nullptr;240 241 if (Value *V = expandBinOp(Opcode, L, R, OpcodeToExpand, Q, MaxRecurse))242 return V;243 if (Value *V = expandBinOp(Opcode, R, L, OpcodeToExpand, Q, MaxRecurse))244 return V;245 return nullptr;246}247 248/// Generic simplifications for associative binary operations.249/// Returns the simpler value, or null if none was found.250static Value *simplifyAssociativeBinOp(Instruction::BinaryOps Opcode,251 Value *LHS, Value *RHS,252 const SimplifyQuery &Q,253 unsigned MaxRecurse) {254 assert(Instruction::isAssociative(Opcode) && "Not an associative operation!");255 256 // Recursion is always used, so bail out at once if we already hit the limit.257 if (!MaxRecurse--)258 return nullptr;259 260 BinaryOperator *Op0 = dyn_cast<BinaryOperator>(LHS);261 BinaryOperator *Op1 = dyn_cast<BinaryOperator>(RHS);262 263 // Transform: "(A op B) op C" ==> "A op (B op C)" if it simplifies completely.264 if (Op0 && Op0->getOpcode() == Opcode) {265 Value *A = Op0->getOperand(0);266 Value *B = Op0->getOperand(1);267 Value *C = RHS;268 269 // Does "B op C" simplify?270 if (Value *V = simplifyBinOp(Opcode, B, C, Q, MaxRecurse)) {271 // It does! Return "A op V" if it simplifies or is already available.272 // If V equals B then "A op V" is just the LHS.273 if (V == B)274 return LHS;275 // Otherwise return "A op V" if it simplifies.276 if (Value *W = simplifyBinOp(Opcode, A, V, Q, MaxRecurse)) {277 ++NumReassoc;278 return W;279 }280 }281 }282 283 // Transform: "A op (B op C)" ==> "(A op B) op C" if it simplifies completely.284 if (Op1 && Op1->getOpcode() == Opcode) {285 Value *A = LHS;286 Value *B = Op1->getOperand(0);287 Value *C = Op1->getOperand(1);288 289 // Does "A op B" simplify?290 if (Value *V = simplifyBinOp(Opcode, A, B, Q, MaxRecurse)) {291 // It does! Return "V op C" if it simplifies or is already available.292 // If V equals B then "V op C" is just the RHS.293 if (V == B)294 return RHS;295 // Otherwise return "V op C" if it simplifies.296 if (Value *W = simplifyBinOp(Opcode, V, C, Q, MaxRecurse)) {297 ++NumReassoc;298 return W;299 }300 }301 }302 303 // The remaining transforms require commutativity as well as associativity.304 if (!Instruction::isCommutative(Opcode))305 return nullptr;306 307 // Transform: "(A op B) op C" ==> "(C op A) op B" if it simplifies completely.308 if (Op0 && Op0->getOpcode() == Opcode) {309 Value *A = Op0->getOperand(0);310 Value *B = Op0->getOperand(1);311 Value *C = RHS;312 313 // Does "C op A" simplify?314 if (Value *V = simplifyBinOp(Opcode, C, A, Q, MaxRecurse)) {315 // It does! Return "V op B" if it simplifies or is already available.316 // If V equals A then "V op B" is just the LHS.317 if (V == A)318 return LHS;319 // Otherwise return "V op B" if it simplifies.320 if (Value *W = simplifyBinOp(Opcode, V, B, Q, MaxRecurse)) {321 ++NumReassoc;322 return W;323 }324 }325 }326 327 // Transform: "A op (B op C)" ==> "B op (C op A)" if it simplifies completely.328 if (Op1 && Op1->getOpcode() == Opcode) {329 Value *A = LHS;330 Value *B = Op1->getOperand(0);331 Value *C = Op1->getOperand(1);332 333 // Does "C op A" simplify?334 if (Value *V = simplifyBinOp(Opcode, C, A, Q, MaxRecurse)) {335 // It does! Return "B op V" if it simplifies or is already available.336 // If V equals C then "B op V" is just the RHS.337 if (V == C)338 return RHS;339 // Otherwise return "B op V" if it simplifies.340 if (Value *W = simplifyBinOp(Opcode, B, V, Q, MaxRecurse)) {341 ++NumReassoc;342 return W;343 }344 }345 }346 347 return nullptr;348}349 350/// In the case of a binary operation with a select instruction as an operand,351/// try to simplify the binop by seeing whether evaluating it on both branches352/// of the select results in the same value. Returns the common value if so,353/// otherwise returns null.354static Value *threadBinOpOverSelect(Instruction::BinaryOps Opcode, Value *LHS,355 Value *RHS, const SimplifyQuery &Q,356 unsigned MaxRecurse) {357 // Recursion is always used, so bail out at once if we already hit the limit.358 if (!MaxRecurse--)359 return nullptr;360 361 SelectInst *SI;362 if (isa<SelectInst>(LHS)) {363 SI = cast<SelectInst>(LHS);364 } else {365 assert(isa<SelectInst>(RHS) && "No select instruction operand!");366 SI = cast<SelectInst>(RHS);367 }368 369 // Evaluate the BinOp on the true and false branches of the select.370 Value *TV;371 Value *FV;372 if (SI == LHS) {373 TV = simplifyBinOp(Opcode, SI->getTrueValue(), RHS, Q, MaxRecurse);374 FV = simplifyBinOp(Opcode, SI->getFalseValue(), RHS, Q, MaxRecurse);375 } else {376 TV = simplifyBinOp(Opcode, LHS, SI->getTrueValue(), Q, MaxRecurse);377 FV = simplifyBinOp(Opcode, LHS, SI->getFalseValue(), Q, MaxRecurse);378 }379 380 // If they simplified to the same value, then return the common value.381 // If they both failed to simplify then return null.382 if (TV == FV)383 return TV;384 385 // If one branch simplified to undef, return the other one.386 if (TV && Q.isUndefValue(TV))387 return FV;388 if (FV && Q.isUndefValue(FV))389 return TV;390 391 // If applying the operation did not change the true and false select values,392 // then the result of the binop is the select itself.393 if (TV == SI->getTrueValue() && FV == SI->getFalseValue())394 return SI;395 396 // If one branch simplified and the other did not, and the simplified397 // value is equal to the unsimplified one, return the simplified value.398 // For example, select (cond, X, X & Z) & Z -> X & Z.399 if ((FV && !TV) || (TV && !FV)) {400 // Check that the simplified value has the form "X op Y" where "op" is the401 // same as the original operation.402 Instruction *Simplified = dyn_cast<Instruction>(FV ? FV : TV);403 if (Simplified && Simplified->getOpcode() == unsigned(Opcode) &&404 !Simplified->hasPoisonGeneratingFlags()) {405 // The value that didn't simplify is "UnsimplifiedLHS op UnsimplifiedRHS".406 // We already know that "op" is the same as for the simplified value. See407 // if the operands match too. If so, return the simplified value.408 Value *UnsimplifiedBranch = FV ? SI->getTrueValue() : SI->getFalseValue();409 Value *UnsimplifiedLHS = SI == LHS ? UnsimplifiedBranch : LHS;410 Value *UnsimplifiedRHS = SI == LHS ? RHS : UnsimplifiedBranch;411 if (Simplified->getOperand(0) == UnsimplifiedLHS &&412 Simplified->getOperand(1) == UnsimplifiedRHS)413 return Simplified;414 if (Simplified->isCommutative() &&415 Simplified->getOperand(1) == UnsimplifiedLHS &&416 Simplified->getOperand(0) == UnsimplifiedRHS)417 return Simplified;418 }419 }420 421 return nullptr;422}423 424/// In the case of a comparison with a select instruction, try to simplify the425/// comparison by seeing whether both branches of the select result in the same426/// value. Returns the common value if so, otherwise returns null.427/// For example, if we have:428/// %tmp = select i1 %cmp, i32 1, i32 2429/// %cmp1 = icmp sle i32 %tmp, 3430/// We can simplify %cmp1 to true, because both branches of select are431/// less than 3. We compose new comparison by substituting %tmp with both432/// branches of select and see if it can be simplified.433static Value *threadCmpOverSelect(CmpPredicate Pred, Value *LHS, Value *RHS,434 const SimplifyQuery &Q, unsigned MaxRecurse) {435 // Recursion is always used, so bail out at once if we already hit the limit.436 if (!MaxRecurse--)437 return nullptr;438 439 // Make sure the select is on the LHS.440 if (!isa<SelectInst>(LHS)) {441 std::swap(LHS, RHS);442 Pred = CmpInst::getSwappedPredicate(Pred);443 }444 assert(isa<SelectInst>(LHS) && "Not comparing with a select instruction!");445 SelectInst *SI = cast<SelectInst>(LHS);446 Value *Cond = SI->getCondition();447 Value *TV = SI->getTrueValue();448 Value *FV = SI->getFalseValue();449 450 // Now that we have "cmp select(Cond, TV, FV), RHS", analyse it.451 // Does "cmp TV, RHS" simplify?452 Value *TCmp = simplifyCmpSelTrueCase(Pred, TV, RHS, Cond, Q, MaxRecurse);453 if (!TCmp)454 return nullptr;455 456 // Does "cmp FV, RHS" simplify?457 Value *FCmp = simplifyCmpSelFalseCase(Pred, FV, RHS, Cond, Q, MaxRecurse);458 if (!FCmp)459 return nullptr;460 461 // If both sides simplified to the same value, then use it as the result of462 // the original comparison.463 if (TCmp == FCmp)464 return TCmp;465 466 // The remaining cases only make sense if the select condition has the same467 // type as the result of the comparison, so bail out if this is not so.468 if (Cond->getType()->isVectorTy() == RHS->getType()->isVectorTy())469 return handleOtherCmpSelSimplifications(TCmp, FCmp, Cond, Q, MaxRecurse);470 471 return nullptr;472}473 474/// In the case of a binary operation with an operand that is a PHI instruction,475/// try to simplify the binop by seeing whether evaluating it on the incoming476/// phi values yields the same result for every value. If so returns the common477/// value, otherwise returns null.478static Value *threadBinOpOverPHI(Instruction::BinaryOps Opcode, Value *LHS,479 Value *RHS, const SimplifyQuery &Q,480 unsigned MaxRecurse) {481 // Recursion is always used, so bail out at once if we already hit the limit.482 if (!MaxRecurse--)483 return nullptr;484 485 PHINode *PI;486 if (isa<PHINode>(LHS)) {487 PI = cast<PHINode>(LHS);488 // Bail out if RHS and the phi may be mutually interdependent due to a loop.489 if (!valueDominatesPHI(RHS, PI, Q.DT))490 return nullptr;491 } else {492 assert(isa<PHINode>(RHS) && "No PHI instruction operand!");493 PI = cast<PHINode>(RHS);494 // Bail out if LHS and the phi may be mutually interdependent due to a loop.495 if (!valueDominatesPHI(LHS, PI, Q.DT))496 return nullptr;497 }498 499 // Evaluate the BinOp on the incoming phi values.500 Value *CommonValue = nullptr;501 for (Use &Incoming : PI->incoming_values()) {502 // If the incoming value is the phi node itself, it can safely be skipped.503 if (Incoming == PI)504 continue;505 Instruction *InTI = PI->getIncomingBlock(Incoming)->getTerminator();506 Value *V = PI == LHS507 ? simplifyBinOp(Opcode, Incoming, RHS,508 Q.getWithInstruction(InTI), MaxRecurse)509 : simplifyBinOp(Opcode, LHS, Incoming,510 Q.getWithInstruction(InTI), MaxRecurse);511 // If the operation failed to simplify, or simplified to a different value512 // to previously, then give up.513 if (!V || (CommonValue && V != CommonValue))514 return nullptr;515 CommonValue = V;516 }517 518 return CommonValue;519}520 521/// In the case of a comparison with a PHI instruction, try to simplify the522/// comparison by seeing whether comparing with all of the incoming phi values523/// yields the same result every time. If so returns the common result,524/// otherwise returns null.525static Value *threadCmpOverPHI(CmpPredicate Pred, Value *LHS, Value *RHS,526 const SimplifyQuery &Q, unsigned MaxRecurse) {527 // Recursion is always used, so bail out at once if we already hit the limit.528 if (!MaxRecurse--)529 return nullptr;530 531 // Make sure the phi is on the LHS.532 if (!isa<PHINode>(LHS)) {533 std::swap(LHS, RHS);534 Pred = CmpInst::getSwappedPredicate(Pred);535 }536 assert(isa<PHINode>(LHS) && "Not comparing with a phi instruction!");537 PHINode *PI = cast<PHINode>(LHS);538 539 // Bail out if RHS and the phi may be mutually interdependent due to a loop.540 if (!valueDominatesPHI(RHS, PI, Q.DT))541 return nullptr;542 543 // Evaluate the BinOp on the incoming phi values.544 Value *CommonValue = nullptr;545 for (unsigned u = 0, e = PI->getNumIncomingValues(); u < e; ++u) {546 Value *Incoming = PI->getIncomingValue(u);547 Instruction *InTI = PI->getIncomingBlock(u)->getTerminator();548 // If the incoming value is the phi node itself, it can safely be skipped.549 if (Incoming == PI)550 continue;551 // Change the context instruction to the "edge" that flows into the phi.552 // This is important because that is where incoming is actually "evaluated"553 // even though it is used later somewhere else.554 Value *V = simplifyCmpInst(Pred, Incoming, RHS, Q.getWithInstruction(InTI),555 MaxRecurse);556 // If the operation failed to simplify, or simplified to a different value557 // to previously, then give up.558 if (!V || (CommonValue && V != CommonValue))559 return nullptr;560 CommonValue = V;561 }562 563 return CommonValue;564}565 566static Constant *foldOrCommuteConstant(Instruction::BinaryOps Opcode,567 Value *&Op0, Value *&Op1,568 const SimplifyQuery &Q) {569 if (auto *CLHS = dyn_cast<Constant>(Op0)) {570 if (auto *CRHS = dyn_cast<Constant>(Op1)) {571 switch (Opcode) {572 default:573 break;574 case Instruction::FAdd:575 case Instruction::FSub:576 case Instruction::FMul:577 case Instruction::FDiv:578 case Instruction::FRem:579 if (Q.CxtI != nullptr)580 return ConstantFoldFPInstOperands(Opcode, CLHS, CRHS, Q.DL, Q.CxtI);581 }582 return ConstantFoldBinaryOpOperands(Opcode, CLHS, CRHS, Q.DL);583 }584 585 // Canonicalize the constant to the RHS if this is a commutative operation.586 if (Instruction::isCommutative(Opcode))587 std::swap(Op0, Op1);588 }589 return nullptr;590}591 592/// Given operands for an Add, see if we can fold the result.593/// If not, this returns null.594static Value *simplifyAddInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,595 const SimplifyQuery &Q, unsigned MaxRecurse) {596 if (Constant *C = foldOrCommuteConstant(Instruction::Add, Op0, Op1, Q))597 return C;598 599 // X + poison -> poison600 if (isa<PoisonValue>(Op1))601 return Op1;602 603 // X + undef -> undef604 if (Q.isUndefValue(Op1))605 return Op1;606 607 // X + 0 -> X608 if (match(Op1, m_Zero()))609 return Op0;610 611 // If two operands are negative, return 0.612 if (isKnownNegation(Op0, Op1))613 return Constant::getNullValue(Op0->getType());614 615 // X + (Y - X) -> Y616 // (Y - X) + X -> Y617 // Eg: X + -X -> 0618 Value *Y = nullptr;619 if (match(Op1, m_Sub(m_Value(Y), m_Specific(Op0))) ||620 match(Op0, m_Sub(m_Value(Y), m_Specific(Op1))))621 return Y;622 623 // X + ~X -> -1 since ~X = -X-1624 Type *Ty = Op0->getType();625 if (match(Op0, m_Not(m_Specific(Op1))) || match(Op1, m_Not(m_Specific(Op0))))626 return Constant::getAllOnesValue(Ty);627 628 // add nsw/nuw (xor Y, signmask), signmask --> Y629 // The no-wrapping add guarantees that the top bit will be set by the add.630 // Therefore, the xor must be clearing the already set sign bit of Y.631 if ((IsNSW || IsNUW) && match(Op1, m_SignMask()) &&632 match(Op0, m_Xor(m_Value(Y), m_SignMask())))633 return Y;634 635 // add nuw %x, -1 -> -1, because %x can only be 0.636 if (IsNUW && match(Op1, m_AllOnes()))637 return Op1; // Which is -1.638 639 /// i1 add -> xor.640 if (MaxRecurse && Op0->getType()->isIntOrIntVectorTy(1))641 if (Value *V = simplifyXorInst(Op0, Op1, Q, MaxRecurse - 1))642 return V;643 644 // Try some generic simplifications for associative operations.645 if (Value *V =646 simplifyAssociativeBinOp(Instruction::Add, Op0, Op1, Q, MaxRecurse))647 return V;648 649 // Threading Add over selects and phi nodes is pointless, so don't bother.650 // Threading over the select in "A + select(cond, B, C)" means evaluating651 // "A+B" and "A+C" and seeing if they are equal; but they are equal if and652 // only if B and C are equal. If B and C are equal then (since we assume653 // that operands have already been simplified) "select(cond, B, C)" should654 // have been simplified to the common value of B and C already. Analysing655 // "A+B" and "A+C" thus gains nothing, but costs compile time. Similarly656 // for threading over phi nodes.657 658 return nullptr;659}660 661Value *llvm::simplifyAddInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,662 const SimplifyQuery &Query) {663 return ::simplifyAddInst(Op0, Op1, IsNSW, IsNUW, Query, RecursionLimit);664}665 666/// Compute the base pointer and cumulative constant offsets for V.667///668/// This strips all constant offsets off of V, leaving it the base pointer, and669/// accumulates the total constant offset applied in the returned constant.670/// It returns zero if there are no constant offsets applied.671///672/// This is very similar to stripAndAccumulateConstantOffsets(), except it673/// normalizes the offset bitwidth to the stripped pointer type, not the674/// original pointer type.675static APInt stripAndComputeConstantOffsets(const DataLayout &DL, Value *&V) {676 assert(V->getType()->isPtrOrPtrVectorTy());677 678 APInt Offset = APInt::getZero(DL.getIndexTypeSizeInBits(V->getType()));679 V = V->stripAndAccumulateConstantOffsets(DL, Offset,680 /*AllowNonInbounds=*/true);681 // As that strip may trace through `addrspacecast`, need to sext or trunc682 // the offset calculated.683 return Offset.sextOrTrunc(DL.getIndexTypeSizeInBits(V->getType()));684}685 686/// Compute the constant difference between two pointer values.687/// If the difference is not a constant, returns zero.688static Constant *computePointerDifference(const DataLayout &DL, Value *LHS,689 Value *RHS) {690 APInt LHSOffset = stripAndComputeConstantOffsets(DL, LHS);691 APInt RHSOffset = stripAndComputeConstantOffsets(DL, RHS);692 693 // If LHS and RHS are not related via constant offsets to the same base694 // value, there is nothing we can do here.695 if (LHS != RHS)696 return nullptr;697 698 // Otherwise, the difference of LHS - RHS can be computed as:699 // LHS - RHS700 // = (LHSOffset + Base) - (RHSOffset + Base)701 // = LHSOffset - RHSOffset702 Constant *Res = ConstantInt::get(LHS->getContext(), LHSOffset - RHSOffset);703 if (auto *VecTy = dyn_cast<VectorType>(LHS->getType()))704 Res = ConstantVector::getSplat(VecTy->getElementCount(), Res);705 return Res;706}707 708/// Test if there is a dominating equivalence condition for the709/// two operands. If there is, try to reduce the binary operation710/// between the two operands.711/// Example: Op0 - Op1 --> 0 when Op0 == Op1712static Value *simplifyByDomEq(unsigned Opcode, Value *Op0, Value *Op1,713 const SimplifyQuery &Q, unsigned MaxRecurse) {714 // Recursive run it can not get any benefit715 if (MaxRecurse != RecursionLimit)716 return nullptr;717 718 std::optional<bool> Imp =719 isImpliedByDomCondition(CmpInst::ICMP_EQ, Op0, Op1, Q.CxtI, Q.DL);720 if (Imp && *Imp) {721 Type *Ty = Op0->getType();722 switch (Opcode) {723 case Instruction::Sub:724 case Instruction::Xor:725 case Instruction::URem:726 case Instruction::SRem:727 return Constant::getNullValue(Ty);728 729 case Instruction::SDiv:730 case Instruction::UDiv:731 return ConstantInt::get(Ty, 1);732 733 case Instruction::And:734 case Instruction::Or:735 // Could be either one - choose Op1 since that's more likely a constant.736 return Op1;737 default:738 break;739 }740 }741 return nullptr;742}743 744/// Given operands for a Sub, see if we can fold the result.745/// If not, this returns null.746static Value *simplifySubInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,747 const SimplifyQuery &Q, unsigned MaxRecurse) {748 if (Constant *C = foldOrCommuteConstant(Instruction::Sub, Op0, Op1, Q))749 return C;750 751 // X - poison -> poison752 // poison - X -> poison753 if (isa<PoisonValue>(Op0) || isa<PoisonValue>(Op1))754 return PoisonValue::get(Op0->getType());755 756 // X - undef -> undef757 // undef - X -> undef758 if (Q.isUndefValue(Op0) || Q.isUndefValue(Op1))759 return UndefValue::get(Op0->getType());760 761 // X - 0 -> X762 if (match(Op1, m_Zero()))763 return Op0;764 765 // X - X -> 0766 if (Op0 == Op1)767 return Constant::getNullValue(Op0->getType());768 769 // Is this a negation?770 if (match(Op0, m_Zero())) {771 // 0 - X -> 0 if the sub is NUW.772 if (IsNUW)773 return Constant::getNullValue(Op0->getType());774 775 KnownBits Known = computeKnownBits(Op1, Q);776 if (Known.Zero.isMaxSignedValue()) {777 // Op1 is either 0 or the minimum signed value. If the sub is NSW, then778 // Op1 must be 0 because negating the minimum signed value is undefined.779 if (IsNSW)780 return Constant::getNullValue(Op0->getType());781 782 // 0 - X -> X if X is 0 or the minimum signed value.783 return Op1;784 }785 }786 787 // (X + Y) - Z -> X + (Y - Z) or Y + (X - Z) if everything simplifies.788 // For example, (X + Y) - Y -> X; (Y + X) - Y -> X789 Value *X = nullptr, *Y = nullptr, *Z = Op1;790 if (MaxRecurse && match(Op0, m_Add(m_Value(X), m_Value(Y)))) { // (X + Y) - Z791 // See if "V === Y - Z" simplifies.792 if (Value *V = simplifyBinOp(Instruction::Sub, Y, Z, Q, MaxRecurse - 1))793 // It does! Now see if "X + V" simplifies.794 if (Value *W = simplifyBinOp(Instruction::Add, X, V, Q, MaxRecurse - 1)) {795 // It does, we successfully reassociated!796 ++NumReassoc;797 return W;798 }799 // See if "V === X - Z" simplifies.800 if (Value *V = simplifyBinOp(Instruction::Sub, X, Z, Q, MaxRecurse - 1))801 // It does! Now see if "Y + V" simplifies.802 if (Value *W = simplifyBinOp(Instruction::Add, Y, V, Q, MaxRecurse - 1)) {803 // It does, we successfully reassociated!804 ++NumReassoc;805 return W;806 }807 }808 809 // X - (Y + Z) -> (X - Y) - Z or (X - Z) - Y if everything simplifies.810 // For example, X - (X + 1) -> -1811 X = Op0;812 if (MaxRecurse && match(Op1, m_Add(m_Value(Y), m_Value(Z)))) { // X - (Y + Z)813 // See if "V === X - Y" simplifies.814 if (Value *V = simplifyBinOp(Instruction::Sub, X, Y, Q, MaxRecurse - 1))815 // It does! Now see if "V - Z" simplifies.816 if (Value *W = simplifyBinOp(Instruction::Sub, V, Z, Q, MaxRecurse - 1)) {817 // It does, we successfully reassociated!818 ++NumReassoc;819 return W;820 }821 // See if "V === X - Z" simplifies.822 if (Value *V = simplifyBinOp(Instruction::Sub, X, Z, Q, MaxRecurse - 1))823 // It does! Now see if "V - Y" simplifies.824 if (Value *W = simplifyBinOp(Instruction::Sub, V, Y, Q, MaxRecurse - 1)) {825 // It does, we successfully reassociated!826 ++NumReassoc;827 return W;828 }829 }830 831 // Z - (X - Y) -> (Z - X) + Y if everything simplifies.832 // For example, X - (X - Y) -> Y.833 Z = Op0;834 if (MaxRecurse && match(Op1, m_Sub(m_Value(X), m_Value(Y)))) // Z - (X - Y)835 // See if "V === Z - X" simplifies.836 if (Value *V = simplifyBinOp(Instruction::Sub, Z, X, Q, MaxRecurse - 1))837 // It does! Now see if "V + Y" simplifies.838 if (Value *W = simplifyBinOp(Instruction::Add, V, Y, Q, MaxRecurse - 1)) {839 // It does, we successfully reassociated!840 ++NumReassoc;841 return W;842 }843 844 // trunc(X) - trunc(Y) -> trunc(X - Y) if everything simplifies.845 if (MaxRecurse && match(Op0, m_Trunc(m_Value(X))) &&846 match(Op1, m_Trunc(m_Value(Y))))847 if (X->getType() == Y->getType())848 // See if "V === X - Y" simplifies.849 if (Value *V = simplifyBinOp(Instruction::Sub, X, Y, Q, MaxRecurse - 1))850 // It does! Now see if "trunc V" simplifies.851 if (Value *W = simplifyCastInst(Instruction::Trunc, V, Op0->getType(),852 Q, MaxRecurse - 1))853 // It does, return the simplified "trunc V".854 return W;855 856 // Variations on GEP(base, I, ...) - GEP(base, i, ...) -> GEP(null, I-i, ...).857 if (match(Op0, m_PtrToIntOrAddr(m_Value(X))) &&858 match(Op1, m_PtrToIntOrAddr(m_Value(Y)))) {859 if (Constant *Result = computePointerDifference(Q.DL, X, Y))860 return ConstantFoldIntegerCast(Result, Op0->getType(), /*IsSigned*/ true,861 Q.DL);862 }863 864 // i1 sub -> xor.865 if (MaxRecurse && Op0->getType()->isIntOrIntVectorTy(1))866 if (Value *V = simplifyXorInst(Op0, Op1, Q, MaxRecurse - 1))867 return V;868 869 // Threading Sub over selects and phi nodes is pointless, so don't bother.870 // Threading over the select in "A - select(cond, B, C)" means evaluating871 // "A-B" and "A-C" and seeing if they are equal; but they are equal if and872 // only if B and C are equal. If B and C are equal then (since we assume873 // that operands have already been simplified) "select(cond, B, C)" should874 // have been simplified to the common value of B and C already. Analysing875 // "A-B" and "A-C" thus gains nothing, but costs compile time. Similarly876 // for threading over phi nodes.877 878 if (Value *V = simplifyByDomEq(Instruction::Sub, Op0, Op1, Q, MaxRecurse))879 return V;880 881 // (sub nuw C_Mask, (xor X, C_Mask)) -> X882 if (IsNUW) {883 Value *X;884 if (match(Op1, m_Xor(m_Value(X), m_Specific(Op0))) &&885 match(Op0, m_LowBitMask()))886 return X;887 }888 889 return nullptr;890}891 892Value *llvm::simplifySubInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,893 const SimplifyQuery &Q) {894 return ::simplifySubInst(Op0, Op1, IsNSW, IsNUW, Q, RecursionLimit);895}896 897/// Given operands for a Mul, see if we can fold the result.898/// If not, this returns null.899static Value *simplifyMulInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,900 const SimplifyQuery &Q, unsigned MaxRecurse) {901 if (Constant *C = foldOrCommuteConstant(Instruction::Mul, Op0, Op1, Q))902 return C;903 904 // X * poison -> poison905 if (isa<PoisonValue>(Op1))906 return Op1;907 908 // X * undef -> 0909 // X * 0 -> 0910 if (Q.isUndefValue(Op1) || match(Op1, m_Zero()))911 return Constant::getNullValue(Op0->getType());912 913 // X * 1 -> X914 if (match(Op1, m_One()))915 return Op0;916 917 // (X / Y) * Y -> X if the division is exact.918 Value *X = nullptr;919 if (Q.IIQ.UseInstrInfo &&920 (match(Op0,921 m_Exact(m_IDiv(m_Value(X), m_Specific(Op1)))) || // (X / Y) * Y922 match(Op1, m_Exact(m_IDiv(m_Value(X), m_Specific(Op0)))))) // Y * (X / Y)923 return X;924 925 if (Op0->getType()->isIntOrIntVectorTy(1)) {926 // mul i1 nsw is a special-case because -1 * -1 is poison (+1 is not927 // representable). All other cases reduce to 0, so just return 0.928 if (IsNSW)929 return ConstantInt::getNullValue(Op0->getType());930 931 // Treat "mul i1" as "and i1".932 if (MaxRecurse)933 if (Value *V = simplifyAndInst(Op0, Op1, Q, MaxRecurse - 1))934 return V;935 }936 937 // Try some generic simplifications for associative operations.938 if (Value *V =939 simplifyAssociativeBinOp(Instruction::Mul, Op0, Op1, Q, MaxRecurse))940 return V;941 942 // Mul distributes over Add. Try some generic simplifications based on this.943 if (Value *V = expandCommutativeBinOp(Instruction::Mul, Op0, Op1,944 Instruction::Add, Q, MaxRecurse))945 return V;946 947 // If the operation is with the result of a select instruction, check whether948 // operating on either branch of the select always yields the same value.949 if (isa<SelectInst>(Op0) || isa<SelectInst>(Op1))950 if (Value *V =951 threadBinOpOverSelect(Instruction::Mul, Op0, Op1, Q, MaxRecurse))952 return V;953 954 // If the operation is with the result of a phi instruction, check whether955 // operating on all incoming values of the phi always yields the same value.956 if (isa<PHINode>(Op0) || isa<PHINode>(Op1))957 if (Value *V =958 threadBinOpOverPHI(Instruction::Mul, Op0, Op1, Q, MaxRecurse))959 return V;960 961 return nullptr;962}963 964Value *llvm::simplifyMulInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,965 const SimplifyQuery &Q) {966 return ::simplifyMulInst(Op0, Op1, IsNSW, IsNUW, Q, RecursionLimit);967}968 969/// Given a predicate and two operands, return true if the comparison is true.970/// This is a helper for div/rem simplification where we return some other value971/// when we can prove a relationship between the operands.972static bool isICmpTrue(CmpPredicate Pred, Value *LHS, Value *RHS,973 const SimplifyQuery &Q, unsigned MaxRecurse) {974 Value *V = simplifyICmpInst(Pred, LHS, RHS, Q, MaxRecurse);975 Constant *C = dyn_cast_or_null<Constant>(V);976 return (C && C->isAllOnesValue());977}978 979/// Return true if we can simplify X / Y to 0. Remainder can adapt that answer980/// to simplify X % Y to X.981static bool isDivZero(Value *X, Value *Y, const SimplifyQuery &Q,982 unsigned MaxRecurse, bool IsSigned) {983 // Recursion is always used, so bail out at once if we already hit the limit.984 if (!MaxRecurse--)985 return false;986 987 if (IsSigned) {988 // (X srem Y) sdiv Y --> 0989 if (match(X, m_SRem(m_Value(), m_Specific(Y))))990 return true;991 992 // |X| / |Y| --> 0993 //994 // We require that 1 operand is a simple constant. That could be extended to995 // 2 variables if we computed the sign bit for each.996 //997 // Make sure that a constant is not the minimum signed value because taking998 // the abs() of that is undefined.999 Type *Ty = X->getType();1000 const APInt *C;1001 if (match(X, m_APInt(C)) && !C->isMinSignedValue()) {1002 // Is the variable divisor magnitude always greater than the constant1003 // dividend magnitude?1004 // |Y| > |C| --> Y < -abs(C) or Y > abs(C)1005 Constant *PosDividendC = ConstantInt::get(Ty, C->abs());1006 Constant *NegDividendC = ConstantInt::get(Ty, -C->abs());1007 if (isICmpTrue(CmpInst::ICMP_SLT, Y, NegDividendC, Q, MaxRecurse) ||1008 isICmpTrue(CmpInst::ICMP_SGT, Y, PosDividendC, Q, MaxRecurse))1009 return true;1010 }1011 if (match(Y, m_APInt(C))) {1012 // Special-case: we can't take the abs() of a minimum signed value. If1013 // that's the divisor, then all we have to do is prove that the dividend1014 // is also not the minimum signed value.1015 if (C->isMinSignedValue())1016 return isICmpTrue(CmpInst::ICMP_NE, X, Y, Q, MaxRecurse);1017 1018 // Is the variable dividend magnitude always less than the constant1019 // divisor magnitude?1020 // |X| < |C| --> X > -abs(C) and X < abs(C)1021 Constant *PosDivisorC = ConstantInt::get(Ty, C->abs());1022 Constant *NegDivisorC = ConstantInt::get(Ty, -C->abs());1023 if (isICmpTrue(CmpInst::ICMP_SGT, X, NegDivisorC, Q, MaxRecurse) &&1024 isICmpTrue(CmpInst::ICMP_SLT, X, PosDivisorC, Q, MaxRecurse))1025 return true;1026 }1027 return false;1028 }1029 1030 // IsSigned == false.1031 1032 // Is the unsigned dividend known to be less than a constant divisor?1033 // TODO: Convert this (and above) to range analysis1034 // ("computeConstantRangeIncludingKnownBits")?1035 const APInt *C;1036 if (match(Y, m_APInt(C)) && computeKnownBits(X, Q).getMaxValue().ult(*C))1037 return true;1038 1039 // Try again for any divisor:1040 // Is the dividend unsigned less than the divisor?1041 return isICmpTrue(ICmpInst::ICMP_ULT, X, Y, Q, MaxRecurse);1042}1043 1044/// Check for common or similar folds of integer division or integer remainder.1045/// This applies to all 4 opcodes (sdiv/udiv/srem/urem).1046static Value *simplifyDivRem(Instruction::BinaryOps Opcode, Value *Op0,1047 Value *Op1, const SimplifyQuery &Q,1048 unsigned MaxRecurse) {1049 bool IsDiv = (Opcode == Instruction::SDiv || Opcode == Instruction::UDiv);1050 bool IsSigned = (Opcode == Instruction::SDiv || Opcode == Instruction::SRem);1051 1052 Type *Ty = Op0->getType();1053 1054 // X / undef -> poison1055 // X % undef -> poison1056 if (Q.isUndefValue(Op1) || isa<PoisonValue>(Op1))1057 return PoisonValue::get(Ty);1058 1059 // X / 0 -> poison1060 // X % 0 -> poison1061 // We don't need to preserve faults!1062 if (match(Op1, m_Zero()))1063 return PoisonValue::get(Ty);1064 1065 // poison / X -> poison1066 // poison % X -> poison1067 if (isa<PoisonValue>(Op0))1068 return Op0;1069 1070 // undef / X -> 01071 // undef % X -> 01072 if (Q.isUndefValue(Op0))1073 return Constant::getNullValue(Ty);1074 1075 // 0 / X -> 01076 // 0 % X -> 01077 if (match(Op0, m_Zero()))1078 return Constant::getNullValue(Op0->getType());1079 1080 // X / X -> 11081 // X % X -> 01082 if (Op0 == Op1)1083 return IsDiv ? ConstantInt::get(Ty, 1) : Constant::getNullValue(Ty);1084 1085 KnownBits Known = computeKnownBits(Op1, Q);1086 // X / 0 -> poison1087 // X % 0 -> poison1088 // If the divisor is known to be zero, just return poison. This can happen in1089 // some cases where its provable indirectly the denominator is zero but it's1090 // not trivially simplifiable (i.e known zero through a phi node).1091 if (Known.isZero())1092 return PoisonValue::get(Ty);1093 1094 // X / 1 -> X1095 // X % 1 -> 01096 // If the divisor can only be zero or one, we can't have division-by-zero1097 // or remainder-by-zero, so assume the divisor is 1.1098 // e.g. 1, zext (i8 X), sdiv X (Y and 1)1099 if (Known.countMinLeadingZeros() == Known.getBitWidth() - 1)1100 return IsDiv ? Op0 : Constant::getNullValue(Ty);1101 1102 // If X * Y does not overflow, then:1103 // X * Y / Y -> X1104 // X * Y % Y -> 01105 Value *X;1106 if (match(Op0, m_c_Mul(m_Value(X), m_Specific(Op1)))) {1107 auto *Mul = cast<OverflowingBinaryOperator>(Op0);1108 // The multiplication can't overflow if it is defined not to, or if1109 // X == A / Y for some A.1110 if ((IsSigned && Q.IIQ.hasNoSignedWrap(Mul)) ||1111 (!IsSigned && Q.IIQ.hasNoUnsignedWrap(Mul)) ||1112 (IsSigned && match(X, m_SDiv(m_Value(), m_Specific(Op1)))) ||1113 (!IsSigned && match(X, m_UDiv(m_Value(), m_Specific(Op1))))) {1114 return IsDiv ? X : Constant::getNullValue(Op0->getType());1115 }1116 }1117 1118 if (isDivZero(Op0, Op1, Q, MaxRecurse, IsSigned))1119 return IsDiv ? Constant::getNullValue(Op0->getType()) : Op0;1120 1121 if (Value *V = simplifyByDomEq(Opcode, Op0, Op1, Q, MaxRecurse))1122 return V;1123 1124 // If the operation is with the result of a select instruction, check whether1125 // operating on either branch of the select always yields the same value.1126 if (isa<SelectInst>(Op0) || isa<SelectInst>(Op1))1127 if (Value *V = threadBinOpOverSelect(Opcode, Op0, Op1, Q, MaxRecurse))1128 return V;1129 1130 // If the operation is with the result of a phi instruction, check whether1131 // operating on all incoming values of the phi always yields the same value.1132 if (isa<PHINode>(Op0) || isa<PHINode>(Op1))1133 if (Value *V = threadBinOpOverPHI(Opcode, Op0, Op1, Q, MaxRecurse))1134 return V;1135 1136 return nullptr;1137}1138 1139/// These are simplifications common to SDiv and UDiv.1140static Value *simplifyDiv(Instruction::BinaryOps Opcode, Value *Op0, Value *Op1,1141 bool IsExact, const SimplifyQuery &Q,1142 unsigned MaxRecurse) {1143 if (Constant *C = foldOrCommuteConstant(Opcode, Op0, Op1, Q))1144 return C;1145 1146 if (Value *V = simplifyDivRem(Opcode, Op0, Op1, Q, MaxRecurse))1147 return V;1148 1149 const APInt *DivC;1150 if (IsExact && match(Op1, m_APInt(DivC))) {1151 // If this is an exact divide by a constant, then the dividend (Op0) must1152 // have at least as many trailing zeros as the divisor to divide evenly. If1153 // it has less trailing zeros, then the result must be poison.1154 if (DivC->countr_zero()) {1155 KnownBits KnownOp0 = computeKnownBits(Op0, Q);1156 if (KnownOp0.countMaxTrailingZeros() < DivC->countr_zero())1157 return PoisonValue::get(Op0->getType());1158 }1159 1160 // udiv exact (mul nsw X, C), C --> X1161 // sdiv exact (mul nuw X, C), C --> X1162 // where C is not a power of 2.1163 Value *X;1164 if (!DivC->isPowerOf2() &&1165 (Opcode == Instruction::UDiv1166 ? match(Op0, m_NSWMul(m_Value(X), m_Specific(Op1)))1167 : match(Op0, m_NUWMul(m_Value(X), m_Specific(Op1)))))1168 return X;1169 }1170 1171 return nullptr;1172}1173 1174/// These are simplifications common to SRem and URem.1175static Value *simplifyRem(Instruction::BinaryOps Opcode, Value *Op0, Value *Op1,1176 const SimplifyQuery &Q, unsigned MaxRecurse) {1177 if (Constant *C = foldOrCommuteConstant(Opcode, Op0, Op1, Q))1178 return C;1179 1180 if (Value *V = simplifyDivRem(Opcode, Op0, Op1, Q, MaxRecurse))1181 return V;1182 1183 // (X << Y) % X -> 01184 if (Q.IIQ.UseInstrInfo) {1185 if ((Opcode == Instruction::SRem &&1186 match(Op0, m_NSWShl(m_Specific(Op1), m_Value()))) ||1187 (Opcode == Instruction::URem &&1188 match(Op0, m_NUWShl(m_Specific(Op1), m_Value()))))1189 return Constant::getNullValue(Op0->getType());1190 1191 const APInt *C0;1192 if (match(Op1, m_APInt(C0))) {1193 // (srem (mul nsw X, C1), C0) -> 0 if C1 s% C0 == 01194 // (urem (mul nuw X, C1), C0) -> 0 if C1 u% C0 == 01195 if (Opcode == Instruction::SRem1196 ? match(Op0,1197 m_NSWMul(m_Value(), m_CheckedInt([C0](const APInt &C) {1198 return C.srem(*C0).isZero();1199 })))1200 : match(Op0,1201 m_NUWMul(m_Value(), m_CheckedInt([C0](const APInt &C) {1202 return C.urem(*C0).isZero();1203 }))))1204 return Constant::getNullValue(Op0->getType());1205 }1206 }1207 return nullptr;1208}1209 1210/// Given operands for an SDiv, see if we can fold the result.1211/// If not, this returns null.1212static Value *simplifySDivInst(Value *Op0, Value *Op1, bool IsExact,1213 const SimplifyQuery &Q, unsigned MaxRecurse) {1214 // If two operands are negated and no signed overflow, return -1.1215 if (isKnownNegation(Op0, Op1, /*NeedNSW=*/true))1216 return Constant::getAllOnesValue(Op0->getType());1217 1218 return simplifyDiv(Instruction::SDiv, Op0, Op1, IsExact, Q, MaxRecurse);1219}1220 1221Value *llvm::simplifySDivInst(Value *Op0, Value *Op1, bool IsExact,1222 const SimplifyQuery &Q) {1223 return ::simplifySDivInst(Op0, Op1, IsExact, Q, RecursionLimit);1224}1225 1226/// Given operands for a UDiv, see if we can fold the result.1227/// If not, this returns null.1228static Value *simplifyUDivInst(Value *Op0, Value *Op1, bool IsExact,1229 const SimplifyQuery &Q, unsigned MaxRecurse) {1230 return simplifyDiv(Instruction::UDiv, Op0, Op1, IsExact, Q, MaxRecurse);1231}1232 1233Value *llvm::simplifyUDivInst(Value *Op0, Value *Op1, bool IsExact,1234 const SimplifyQuery &Q) {1235 return ::simplifyUDivInst(Op0, Op1, IsExact, Q, RecursionLimit);1236}1237 1238/// Given operands for an SRem, see if we can fold the result.1239/// If not, this returns null.1240static Value *simplifySRemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,1241 unsigned MaxRecurse) {1242 // If the divisor is 0, the result is undefined, so assume the divisor is -1.1243 // srem Op0, (sext i1 X) --> srem Op0, -1 --> 01244 Value *X;1245 if (match(Op1, m_SExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1))1246 return ConstantInt::getNullValue(Op0->getType());1247 1248 // If the two operands are negated, return 0.1249 if (isKnownNegation(Op0, Op1))1250 return ConstantInt::getNullValue(Op0->getType());1251 1252 return simplifyRem(Instruction::SRem, Op0, Op1, Q, MaxRecurse);1253}1254 1255Value *llvm::simplifySRemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) {1256 return ::simplifySRemInst(Op0, Op1, Q, RecursionLimit);1257}1258 1259/// Given operands for a URem, see if we can fold the result.1260/// If not, this returns null.1261static Value *simplifyURemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,1262 unsigned MaxRecurse) {1263 return simplifyRem(Instruction::URem, Op0, Op1, Q, MaxRecurse);1264}1265 1266Value *llvm::simplifyURemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) {1267 return ::simplifyURemInst(Op0, Op1, Q, RecursionLimit);1268}1269 1270/// Returns true if a shift by \c Amount always yields poison.1271static bool isPoisonShift(Value *Amount, const SimplifyQuery &Q) {1272 Constant *C = dyn_cast<Constant>(Amount);1273 if (!C)1274 return false;1275 1276 // X shift by undef -> poison because it may shift by the bitwidth.1277 if (Q.isUndefValue(C))1278 return true;1279 1280 // Shifting by the bitwidth or more is poison. This covers scalars and1281 // fixed/scalable vectors with splat constants.1282 const APInt *AmountC;1283 if (match(C, m_APInt(AmountC)) && AmountC->uge(AmountC->getBitWidth()))1284 return true;1285 1286 // Try harder for fixed-length vectors:1287 // If all lanes of a vector shift are poison, the whole shift is poison.1288 if (isa<ConstantVector>(C) || isa<ConstantDataVector>(C)) {1289 for (unsigned I = 0,1290 E = cast<FixedVectorType>(C->getType())->getNumElements();1291 I != E; ++I)1292 if (!isPoisonShift(C->getAggregateElement(I), Q))1293 return false;1294 return true;1295 }1296 1297 return false;1298}1299 1300/// Given operands for an Shl, LShr or AShr, see if we can fold the result.1301/// If not, this returns null.1302static Value *simplifyShift(Instruction::BinaryOps Opcode, Value *Op0,1303 Value *Op1, bool IsNSW, const SimplifyQuery &Q,1304 unsigned MaxRecurse) {1305 if (Constant *C = foldOrCommuteConstant(Opcode, Op0, Op1, Q))1306 return C;1307 1308 // poison shift by X -> poison1309 if (isa<PoisonValue>(Op0))1310 return Op0;1311 1312 // 0 shift by X -> 01313 if (match(Op0, m_Zero()))1314 return Constant::getNullValue(Op0->getType());1315 1316 // X shift by 0 -> X1317 // Shift-by-sign-extended bool must be shift-by-0 because shift-by-all-ones1318 // would be poison.1319 Value *X;1320 if (match(Op1, m_Zero()) ||1321 (match(Op1, m_SExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1)))1322 return Op0;1323 1324 // Fold undefined shifts.1325 if (isPoisonShift(Op1, Q))1326 return PoisonValue::get(Op0->getType());1327 1328 // If the operation is with the result of a select instruction, check whether1329 // operating on either branch of the select always yields the same value.1330 if (isa<SelectInst>(Op0) || isa<SelectInst>(Op1))1331 if (Value *V = threadBinOpOverSelect(Opcode, Op0, Op1, Q, MaxRecurse))1332 return V;1333 1334 // If the operation is with the result of a phi instruction, check whether1335 // operating on all incoming values of the phi always yields the same value.1336 if (isa<PHINode>(Op0) || isa<PHINode>(Op1))1337 if (Value *V = threadBinOpOverPHI(Opcode, Op0, Op1, Q, MaxRecurse))1338 return V;1339 1340 // If any bits in the shift amount make that value greater than or equal to1341 // the number of bits in the type, the shift is undefined.1342 KnownBits KnownAmt = computeKnownBits(Op1, Q);1343 if (KnownAmt.getMinValue().uge(KnownAmt.getBitWidth()))1344 return PoisonValue::get(Op0->getType());1345 1346 // If all valid bits in the shift amount are known zero, the first operand is1347 // unchanged.1348 unsigned NumValidShiftBits = Log2_32_Ceil(KnownAmt.getBitWidth());1349 if (KnownAmt.countMinTrailingZeros() >= NumValidShiftBits)1350 return Op0;1351 1352 // Check for nsw shl leading to a poison value.1353 if (IsNSW) {1354 assert(Opcode == Instruction::Shl && "Expected shl for nsw instruction");1355 KnownBits KnownVal = computeKnownBits(Op0, Q);1356 KnownBits KnownShl = KnownBits::shl(KnownVal, KnownAmt);1357 1358 if (KnownVal.Zero.isSignBitSet())1359 KnownShl.Zero.setSignBit();1360 if (KnownVal.One.isSignBitSet())1361 KnownShl.One.setSignBit();1362 1363 if (KnownShl.hasConflict())1364 return PoisonValue::get(Op0->getType());1365 }1366 1367 return nullptr;1368}1369 1370/// Given operands for an LShr or AShr, see if we can fold the result. If not,1371/// this returns null.1372static Value *simplifyRightShift(Instruction::BinaryOps Opcode, Value *Op0,1373 Value *Op1, bool IsExact,1374 const SimplifyQuery &Q, unsigned MaxRecurse) {1375 if (Value *V =1376 simplifyShift(Opcode, Op0, Op1, /*IsNSW*/ false, Q, MaxRecurse))1377 return V;1378 1379 // X >> X -> 01380 if (Op0 == Op1)1381 return Constant::getNullValue(Op0->getType());1382 1383 // undef >> X -> 01384 // undef >> X -> undef (if it's exact)1385 if (Q.isUndefValue(Op0))1386 return IsExact ? Op0 : Constant::getNullValue(Op0->getType());1387 1388 // The low bit cannot be shifted out of an exact shift if it is set.1389 // TODO: Generalize by counting trailing zeros (see fold for exact division).1390 if (IsExact) {1391 KnownBits Op0Known = computeKnownBits(Op0, Q);1392 if (Op0Known.One[0])1393 return Op0;1394 }1395 1396 return nullptr;1397}1398 1399/// Given operands for an Shl, see if we can fold the result.1400/// If not, this returns null.1401static Value *simplifyShlInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,1402 const SimplifyQuery &Q, unsigned MaxRecurse) {1403 if (Value *V =1404 simplifyShift(Instruction::Shl, Op0, Op1, IsNSW, Q, MaxRecurse))1405 return V;1406 1407 Type *Ty = Op0->getType();1408 // undef << X -> 01409 // undef << X -> undef if (if it's NSW/NUW)1410 if (Q.isUndefValue(Op0))1411 return IsNSW || IsNUW ? Op0 : Constant::getNullValue(Ty);1412 1413 // (X >> A) << A -> X1414 Value *X;1415 if (Q.IIQ.UseInstrInfo &&1416 match(Op0, m_Exact(m_Shr(m_Value(X), m_Specific(Op1)))))1417 return X;1418 1419 // shl nuw i8 C, %x -> C iff C has sign bit set.1420 if (IsNUW && match(Op0, m_Negative()))1421 return Op0;1422 // NOTE: could use computeKnownBits() / LazyValueInfo,1423 // but the cost-benefit analysis suggests it isn't worth it.1424 1425 // "nuw" guarantees that only zeros are shifted out, and "nsw" guarantees1426 // that the sign-bit does not change, so the only input that does not1427 // produce poison is 0, and "0 << (bitwidth-1) --> 0".1428 if (IsNSW && IsNUW &&1429 match(Op1, m_SpecificInt(Ty->getScalarSizeInBits() - 1)))1430 return Constant::getNullValue(Ty);1431 1432 return nullptr;1433}1434 1435Value *llvm::simplifyShlInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW,1436 const SimplifyQuery &Q) {1437 return ::simplifyShlInst(Op0, Op1, IsNSW, IsNUW, Q, RecursionLimit);1438}1439 1440/// Given operands for an LShr, see if we can fold the result.1441/// If not, this returns null.1442static Value *simplifyLShrInst(Value *Op0, Value *Op1, bool IsExact,1443 const SimplifyQuery &Q, unsigned MaxRecurse) {1444 if (Value *V = simplifyRightShift(Instruction::LShr, Op0, Op1, IsExact, Q,1445 MaxRecurse))1446 return V;1447 1448 // (X << A) >> A -> X1449 Value *X;1450 if (Q.IIQ.UseInstrInfo && match(Op0, m_NUWShl(m_Value(X), m_Specific(Op1))))1451 return X;1452 1453 // ((X << A) | Y) >> A -> X if effective width of Y is not larger than A.1454 // We can return X as we do in the above case since OR alters no bits in X.1455 // SimplifyDemandedBits in InstCombine can do more general optimization for1456 // bit manipulation. This pattern aims to provide opportunities for other1457 // optimizers by supporting a simple but common case in InstSimplify.1458 Value *Y;1459 const APInt *ShRAmt, *ShLAmt;1460 if (Q.IIQ.UseInstrInfo && match(Op1, m_APInt(ShRAmt)) &&1461 match(Op0, m_c_Or(m_NUWShl(m_Value(X), m_APInt(ShLAmt)), m_Value(Y))) &&1462 *ShRAmt == *ShLAmt) {1463 const KnownBits YKnown = computeKnownBits(Y, Q);1464 const unsigned EffWidthY = YKnown.countMaxActiveBits();1465 if (ShRAmt->uge(EffWidthY))1466 return X;1467 }1468 1469 return nullptr;1470}1471 1472Value *llvm::simplifyLShrInst(Value *Op0, Value *Op1, bool IsExact,1473 const SimplifyQuery &Q) {1474 return ::simplifyLShrInst(Op0, Op1, IsExact, Q, RecursionLimit);1475}1476 1477/// Given operands for an AShr, see if we can fold the result.1478/// If not, this returns null.1479static Value *simplifyAShrInst(Value *Op0, Value *Op1, bool IsExact,1480 const SimplifyQuery &Q, unsigned MaxRecurse) {1481 if (Value *V = simplifyRightShift(Instruction::AShr, Op0, Op1, IsExact, Q,1482 MaxRecurse))1483 return V;1484 1485 // -1 >>a X --> -11486 // (-1 << X) a>> X --> -11487 // We could return the original -1 constant to preserve poison elements.1488 if (match(Op0, m_AllOnes()) ||1489 match(Op0, m_Shl(m_AllOnes(), m_Specific(Op1))))1490 return Constant::getAllOnesValue(Op0->getType());1491 1492 // (X << A) >> A -> X1493 Value *X;1494 if (Q.IIQ.UseInstrInfo && match(Op0, m_NSWShl(m_Value(X), m_Specific(Op1))))1495 return X;1496 1497 // Arithmetic shifting an all-sign-bit value is a no-op.1498 unsigned NumSignBits = ComputeNumSignBits(Op0, Q.DL, Q.AC, Q.CxtI, Q.DT);1499 if (NumSignBits == Op0->getType()->getScalarSizeInBits())1500 return Op0;1501 1502 return nullptr;1503}1504 1505Value *llvm::simplifyAShrInst(Value *Op0, Value *Op1, bool IsExact,1506 const SimplifyQuery &Q) {1507 return ::simplifyAShrInst(Op0, Op1, IsExact, Q, RecursionLimit);1508}1509 1510/// Commuted variants are assumed to be handled by calling this function again1511/// with the parameters swapped.1512static Value *simplifyUnsignedRangeCheck(ICmpInst *ZeroICmp,1513 ICmpInst *UnsignedICmp, bool IsAnd,1514 const SimplifyQuery &Q) {1515 Value *X, *Y;1516 1517 CmpPredicate EqPred;1518 if (!match(ZeroICmp, m_ICmp(EqPred, m_Value(Y), m_Zero())) ||1519 !ICmpInst::isEquality(EqPred))1520 return nullptr;1521 1522 CmpPredicate UnsignedPred;1523 1524 Value *A, *B;1525 // Y = (A - B);1526 if (match(Y, m_Sub(m_Value(A), m_Value(B)))) {1527 if (match(UnsignedICmp,1528 m_c_ICmp(UnsignedPred, m_Specific(A), m_Specific(B))) &&1529 ICmpInst::isUnsigned(UnsignedPred)) {1530 // A >=/<= B || (A - B) != 0 <--> true1531 if ((UnsignedPred == ICmpInst::ICMP_UGE ||1532 UnsignedPred == ICmpInst::ICMP_ULE) &&1533 EqPred == ICmpInst::ICMP_NE && !IsAnd)1534 return ConstantInt::getTrue(UnsignedICmp->getType());1535 // A </> B && (A - B) == 0 <--> false1536 if ((UnsignedPred == ICmpInst::ICMP_ULT ||1537 UnsignedPred == ICmpInst::ICMP_UGT) &&1538 EqPred == ICmpInst::ICMP_EQ && IsAnd)1539 return ConstantInt::getFalse(UnsignedICmp->getType());1540 1541 // A </> B && (A - B) != 0 <--> A </> B1542 // A </> B || (A - B) != 0 <--> (A - B) != 01543 if (EqPred == ICmpInst::ICMP_NE && (UnsignedPred == ICmpInst::ICMP_ULT ||1544 UnsignedPred == ICmpInst::ICMP_UGT))1545 return IsAnd ? UnsignedICmp : ZeroICmp;1546 1547 // A <=/>= B && (A - B) == 0 <--> (A - B) == 01548 // A <=/>= B || (A - B) == 0 <--> A <=/>= B1549 if (EqPred == ICmpInst::ICMP_EQ && (UnsignedPred == ICmpInst::ICMP_ULE ||1550 UnsignedPred == ICmpInst::ICMP_UGE))1551 return IsAnd ? ZeroICmp : UnsignedICmp;1552 }1553 1554 // Given Y = (A - B)1555 // Y >= A && Y != 0 --> Y >= A iff B != 01556 // Y < A || Y == 0 --> Y < A iff B != 01557 if (match(UnsignedICmp,1558 m_c_ICmp(UnsignedPred, m_Specific(Y), m_Specific(A)))) {1559 if (UnsignedPred == ICmpInst::ICMP_UGE && IsAnd &&1560 EqPred == ICmpInst::ICMP_NE && isKnownNonZero(B, Q))1561 return UnsignedICmp;1562 if (UnsignedPred == ICmpInst::ICMP_ULT && !IsAnd &&1563 EqPred == ICmpInst::ICMP_EQ && isKnownNonZero(B, Q))1564 return UnsignedICmp;1565 }1566 }1567 1568 if (match(UnsignedICmp, m_ICmp(UnsignedPred, m_Value(X), m_Specific(Y))) &&1569 ICmpInst::isUnsigned(UnsignedPred))1570 ;1571 else if (match(UnsignedICmp,1572 m_ICmp(UnsignedPred, m_Specific(Y), m_Value(X))) &&1573 ICmpInst::isUnsigned(UnsignedPred))1574 UnsignedPred = ICmpInst::getSwappedPredicate(UnsignedPred);1575 else1576 return nullptr;1577 1578 // X > Y && Y == 0 --> Y == 0 iff X != 01579 // X > Y || Y == 0 --> X > Y iff X != 01580 if (UnsignedPred == ICmpInst::ICMP_UGT && EqPred == ICmpInst::ICMP_EQ &&1581 isKnownNonZero(X, Q))1582 return IsAnd ? ZeroICmp : UnsignedICmp;1583 1584 // X <= Y && Y != 0 --> X <= Y iff X != 01585 // X <= Y || Y != 0 --> Y != 0 iff X != 01586 if (UnsignedPred == ICmpInst::ICMP_ULE && EqPred == ICmpInst::ICMP_NE &&1587 isKnownNonZero(X, Q))1588 return IsAnd ? UnsignedICmp : ZeroICmp;1589 1590 // The transforms below here are expected to be handled more generally with1591 // simplifyAndOrOfICmpsWithLimitConst() or in InstCombine's1592 // foldAndOrOfICmpsWithConstEq(). If we are looking to trim optimizer overlap,1593 // these are candidates for removal.1594 1595 // X < Y && Y != 0 --> X < Y1596 // X < Y || Y != 0 --> Y != 01597 if (UnsignedPred == ICmpInst::ICMP_ULT && EqPred == ICmpInst::ICMP_NE)1598 return IsAnd ? UnsignedICmp : ZeroICmp;1599 1600 // X >= Y && Y == 0 --> Y == 01601 // X >= Y || Y == 0 --> X >= Y1602 if (UnsignedPred == ICmpInst::ICMP_UGE && EqPred == ICmpInst::ICMP_EQ)1603 return IsAnd ? ZeroICmp : UnsignedICmp;1604 1605 // X < Y && Y == 0 --> false1606 if (UnsignedPred == ICmpInst::ICMP_ULT && EqPred == ICmpInst::ICMP_EQ &&1607 IsAnd)1608 return getFalse(UnsignedICmp->getType());1609 1610 // X >= Y || Y != 0 --> true1611 if (UnsignedPred == ICmpInst::ICMP_UGE && EqPred == ICmpInst::ICMP_NE &&1612 !IsAnd)1613 return getTrue(UnsignedICmp->getType());1614 1615 return nullptr;1616}1617 1618/// Test if a pair of compares with a shared operand and 2 constants has an1619/// empty set intersection, full set union, or if one compare is a superset of1620/// the other.1621static Value *simplifyAndOrOfICmpsWithConstants(ICmpInst *Cmp0, ICmpInst *Cmp1,1622 bool IsAnd) {1623 // Look for this pattern: {and/or} (icmp X, C0), (icmp X, C1)).1624 if (Cmp0->getOperand(0) != Cmp1->getOperand(0))1625 return nullptr;1626 1627 const APInt *C0, *C1;1628 if (!match(Cmp0->getOperand(1), m_APInt(C0)) ||1629 !match(Cmp1->getOperand(1), m_APInt(C1)))1630 return nullptr;1631 1632 auto Range0 = ConstantRange::makeExactICmpRegion(Cmp0->getPredicate(), *C0);1633 auto Range1 = ConstantRange::makeExactICmpRegion(Cmp1->getPredicate(), *C1);1634 1635 // For and-of-compares, check if the intersection is empty:1636 // (icmp X, C0) && (icmp X, C1) --> empty set --> false1637 if (IsAnd && Range0.intersectWith(Range1).isEmptySet())1638 return getFalse(Cmp0->getType());1639 1640 // For or-of-compares, check if the union is full:1641 // (icmp X, C0) || (icmp X, C1) --> full set --> true1642 if (!IsAnd && Range0.unionWith(Range1).isFullSet())1643 return getTrue(Cmp0->getType());1644 1645 // Is one range a superset of the other?1646 // If this is and-of-compares, take the smaller set:1647 // (icmp sgt X, 4) && (icmp sgt X, 42) --> icmp sgt X, 421648 // If this is or-of-compares, take the larger set:1649 // (icmp sgt X, 4) || (icmp sgt X, 42) --> icmp sgt X, 41650 if (Range0.contains(Range1))1651 return IsAnd ? Cmp1 : Cmp0;1652 if (Range1.contains(Range0))1653 return IsAnd ? Cmp0 : Cmp1;1654 1655 return nullptr;1656}1657 1658static Value *simplifyAndOfICmpsWithAdd(ICmpInst *Op0, ICmpInst *Op1,1659 const InstrInfoQuery &IIQ) {1660 // (icmp (add V, C0), C1) & (icmp V, C0)1661 CmpPredicate Pred0, Pred1;1662 const APInt *C0, *C1;1663 Value *V;1664 if (!match(Op0, m_ICmp(Pred0, m_Add(m_Value(V), m_APInt(C0)), m_APInt(C1))))1665 return nullptr;1666 1667 if (!match(Op1, m_ICmp(Pred1, m_Specific(V), m_Value())))1668 return nullptr;1669 1670 auto *AddInst = cast<OverflowingBinaryOperator>(Op0->getOperand(0));1671 if (AddInst->getOperand(1) != Op1->getOperand(1))1672 return nullptr;1673 1674 Type *ITy = Op0->getType();1675 bool IsNSW = IIQ.hasNoSignedWrap(AddInst);1676 bool IsNUW = IIQ.hasNoUnsignedWrap(AddInst);1677 1678 const APInt Delta = *C1 - *C0;1679 if (C0->isStrictlyPositive()) {1680 if (Delta == 2) {1681 if (Pred0 == ICmpInst::ICMP_ULT && Pred1 == ICmpInst::ICMP_SGT)1682 return getFalse(ITy);1683 if (Pred0 == ICmpInst::ICMP_SLT && Pred1 == ICmpInst::ICMP_SGT && IsNSW)1684 return getFalse(ITy);1685 }1686 if (Delta == 1) {1687 if (Pred0 == ICmpInst::ICMP_ULE && Pred1 == ICmpInst::ICMP_SGT)1688 return getFalse(ITy);1689 if (Pred0 == ICmpInst::ICMP_SLE && Pred1 == ICmpInst::ICMP_SGT && IsNSW)1690 return getFalse(ITy);1691 }1692 }1693 if (C0->getBoolValue() && IsNUW) {1694 if (Delta == 2)1695 if (Pred0 == ICmpInst::ICMP_ULT && Pred1 == ICmpInst::ICMP_UGT)1696 return getFalse(ITy);1697 if (Delta == 1)1698 if (Pred0 == ICmpInst::ICMP_ULE && Pred1 == ICmpInst::ICMP_UGT)1699 return getFalse(ITy);1700 }1701 1702 return nullptr;1703}1704 1705/// Try to simplify and/or of icmp with ctpop intrinsic.1706static Value *simplifyAndOrOfICmpsWithCtpop(ICmpInst *Cmp0, ICmpInst *Cmp1,1707 bool IsAnd) {1708 CmpPredicate Pred0, Pred1;1709 Value *X;1710 const APInt *C;1711 if (!match(Cmp0, m_ICmp(Pred0, m_Intrinsic<Intrinsic::ctpop>(m_Value(X)),1712 m_APInt(C))) ||1713 !match(Cmp1, m_ICmp(Pred1, m_Specific(X), m_ZeroInt())) || C->isZero())1714 return nullptr;1715 1716 // (ctpop(X) == C) || (X != 0) --> X != 0 where C > 01717 if (!IsAnd && Pred0 == ICmpInst::ICMP_EQ && Pred1 == ICmpInst::ICMP_NE)1718 return Cmp1;1719 // (ctpop(X) != C) && (X == 0) --> X == 0 where C > 01720 if (IsAnd && Pred0 == ICmpInst::ICMP_NE && Pred1 == ICmpInst::ICMP_EQ)1721 return Cmp1;1722 1723 return nullptr;1724}1725 1726static Value *simplifyAndOfICmps(ICmpInst *Op0, ICmpInst *Op1,1727 const SimplifyQuery &Q) {1728 if (Value *X = simplifyUnsignedRangeCheck(Op0, Op1, /*IsAnd=*/true, Q))1729 return X;1730 if (Value *X = simplifyUnsignedRangeCheck(Op1, Op0, /*IsAnd=*/true, Q))1731 return X;1732 1733 if (Value *X = simplifyAndOrOfICmpsWithConstants(Op0, Op1, true))1734 return X;1735 1736 if (Value *X = simplifyAndOrOfICmpsWithCtpop(Op0, Op1, true))1737 return X;1738 if (Value *X = simplifyAndOrOfICmpsWithCtpop(Op1, Op0, true))1739 return X;1740 1741 if (Value *X = simplifyAndOfICmpsWithAdd(Op0, Op1, Q.IIQ))1742 return X;1743 if (Value *X = simplifyAndOfICmpsWithAdd(Op1, Op0, Q.IIQ))1744 return X;1745 1746 return nullptr;1747}1748 1749static Value *simplifyOrOfICmpsWithAdd(ICmpInst *Op0, ICmpInst *Op1,1750 const InstrInfoQuery &IIQ) {1751 // (icmp (add V, C0), C1) | (icmp V, C0)1752 CmpPredicate Pred0, Pred1;1753 const APInt *C0, *C1;1754 Value *V;1755 if (!match(Op0, m_ICmp(Pred0, m_Add(m_Value(V), m_APInt(C0)), m_APInt(C1))))1756 return nullptr;1757 1758 if (!match(Op1, m_ICmp(Pred1, m_Specific(V), m_Value())))1759 return nullptr;1760 1761 auto *AddInst = cast<BinaryOperator>(Op0->getOperand(0));1762 if (AddInst->getOperand(1) != Op1->getOperand(1))1763 return nullptr;1764 1765 Type *ITy = Op0->getType();1766 bool IsNSW = IIQ.hasNoSignedWrap(AddInst);1767 bool IsNUW = IIQ.hasNoUnsignedWrap(AddInst);1768 1769 const APInt Delta = *C1 - *C0;1770 if (C0->isStrictlyPositive()) {1771 if (Delta == 2) {1772 if (Pred0 == ICmpInst::ICMP_UGE && Pred1 == ICmpInst::ICMP_SLE)1773 return getTrue(ITy);1774 if (Pred0 == ICmpInst::ICMP_SGE && Pred1 == ICmpInst::ICMP_SLE && IsNSW)1775 return getTrue(ITy);1776 }1777 if (Delta == 1) {1778 if (Pred0 == ICmpInst::ICMP_UGT && Pred1 == ICmpInst::ICMP_SLE)1779 return getTrue(ITy);1780 if (Pred0 == ICmpInst::ICMP_SGT && Pred1 == ICmpInst::ICMP_SLE && IsNSW)1781 return getTrue(ITy);1782 }1783 }1784 if (C0->getBoolValue() && IsNUW) {1785 if (Delta == 2)1786 if (Pred0 == ICmpInst::ICMP_UGE && Pred1 == ICmpInst::ICMP_ULE)1787 return getTrue(ITy);1788 if (Delta == 1)1789 if (Pred0 == ICmpInst::ICMP_UGT && Pred1 == ICmpInst::ICMP_ULE)1790 return getTrue(ITy);1791 }1792 1793 return nullptr;1794}1795 1796static Value *simplifyOrOfICmps(ICmpInst *Op0, ICmpInst *Op1,1797 const SimplifyQuery &Q) {1798 if (Value *X = simplifyUnsignedRangeCheck(Op0, Op1, /*IsAnd=*/false, Q))1799 return X;1800 if (Value *X = simplifyUnsignedRangeCheck(Op1, Op0, /*IsAnd=*/false, Q))1801 return X;1802 1803 if (Value *X = simplifyAndOrOfICmpsWithConstants(Op0, Op1, false))1804 return X;1805 1806 if (Value *X = simplifyAndOrOfICmpsWithCtpop(Op0, Op1, false))1807 return X;1808 if (Value *X = simplifyAndOrOfICmpsWithCtpop(Op1, Op0, false))1809 return X;1810 1811 if (Value *X = simplifyOrOfICmpsWithAdd(Op0, Op1, Q.IIQ))1812 return X;1813 if (Value *X = simplifyOrOfICmpsWithAdd(Op1, Op0, Q.IIQ))1814 return X;1815 1816 return nullptr;1817}1818 1819/// Test if a pair of compares with a shared operand and 2 constants has an1820/// empty set intersection, full set union, or if one compare is a superset of1821/// the other.1822static Value *simplifyAndOrOfFCmpsWithConstants(FCmpInst *Cmp0, FCmpInst *Cmp1,1823 bool IsAnd) {1824 // Look for this pattern: {and/or} (fcmp X, C0), (fcmp X, C1)).1825 if (Cmp0->getOperand(0) != Cmp1->getOperand(0))1826 return nullptr;1827 1828 const APFloat *C0, *C1;1829 if (!match(Cmp0->getOperand(1), m_APFloat(C0)) ||1830 !match(Cmp1->getOperand(1), m_APFloat(C1)))1831 return nullptr;1832 1833 auto Range0 = ConstantFPRange::makeExactFCmpRegion(1834 IsAnd ? Cmp0->getPredicate() : Cmp0->getInversePredicate(), *C0);1835 auto Range1 = ConstantFPRange::makeExactFCmpRegion(1836 IsAnd ? Cmp1->getPredicate() : Cmp1->getInversePredicate(), *C1);1837 1838 if (!Range0 || !Range1)1839 return nullptr;1840 1841 // For and-of-compares, check if the intersection is empty:1842 // (fcmp X, C0) && (fcmp X, C1) --> empty set --> false1843 if (Range0->intersectWith(*Range1).isEmptySet())1844 return ConstantInt::getBool(Cmp0->getType(), !IsAnd);1845 1846 // Is one range a superset of the other?1847 // If this is and-of-compares, take the smaller set:1848 // (fcmp ogt X, 4) && (fcmp ogt X, 42) --> fcmp ogt X, 421849 // If this is or-of-compares, take the larger set:1850 // (fcmp ogt X, 4) || (fcmp ogt X, 42) --> fcmp ogt X, 41851 if (Range0->contains(*Range1))1852 return Cmp1;1853 if (Range1->contains(*Range0))1854 return Cmp0;1855 1856 return nullptr;1857}1858 1859static Value *simplifyAndOrOfFCmps(const SimplifyQuery &Q, FCmpInst *LHS,1860 FCmpInst *RHS, bool IsAnd) {1861 Value *LHS0 = LHS->getOperand(0), *LHS1 = LHS->getOperand(1);1862 Value *RHS0 = RHS->getOperand(0), *RHS1 = RHS->getOperand(1);1863 if (LHS0->getType() != RHS0->getType())1864 return nullptr;1865 1866 FCmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();1867 auto AbsOrSelfLHS0 = m_CombineOr(m_Specific(LHS0), m_FAbs(m_Specific(LHS0)));1868 if ((PredL == FCmpInst::FCMP_ORD || PredL == FCmpInst::FCMP_UNO) &&1869 ((FCmpInst::isOrdered(PredR) && IsAnd) ||1870 (FCmpInst::isUnordered(PredR) && !IsAnd))) {1871 // (fcmp ord X, 0) & (fcmp o** X/abs(X), Y) --> fcmp o** X/abs(X), Y1872 // (fcmp uno X, 0) & (fcmp o** X/abs(X), Y) --> false1873 // (fcmp uno X, 0) | (fcmp u** X/abs(X), Y) --> fcmp u** X/abs(X), Y1874 // (fcmp ord X, 0) | (fcmp u** X/abs(X), Y) --> true1875 if ((match(RHS0, AbsOrSelfLHS0) || match(RHS1, AbsOrSelfLHS0)) &&1876 match(LHS1, m_PosZeroFP()))1877 return FCmpInst::isOrdered(PredL) == FCmpInst::isOrdered(PredR)1878 ? static_cast<Value *>(RHS)1879 : ConstantInt::getBool(LHS->getType(), !IsAnd);1880 }1881 1882 auto AbsOrSelfRHS0 = m_CombineOr(m_Specific(RHS0), m_FAbs(m_Specific(RHS0)));1883 if ((PredR == FCmpInst::FCMP_ORD || PredR == FCmpInst::FCMP_UNO) &&1884 ((FCmpInst::isOrdered(PredL) && IsAnd) ||1885 (FCmpInst::isUnordered(PredL) && !IsAnd))) {1886 // (fcmp o** X/abs(X), Y) & (fcmp ord X, 0) --> fcmp o** X/abs(X), Y1887 // (fcmp o** X/abs(X), Y) & (fcmp uno X, 0) --> false1888 // (fcmp u** X/abs(X), Y) | (fcmp uno X, 0) --> fcmp u** X/abs(X), Y1889 // (fcmp u** X/abs(X), Y) | (fcmp ord X, 0) --> true1890 if ((match(LHS0, AbsOrSelfRHS0) || match(LHS1, AbsOrSelfRHS0)) &&1891 match(RHS1, m_PosZeroFP()))1892 return FCmpInst::isOrdered(PredL) == FCmpInst::isOrdered(PredR)1893 ? static_cast<Value *>(LHS)1894 : ConstantInt::getBool(LHS->getType(), !IsAnd);1895 }1896 1897 if (auto *V = simplifyAndOrOfFCmpsWithConstants(LHS, RHS, IsAnd))1898 return V;1899 1900 return nullptr;1901}1902 1903static Value *simplifyAndOrOfCmps(const SimplifyQuery &Q, Value *Op0,1904 Value *Op1, bool IsAnd) {1905 // Look through casts of the 'and' operands to find compares.1906 auto *Cast0 = dyn_cast<CastInst>(Op0);1907 auto *Cast1 = dyn_cast<CastInst>(Op1);1908 if (Cast0 && Cast1 && Cast0->getOpcode() == Cast1->getOpcode() &&1909 Cast0->getSrcTy() == Cast1->getSrcTy()) {1910 Op0 = Cast0->getOperand(0);1911 Op1 = Cast1->getOperand(0);1912 }1913 1914 Value *V = nullptr;1915 auto *ICmp0 = dyn_cast<ICmpInst>(Op0);1916 auto *ICmp1 = dyn_cast<ICmpInst>(Op1);1917 if (ICmp0 && ICmp1)1918 V = IsAnd ? simplifyAndOfICmps(ICmp0, ICmp1, Q)1919 : simplifyOrOfICmps(ICmp0, ICmp1, Q);1920 1921 auto *FCmp0 = dyn_cast<FCmpInst>(Op0);1922 auto *FCmp1 = dyn_cast<FCmpInst>(Op1);1923 if (FCmp0 && FCmp1)1924 V = simplifyAndOrOfFCmps(Q, FCmp0, FCmp1, IsAnd);1925 1926 if (!V)1927 return nullptr;1928 if (!Cast0)1929 return V;1930 1931 // If we looked through casts, we can only handle a constant simplification1932 // because we are not allowed to create a cast instruction here.1933 if (auto *C = dyn_cast<Constant>(V))1934 return ConstantFoldCastOperand(Cast0->getOpcode(), C, Cast0->getType(),1935 Q.DL);1936 1937 return nullptr;1938}1939 1940static Value *simplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,1941 const SimplifyQuery &Q,1942 bool AllowRefinement,1943 SmallVectorImpl<Instruction *> *DropFlags,1944 unsigned MaxRecurse);1945 1946static Value *simplifyAndOrWithICmpEq(unsigned Opcode, Value *Op0, Value *Op1,1947 const SimplifyQuery &Q,1948 unsigned MaxRecurse) {1949 assert((Opcode == Instruction::And || Opcode == Instruction::Or) &&1950 "Must be and/or");1951 CmpPredicate Pred;1952 Value *A, *B;1953 if (!match(Op0, m_ICmp(Pred, m_Value(A), m_Value(B))) ||1954 !ICmpInst::isEquality(Pred))1955 return nullptr;1956 1957 auto Simplify = [&](Value *Res) -> Value * {1958 Constant *Absorber = ConstantExpr::getBinOpAbsorber(Opcode, Res->getType());1959 1960 // and (icmp eq a, b), x implies (a==b) inside x.1961 // or (icmp ne a, b), x implies (a==b) inside x.1962 // If x simplifies to true/false, we can simplify the and/or.1963 if (Pred ==1964 (Opcode == Instruction::And ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE)) {1965 if (Res == Absorber)1966 return Absorber;1967 if (Res == ConstantExpr::getBinOpIdentity(Opcode, Res->getType()))1968 return Op0;1969 return nullptr;1970 }1971 1972 // If we have and (icmp ne a, b), x and for a==b we can simplify x to false,1973 // then we can drop the icmp, as x will already be false in the case where1974 // the icmp is false. Similar for or and true.1975 if (Res == Absorber)1976 return Op1;1977 return nullptr;1978 };1979 1980 // In the final case (Res == Absorber with inverted predicate), it is safe to1981 // refine poison during simplification, but not undef. For simplicity always1982 // disable undef-based folds here.1983 if (Value *Res = simplifyWithOpReplaced(Op1, A, B, Q.getWithoutUndef(),1984 /* AllowRefinement */ true,1985 /* DropFlags */ nullptr, MaxRecurse))1986 return Simplify(Res);1987 if (Value *Res = simplifyWithOpReplaced(Op1, B, A, Q.getWithoutUndef(),1988 /* AllowRefinement */ true,1989 /* DropFlags */ nullptr, MaxRecurse))1990 return Simplify(Res);1991 1992 return nullptr;1993}1994 1995/// Given a bitwise logic op, check if the operands are add/sub with a common1996/// source value and inverted constant (identity: C - X -> ~(X + ~C)).1997static Value *simplifyLogicOfAddSub(Value *Op0, Value *Op1,1998 Instruction::BinaryOps Opcode) {1999 assert(Op0->getType() == Op1->getType() && "Mismatched binop types");2000 assert(BinaryOperator::isBitwiseLogicOp(Opcode) && "Expected logic op");2001 Value *X;2002 Constant *C1, *C2;2003 if ((match(Op0, m_Add(m_Value(X), m_Constant(C1))) &&2004 match(Op1, m_Sub(m_Constant(C2), m_Specific(X)))) ||2005 (match(Op1, m_Add(m_Value(X), m_Constant(C1))) &&2006 match(Op0, m_Sub(m_Constant(C2), m_Specific(X))))) {2007 if (ConstantExpr::getNot(C1) == C2) {2008 // (X + C) & (~C - X) --> (X + C) & ~(X + C) --> 02009 // (X + C) | (~C - X) --> (X + C) | ~(X + C) --> -12010 // (X + C) ^ (~C - X) --> (X + C) ^ ~(X + C) --> -12011 Type *Ty = Op0->getType();2012 return Opcode == Instruction::And ? ConstantInt::getNullValue(Ty)2013 : ConstantInt::getAllOnesValue(Ty);2014 }2015 }2016 return nullptr;2017}2018 2019// Commutative patterns for and that will be tried with both operand orders.2020static Value *simplifyAndCommutative(Value *Op0, Value *Op1,2021 const SimplifyQuery &Q,2022 unsigned MaxRecurse) {2023 // ~A & A = 02024 if (match(Op0, m_Not(m_Specific(Op1))))2025 return Constant::getNullValue(Op0->getType());2026 2027 // (A | ?) & A = A2028 if (match(Op0, m_c_Or(m_Specific(Op1), m_Value())))2029 return Op1;2030 2031 // (X | ~Y) & (X | Y) --> X2032 Value *X, *Y;2033 if (match(Op0, m_c_Or(m_Value(X), m_Not(m_Value(Y)))) &&2034 match(Op1, m_c_Or(m_Specific(X), m_Specific(Y))))2035 return X;2036 2037 // If we have a multiplication overflow check that is being 'and'ed with a2038 // check that one of the multipliers is not zero, we can omit the 'and', and2039 // only keep the overflow check.2040 if (isCheckForZeroAndMulWithOverflow(Op0, Op1, true))2041 return Op1;2042 2043 // -A & A = A if A is a power of two or zero.2044 if (match(Op0, m_Neg(m_Specific(Op1))) &&2045 isKnownToBeAPowerOfTwo(Op1, Q.DL, /*OrZero*/ true, Q.AC, Q.CxtI, Q.DT))2046 return Op1;2047 2048 // This is a similar pattern used for checking if a value is a power-of-2:2049 // (A - 1) & A --> 0 (if A is a power-of-2 or 0)2050 if (match(Op0, m_Add(m_Specific(Op1), m_AllOnes())) &&2051 isKnownToBeAPowerOfTwo(Op1, Q.DL, /*OrZero*/ true, Q.AC, Q.CxtI, Q.DT))2052 return Constant::getNullValue(Op1->getType());2053 2054 // (x << N) & ((x << M) - 1) --> 0, where x is known to be a power of 2 and2055 // M <= N.2056 const APInt *Shift1, *Shift2;2057 if (match(Op0, m_Shl(m_Value(X), m_APInt(Shift1))) &&2058 match(Op1, m_Add(m_Shl(m_Specific(X), m_APInt(Shift2)), m_AllOnes())) &&2059 isKnownToBeAPowerOfTwo(X, Q.DL, /*OrZero*/ true, Q.AC, Q.CxtI) &&2060 Shift1->uge(*Shift2))2061 return Constant::getNullValue(Op0->getType());2062 2063 if (Value *V =2064 simplifyAndOrWithICmpEq(Instruction::And, Op0, Op1, Q, MaxRecurse))2065 return V;2066 2067 return nullptr;2068}2069 2070/// Given operands for an And, see if we can fold the result.2071/// If not, this returns null.2072static Value *simplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,2073 unsigned MaxRecurse) {2074 if (Constant *C = foldOrCommuteConstant(Instruction::And, Op0, Op1, Q))2075 return C;2076 2077 // X & poison -> poison2078 if (isa<PoisonValue>(Op1))2079 return Op1;2080 2081 // X & undef -> 02082 if (Q.isUndefValue(Op1))2083 return Constant::getNullValue(Op0->getType());2084 2085 // X & X = X2086 if (Op0 == Op1)2087 return Op0;2088 2089 // X & 0 = 02090 if (match(Op1, m_Zero()))2091 return Constant::getNullValue(Op0->getType());2092 2093 // X & -1 = X2094 if (match(Op1, m_AllOnes()))2095 return Op0;2096 2097 if (Value *Res = simplifyAndCommutative(Op0, Op1, Q, MaxRecurse))2098 return Res;2099 if (Value *Res = simplifyAndCommutative(Op1, Op0, Q, MaxRecurse))2100 return Res;2101 2102 if (Value *V = simplifyLogicOfAddSub(Op0, Op1, Instruction::And))2103 return V;2104 2105 // A mask that only clears known zeros of a shifted value is a no-op.2106 const APInt *Mask;2107 const APInt *ShAmt;2108 Value *X, *Y;2109 if (match(Op1, m_APInt(Mask))) {2110 // If all bits in the inverted and shifted mask are clear:2111 // and (shl X, ShAmt), Mask --> shl X, ShAmt2112 if (match(Op0, m_Shl(m_Value(X), m_APInt(ShAmt))) &&2113 (~(*Mask)).lshr(*ShAmt).isZero())2114 return Op0;2115 2116 // If all bits in the inverted and shifted mask are clear:2117 // and (lshr X, ShAmt), Mask --> lshr X, ShAmt2118 if (match(Op0, m_LShr(m_Value(X), m_APInt(ShAmt))) &&2119 (~(*Mask)).shl(*ShAmt).isZero())2120 return Op0;2121 }2122 2123 // and 2^x-1, 2^C --> 0 where x <= C.2124 const APInt *PowerC;2125 Value *Shift;2126 if (match(Op1, m_Power2(PowerC)) &&2127 match(Op0, m_Add(m_Value(Shift), m_AllOnes())) &&2128 isKnownToBeAPowerOfTwo(Shift, Q.DL, /*OrZero*/ false, Q.AC, Q.CxtI,2129 Q.DT)) {2130 KnownBits Known = computeKnownBits(Shift, Q);2131 // Use getActiveBits() to make use of the additional power of two knowledge2132 if (PowerC->getActiveBits() >= Known.getMaxValue().getActiveBits())2133 return ConstantInt::getNullValue(Op1->getType());2134 }2135 2136 if (Value *V = simplifyAndOrOfCmps(Q, Op0, Op1, true))2137 return V;2138 2139 // Try some generic simplifications for associative operations.2140 if (Value *V =2141 simplifyAssociativeBinOp(Instruction::And, Op0, Op1, Q, MaxRecurse))2142 return V;2143 2144 // And distributes over Or. Try some generic simplifications based on this.2145 if (Value *V = expandCommutativeBinOp(Instruction::And, Op0, Op1,2146 Instruction::Or, Q, MaxRecurse))2147 return V;2148 2149 // And distributes over Xor. Try some generic simplifications based on this.2150 if (Value *V = expandCommutativeBinOp(Instruction::And, Op0, Op1,2151 Instruction::Xor, Q, MaxRecurse))2152 return V;2153 2154 if (isa<SelectInst>(Op0) || isa<SelectInst>(Op1)) {2155 if (Op0->getType()->isIntOrIntVectorTy(1)) {2156 // A & (A && B) -> A && B2157 if (match(Op1, m_Select(m_Specific(Op0), m_Value(), m_Zero())))2158 return Op1;2159 else if (match(Op0, m_Select(m_Specific(Op1), m_Value(), m_Zero())))2160 return Op0;2161 }2162 // If the operation is with the result of a select instruction, check2163 // whether operating on either branch of the select always yields the same2164 // value.2165 if (Value *V =2166 threadBinOpOverSelect(Instruction::And, Op0, Op1, Q, MaxRecurse))2167 return V;2168 }2169 2170 // If the operation is with the result of a phi instruction, check whether2171 // operating on all incoming values of the phi always yields the same value.2172 if (isa<PHINode>(Op0) || isa<PHINode>(Op1))2173 if (Value *V =2174 threadBinOpOverPHI(Instruction::And, Op0, Op1, Q, MaxRecurse))2175 return V;2176 2177 // Assuming the effective width of Y is not larger than A, i.e. all bits2178 // from X and Y are disjoint in (X << A) | Y,2179 // if the mask of this AND op covers all bits of X or Y, while it covers2180 // no bits from the other, we can bypass this AND op. E.g.,2181 // ((X << A) | Y) & Mask -> Y,2182 // if Mask = ((1 << effective_width_of(Y)) - 1)2183 // ((X << A) | Y) & Mask -> X << A,2184 // if Mask = ((1 << effective_width_of(X)) - 1) << A2185 // SimplifyDemandedBits in InstCombine can optimize the general case.2186 // This pattern aims to help other passes for a common case.2187 Value *XShifted;2188 if (Q.IIQ.UseInstrInfo && match(Op1, m_APInt(Mask)) &&2189 match(Op0, m_c_Or(m_CombineAnd(m_NUWShl(m_Value(X), m_APInt(ShAmt)),2190 m_Value(XShifted)),2191 m_Value(Y)))) {2192 const unsigned Width = Op0->getType()->getScalarSizeInBits();2193 const unsigned ShftCnt = ShAmt->getLimitedValue(Width);2194 const KnownBits YKnown = computeKnownBits(Y, Q);2195 const unsigned EffWidthY = YKnown.countMaxActiveBits();2196 if (EffWidthY <= ShftCnt) {2197 const KnownBits XKnown = computeKnownBits(X, Q);2198 const unsigned EffWidthX = XKnown.countMaxActiveBits();2199 const APInt EffBitsY = APInt::getLowBitsSet(Width, EffWidthY);2200 const APInt EffBitsX = APInt::getLowBitsSet(Width, EffWidthX) << ShftCnt;2201 // If the mask is extracting all bits from X or Y as is, we can skip2202 // this AND op.2203 if (EffBitsY.isSubsetOf(*Mask) && !EffBitsX.intersects(*Mask))2204 return Y;2205 if (EffBitsX.isSubsetOf(*Mask) && !EffBitsY.intersects(*Mask))2206 return XShifted;2207 }2208 }2209 2210 // ((X | Y) ^ X ) & ((X | Y) ^ Y) --> 02211 // ((X | Y) ^ Y ) & ((X | Y) ^ X) --> 02212 BinaryOperator *Or;2213 if (match(Op0, m_c_Xor(m_Value(X),2214 m_CombineAnd(m_BinOp(Or),2215 m_c_Or(m_Deferred(X), m_Value(Y))))) &&2216 match(Op1, m_c_Xor(m_Specific(Or), m_Specific(Y))))2217 return Constant::getNullValue(Op0->getType());2218 2219 const APInt *C1;2220 Value *A;2221 // (A ^ C) & (A ^ ~C) -> 02222 if (match(Op0, m_Xor(m_Value(A), m_APInt(C1))) &&2223 match(Op1, m_Xor(m_Specific(A), m_SpecificInt(~*C1))))2224 return Constant::getNullValue(Op0->getType());2225 2226 if (Op0->getType()->isIntOrIntVectorTy(1)) {2227 if (std::optional<bool> Implied = isImpliedCondition(Op0, Op1, Q.DL)) {2228 // If Op0 is true implies Op1 is true, then Op0 is a subset of Op1.2229 if (*Implied == true)2230 return Op0;2231 // If Op0 is true implies Op1 is false, then they are not true together.2232 if (*Implied == false)2233 return ConstantInt::getFalse(Op0->getType());2234 }2235 if (std::optional<bool> Implied = isImpliedCondition(Op1, Op0, Q.DL)) {2236 // If Op1 is true implies Op0 is true, then Op1 is a subset of Op0.2237 if (*Implied)2238 return Op1;2239 // If Op1 is true implies Op0 is false, then they are not true together.2240 if (!*Implied)2241 return ConstantInt::getFalse(Op1->getType());2242 }2243 }2244 2245 if (Value *V = simplifyByDomEq(Instruction::And, Op0, Op1, Q, MaxRecurse))2246 return V;2247 2248 return nullptr;2249}2250 2251Value *llvm::simplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) {2252 return ::simplifyAndInst(Op0, Op1, Q, RecursionLimit);2253}2254 2255// TODO: Many of these folds could use LogicalAnd/LogicalOr.2256static Value *simplifyOrLogic(Value *X, Value *Y) {2257 assert(X->getType() == Y->getType() && "Expected same type for 'or' ops");2258 Type *Ty = X->getType();2259 2260 // X | ~X --> -12261 if (match(Y, m_Not(m_Specific(X))))2262 return ConstantInt::getAllOnesValue(Ty);2263 2264 // X | ~(X & ?) = -12265 if (match(Y, m_Not(m_c_And(m_Specific(X), m_Value()))))2266 return ConstantInt::getAllOnesValue(Ty);2267 2268 // X | (X & ?) --> X2269 if (match(Y, m_c_And(m_Specific(X), m_Value())))2270 return X;2271 2272 Value *A, *B;2273 2274 // (A ^ B) | (A | B) --> A | B2275 // (A ^ B) | (B | A) --> B | A2276 if (match(X, m_Xor(m_Value(A), m_Value(B))) &&2277 match(Y, m_c_Or(m_Specific(A), m_Specific(B))))2278 return Y;2279 2280 // ~(A ^ B) | (A | B) --> -12281 // ~(A ^ B) | (B | A) --> -12282 if (match(X, m_Not(m_Xor(m_Value(A), m_Value(B)))) &&2283 match(Y, m_c_Or(m_Specific(A), m_Specific(B))))2284 return ConstantInt::getAllOnesValue(Ty);2285 2286 // (A & ~B) | (A ^ B) --> A ^ B2287 // (~B & A) | (A ^ B) --> A ^ B2288 // (A & ~B) | (B ^ A) --> B ^ A2289 // (~B & A) | (B ^ A) --> B ^ A2290 if (match(X, m_c_And(m_Value(A), m_Not(m_Value(B)))) &&2291 match(Y, m_c_Xor(m_Specific(A), m_Specific(B))))2292 return Y;2293 2294 // (~A ^ B) | (A & B) --> ~A ^ B2295 // (B ^ ~A) | (A & B) --> B ^ ~A2296 // (~A ^ B) | (B & A) --> ~A ^ B2297 // (B ^ ~A) | (B & A) --> B ^ ~A2298 if (match(X, m_c_Xor(m_Not(m_Value(A)), m_Value(B))) &&2299 match(Y, m_c_And(m_Specific(A), m_Specific(B))))2300 return X;2301 2302 // (~A | B) | (A ^ B) --> -12303 // (~A | B) | (B ^ A) --> -12304 // (B | ~A) | (A ^ B) --> -12305 // (B | ~A) | (B ^ A) --> -12306 if (match(X, m_c_Or(m_Not(m_Value(A)), m_Value(B))) &&2307 match(Y, m_c_Xor(m_Specific(A), m_Specific(B))))2308 return ConstantInt::getAllOnesValue(Ty);2309 2310 // (~A & B) | ~(A | B) --> ~A2311 // (~A & B) | ~(B | A) --> ~A2312 // (B & ~A) | ~(A | B) --> ~A2313 // (B & ~A) | ~(B | A) --> ~A2314 Value *NotA;2315 if (match(X, m_c_And(m_CombineAnd(m_Value(NotA), m_Not(m_Value(A))),2316 m_Value(B))) &&2317 match(Y, m_Not(m_c_Or(m_Specific(A), m_Specific(B)))))2318 return NotA;2319 // The same is true of Logical And2320 // TODO: This could share the logic of the version above if there was a2321 // version of LogicalAnd that allowed more than just i1 types.2322 if (match(X, m_c_LogicalAnd(m_CombineAnd(m_Value(NotA), m_Not(m_Value(A))),2323 m_Value(B))) &&2324 match(Y, m_Not(m_c_LogicalOr(m_Specific(A), m_Specific(B)))))2325 return NotA;2326 2327 // ~(A ^ B) | (A & B) --> ~(A ^ B)2328 // ~(A ^ B) | (B & A) --> ~(A ^ B)2329 Value *NotAB;2330 if (match(X, m_CombineAnd(m_Not(m_Xor(m_Value(A), m_Value(B))),2331 m_Value(NotAB))) &&2332 match(Y, m_c_And(m_Specific(A), m_Specific(B))))2333 return NotAB;2334 2335 // ~(A & B) | (A ^ B) --> ~(A & B)2336 // ~(A & B) | (B ^ A) --> ~(A & B)2337 if (match(X, m_CombineAnd(m_Not(m_And(m_Value(A), m_Value(B))),2338 m_Value(NotAB))) &&2339 match(Y, m_c_Xor(m_Specific(A), m_Specific(B))))2340 return NotAB;2341 2342 return nullptr;2343}2344 2345/// Given operands for an Or, see if we can fold the result.2346/// If not, this returns null.2347static Value *simplifyOrInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,2348 unsigned MaxRecurse) {2349 if (Constant *C = foldOrCommuteConstant(Instruction::Or, Op0, Op1, Q))2350 return C;2351 2352 // X | poison -> poison2353 if (isa<PoisonValue>(Op1))2354 return Op1;2355 2356 // X | undef -> -12357 // X | -1 = -12358 // Do not return Op1 because it may contain undef elements if it's a vector.2359 if (Q.isUndefValue(Op1) || match(Op1, m_AllOnes()))2360 return Constant::getAllOnesValue(Op0->getType());2361 2362 // X | X = X2363 // X | 0 = X2364 if (Op0 == Op1 || match(Op1, m_Zero()))2365 return Op0;2366 2367 if (Value *R = simplifyOrLogic(Op0, Op1))2368 return R;2369 if (Value *R = simplifyOrLogic(Op1, Op0))2370 return R;2371 2372 if (Value *V = simplifyLogicOfAddSub(Op0, Op1, Instruction::Or))2373 return V;2374 2375 // Rotated -1 is still -1:2376 // (-1 << X) | (-1 >> (C - X)) --> -12377 // (-1 >> X) | (-1 << (C - X)) --> -12378 // ...with C <= bitwidth (and commuted variants).2379 Value *X, *Y;2380 if ((match(Op0, m_Shl(m_AllOnes(), m_Value(X))) &&2381 match(Op1, m_LShr(m_AllOnes(), m_Value(Y)))) ||2382 (match(Op1, m_Shl(m_AllOnes(), m_Value(X))) &&2383 match(Op0, m_LShr(m_AllOnes(), m_Value(Y))))) {2384 const APInt *C;2385 if ((match(X, m_Sub(m_APInt(C), m_Specific(Y))) ||2386 match(Y, m_Sub(m_APInt(C), m_Specific(X)))) &&2387 C->ule(X->getType()->getScalarSizeInBits())) {2388 return ConstantInt::getAllOnesValue(X->getType());2389 }2390 }2391 2392 // A funnel shift (rotate) can be decomposed into simpler shifts. See if we2393 // are mixing in another shift that is redundant with the funnel shift.2394 2395 // (fshl X, ?, Y) | (shl X, Y) --> fshl X, ?, Y2396 // (shl X, Y) | (fshl X, ?, Y) --> fshl X, ?, Y2397 if (match(Op0,2398 m_Intrinsic<Intrinsic::fshl>(m_Value(X), m_Value(), m_Value(Y))) &&2399 match(Op1, m_Shl(m_Specific(X), m_Specific(Y))))2400 return Op0;2401 if (match(Op1,2402 m_Intrinsic<Intrinsic::fshl>(m_Value(X), m_Value(), m_Value(Y))) &&2403 match(Op0, m_Shl(m_Specific(X), m_Specific(Y))))2404 return Op1;2405 2406 // (fshr ?, X, Y) | (lshr X, Y) --> fshr ?, X, Y2407 // (lshr X, Y) | (fshr ?, X, Y) --> fshr ?, X, Y2408 if (match(Op0,2409 m_Intrinsic<Intrinsic::fshr>(m_Value(), m_Value(X), m_Value(Y))) &&2410 match(Op1, m_LShr(m_Specific(X), m_Specific(Y))))2411 return Op0;2412 if (match(Op1,2413 m_Intrinsic<Intrinsic::fshr>(m_Value(), m_Value(X), m_Value(Y))) &&2414 match(Op0, m_LShr(m_Specific(X), m_Specific(Y))))2415 return Op1;2416 2417 if (Value *V =2418 simplifyAndOrWithICmpEq(Instruction::Or, Op0, Op1, Q, MaxRecurse))2419 return V;2420 if (Value *V =2421 simplifyAndOrWithICmpEq(Instruction::Or, Op1, Op0, Q, MaxRecurse))2422 return V;2423 2424 if (Value *V = simplifyAndOrOfCmps(Q, Op0, Op1, false))2425 return V;2426 2427 // If we have a multiplication overflow check that is being 'and'ed with a2428 // check that one of the multipliers is not zero, we can omit the 'and', and2429 // only keep the overflow check.2430 if (isCheckForZeroAndMulWithOverflow(Op0, Op1, false))2431 return Op1;2432 if (isCheckForZeroAndMulWithOverflow(Op1, Op0, false))2433 return Op0;2434 2435 // Try some generic simplifications for associative operations.2436 if (Value *V =2437 simplifyAssociativeBinOp(Instruction::Or, Op0, Op1, Q, MaxRecurse))2438 return V;2439 2440 // Or distributes over And. Try some generic simplifications based on this.2441 if (Value *V = expandCommutativeBinOp(Instruction::Or, Op0, Op1,2442 Instruction::And, Q, MaxRecurse))2443 return V;2444 2445 if (isa<SelectInst>(Op0) || isa<SelectInst>(Op1)) {2446 if (Op0->getType()->isIntOrIntVectorTy(1)) {2447 // A | (A || B) -> A || B2448 if (match(Op1, m_Select(m_Specific(Op0), m_One(), m_Value())))2449 return Op1;2450 else if (match(Op0, m_Select(m_Specific(Op1), m_One(), m_Value())))2451 return Op0;2452 }2453 // If the operation is with the result of a select instruction, check2454 // whether operating on either branch of the select always yields the same2455 // value.2456 if (Value *V =2457 threadBinOpOverSelect(Instruction::Or, Op0, Op1, Q, MaxRecurse))2458 return V;2459 }2460 2461 // (A & C1)|(B & C2)2462 Value *A, *B;2463 const APInt *C1, *C2;2464 if (match(Op0, m_And(m_Value(A), m_APInt(C1))) &&2465 match(Op1, m_And(m_Value(B), m_APInt(C2)))) {2466 if (*C1 == ~*C2) {2467 // (A & C1)|(B & C2)2468 // If we have: ((V + N) & C1) | (V & C2)2469 // .. and C2 = ~C1 and C2 is 0+1+ and (N & C2) == 02470 // replace with V+N.2471 Value *N;2472 if (C2->isMask() && // C2 == 0+1+2473 match(A, m_c_Add(m_Specific(B), m_Value(N)))) {2474 // Add commutes, try both ways.2475 if (MaskedValueIsZero(N, *C2, Q))2476 return A;2477 }2478 // Or commutes, try both ways.2479 if (C1->isMask() && match(B, m_c_Add(m_Specific(A), m_Value(N)))) {2480 // Add commutes, try both ways.2481 if (MaskedValueIsZero(N, *C1, Q))2482 return B;2483 }2484 }2485 }2486 2487 // If the operation is with the result of a phi instruction, check whether2488 // operating on all incoming values of the phi always yields the same value.2489 if (isa<PHINode>(Op0) || isa<PHINode>(Op1))2490 if (Value *V = threadBinOpOverPHI(Instruction::Or, Op0, Op1, Q, MaxRecurse))2491 return V;2492 2493 // (A ^ C) | (A ^ ~C) -> -1, i.e. all bits set to one.2494 if (match(Op0, m_Xor(m_Value(A), m_APInt(C1))) &&2495 match(Op1, m_Xor(m_Specific(A), m_SpecificInt(~*C1))))2496 return Constant::getAllOnesValue(Op0->getType());2497 2498 if (Op0->getType()->isIntOrIntVectorTy(1)) {2499 if (std::optional<bool> Implied =2500 isImpliedCondition(Op0, Op1, Q.DL, false)) {2501 // If Op0 is false implies Op1 is false, then Op1 is a subset of Op0.2502 if (*Implied == false)2503 return Op0;2504 // If Op0 is false implies Op1 is true, then at least one is always true.2505 if (*Implied == true)2506 return ConstantInt::getTrue(Op0->getType());2507 }2508 if (std::optional<bool> Implied =2509 isImpliedCondition(Op1, Op0, Q.DL, false)) {2510 // If Op1 is false implies Op0 is false, then Op0 is a subset of Op1.2511 if (*Implied == false)2512 return Op1;2513 // If Op1 is false implies Op0 is true, then at least one is always true.2514 if (*Implied == true)2515 return ConstantInt::getTrue(Op1->getType());2516 }2517 }2518 2519 if (Value *V = simplifyByDomEq(Instruction::Or, Op0, Op1, Q, MaxRecurse))2520 return V;2521 2522 return nullptr;2523}2524 2525Value *llvm::simplifyOrInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) {2526 return ::simplifyOrInst(Op0, Op1, Q, RecursionLimit);2527}2528 2529/// Given operands for a Xor, see if we can fold the result.2530/// If not, this returns null.2531static Value *simplifyXorInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,2532 unsigned MaxRecurse) {2533 if (Constant *C = foldOrCommuteConstant(Instruction::Xor, Op0, Op1, Q))2534 return C;2535 2536 // X ^ poison -> poison2537 if (isa<PoisonValue>(Op1))2538 return Op1;2539 2540 // A ^ undef -> undef2541 if (Q.isUndefValue(Op1))2542 return Op1;2543 2544 // A ^ 0 = A2545 if (match(Op1, m_Zero()))2546 return Op0;2547 2548 // A ^ A = 02549 if (Op0 == Op1)2550 return Constant::getNullValue(Op0->getType());2551 2552 // A ^ ~A = ~A ^ A = -12553 if (match(Op0, m_Not(m_Specific(Op1))) || match(Op1, m_Not(m_Specific(Op0))))2554 return Constant::getAllOnesValue(Op0->getType());2555 2556 auto foldAndOrNot = [](Value *X, Value *Y) -> Value * {2557 Value *A, *B;2558 // (~A & B) ^ (A | B) --> A -- There are 8 commuted variants.2559 if (match(X, m_c_And(m_Not(m_Value(A)), m_Value(B))) &&2560 match(Y, m_c_Or(m_Specific(A), m_Specific(B))))2561 return A;2562 2563 // (~A | B) ^ (A & B) --> ~A -- There are 8 commuted variants.2564 // The 'not' op must contain a complete -1 operand (no undef elements for2565 // vector) for the transform to be safe.2566 Value *NotA;2567 if (match(X, m_c_Or(m_CombineAnd(m_Not(m_Value(A)), m_Value(NotA)),2568 m_Value(B))) &&2569 match(Y, m_c_And(m_Specific(A), m_Specific(B))))2570 return NotA;2571 2572 return nullptr;2573 };2574 if (Value *R = foldAndOrNot(Op0, Op1))2575 return R;2576 if (Value *R = foldAndOrNot(Op1, Op0))2577 return R;2578 2579 if (Value *V = simplifyLogicOfAddSub(Op0, Op1, Instruction::Xor))2580 return V;2581 2582 // Try some generic simplifications for associative operations.2583 if (Value *V =2584 simplifyAssociativeBinOp(Instruction::Xor, Op0, Op1, Q, MaxRecurse))2585 return V;2586 2587 // Threading Xor over selects and phi nodes is pointless, so don't bother.2588 // Threading over the select in "A ^ select(cond, B, C)" means evaluating2589 // "A^B" and "A^C" and seeing if they are equal; but they are equal if and2590 // only if B and C are equal. If B and C are equal then (since we assume2591 // that operands have already been simplified) "select(cond, B, C)" should2592 // have been simplified to the common value of B and C already. Analysing2593 // "A^B" and "A^C" thus gains nothing, but costs compile time. Similarly2594 // for threading over phi nodes.2595 2596 if (Value *V = simplifyByDomEq(Instruction::Xor, Op0, Op1, Q, MaxRecurse))2597 return V;2598 2599 // (xor (sub nuw C_Mask, X), C_Mask) -> X2600 {2601 Value *X;2602 if (match(Op0, m_NUWSub(m_Specific(Op1), m_Value(X))) &&2603 match(Op1, m_LowBitMask()))2604 return X;2605 }2606 2607 return nullptr;2608}2609 2610Value *llvm::simplifyXorInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) {2611 return ::simplifyXorInst(Op0, Op1, Q, RecursionLimit);2612}2613 2614static Type *getCompareTy(Value *Op) {2615 return CmpInst::makeCmpResultType(Op->getType());2616}2617 2618/// Rummage around inside V looking for something equivalent to the comparison2619/// "LHS Pred RHS". Return such a value if found, otherwise return null.2620/// Helper function for analyzing max/min idioms.2621static Value *extractEquivalentCondition(Value *V, CmpPredicate Pred,2622 Value *LHS, Value *RHS) {2623 SelectInst *SI = dyn_cast<SelectInst>(V);2624 if (!SI)2625 return nullptr;2626 CmpInst *Cmp = dyn_cast<CmpInst>(SI->getCondition());2627 if (!Cmp)2628 return nullptr;2629 Value *CmpLHS = Cmp->getOperand(0), *CmpRHS = Cmp->getOperand(1);2630 if (Pred == Cmp->getPredicate() && LHS == CmpLHS && RHS == CmpRHS)2631 return Cmp;2632 if (Pred == CmpInst::getSwappedPredicate(Cmp->getPredicate()) &&2633 LHS == CmpRHS && RHS == CmpLHS)2634 return Cmp;2635 return nullptr;2636}2637 2638/// Return true if the underlying object (storage) must be disjoint from2639/// storage returned by any noalias return call.2640static bool isAllocDisjoint(const Value *V) {2641 // For allocas, we consider only static ones (dynamic2642 // allocas might be transformed into calls to malloc not simultaneously2643 // live with the compared-to allocation). For globals, we exclude symbols2644 // that might be resolve lazily to symbols in another dynamically-loaded2645 // library (and, thus, could be malloc'ed by the implementation).2646 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V))2647 return AI->isStaticAlloca();2648 if (const GlobalValue *GV = dyn_cast<GlobalValue>(V))2649 return (GV->hasLocalLinkage() || GV->hasHiddenVisibility() ||2650 GV->hasProtectedVisibility() || GV->hasGlobalUnnamedAddr()) &&2651 !GV->isThreadLocal();2652 if (const Argument *A = dyn_cast<Argument>(V))2653 return A->hasByValAttr();2654 return false;2655}2656 2657/// Return true if V1 and V2 are each the base of some distict storage region2658/// [V, object_size(V)] which do not overlap. Note that zero sized regions2659/// *are* possible, and that zero sized regions do not overlap with any other.2660static bool haveNonOverlappingStorage(const Value *V1, const Value *V2) {2661 // Global variables always exist, so they always exist during the lifetime2662 // of each other and all allocas. Global variables themselves usually have2663 // non-overlapping storage, but since their addresses are constants, the2664 // case involving two globals does not reach here and is instead handled in2665 // constant folding.2666 //2667 // Two different allocas usually have different addresses...2668 //2669 // However, if there's an @llvm.stackrestore dynamically in between two2670 // allocas, they may have the same address. It's tempting to reduce the2671 // scope of the problem by only looking at *static* allocas here. That would2672 // cover the majority of allocas while significantly reducing the likelihood2673 // of having an @llvm.stackrestore pop up in the middle. However, it's not2674 // actually impossible for an @llvm.stackrestore to pop up in the middle of2675 // an entry block. Also, if we have a block that's not attached to a2676 // function, we can't tell if it's "static" under the current definition.2677 // Theoretically, this problem could be fixed by creating a new kind of2678 // instruction kind specifically for static allocas. Such a new instruction2679 // could be required to be at the top of the entry block, thus preventing it2680 // from being subject to a @llvm.stackrestore. Instcombine could even2681 // convert regular allocas into these special allocas. It'd be nifty.2682 // However, until then, this problem remains open.2683 //2684 // So, we'll assume that two non-empty allocas have different addresses2685 // for now.2686 auto isByValArg = [](const Value *V) {2687 const Argument *A = dyn_cast<Argument>(V);2688 return A && A->hasByValAttr();2689 };2690 2691 // Byval args are backed by store which does not overlap with each other,2692 // allocas, or globals.2693 if (isByValArg(V1))2694 return isa<AllocaInst>(V2) || isa<GlobalVariable>(V2) || isByValArg(V2);2695 if (isByValArg(V2))2696 return isa<AllocaInst>(V1) || isa<GlobalVariable>(V1) || isByValArg(V1);2697 2698 return isa<AllocaInst>(V1) &&2699 (isa<AllocaInst>(V2) || isa<GlobalVariable>(V2));2700}2701 2702// A significant optimization not implemented here is assuming that alloca2703// addresses are not equal to incoming argument values. They don't *alias*,2704// as we say, but that doesn't mean they aren't equal, so we take a2705// conservative approach.2706//2707// This is inspired in part by C++11 5.10p1:2708// "Two pointers of the same type compare equal if and only if they are both2709// null, both point to the same function, or both represent the same2710// address."2711//2712// This is pretty permissive.2713//2714// It's also partly due to C11 6.5.9p6:2715// "Two pointers compare equal if and only if both are null pointers, both are2716// pointers to the same object (including a pointer to an object and a2717// subobject at its beginning) or function, both are pointers to one past the2718// last element of the same array object, or one is a pointer to one past the2719// end of one array object and the other is a pointer to the start of a2720// different array object that happens to immediately follow the first array2721// object in the address space.)2722//2723// C11's version is more restrictive, however there's no reason why an argument2724// couldn't be a one-past-the-end value for a stack object in the caller and be2725// equal to the beginning of a stack object in the callee.2726//2727// If the C and C++ standards are ever made sufficiently restrictive in this2728// area, it may be possible to update LLVM's semantics accordingly and reinstate2729// this optimization.2730static Constant *computePointerICmp(CmpPredicate Pred, Value *LHS, Value *RHS,2731 const SimplifyQuery &Q) {2732 assert(LHS->getType() == RHS->getType() && "Must have same types");2733 const DataLayout &DL = Q.DL;2734 const TargetLibraryInfo *TLI = Q.TLI;2735 2736 // We fold equality and unsigned predicates on pointer comparisons, but forbid2737 // signed predicates since a GEP with inbounds could cross the sign boundary.2738 if (CmpInst::isSigned(Pred))2739 return nullptr;2740 2741 // We have to switch to a signed predicate to handle negative indices from2742 // the base pointer.2743 Pred = ICmpInst::getSignedPredicate(Pred);2744 2745 // Strip off any constant offsets so that we can reason about them.2746 // It's tempting to use getUnderlyingObject or even just stripInBoundsOffsets2747 // here and compare base addresses like AliasAnalysis does, however there are2748 // numerous hazards. AliasAnalysis and its utilities rely on special rules2749 // governing loads and stores which don't apply to icmps. Also, AliasAnalysis2750 // doesn't need to guarantee pointer inequality when it says NoAlias.2751 2752 // Even if an non-inbounds GEP occurs along the path we can still optimize2753 // equality comparisons concerning the result.2754 bool AllowNonInbounds = ICmpInst::isEquality(Pred);2755 unsigned IndexSize = DL.getIndexTypeSizeInBits(LHS->getType());2756 APInt LHSOffset(IndexSize, 0), RHSOffset(IndexSize, 0);2757 LHS = LHS->stripAndAccumulateConstantOffsets(DL, LHSOffset, AllowNonInbounds);2758 RHS = RHS->stripAndAccumulateConstantOffsets(DL, RHSOffset, AllowNonInbounds);2759 2760 // If LHS and RHS are related via constant offsets to the same base2761 // value, we can replace it with an icmp which just compares the offsets.2762 if (LHS == RHS)2763 return ConstantInt::get(getCompareTy(LHS),2764 ICmpInst::compare(LHSOffset, RHSOffset, Pred));2765 2766 // Various optimizations for (in)equality comparisons.2767 if (ICmpInst::isEquality(Pred)) {2768 // Different non-empty allocations that exist at the same time have2769 // different addresses (if the program can tell). If the offsets are2770 // within the bounds of their allocations (and not one-past-the-end!2771 // so we can't use inbounds!), and their allocations aren't the same,2772 // the pointers are not equal.2773 if (haveNonOverlappingStorage(LHS, RHS)) {2774 uint64_t LHSSize, RHSSize;2775 ObjectSizeOpts Opts;2776 Opts.EvalMode = ObjectSizeOpts::Mode::Min;2777 auto *F = [](Value *V) -> Function * {2778 if (auto *I = dyn_cast<Instruction>(V))2779 return I->getFunction();2780 if (auto *A = dyn_cast<Argument>(V))2781 return A->getParent();2782 return nullptr;2783 }(LHS);2784 Opts.NullIsUnknownSize = F ? NullPointerIsDefined(F) : true;2785 if (getObjectSize(LHS, LHSSize, DL, TLI, Opts) && LHSSize != 0 &&2786 getObjectSize(RHS, RHSSize, DL, TLI, Opts) && RHSSize != 0) {2787 APInt Dist = LHSOffset - RHSOffset;2788 if (Dist.isNonNegative() ? Dist.ult(LHSSize) : (-Dist).ult(RHSSize))2789 return ConstantInt::get(getCompareTy(LHS),2790 !CmpInst::isTrueWhenEqual(Pred));2791 }2792 }2793 2794 // If one side of the equality comparison must come from a noalias call2795 // (meaning a system memory allocation function), and the other side must2796 // come from a pointer that cannot overlap with dynamically-allocated2797 // memory within the lifetime of the current function (allocas, byval2798 // arguments, globals), then determine the comparison result here.2799 SmallVector<const Value *, 8> LHSUObjs, RHSUObjs;2800 getUnderlyingObjects(LHS, LHSUObjs);2801 getUnderlyingObjects(RHS, RHSUObjs);2802 2803 // Is the set of underlying objects all noalias calls?2804 auto IsNAC = [](ArrayRef<const Value *> Objects) {2805 return all_of(Objects, isNoAliasCall);2806 };2807 2808 // Is the set of underlying objects all things which must be disjoint from2809 // noalias calls. We assume that indexing from such disjoint storage2810 // into the heap is undefined, and thus offsets can be safely ignored.2811 auto IsAllocDisjoint = [](ArrayRef<const Value *> Objects) {2812 return all_of(Objects, ::isAllocDisjoint);2813 };2814 2815 if ((IsNAC(LHSUObjs) && IsAllocDisjoint(RHSUObjs)) ||2816 (IsNAC(RHSUObjs) && IsAllocDisjoint(LHSUObjs)))2817 return ConstantInt::get(getCompareTy(LHS),2818 !CmpInst::isTrueWhenEqual(Pred));2819 2820 // Fold comparisons for non-escaping pointer even if the allocation call2821 // cannot be elided. We cannot fold malloc comparison to null. Also, the2822 // dynamic allocation call could be either of the operands. Note that2823 // the other operand can not be based on the alloc - if it were, then2824 // the cmp itself would be a capture.2825 Value *MI = nullptr;2826 if (isAllocLikeFn(LHS, TLI) && llvm::isKnownNonZero(RHS, Q))2827 MI = LHS;2828 else if (isAllocLikeFn(RHS, TLI) && llvm::isKnownNonZero(LHS, Q))2829 MI = RHS;2830 if (MI) {2831 // FIXME: This is incorrect, see PR54002. While we can assume that the2832 // allocation is at an address that makes the comparison false, this2833 // requires that *all* comparisons to that address be false, which2834 // InstSimplify cannot guarantee.2835 struct CustomCaptureTracker : public CaptureTracker {2836 bool Captured = false;2837 void tooManyUses() override { Captured = true; }2838 Action captured(const Use *U, UseCaptureInfo CI) override {2839 // TODO(captures): Use UseCaptureInfo.2840 if (auto *ICmp = dyn_cast<ICmpInst>(U->getUser())) {2841 // Comparison against value stored in global variable. Given the2842 // pointer does not escape, its value cannot be guessed and stored2843 // separately in a global variable.2844 unsigned OtherIdx = 1 - U->getOperandNo();2845 auto *LI = dyn_cast<LoadInst>(ICmp->getOperand(OtherIdx));2846 if (LI && isa<GlobalVariable>(LI->getPointerOperand()))2847 return Continue;2848 }2849 2850 Captured = true;2851 return Stop;2852 }2853 };2854 CustomCaptureTracker Tracker;2855 PointerMayBeCaptured(MI, &Tracker);2856 if (!Tracker.Captured)2857 return ConstantInt::get(getCompareTy(LHS),2858 CmpInst::isFalseWhenEqual(Pred));2859 }2860 }2861 2862 // Otherwise, fail.2863 return nullptr;2864}2865 2866/// Fold an icmp when its operands have i1 scalar type.2867static Value *simplifyICmpOfBools(CmpPredicate Pred, Value *LHS, Value *RHS,2868 const SimplifyQuery &Q) {2869 Type *ITy = getCompareTy(LHS); // The return type.2870 Type *OpTy = LHS->getType(); // The operand type.2871 if (!OpTy->isIntOrIntVectorTy(1))2872 return nullptr;2873 2874 // A boolean compared to true/false can be reduced in 14 out of the 202875 // (10 predicates * 2 constants) possible combinations. The other2876 // 6 cases require a 'not' of the LHS.2877 2878 auto ExtractNotLHS = [](Value *V) -> Value * {2879 Value *X;2880 if (match(V, m_Not(m_Value(X))))2881 return X;2882 return nullptr;2883 };2884 2885 if (match(RHS, m_Zero())) {2886 switch (Pred) {2887 case CmpInst::ICMP_NE: // X != 0 -> X2888 case CmpInst::ICMP_UGT: // X >u 0 -> X2889 case CmpInst::ICMP_SLT: // X <s 0 -> X2890 return LHS;2891 2892 case CmpInst::ICMP_EQ: // not(X) == 0 -> X != 0 -> X2893 case CmpInst::ICMP_ULE: // not(X) <=u 0 -> X >u 0 -> X2894 case CmpInst::ICMP_SGE: // not(X) >=s 0 -> X <s 0 -> X2895 if (Value *X = ExtractNotLHS(LHS))2896 return X;2897 break;2898 2899 case CmpInst::ICMP_ULT: // X <u 0 -> false2900 case CmpInst::ICMP_SGT: // X >s 0 -> false2901 return getFalse(ITy);2902 2903 case CmpInst::ICMP_UGE: // X >=u 0 -> true2904 case CmpInst::ICMP_SLE: // X <=s 0 -> true2905 return getTrue(ITy);2906 2907 default:2908 break;2909 }2910 } else if (match(RHS, m_One())) {2911 switch (Pred) {2912 case CmpInst::ICMP_EQ: // X == 1 -> X2913 case CmpInst::ICMP_UGE: // X >=u 1 -> X2914 case CmpInst::ICMP_SLE: // X <=s -1 -> X2915 return LHS;2916 2917 case CmpInst::ICMP_NE: // not(X) != 1 -> X == 1 -> X2918 case CmpInst::ICMP_ULT: // not(X) <=u 1 -> X >=u 1 -> X2919 case CmpInst::ICMP_SGT: // not(X) >s 1 -> X <=s -1 -> X2920 if (Value *X = ExtractNotLHS(LHS))2921 return X;2922 break;2923 2924 case CmpInst::ICMP_UGT: // X >u 1 -> false2925 case CmpInst::ICMP_SLT: // X <s -1 -> false2926 return getFalse(ITy);2927 2928 case CmpInst::ICMP_ULE: // X <=u 1 -> true2929 case CmpInst::ICMP_SGE: // X >=s -1 -> true2930 return getTrue(ITy);2931 2932 default:2933 break;2934 }2935 }2936 2937 switch (Pred) {2938 default:2939 break;2940 case ICmpInst::ICMP_UGE:2941 if (isImpliedCondition(RHS, LHS, Q.DL).value_or(false))2942 return getTrue(ITy);2943 break;2944 case ICmpInst::ICMP_SGE:2945 /// For signed comparison, the values for an i1 are 0 and -12946 /// respectively. This maps into a truth table of:2947 /// LHS | RHS | LHS >=s RHS | LHS implies RHS2948 /// 0 | 0 | 1 (0 >= 0) | 12949 /// 0 | 1 | 1 (0 >= -1) | 12950 /// 1 | 0 | 0 (-1 >= 0) | 02951 /// 1 | 1 | 1 (-1 >= -1) | 12952 if (isImpliedCondition(LHS, RHS, Q.DL).value_or(false))2953 return getTrue(ITy);2954 break;2955 case ICmpInst::ICMP_ULE:2956 if (isImpliedCondition(LHS, RHS, Q.DL).value_or(false))2957 return getTrue(ITy);2958 break;2959 case ICmpInst::ICMP_SLE:2960 /// SLE follows the same logic as SGE with the LHS and RHS swapped.2961 if (isImpliedCondition(RHS, LHS, Q.DL).value_or(false))2962 return getTrue(ITy);2963 break;2964 }2965 2966 return nullptr;2967}2968 2969/// Try hard to fold icmp with zero RHS because this is a common case.2970static Value *simplifyICmpWithZero(CmpPredicate Pred, Value *LHS, Value *RHS,2971 const SimplifyQuery &Q) {2972 if (!match(RHS, m_Zero()))2973 return nullptr;2974 2975 Type *ITy = getCompareTy(LHS); // The return type.2976 switch (Pred) {2977 default:2978 llvm_unreachable("Unknown ICmp predicate!");2979 case ICmpInst::ICMP_ULT:2980 return getFalse(ITy);2981 case ICmpInst::ICMP_UGE:2982 return getTrue(ITy);2983 case ICmpInst::ICMP_EQ:2984 case ICmpInst::ICMP_ULE:2985 if (isKnownNonZero(LHS, Q))2986 return getFalse(ITy);2987 break;2988 case ICmpInst::ICMP_NE:2989 case ICmpInst::ICMP_UGT:2990 if (isKnownNonZero(LHS, Q))2991 return getTrue(ITy);2992 break;2993 case ICmpInst::ICMP_SLT: {2994 KnownBits LHSKnown = computeKnownBits(LHS, Q);2995 if (LHSKnown.isNegative())2996 return getTrue(ITy);2997 if (LHSKnown.isNonNegative())2998 return getFalse(ITy);2999 break;3000 }3001 case ICmpInst::ICMP_SLE: {3002 KnownBits LHSKnown = computeKnownBits(LHS, Q);3003 if (LHSKnown.isNegative())3004 return getTrue(ITy);3005 if (LHSKnown.isNonNegative() && isKnownNonZero(LHS, Q))3006 return getFalse(ITy);3007 break;3008 }3009 case ICmpInst::ICMP_SGE: {3010 KnownBits LHSKnown = computeKnownBits(LHS, Q);3011 if (LHSKnown.isNegative())3012 return getFalse(ITy);3013 if (LHSKnown.isNonNegative())3014 return getTrue(ITy);3015 break;3016 }3017 case ICmpInst::ICMP_SGT: {3018 KnownBits LHSKnown = computeKnownBits(LHS, Q);3019 if (LHSKnown.isNegative())3020 return getFalse(ITy);3021 if (LHSKnown.isNonNegative() && isKnownNonZero(LHS, Q))3022 return getTrue(ITy);3023 break;3024 }3025 }3026 3027 return nullptr;3028}3029 3030static Value *simplifyICmpWithConstant(CmpPredicate Pred, Value *LHS,3031 Value *RHS, const SimplifyQuery &Q) {3032 Type *ITy = getCompareTy(RHS); // The return type.3033 3034 Value *X;3035 const APInt *C;3036 if (!match(RHS, m_APIntAllowPoison(C)))3037 return nullptr;3038 3039 // Sign-bit checks can be optimized to true/false after unsigned3040 // floating-point casts:3041 // icmp slt (bitcast (uitofp X)), 0 --> false3042 // icmp sgt (bitcast (uitofp X)), -1 --> true3043 if (match(LHS, m_ElementWiseBitCast(m_UIToFP(m_Value(X))))) {3044 bool TrueIfSigned;3045 if (isSignBitCheck(Pred, *C, TrueIfSigned))3046 return ConstantInt::getBool(ITy, !TrueIfSigned);3047 }3048 3049 // Rule out tautological comparisons (eg., ult 0 or uge 0).3050 ConstantRange RHS_CR = ConstantRange::makeExactICmpRegion(Pred, *C);3051 if (RHS_CR.isEmptySet())3052 return ConstantInt::getFalse(ITy);3053 if (RHS_CR.isFullSet())3054 return ConstantInt::getTrue(ITy);3055 3056 ConstantRange LHS_CR =3057 computeConstantRange(LHS, CmpInst::isSigned(Pred), Q.IIQ.UseInstrInfo);3058 if (!LHS_CR.isFullSet()) {3059 if (RHS_CR.contains(LHS_CR))3060 return ConstantInt::getTrue(ITy);3061 if (RHS_CR.inverse().contains(LHS_CR))3062 return ConstantInt::getFalse(ITy);3063 }3064 3065 // (mul nuw/nsw X, MulC) != C --> true (if C is not a multiple of MulC)3066 // (mul nuw/nsw X, MulC) == C --> false (if C is not a multiple of MulC)3067 const APInt *MulC;3068 if (Q.IIQ.UseInstrInfo && ICmpInst::isEquality(Pred) &&3069 ((match(LHS, m_NUWMul(m_Value(), m_APIntAllowPoison(MulC))) &&3070 *MulC != 0 && C->urem(*MulC) != 0) ||3071 (match(LHS, m_NSWMul(m_Value(), m_APIntAllowPoison(MulC))) &&3072 *MulC != 0 && C->srem(*MulC) != 0)))3073 return ConstantInt::get(ITy, Pred == ICmpInst::ICMP_NE);3074 3075 if (Pred == ICmpInst::ICMP_UGE && C->isOne() && isKnownNonZero(LHS, Q))3076 return ConstantInt::getTrue(ITy);3077 3078 return nullptr;3079}3080 3081enum class MonotonicType { GreaterEq, LowerEq };3082 3083/// Get values V_i such that V uge V_i (GreaterEq) or V ule V_i (LowerEq).3084static void getUnsignedMonotonicValues(SmallPtrSetImpl<Value *> &Res, Value *V,3085 MonotonicType Type,3086 const SimplifyQuery &Q,3087 unsigned Depth = 0) {3088 if (!Res.insert(V).second)3089 return;3090 3091 // Can be increased if useful.3092 if (++Depth > 1)3093 return;3094 3095 auto *I = dyn_cast<Instruction>(V);3096 if (!I)3097 return;3098 3099 Value *X, *Y;3100 if (Type == MonotonicType::GreaterEq) {3101 if (match(I, m_Or(m_Value(X), m_Value(Y))) ||3102 match(I, m_Intrinsic<Intrinsic::uadd_sat>(m_Value(X), m_Value(Y)))) {3103 getUnsignedMonotonicValues(Res, X, Type, Q, Depth);3104 getUnsignedMonotonicValues(Res, Y, Type, Q, Depth);3105 }3106 // X * Y >= X --> true3107 if (match(I, m_NUWMul(m_Value(X), m_Value(Y)))) {3108 if (isKnownNonZero(X, Q))3109 getUnsignedMonotonicValues(Res, Y, Type, Q, Depth);3110 if (isKnownNonZero(Y, Q))3111 getUnsignedMonotonicValues(Res, X, Type, Q, Depth);3112 }3113 } else {3114 assert(Type == MonotonicType::LowerEq);3115 switch (I->getOpcode()) {3116 case Instruction::And:3117 getUnsignedMonotonicValues(Res, I->getOperand(0), Type, Q, Depth);3118 getUnsignedMonotonicValues(Res, I->getOperand(1), Type, Q, Depth);3119 break;3120 case Instruction::URem:3121 case Instruction::UDiv:3122 case Instruction::LShr:3123 getUnsignedMonotonicValues(Res, I->getOperand(0), Type, Q, Depth);3124 break;3125 case Instruction::Call:3126 if (match(I, m_Intrinsic<Intrinsic::usub_sat>(m_Value(X))))3127 getUnsignedMonotonicValues(Res, X, Type, Q, Depth);3128 break;3129 default:3130 break;3131 }3132 }3133}3134 3135static Value *simplifyICmpUsingMonotonicValues(CmpPredicate Pred, Value *LHS,3136 Value *RHS,3137 const SimplifyQuery &Q) {3138 if (Pred != ICmpInst::ICMP_UGE && Pred != ICmpInst::ICMP_ULT)3139 return nullptr;3140 3141 // We have LHS uge GreaterValues and LowerValues uge RHS. If any of the3142 // GreaterValues and LowerValues are the same, it follows that LHS uge RHS.3143 SmallPtrSet<Value *, 4> GreaterValues;3144 SmallPtrSet<Value *, 4> LowerValues;3145 getUnsignedMonotonicValues(GreaterValues, LHS, MonotonicType::GreaterEq, Q);3146 getUnsignedMonotonicValues(LowerValues, RHS, MonotonicType::LowerEq, Q);3147 for (Value *GV : GreaterValues)3148 if (LowerValues.contains(GV))3149 return ConstantInt::getBool(getCompareTy(LHS),3150 Pred == ICmpInst::ICMP_UGE);3151 return nullptr;3152}3153 3154static Value *simplifyICmpWithBinOpOnLHS(CmpPredicate Pred, BinaryOperator *LBO,3155 Value *RHS, const SimplifyQuery &Q,3156 unsigned MaxRecurse) {3157 Type *ITy = getCompareTy(RHS); // The return type.3158 3159 Value *Y = nullptr;3160 // icmp pred (or X, Y), X3161 if (match(LBO, m_c_Or(m_Value(Y), m_Specific(RHS)))) {3162 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) {3163 KnownBits RHSKnown = computeKnownBits(RHS, Q);3164 KnownBits YKnown = computeKnownBits(Y, Q);3165 if (RHSKnown.isNonNegative() && YKnown.isNegative())3166 return Pred == ICmpInst::ICMP_SLT ? getTrue(ITy) : getFalse(ITy);3167 if (RHSKnown.isNegative() || YKnown.isNonNegative())3168 return Pred == ICmpInst::ICMP_SLT ? getFalse(ITy) : getTrue(ITy);3169 }3170 }3171 3172 // icmp pred (urem X, Y), Y3173 if (match(LBO, m_URem(m_Value(), m_Specific(RHS)))) {3174 switch (Pred) {3175 default:3176 break;3177 case ICmpInst::ICMP_SGT:3178 case ICmpInst::ICMP_SGE: {3179 KnownBits Known = computeKnownBits(RHS, Q);3180 if (!Known.isNonNegative())3181 break;3182 [[fallthrough]];3183 }3184 case ICmpInst::ICMP_EQ:3185 case ICmpInst::ICMP_UGT:3186 case ICmpInst::ICMP_UGE:3187 return getFalse(ITy);3188 case ICmpInst::ICMP_SLT:3189 case ICmpInst::ICMP_SLE: {3190 KnownBits Known = computeKnownBits(RHS, Q);3191 if (!Known.isNonNegative())3192 break;3193 [[fallthrough]];3194 }3195 case ICmpInst::ICMP_NE:3196 case ICmpInst::ICMP_ULT:3197 case ICmpInst::ICMP_ULE:3198 return getTrue(ITy);3199 }3200 }3201 3202 // If x is nonzero:3203 // x >>u C <u x --> true for C != 0.3204 // x >>u C != x --> true for C != 0.3205 // x >>u C >=u x --> false for C != 0.3206 // x >>u C == x --> false for C != 0.3207 // x udiv C <u x --> true for C != 1.3208 // x udiv C != x --> true for C != 1.3209 // x udiv C >=u x --> false for C != 1.3210 // x udiv C == x --> false for C != 1.3211 // TODO: allow non-constant shift amount/divisor3212 const APInt *C;3213 if ((match(LBO, m_LShr(m_Specific(RHS), m_APInt(C))) && *C != 0) ||3214 (match(LBO, m_UDiv(m_Specific(RHS), m_APInt(C))) && *C != 1)) {3215 if (isKnownNonZero(RHS, Q)) {3216 switch (Pred) {3217 default:3218 break;3219 case ICmpInst::ICMP_EQ:3220 case ICmpInst::ICMP_UGE:3221 case ICmpInst::ICMP_UGT:3222 return getFalse(ITy);3223 case ICmpInst::ICMP_NE:3224 case ICmpInst::ICMP_ULT:3225 case ICmpInst::ICMP_ULE:3226 return getTrue(ITy);3227 }3228 }3229 }3230 3231 // (x*C1)/C2 <= x for C1 <= C2.3232 // This holds even if the multiplication overflows: Assume that x != 0 and3233 // arithmetic is modulo M. For overflow to occur we must have C1 >= M/x and3234 // thus C2 >= M/x. It follows that (x*C1)/C2 <= (M-1)/C2 <= ((M-1)*x)/M < x.3235 //3236 // Additionally, either the multiplication and division might be represented3237 // as shifts:3238 // (x*C1)>>C2 <= x for C1 < 2**C2.3239 // (x<<C1)/C2 <= x for 2**C1 < C2.3240 const APInt *C1, *C2;3241 if ((match(LBO, m_UDiv(m_Mul(m_Specific(RHS), m_APInt(C1)), m_APInt(C2))) &&3242 C1->ule(*C2)) ||3243 (match(LBO, m_LShr(m_Mul(m_Specific(RHS), m_APInt(C1)), m_APInt(C2))) &&3244 C1->ule(APInt(C2->getBitWidth(), 1) << *C2)) ||3245 (match(LBO, m_UDiv(m_Shl(m_Specific(RHS), m_APInt(C1)), m_APInt(C2))) &&3246 (APInt(C1->getBitWidth(), 1) << *C1).ule(*C2))) {3247 if (Pred == ICmpInst::ICMP_UGT)3248 return getFalse(ITy);3249 if (Pred == ICmpInst::ICMP_ULE)3250 return getTrue(ITy);3251 }3252 3253 // (sub C, X) == X, C is odd --> false3254 // (sub C, X) != X, C is odd --> true3255 if (match(LBO, m_Sub(m_APIntAllowPoison(C), m_Specific(RHS))) &&3256 (*C & 1) == 1 && ICmpInst::isEquality(Pred))3257 return (Pred == ICmpInst::ICMP_EQ) ? getFalse(ITy) : getTrue(ITy);3258 3259 return nullptr;3260}3261 3262// If only one of the icmp's operands has NSW flags, try to prove that:3263//3264// icmp slt/sgt/sle/sge (x + C1), (x +nsw C2)3265//3266// is equivalent to:3267//3268// icmp slt/sgt/sle/sge C1, C23269//3270// which is true if x + C2 has the NSW flags set and:3271// *) C1 <= C2 && C1 >= 0, or3272// *) C2 <= C1 && C1 <= 0.3273//3274static bool trySimplifyICmpWithAdds(CmpPredicate Pred, Value *LHS, Value *RHS,3275 const InstrInfoQuery &IIQ) {3276 // TODO: support other predicates.3277 if (!ICmpInst::isSigned(Pred) || !IIQ.UseInstrInfo)3278 return false;3279 3280 // Canonicalize nsw add as RHS.3281 if (!match(RHS, m_NSWAdd(m_Value(), m_Value())))3282 std::swap(LHS, RHS);3283 if (!match(RHS, m_NSWAdd(m_Value(), m_Value())))3284 return false;3285 3286 Value *X;3287 const APInt *C1, *C2;3288 if (!match(LHS, m_Add(m_Value(X), m_APInt(C1))) ||3289 !match(RHS, m_Add(m_Specific(X), m_APInt(C2))))3290 return false;3291 3292 return (C1->sle(*C2) && C1->isNonNegative()) ||3293 (C2->sle(*C1) && C1->isNonPositive());3294}3295 3296/// TODO: A large part of this logic is duplicated in InstCombine's3297/// foldICmpBinOp(). We should be able to share that and avoid the code3298/// duplication.3299static Value *simplifyICmpWithBinOp(CmpPredicate Pred, Value *LHS, Value *RHS,3300 const SimplifyQuery &Q,3301 unsigned MaxRecurse) {3302 BinaryOperator *LBO = dyn_cast<BinaryOperator>(LHS);3303 BinaryOperator *RBO = dyn_cast<BinaryOperator>(RHS);3304 if (MaxRecurse && (LBO || RBO)) {3305 // Analyze the case when either LHS or RHS is an add instruction.3306 Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr;3307 // LHS = A + B (or A and B are null); RHS = C + D (or C and D are null).3308 bool NoLHSWrapProblem = false, NoRHSWrapProblem = false;3309 if (LBO && LBO->getOpcode() == Instruction::Add) {3310 A = LBO->getOperand(0);3311 B = LBO->getOperand(1);3312 NoLHSWrapProblem =3313 ICmpInst::isEquality(Pred) ||3314 (CmpInst::isUnsigned(Pred) &&3315 Q.IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(LBO))) ||3316 (CmpInst::isSigned(Pred) &&3317 Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(LBO)));3318 }3319 if (RBO && RBO->getOpcode() == Instruction::Add) {3320 C = RBO->getOperand(0);3321 D = RBO->getOperand(1);3322 NoRHSWrapProblem =3323 ICmpInst::isEquality(Pred) ||3324 (CmpInst::isUnsigned(Pred) &&3325 Q.IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(RBO))) ||3326 (CmpInst::isSigned(Pred) &&3327 Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(RBO)));3328 }3329 3330 // icmp (X+Y), X -> icmp Y, 0 for equalities or if there is no overflow.3331 if ((A == RHS || B == RHS) && NoLHSWrapProblem)3332 if (Value *V = simplifyICmpInst(Pred, A == RHS ? B : A,3333 Constant::getNullValue(RHS->getType()), Q,3334 MaxRecurse - 1))3335 return V;3336 3337 // icmp X, (X+Y) -> icmp 0, Y for equalities or if there is no overflow.3338 if ((C == LHS || D == LHS) && NoRHSWrapProblem)3339 if (Value *V =3340 simplifyICmpInst(Pred, Constant::getNullValue(LHS->getType()),3341 C == LHS ? D : C, Q, MaxRecurse - 1))3342 return V;3343 3344 // icmp (X+Y), (X+Z) -> icmp Y,Z for equalities or if there is no overflow.3345 bool CanSimplify = (NoLHSWrapProblem && NoRHSWrapProblem) ||3346 trySimplifyICmpWithAdds(Pred, LHS, RHS, Q.IIQ);3347 if (A && C && (A == C || A == D || B == C || B == D) && CanSimplify) {3348 // Determine Y and Z in the form icmp (X+Y), (X+Z).3349 Value *Y, *Z;3350 if (A == C) {3351 // C + B == C + D -> B == D3352 Y = B;3353 Z = D;3354 } else if (A == D) {3355 // D + B == C + D -> B == C3356 Y = B;3357 Z = C;3358 } else if (B == C) {3359 // A + C == C + D -> A == D3360 Y = A;3361 Z = D;3362 } else {3363 assert(B == D);3364 // A + D == C + D -> A == C3365 Y = A;3366 Z = C;3367 }3368 if (Value *V = simplifyICmpInst(Pred, Y, Z, Q, MaxRecurse - 1))3369 return V;3370 }3371 }3372 3373 if (LBO)3374 if (Value *V = simplifyICmpWithBinOpOnLHS(Pred, LBO, RHS, Q, MaxRecurse))3375 return V;3376 3377 if (RBO)3378 if (Value *V = simplifyICmpWithBinOpOnLHS(3379 ICmpInst::getSwappedPredicate(Pred), RBO, LHS, Q, MaxRecurse))3380 return V;3381 3382 // 0 - (zext X) pred C3383 if (!CmpInst::isUnsigned(Pred) && match(LHS, m_Neg(m_ZExt(m_Value())))) {3384 const APInt *C;3385 if (match(RHS, m_APInt(C))) {3386 if (C->isStrictlyPositive()) {3387 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_NE)3388 return ConstantInt::getTrue(getCompareTy(RHS));3389 if (Pred == ICmpInst::ICMP_SGE || Pred == ICmpInst::ICMP_EQ)3390 return ConstantInt::getFalse(getCompareTy(RHS));3391 }3392 if (C->isNonNegative()) {3393 if (Pred == ICmpInst::ICMP_SLE)3394 return ConstantInt::getTrue(getCompareTy(RHS));3395 if (Pred == ICmpInst::ICMP_SGT)3396 return ConstantInt::getFalse(getCompareTy(RHS));3397 }3398 }3399 }3400 3401 // If C2 is a power-of-2 and C is not:3402 // (C2 << X) == C --> false3403 // (C2 << X) != C --> true3404 const APInt *C;3405 if (match(LHS, m_Shl(m_Power2(), m_Value())) &&3406 match(RHS, m_APIntAllowPoison(C)) && !C->isPowerOf2()) {3407 // C2 << X can equal zero in some circumstances.3408 // This simplification might be unsafe if C is zero.3409 //3410 // We know it is safe if:3411 // - The shift is nsw. We can't shift out the one bit.3412 // - The shift is nuw. We can't shift out the one bit.3413 // - C2 is one.3414 // - C isn't zero.3415 if (Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(LBO)) ||3416 Q.IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(LBO)) ||3417 match(LHS, m_Shl(m_One(), m_Value())) || !C->isZero()) {3418 if (Pred == ICmpInst::ICMP_EQ)3419 return ConstantInt::getFalse(getCompareTy(RHS));3420 if (Pred == ICmpInst::ICMP_NE)3421 return ConstantInt::getTrue(getCompareTy(RHS));3422 }3423 }3424 3425 // If C is a power-of-2:3426 // (C << X) >u 0x8000 --> false3427 // (C << X) <=u 0x8000 --> true3428 if (match(LHS, m_Shl(m_Power2(), m_Value())) && match(RHS, m_SignMask())) {3429 if (Pred == ICmpInst::ICMP_UGT)3430 return ConstantInt::getFalse(getCompareTy(RHS));3431 if (Pred == ICmpInst::ICMP_ULE)3432 return ConstantInt::getTrue(getCompareTy(RHS));3433 }3434 3435 if (!MaxRecurse || !LBO || !RBO || LBO->getOpcode() != RBO->getOpcode())3436 return nullptr;3437 3438 if (LBO->getOperand(0) == RBO->getOperand(0)) {3439 switch (LBO->getOpcode()) {3440 default:3441 break;3442 case Instruction::Shl: {3443 bool NUW = Q.IIQ.hasNoUnsignedWrap(LBO) && Q.IIQ.hasNoUnsignedWrap(RBO);3444 bool NSW = Q.IIQ.hasNoSignedWrap(LBO) && Q.IIQ.hasNoSignedWrap(RBO);3445 if (!NUW || (ICmpInst::isSigned(Pred) && !NSW) ||3446 !isKnownNonZero(LBO->getOperand(0), Q))3447 break;3448 if (Value *V = simplifyICmpInst(Pred, LBO->getOperand(1),3449 RBO->getOperand(1), Q, MaxRecurse - 1))3450 return V;3451 break;3452 }3453 // If C1 & C2 == C1, A = X and/or C1, B = X and/or C2:3454 // icmp ule A, B -> true3455 // icmp ugt A, B -> false3456 // icmp sle A, B -> true (C1 and C2 are the same sign)3457 // icmp sgt A, B -> false (C1 and C2 are the same sign)3458 case Instruction::And:3459 case Instruction::Or: {3460 const APInt *C1, *C2;3461 if (ICmpInst::isRelational(Pred) &&3462 match(LBO->getOperand(1), m_APInt(C1)) &&3463 match(RBO->getOperand(1), m_APInt(C2))) {3464 if (!C1->isSubsetOf(*C2)) {3465 std::swap(C1, C2);3466 Pred = ICmpInst::getSwappedPredicate(Pred);3467 }3468 if (C1->isSubsetOf(*C2)) {3469 if (Pred == ICmpInst::ICMP_ULE)3470 return ConstantInt::getTrue(getCompareTy(LHS));3471 if (Pred == ICmpInst::ICMP_UGT)3472 return ConstantInt::getFalse(getCompareTy(LHS));3473 if (C1->isNonNegative() == C2->isNonNegative()) {3474 if (Pred == ICmpInst::ICMP_SLE)3475 return ConstantInt::getTrue(getCompareTy(LHS));3476 if (Pred == ICmpInst::ICMP_SGT)3477 return ConstantInt::getFalse(getCompareTy(LHS));3478 }3479 }3480 }3481 break;3482 }3483 }3484 }3485 3486 if (LBO->getOperand(1) == RBO->getOperand(1)) {3487 switch (LBO->getOpcode()) {3488 default:3489 break;3490 case Instruction::UDiv:3491 case Instruction::LShr:3492 if (ICmpInst::isSigned(Pred) || !Q.IIQ.isExact(LBO) ||3493 !Q.IIQ.isExact(RBO))3494 break;3495 if (Value *V = simplifyICmpInst(Pred, LBO->getOperand(0),3496 RBO->getOperand(0), Q, MaxRecurse - 1))3497 return V;3498 break;3499 case Instruction::SDiv:3500 if (!ICmpInst::isEquality(Pred) || !Q.IIQ.isExact(LBO) ||3501 !Q.IIQ.isExact(RBO))3502 break;3503 if (Value *V = simplifyICmpInst(Pred, LBO->getOperand(0),3504 RBO->getOperand(0), Q, MaxRecurse - 1))3505 return V;3506 break;3507 case Instruction::AShr:3508 if (!Q.IIQ.isExact(LBO) || !Q.IIQ.isExact(RBO))3509 break;3510 if (Value *V = simplifyICmpInst(Pred, LBO->getOperand(0),3511 RBO->getOperand(0), Q, MaxRecurse - 1))3512 return V;3513 break;3514 case Instruction::Shl: {3515 bool NUW = Q.IIQ.hasNoUnsignedWrap(LBO) && Q.IIQ.hasNoUnsignedWrap(RBO);3516 bool NSW = Q.IIQ.hasNoSignedWrap(LBO) && Q.IIQ.hasNoSignedWrap(RBO);3517 if (!NUW && !NSW)3518 break;3519 if (!NSW && ICmpInst::isSigned(Pred))3520 break;3521 if (Value *V = simplifyICmpInst(Pred, LBO->getOperand(0),3522 RBO->getOperand(0), Q, MaxRecurse - 1))3523 return V;3524 break;3525 }3526 }3527 }3528 return nullptr;3529}3530 3531/// simplify integer comparisons where at least one operand of the compare3532/// matches an integer min/max idiom.3533static Value *simplifyICmpWithMinMax(CmpPredicate Pred, Value *LHS, Value *RHS,3534 const SimplifyQuery &Q,3535 unsigned MaxRecurse) {3536 Type *ITy = getCompareTy(LHS); // The return type.3537 Value *A, *B;3538 CmpInst::Predicate P = CmpInst::BAD_ICMP_PREDICATE;3539 CmpInst::Predicate EqP; // Chosen so that "A == max/min(A,B)" iff "A EqP B".3540 3541 // Signed variants on "max(a,b)>=a -> true".3542 if (match(LHS, m_SMax(m_Value(A), m_Value(B))) && (A == RHS || B == RHS)) {3543 if (A != RHS)3544 std::swap(A, B); // smax(A, B) pred A.3545 EqP = CmpInst::ICMP_SGE; // "A == smax(A, B)" iff "A sge B".3546 // We analyze this as smax(A, B) pred A.3547 P = Pred;3548 } else if (match(RHS, m_SMax(m_Value(A), m_Value(B))) &&3549 (A == LHS || B == LHS)) {3550 if (A != LHS)3551 std::swap(A, B); // A pred smax(A, B).3552 EqP = CmpInst::ICMP_SGE; // "A == smax(A, B)" iff "A sge B".3553 // We analyze this as smax(A, B) swapped-pred A.3554 P = CmpInst::getSwappedPredicate(Pred);3555 } else if (match(LHS, m_SMin(m_Value(A), m_Value(B))) &&3556 (A == RHS || B == RHS)) {3557 if (A != RHS)3558 std::swap(A, B); // smin(A, B) pred A.3559 EqP = CmpInst::ICMP_SLE; // "A == smin(A, B)" iff "A sle B".3560 // We analyze this as smax(-A, -B) swapped-pred -A.3561 // Note that we do not need to actually form -A or -B thanks to EqP.3562 P = CmpInst::getSwappedPredicate(Pred);3563 } else if (match(RHS, m_SMin(m_Value(A), m_Value(B))) &&3564 (A == LHS || B == LHS)) {3565 if (A != LHS)3566 std::swap(A, B); // A pred smin(A, B).3567 EqP = CmpInst::ICMP_SLE; // "A == smin(A, B)" iff "A sle B".3568 // We analyze this as smax(-A, -B) pred -A.3569 // Note that we do not need to actually form -A or -B thanks to EqP.3570 P = Pred;3571 }3572 if (P != CmpInst::BAD_ICMP_PREDICATE) {3573 // Cases correspond to "max(A, B) p A".3574 switch (P) {3575 default:3576 break;3577 case CmpInst::ICMP_EQ:3578 case CmpInst::ICMP_SLE:3579 // Equivalent to "A EqP B". This may be the same as the condition tested3580 // in the max/min; if so, we can just return that.3581 if (Value *V = extractEquivalentCondition(LHS, EqP, A, B))3582 return V;3583 if (Value *V = extractEquivalentCondition(RHS, EqP, A, B))3584 return V;3585 // Otherwise, see if "A EqP B" simplifies.3586 if (MaxRecurse)3587 if (Value *V = simplifyICmpInst(EqP, A, B, Q, MaxRecurse - 1))3588 return V;3589 break;3590 case CmpInst::ICMP_NE:3591 case CmpInst::ICMP_SGT: {3592 CmpInst::Predicate InvEqP = CmpInst::getInversePredicate(EqP);3593 // Equivalent to "A InvEqP B". This may be the same as the condition3594 // tested in the max/min; if so, we can just return that.3595 if (Value *V = extractEquivalentCondition(LHS, InvEqP, A, B))3596 return V;3597 if (Value *V = extractEquivalentCondition(RHS, InvEqP, A, B))3598 return V;3599 // Otherwise, see if "A InvEqP B" simplifies.3600 if (MaxRecurse)3601 if (Value *V = simplifyICmpInst(InvEqP, A, B, Q, MaxRecurse - 1))3602 return V;3603 break;3604 }3605 case CmpInst::ICMP_SGE:3606 // Always true.3607 return getTrue(ITy);3608 case CmpInst::ICMP_SLT:3609 // Always false.3610 return getFalse(ITy);3611 }3612 }3613 3614 // Unsigned variants on "max(a,b)>=a -> true".3615 P = CmpInst::BAD_ICMP_PREDICATE;3616 if (match(LHS, m_UMax(m_Value(A), m_Value(B))) && (A == RHS || B == RHS)) {3617 if (A != RHS)3618 std::swap(A, B); // umax(A, B) pred A.3619 EqP = CmpInst::ICMP_UGE; // "A == umax(A, B)" iff "A uge B".3620 // We analyze this as umax(A, B) pred A.3621 P = Pred;3622 } else if (match(RHS, m_UMax(m_Value(A), m_Value(B))) &&3623 (A == LHS || B == LHS)) {3624 if (A != LHS)3625 std::swap(A, B); // A pred umax(A, B).3626 EqP = CmpInst::ICMP_UGE; // "A == umax(A, B)" iff "A uge B".3627 // We analyze this as umax(A, B) swapped-pred A.3628 P = CmpInst::getSwappedPredicate(Pred);3629 } else if (match(LHS, m_UMin(m_Value(A), m_Value(B))) &&3630 (A == RHS || B == RHS)) {3631 if (A != RHS)3632 std::swap(A, B); // umin(A, B) pred A.3633 EqP = CmpInst::ICMP_ULE; // "A == umin(A, B)" iff "A ule B".3634 // We analyze this as umax(-A, -B) swapped-pred -A.3635 // Note that we do not need to actually form -A or -B thanks to EqP.3636 P = CmpInst::getSwappedPredicate(Pred);3637 } else if (match(RHS, m_UMin(m_Value(A), m_Value(B))) &&3638 (A == LHS || B == LHS)) {3639 if (A != LHS)3640 std::swap(A, B); // A pred umin(A, B).3641 EqP = CmpInst::ICMP_ULE; // "A == umin(A, B)" iff "A ule B".3642 // We analyze this as umax(-A, -B) pred -A.3643 // Note that we do not need to actually form -A or -B thanks to EqP.3644 P = Pred;3645 }3646 if (P != CmpInst::BAD_ICMP_PREDICATE) {3647 // Cases correspond to "max(A, B) p A".3648 switch (P) {3649 default:3650 break;3651 case CmpInst::ICMP_EQ:3652 case CmpInst::ICMP_ULE:3653 // Equivalent to "A EqP B". This may be the same as the condition tested3654 // in the max/min; if so, we can just return that.3655 if (Value *V = extractEquivalentCondition(LHS, EqP, A, B))3656 return V;3657 if (Value *V = extractEquivalentCondition(RHS, EqP, A, B))3658 return V;3659 // Otherwise, see if "A EqP B" simplifies.3660 if (MaxRecurse)3661 if (Value *V = simplifyICmpInst(EqP, A, B, Q, MaxRecurse - 1))3662 return V;3663 break;3664 case CmpInst::ICMP_NE:3665 case CmpInst::ICMP_UGT: {3666 CmpInst::Predicate InvEqP = CmpInst::getInversePredicate(EqP);3667 // Equivalent to "A InvEqP B". This may be the same as the condition3668 // tested in the max/min; if so, we can just return that.3669 if (Value *V = extractEquivalentCondition(LHS, InvEqP, A, B))3670 return V;3671 if (Value *V = extractEquivalentCondition(RHS, InvEqP, A, B))3672 return V;3673 // Otherwise, see if "A InvEqP B" simplifies.3674 if (MaxRecurse)3675 if (Value *V = simplifyICmpInst(InvEqP, A, B, Q, MaxRecurse - 1))3676 return V;3677 break;3678 }3679 case CmpInst::ICMP_UGE:3680 return getTrue(ITy);3681 case CmpInst::ICMP_ULT:3682 return getFalse(ITy);3683 }3684 }3685 3686 // Comparing 1 each of min/max with a common operand?3687 // Canonicalize min operand to RHS.3688 if (match(LHS, m_UMin(m_Value(), m_Value())) ||3689 match(LHS, m_SMin(m_Value(), m_Value()))) {3690 std::swap(LHS, RHS);3691 Pred = ICmpInst::getSwappedPredicate(Pred);3692 }3693 3694 Value *C, *D;3695 if (match(LHS, m_SMax(m_Value(A), m_Value(B))) &&3696 match(RHS, m_SMin(m_Value(C), m_Value(D))) &&3697 (A == C || A == D || B == C || B == D)) {3698 // smax(A, B) >=s smin(A, D) --> true3699 if (Pred == CmpInst::ICMP_SGE)3700 return getTrue(ITy);3701 // smax(A, B) <s smin(A, D) --> false3702 if (Pred == CmpInst::ICMP_SLT)3703 return getFalse(ITy);3704 } else if (match(LHS, m_UMax(m_Value(A), m_Value(B))) &&3705 match(RHS, m_UMin(m_Value(C), m_Value(D))) &&3706 (A == C || A == D || B == C || B == D)) {3707 // umax(A, B) >=u umin(A, D) --> true3708 if (Pred == CmpInst::ICMP_UGE)3709 return getTrue(ITy);3710 // umax(A, B) <u umin(A, D) --> false3711 if (Pred == CmpInst::ICMP_ULT)3712 return getFalse(ITy);3713 }3714 3715 return nullptr;3716}3717 3718static Value *simplifyICmpWithDominatingAssume(CmpPredicate Predicate,3719 Value *LHS, Value *RHS,3720 const SimplifyQuery &Q) {3721 // Gracefully handle instructions that have not been inserted yet.3722 if (!Q.AC || !Q.CxtI)3723 return nullptr;3724 3725 for (Value *AssumeBaseOp : {LHS, RHS}) {3726 for (auto &AssumeVH : Q.AC->assumptionsFor(AssumeBaseOp)) {3727 if (!AssumeVH)3728 continue;3729 3730 CallInst *Assume = cast<CallInst>(AssumeVH);3731 if (std::optional<bool> Imp = isImpliedCondition(3732 Assume->getArgOperand(0), Predicate, LHS, RHS, Q.DL))3733 if (isValidAssumeForContext(Assume, Q.CxtI, Q.DT))3734 return ConstantInt::get(getCompareTy(LHS), *Imp);3735 }3736 }3737 3738 return nullptr;3739}3740 3741static Value *simplifyICmpWithIntrinsicOnLHS(CmpPredicate Pred, Value *LHS,3742 Value *RHS) {3743 auto *II = dyn_cast<IntrinsicInst>(LHS);3744 if (!II)3745 return nullptr;3746 3747 switch (II->getIntrinsicID()) {3748 case Intrinsic::uadd_sat:3749 // uadd.sat(X, Y) uge X + Y3750 if (match(RHS, m_c_Add(m_Specific(II->getArgOperand(0)),3751 m_Specific(II->getArgOperand(1))))) {3752 if (Pred == ICmpInst::ICMP_UGE)3753 return ConstantInt::getTrue(getCompareTy(II));3754 if (Pred == ICmpInst::ICMP_ULT)3755 return ConstantInt::getFalse(getCompareTy(II));3756 }3757 return nullptr;3758 case Intrinsic::usub_sat:3759 // usub.sat(X, Y) ule X - Y3760 if (match(RHS, m_Sub(m_Specific(II->getArgOperand(0)),3761 m_Specific(II->getArgOperand(1))))) {3762 if (Pred == ICmpInst::ICMP_ULE)3763 return ConstantInt::getTrue(getCompareTy(II));3764 if (Pred == ICmpInst::ICMP_UGT)3765 return ConstantInt::getFalse(getCompareTy(II));3766 }3767 return nullptr;3768 default:3769 return nullptr;3770 }3771}3772 3773/// Helper method to get range from metadata or attribute.3774static std::optional<ConstantRange> getRange(Value *V,3775 const InstrInfoQuery &IIQ) {3776 if (Instruction *I = dyn_cast<Instruction>(V))3777 if (MDNode *MD = IIQ.getMetadata(I, LLVMContext::MD_range))3778 return getConstantRangeFromMetadata(*MD);3779 3780 if (const Argument *A = dyn_cast<Argument>(V))3781 return A->getRange();3782 else if (const CallBase *CB = dyn_cast<CallBase>(V))3783 return CB->getRange();3784 3785 return std::nullopt;3786}3787 3788/// Given operands for an ICmpInst, see if we can fold the result.3789/// If not, this returns null.3790static Value *simplifyICmpInst(CmpPredicate Pred, Value *LHS, Value *RHS,3791 const SimplifyQuery &Q, unsigned MaxRecurse) {3792 assert(CmpInst::isIntPredicate(Pred) && "Not an integer compare!");3793 3794 if (Constant *CLHS = dyn_cast<Constant>(LHS)) {3795 if (Constant *CRHS = dyn_cast<Constant>(RHS))3796 return ConstantFoldCompareInstOperands(Pred, CLHS, CRHS, Q.DL, Q.TLI);3797 3798 // If we have a constant, make sure it is on the RHS.3799 std::swap(LHS, RHS);3800 Pred = CmpInst::getSwappedPredicate(Pred);3801 }3802 assert(!isa<UndefValue>(LHS) && "Unexpected icmp undef,%X");3803 3804 Type *ITy = getCompareTy(LHS); // The return type.3805 3806 // icmp poison, X -> poison3807 if (isa<PoisonValue>(RHS))3808 return PoisonValue::get(ITy);3809 3810 // For EQ and NE, we can always pick a value for the undef to make the3811 // predicate pass or fail, so we can return undef.3812 // Matches behavior in llvm::ConstantFoldCompareInstruction.3813 if (Q.isUndefValue(RHS) && ICmpInst::isEquality(Pred))3814 return UndefValue::get(ITy);3815 3816 // icmp X, X -> true/false3817 // icmp X, undef -> true/false because undef could be X.3818 if (LHS == RHS || Q.isUndefValue(RHS))3819 return ConstantInt::get(ITy, CmpInst::isTrueWhenEqual(Pred));3820 3821 if (Value *V = simplifyICmpOfBools(Pred, LHS, RHS, Q))3822 return V;3823 3824 // TODO: Sink/common this with other potentially expensive calls that use3825 // ValueTracking? See comment below for isKnownNonEqual().3826 if (Value *V = simplifyICmpWithZero(Pred, LHS, RHS, Q))3827 return V;3828 3829 if (Value *V = simplifyICmpWithConstant(Pred, LHS, RHS, Q))3830 return V;3831 3832 // If both operands have range metadata, use the metadata3833 // to simplify the comparison.3834 if (std::optional<ConstantRange> RhsCr = getRange(RHS, Q.IIQ))3835 if (std::optional<ConstantRange> LhsCr = getRange(LHS, Q.IIQ)) {3836 if (LhsCr->icmp(Pred, *RhsCr))3837 return ConstantInt::getTrue(ITy);3838 3839 if (LhsCr->icmp(CmpInst::getInversePredicate(Pred), *RhsCr))3840 return ConstantInt::getFalse(ITy);3841 }3842 3843 // Compare of cast, for example (zext X) != 0 -> X != 03844 if (isa<CastInst>(LHS) && (isa<Constant>(RHS) || isa<CastInst>(RHS))) {3845 Instruction *LI = cast<CastInst>(LHS);3846 Value *SrcOp = LI->getOperand(0);3847 Type *SrcTy = SrcOp->getType();3848 Type *DstTy = LI->getType();3849 3850 // Turn icmp (ptrtoint x), (ptrtoint/constant) into a compare of the input3851 // if the integer type is the same size as the pointer type.3852 if (MaxRecurse && isa<PtrToIntInst>(LI) &&3853 Q.DL.getTypeSizeInBits(SrcTy) == DstTy->getPrimitiveSizeInBits()) {3854 if (Constant *RHSC = dyn_cast<Constant>(RHS)) {3855 // Transfer the cast to the constant.3856 if (Value *V = simplifyICmpInst(Pred, SrcOp,3857 ConstantExpr::getIntToPtr(RHSC, SrcTy),3858 Q, MaxRecurse - 1))3859 return V;3860 } else if (PtrToIntInst *RI = dyn_cast<PtrToIntInst>(RHS)) {3861 if (RI->getOperand(0)->getType() == SrcTy)3862 // Compare without the cast.3863 if (Value *V = simplifyICmpInst(Pred, SrcOp, RI->getOperand(0), Q,3864 MaxRecurse - 1))3865 return V;3866 }3867 }3868 3869 if (isa<ZExtInst>(LHS)) {3870 // Turn icmp (zext X), (zext Y) into a compare of X and Y if they have the3871 // same type.3872 if (ZExtInst *RI = dyn_cast<ZExtInst>(RHS)) {3873 if (MaxRecurse && SrcTy == RI->getOperand(0)->getType())3874 // Compare X and Y. Note that signed predicates become unsigned.3875 if (Value *V =3876 simplifyICmpInst(ICmpInst::getUnsignedPredicate(Pred), SrcOp,3877 RI->getOperand(0), Q, MaxRecurse - 1))3878 return V;3879 }3880 // Fold (zext X) ule (sext X), (zext X) sge (sext X) to true.3881 else if (SExtInst *RI = dyn_cast<SExtInst>(RHS)) {3882 if (SrcOp == RI->getOperand(0)) {3883 if (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_SGE)3884 return ConstantInt::getTrue(ITy);3885 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_SLT)3886 return ConstantInt::getFalse(ITy);3887 }3888 }3889 // Turn icmp (zext X), Cst into a compare of X and Cst if Cst is extended3890 // too. If not, then try to deduce the result of the comparison.3891 else if (match(RHS, m_ImmConstant())) {3892 Constant *C = dyn_cast<Constant>(RHS);3893 assert(C != nullptr);3894 3895 // Compute the constant that would happen if we truncated to SrcTy then3896 // reextended to DstTy.3897 Constant *Trunc =3898 ConstantFoldCastOperand(Instruction::Trunc, C, SrcTy, Q.DL);3899 assert(Trunc && "Constant-fold of ImmConstant should not fail");3900 Constant *RExt =3901 ConstantFoldCastOperand(CastInst::ZExt, Trunc, DstTy, Q.DL);3902 assert(RExt && "Constant-fold of ImmConstant should not fail");3903 Constant *AnyEq =3904 ConstantFoldCompareInstOperands(ICmpInst::ICMP_EQ, RExt, C, Q.DL);3905 assert(AnyEq && "Constant-fold of ImmConstant should not fail");3906 3907 // If the re-extended constant didn't change any of the elements then3908 // this is effectively also a case of comparing two zero-extended3909 // values.3910 if (AnyEq->isAllOnesValue() && MaxRecurse)3911 if (Value *V = simplifyICmpInst(ICmpInst::getUnsignedPredicate(Pred),3912 SrcOp, Trunc, Q, MaxRecurse - 1))3913 return V;3914 3915 // Otherwise the upper bits of LHS are zero while RHS has a non-zero bit3916 // there. Use this to work out the result of the comparison.3917 if (AnyEq->isNullValue()) {3918 switch (Pred) {3919 default:3920 llvm_unreachable("Unknown ICmp predicate!");3921 // LHS <u RHS.3922 case ICmpInst::ICMP_EQ:3923 case ICmpInst::ICMP_UGT:3924 case ICmpInst::ICMP_UGE:3925 return Constant::getNullValue(ITy);3926 3927 case ICmpInst::ICMP_NE:3928 case ICmpInst::ICMP_ULT:3929 case ICmpInst::ICMP_ULE:3930 return Constant::getAllOnesValue(ITy);3931 3932 // LHS is non-negative. If RHS is negative then LHS >s LHS. If RHS3933 // is non-negative then LHS <s RHS.3934 case ICmpInst::ICMP_SGT:3935 case ICmpInst::ICMP_SGE:3936 return ConstantFoldCompareInstOperands(3937 ICmpInst::ICMP_SLT, C, Constant::getNullValue(C->getType()),3938 Q.DL);3939 case ICmpInst::ICMP_SLT:3940 case ICmpInst::ICMP_SLE:3941 return ConstantFoldCompareInstOperands(3942 ICmpInst::ICMP_SGE, C, Constant::getNullValue(C->getType()),3943 Q.DL);3944 }3945 }3946 }3947 }3948 3949 if (isa<SExtInst>(LHS)) {3950 // Turn icmp (sext X), (sext Y) into a compare of X and Y if they have the3951 // same type.3952 if (SExtInst *RI = dyn_cast<SExtInst>(RHS)) {3953 if (MaxRecurse && SrcTy == RI->getOperand(0)->getType())3954 // Compare X and Y. Note that the predicate does not change.3955 if (Value *V = simplifyICmpInst(Pred, SrcOp, RI->getOperand(0), Q,3956 MaxRecurse - 1))3957 return V;3958 }3959 // Fold (sext X) uge (zext X), (sext X) sle (zext X) to true.3960 else if (ZExtInst *RI = dyn_cast<ZExtInst>(RHS)) {3961 if (SrcOp == RI->getOperand(0)) {3962 if (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_SLE)3963 return ConstantInt::getTrue(ITy);3964 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_SGT)3965 return ConstantInt::getFalse(ITy);3966 }3967 }3968 // Turn icmp (sext X), Cst into a compare of X and Cst if Cst is extended3969 // too. If not, then try to deduce the result of the comparison.3970 else if (match(RHS, m_ImmConstant())) {3971 Constant *C = cast<Constant>(RHS);3972 3973 // Compute the constant that would happen if we truncated to SrcTy then3974 // reextended to DstTy.3975 Constant *Trunc =3976 ConstantFoldCastOperand(Instruction::Trunc, C, SrcTy, Q.DL);3977 assert(Trunc && "Constant-fold of ImmConstant should not fail");3978 Constant *RExt =3979 ConstantFoldCastOperand(CastInst::SExt, Trunc, DstTy, Q.DL);3980 assert(RExt && "Constant-fold of ImmConstant should not fail");3981 Constant *AnyEq =3982 ConstantFoldCompareInstOperands(ICmpInst::ICMP_EQ, RExt, C, Q.DL);3983 assert(AnyEq && "Constant-fold of ImmConstant should not fail");3984 3985 // If the re-extended constant didn't change then this is effectively3986 // also a case of comparing two sign-extended values.3987 if (AnyEq->isAllOnesValue() && MaxRecurse)3988 if (Value *V =3989 simplifyICmpInst(Pred, SrcOp, Trunc, Q, MaxRecurse - 1))3990 return V;3991 3992 // Otherwise the upper bits of LHS are all equal, while RHS has varying3993 // bits there. Use this to work out the result of the comparison.3994 if (AnyEq->isNullValue()) {3995 switch (Pred) {3996 default:3997 llvm_unreachable("Unknown ICmp predicate!");3998 case ICmpInst::ICMP_EQ:3999 return Constant::getNullValue(ITy);4000 case ICmpInst::ICMP_NE:4001 return Constant::getAllOnesValue(ITy);4002 4003 // If RHS is non-negative then LHS <s RHS. If RHS is negative then4004 // LHS >s RHS.4005 case ICmpInst::ICMP_SGT:4006 case ICmpInst::ICMP_SGE:4007 return ConstantFoldCompareInstOperands(4008 ICmpInst::ICMP_SLT, C, Constant::getNullValue(C->getType()),4009 Q.DL);4010 case ICmpInst::ICMP_SLT:4011 case ICmpInst::ICMP_SLE:4012 return ConstantFoldCompareInstOperands(4013 ICmpInst::ICMP_SGE, C, Constant::getNullValue(C->getType()),4014 Q.DL);4015 4016 // If LHS is non-negative then LHS <u RHS. If LHS is negative then4017 // LHS >u RHS.4018 case ICmpInst::ICMP_UGT:4019 case ICmpInst::ICMP_UGE:4020 // Comparison is true iff the LHS <s 0.4021 if (MaxRecurse)4022 if (Value *V = simplifyICmpInst(ICmpInst::ICMP_SLT, SrcOp,4023 Constant::getNullValue(SrcTy), Q,4024 MaxRecurse - 1))4025 return V;4026 break;4027 case ICmpInst::ICMP_ULT:4028 case ICmpInst::ICMP_ULE:4029 // Comparison is true iff the LHS >=s 0.4030 if (MaxRecurse)4031 if (Value *V = simplifyICmpInst(ICmpInst::ICMP_SGE, SrcOp,4032 Constant::getNullValue(SrcTy), Q,4033 MaxRecurse - 1))4034 return V;4035 break;4036 }4037 }4038 }4039 }4040 }4041 4042 // icmp eq|ne X, Y -> false|true if X != Y4043 // This is potentially expensive, and we have already computedKnownBits for4044 // compares with 0 above here, so only try this for a non-zero compare.4045 if (ICmpInst::isEquality(Pred) && !match(RHS, m_Zero()) &&4046 isKnownNonEqual(LHS, RHS, Q)) {4047 return Pred == ICmpInst::ICMP_NE ? getTrue(ITy) : getFalse(ITy);4048 }4049 4050 if (Value *V = simplifyICmpWithBinOp(Pred, LHS, RHS, Q, MaxRecurse))4051 return V;4052 4053 if (Value *V = simplifyICmpWithMinMax(Pred, LHS, RHS, Q, MaxRecurse))4054 return V;4055 4056 if (Value *V = simplifyICmpWithIntrinsicOnLHS(Pred, LHS, RHS))4057 return V;4058 if (Value *V = simplifyICmpWithIntrinsicOnLHS(4059 ICmpInst::getSwappedPredicate(Pred), RHS, LHS))4060 return V;4061 4062 if (Value *V = simplifyICmpUsingMonotonicValues(Pred, LHS, RHS, Q))4063 return V;4064 if (Value *V = simplifyICmpUsingMonotonicValues(4065 ICmpInst::getSwappedPredicate(Pred), RHS, LHS, Q))4066 return V;4067 4068 if (Value *V = simplifyICmpWithDominatingAssume(Pred, LHS, RHS, Q))4069 return V;4070 4071 if (std::optional<bool> Res =4072 isImpliedByDomCondition(Pred, LHS, RHS, Q.CxtI, Q.DL))4073 return ConstantInt::getBool(ITy, *Res);4074 4075 // Simplify comparisons of related pointers using a powerful, recursive4076 // GEP-walk when we have target data available..4077 if (LHS->getType()->isPointerTy())4078 if (auto *C = computePointerICmp(Pred, LHS, RHS, Q))4079 return C;4080 if (auto *CLHS = dyn_cast<PtrToIntOperator>(LHS))4081 if (auto *CRHS = dyn_cast<PtrToIntOperator>(RHS))4082 if (CLHS->getPointerOperandType() == CRHS->getPointerOperandType() &&4083 Q.DL.getTypeSizeInBits(CLHS->getPointerOperandType()) ==4084 Q.DL.getTypeSizeInBits(CLHS->getType()))4085 if (auto *C = computePointerICmp(Pred, CLHS->getPointerOperand(),4086 CRHS->getPointerOperand(), Q))4087 return C;4088 4089 // If the comparison is with the result of a select instruction, check whether4090 // comparing with either branch of the select always yields the same value.4091 if (isa<SelectInst>(LHS) || isa<SelectInst>(RHS))4092 if (Value *V = threadCmpOverSelect(Pred, LHS, RHS, Q, MaxRecurse))4093 return V;4094 4095 // If the comparison is with the result of a phi instruction, check whether4096 // doing the compare with each incoming phi value yields a common result.4097 if (isa<PHINode>(LHS) || isa<PHINode>(RHS))4098 if (Value *V = threadCmpOverPHI(Pred, LHS, RHS, Q, MaxRecurse))4099 return V;4100 4101 return nullptr;4102}4103 4104Value *llvm::simplifyICmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS,4105 const SimplifyQuery &Q) {4106 return ::simplifyICmpInst(Predicate, LHS, RHS, Q, RecursionLimit);4107}4108 4109/// Given operands for an FCmpInst, see if we can fold the result.4110/// If not, this returns null.4111static Value *simplifyFCmpInst(CmpPredicate Pred, Value *LHS, Value *RHS,4112 FastMathFlags FMF, const SimplifyQuery &Q,4113 unsigned MaxRecurse) {4114 assert(CmpInst::isFPPredicate(Pred) && "Not an FP compare!");4115 4116 if (Constant *CLHS = dyn_cast<Constant>(LHS)) {4117 if (Constant *CRHS = dyn_cast<Constant>(RHS))4118 return ConstantFoldCompareInstOperands(Pred, CLHS, CRHS, Q.DL, Q.TLI,4119 Q.CxtI);4120 4121 // If we have a constant, make sure it is on the RHS.4122 std::swap(LHS, RHS);4123 Pred = CmpInst::getSwappedPredicate(Pred);4124 }4125 4126 // Fold trivial predicates.4127 Type *RetTy = getCompareTy(LHS);4128 if (Pred == FCmpInst::FCMP_FALSE)4129 return getFalse(RetTy);4130 if (Pred == FCmpInst::FCMP_TRUE)4131 return getTrue(RetTy);4132 4133 // fcmp pred x, poison and fcmp pred poison, x4134 // fold to poison4135 if (isa<PoisonValue>(LHS) || isa<PoisonValue>(RHS))4136 return PoisonValue::get(RetTy);4137 4138 // fcmp pred x, undef and fcmp pred undef, x4139 // fold to true if unordered, false if ordered4140 if (Q.isUndefValue(LHS) || Q.isUndefValue(RHS)) {4141 // Choosing NaN for the undef will always make unordered comparison succeed4142 // and ordered comparison fail.4143 return ConstantInt::get(RetTy, CmpInst::isUnordered(Pred));4144 }4145 4146 // fcmp x,x -> true/false. Not all compares are foldable.4147 if (LHS == RHS) {4148 if (CmpInst::isTrueWhenEqual(Pred))4149 return getTrue(RetTy);4150 if (CmpInst::isFalseWhenEqual(Pred))4151 return getFalse(RetTy);4152 }4153 4154 // Fold (un)ordered comparison if we can determine there are no NaNs.4155 //4156 // This catches the 2 variable input case, constants are handled below as a4157 // class-like compare.4158 if (Pred == FCmpInst::FCMP_ORD || Pred == FCmpInst::FCMP_UNO) {4159 KnownFPClass RHSClass = computeKnownFPClass(RHS, fcAllFlags, Q);4160 KnownFPClass LHSClass = computeKnownFPClass(LHS, fcAllFlags, Q);4161 4162 if (FMF.noNaNs() ||4163 (RHSClass.isKnownNeverNaN() && LHSClass.isKnownNeverNaN()))4164 return ConstantInt::get(RetTy, Pred == FCmpInst::FCMP_ORD);4165 4166 if (RHSClass.isKnownAlwaysNaN() || LHSClass.isKnownAlwaysNaN())4167 return ConstantInt::get(RetTy, Pred == CmpInst::FCMP_UNO);4168 }4169 4170 if (std::optional<bool> Res =4171 isImpliedByDomCondition(Pred, LHS, RHS, Q.CxtI, Q.DL))4172 return ConstantInt::getBool(RetTy, *Res);4173 4174 const APFloat *C = nullptr;4175 match(RHS, m_APFloatAllowPoison(C));4176 std::optional<KnownFPClass> FullKnownClassLHS;4177 4178 // Lazily compute the possible classes for LHS. Avoid computing it twice if4179 // RHS is a 0.4180 auto computeLHSClass = [=, &FullKnownClassLHS](FPClassTest InterestedFlags =4181 fcAllFlags) {4182 if (FullKnownClassLHS)4183 return *FullKnownClassLHS;4184 return computeKnownFPClass(LHS, FMF, InterestedFlags, Q);4185 };4186 4187 if (C && Q.CxtI) {4188 // Fold out compares that express a class test.4189 //4190 // FIXME: Should be able to perform folds without context4191 // instruction. Always pass in the context function?4192 4193 const Function *ParentF = Q.CxtI->getFunction();4194 auto [ClassVal, ClassTest] = fcmpToClassTest(Pred, *ParentF, LHS, C);4195 if (ClassVal) {4196 FullKnownClassLHS = computeLHSClass();4197 if ((FullKnownClassLHS->KnownFPClasses & ClassTest) == fcNone)4198 return getFalse(RetTy);4199 if ((FullKnownClassLHS->KnownFPClasses & ~ClassTest) == fcNone)4200 return getTrue(RetTy);4201 }4202 }4203 4204 // Handle fcmp with constant RHS.4205 if (C) {4206 // TODO: If we always required a context function, we wouldn't need to4207 // special case nans.4208 if (C->isNaN())4209 return ConstantInt::get(RetTy, CmpInst::isUnordered(Pred));4210 4211 // TODO: Need version fcmpToClassTest which returns implied class when the4212 // compare isn't a complete class test. e.g. > 1.0 implies fcPositive, but4213 // isn't implementable as a class call.4214 if (C->isNegative() && !C->isNegZero()) {4215 FPClassTest Interested = KnownFPClass::OrderedLessThanZeroMask;4216 4217 // TODO: We can catch more cases by using a range check rather than4218 // relying on CannotBeOrderedLessThanZero.4219 switch (Pred) {4220 case FCmpInst::FCMP_UGE:4221 case FCmpInst::FCMP_UGT:4222 case FCmpInst::FCMP_UNE: {4223 KnownFPClass KnownClass = computeLHSClass(Interested);4224 4225 // (X >= 0) implies (X > C) when (C < 0)4226 if (KnownClass.cannotBeOrderedLessThanZero())4227 return getTrue(RetTy);4228 break;4229 }4230 case FCmpInst::FCMP_OEQ:4231 case FCmpInst::FCMP_OLE:4232 case FCmpInst::FCMP_OLT: {4233 KnownFPClass KnownClass = computeLHSClass(Interested);4234 4235 // (X >= 0) implies !(X < C) when (C < 0)4236 if (KnownClass.cannotBeOrderedLessThanZero())4237 return getFalse(RetTy);4238 break;4239 }4240 default:4241 break;4242 }4243 }4244 // Check comparison of [minnum/maxnum with constant] with other constant.4245 const APFloat *C2;4246 if ((match(LHS, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_APFloat(C2))) &&4247 *C2 < *C) ||4248 (match(LHS, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_APFloat(C2))) &&4249 *C2 > *C)) {4250 bool IsMaxNum =4251 cast<IntrinsicInst>(LHS)->getIntrinsicID() == Intrinsic::maxnum;4252 // The ordered relationship and minnum/maxnum guarantee that we do not4253 // have NaN constants, so ordered/unordered preds are handled the same.4254 switch (Pred) {4255 case FCmpInst::FCMP_OEQ:4256 case FCmpInst::FCMP_UEQ:4257 // minnum(X, LesserC) == C --> false4258 // maxnum(X, GreaterC) == C --> false4259 return getFalse(RetTy);4260 case FCmpInst::FCMP_ONE:4261 case FCmpInst::FCMP_UNE:4262 // minnum(X, LesserC) != C --> true4263 // maxnum(X, GreaterC) != C --> true4264 return getTrue(RetTy);4265 case FCmpInst::FCMP_OGE:4266 case FCmpInst::FCMP_UGE:4267 case FCmpInst::FCMP_OGT:4268 case FCmpInst::FCMP_UGT:4269 // minnum(X, LesserC) >= C --> false4270 // minnum(X, LesserC) > C --> false4271 // maxnum(X, GreaterC) >= C --> true4272 // maxnum(X, GreaterC) > C --> true4273 return ConstantInt::get(RetTy, IsMaxNum);4274 case FCmpInst::FCMP_OLE:4275 case FCmpInst::FCMP_ULE:4276 case FCmpInst::FCMP_OLT:4277 case FCmpInst::FCMP_ULT:4278 // minnum(X, LesserC) <= C --> true4279 // minnum(X, LesserC) < C --> true4280 // maxnum(X, GreaterC) <= C --> false4281 // maxnum(X, GreaterC) < C --> false4282 return ConstantInt::get(RetTy, !IsMaxNum);4283 default:4284 // TRUE/FALSE/ORD/UNO should be handled before this.4285 llvm_unreachable("Unexpected fcmp predicate");4286 }4287 }4288 }4289 4290 // TODO: Could fold this with above if there were a matcher which returned all4291 // classes in a non-splat vector.4292 if (match(RHS, m_AnyZeroFP())) {4293 switch (Pred) {4294 case FCmpInst::FCMP_OGE:4295 case FCmpInst::FCMP_ULT: {4296 FPClassTest Interested = KnownFPClass::OrderedLessThanZeroMask;4297 if (!FMF.noNaNs())4298 Interested |= fcNan;4299 4300 KnownFPClass Known = computeLHSClass(Interested);4301 4302 // Positive or zero X >= 0.0 --> true4303 // Positive or zero X < 0.0 --> false4304 if ((FMF.noNaNs() || Known.isKnownNeverNaN()) &&4305 Known.cannotBeOrderedLessThanZero())4306 return Pred == FCmpInst::FCMP_OGE ? getTrue(RetTy) : getFalse(RetTy);4307 break;4308 }4309 case FCmpInst::FCMP_UGE:4310 case FCmpInst::FCMP_OLT: {4311 FPClassTest Interested = KnownFPClass::OrderedLessThanZeroMask;4312 KnownFPClass Known = computeLHSClass(Interested);4313 4314 // Positive or zero or nan X >= 0.0 --> true4315 // Positive or zero or nan X < 0.0 --> false4316 if (Known.cannotBeOrderedLessThanZero())4317 return Pred == FCmpInst::FCMP_UGE ? getTrue(RetTy) : getFalse(RetTy);4318 break;4319 }4320 default:4321 break;4322 }4323 }4324 4325 // If the comparison is with the result of a select instruction, check whether4326 // comparing with either branch of the select always yields the same value.4327 if (isa<SelectInst>(LHS) || isa<SelectInst>(RHS))4328 if (Value *V = threadCmpOverSelect(Pred, LHS, RHS, Q, MaxRecurse))4329 return V;4330 4331 // If the comparison is with the result of a phi instruction, check whether4332 // doing the compare with each incoming phi value yields a common result.4333 if (isa<PHINode>(LHS) || isa<PHINode>(RHS))4334 if (Value *V = threadCmpOverPHI(Pred, LHS, RHS, Q, MaxRecurse))4335 return V;4336 4337 return nullptr;4338}4339 4340Value *llvm::simplifyFCmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS,4341 FastMathFlags FMF, const SimplifyQuery &Q) {4342 return ::simplifyFCmpInst(Predicate, LHS, RHS, FMF, Q, RecursionLimit);4343}4344 4345static Value *simplifyWithOpsReplaced(Value *V,4346 ArrayRef<std::pair<Value *, Value *>> Ops,4347 const SimplifyQuery &Q,4348 bool AllowRefinement,4349 SmallVectorImpl<Instruction *> *DropFlags,4350 unsigned MaxRecurse) {4351 assert((AllowRefinement || !Q.CanUseUndef) &&4352 "If AllowRefinement=false then CanUseUndef=false");4353 for (const auto &OpAndRepOp : Ops) {4354 // We cannot replace a constant, and shouldn't even try.4355 if (isa<Constant>(OpAndRepOp.first))4356 return nullptr;4357 4358 // Trivial replacement.4359 if (V == OpAndRepOp.first)4360 return OpAndRepOp.second;4361 }4362 4363 if (!MaxRecurse--)4364 return nullptr;4365 4366 auto *I = dyn_cast<Instruction>(V);4367 if (!I)4368 return nullptr;4369 4370 // The arguments of a phi node might refer to a value from a previous4371 // cycle iteration.4372 if (isa<PHINode>(I))4373 return nullptr;4374 4375 // Don't fold away llvm.is.constant checks based on assumptions.4376 if (match(I, m_Intrinsic<Intrinsic::is_constant>()))4377 return nullptr;4378 4379 // Don't simplify freeze.4380 if (isa<FreezeInst>(I))4381 return nullptr;4382 4383 for (const auto &OpAndRepOp : Ops) {4384 // For vector types, the simplification must hold per-lane, so forbid4385 // potentially cross-lane operations like shufflevector.4386 if (OpAndRepOp.first->getType()->isVectorTy() &&4387 !isNotCrossLaneOperation(I))4388 return nullptr;4389 }4390 4391 // Replace Op with RepOp in instruction operands.4392 SmallVector<Value *, 8> NewOps;4393 bool AnyReplaced = false;4394 for (Value *InstOp : I->operands()) {4395 if (Value *NewInstOp = simplifyWithOpsReplaced(4396 InstOp, Ops, Q, AllowRefinement, DropFlags, MaxRecurse)) {4397 NewOps.push_back(NewInstOp);4398 AnyReplaced = InstOp != NewInstOp;4399 } else {4400 NewOps.push_back(InstOp);4401 }4402 4403 // Bail out if any operand is undef and SimplifyQuery disables undef4404 // simplification. Constant folding currently doesn't respect this option.4405 if (isa<UndefValue>(NewOps.back()) && !Q.CanUseUndef)4406 return nullptr;4407 }4408 4409 if (!AnyReplaced)4410 return nullptr;4411 4412 if (!AllowRefinement) {4413 // General InstSimplify functions may refine the result, e.g. by returning4414 // a constant for a potentially poison value. To avoid this, implement only4415 // a few non-refining but profitable transforms here.4416 4417 if (auto *BO = dyn_cast<BinaryOperator>(I)) {4418 unsigned Opcode = BO->getOpcode();4419 // id op x -> x, x op id -> x4420 // Exclude floats, because x op id may produce a different NaN value.4421 if (!BO->getType()->isFPOrFPVectorTy()) {4422 if (NewOps[0] == ConstantExpr::getBinOpIdentity(Opcode, I->getType()))4423 return NewOps[1];4424 if (NewOps[1] == ConstantExpr::getBinOpIdentity(Opcode, I->getType(),4425 /* RHS */ true))4426 return NewOps[0];4427 }4428 4429 // x & x -> x, x | x -> x4430 if ((Opcode == Instruction::And || Opcode == Instruction::Or) &&4431 NewOps[0] == NewOps[1]) {4432 // or disjoint x, x results in poison.4433 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(BO)) {4434 if (PDI->isDisjoint()) {4435 if (!DropFlags)4436 return nullptr;4437 DropFlags->push_back(BO);4438 }4439 }4440 return NewOps[0];4441 }4442 4443 // x - x -> 0, x ^ x -> 0. This is non-refining, because x is non-poison4444 // by assumption and this case never wraps, so nowrap flags can be4445 // ignored.4446 if ((Opcode == Instruction::Sub || Opcode == Instruction::Xor) &&4447 NewOps[0] == NewOps[1] &&4448 any_of(Ops, [=](const auto &Rep) { return NewOps[0] == Rep.second; }))4449 return Constant::getNullValue(I->getType());4450 4451 // If we are substituting an absorber constant into a binop and extra4452 // poison can't leak if we remove the select -- because both operands of4453 // the binop are based on the same value -- then it may be safe to replace4454 // the value with the absorber constant. Examples:4455 // (Op == 0) ? 0 : (Op & -Op) --> Op & -Op4456 // (Op == 0) ? 0 : (Op * (binop Op, C)) --> Op * (binop Op, C)4457 // (Op == -1) ? -1 : (Op | (binop C, Op) --> Op | (binop C, Op)4458 Constant *Absorber = ConstantExpr::getBinOpAbsorber(Opcode, I->getType());4459 if ((NewOps[0] == Absorber || NewOps[1] == Absorber) &&4460 any_of(Ops,4461 [=](const auto &Rep) { return impliesPoison(BO, Rep.first); }))4462 return Absorber;4463 }4464 4465 if (isa<GetElementPtrInst>(I)) {4466 // getelementptr x, 0 -> x.4467 // This never returns poison, even if inbounds is set.4468 if (NewOps.size() == 2 && match(NewOps[1], m_Zero()))4469 return NewOps[0];4470 }4471 } else {4472 // The simplification queries below may return the original value. Consider:4473 // %div = udiv i32 %arg, %arg24474 // %mul = mul nsw i32 %div, %arg24475 // %cmp = icmp eq i32 %mul, %arg4476 // %sel = select i1 %cmp, i32 %div, i32 undef4477 // Replacing %arg by %mul, %div becomes "udiv i32 %mul, %arg2", which4478 // simplifies back to %arg. This can only happen because %mul does not4479 // dominate %div. To ensure a consistent return value contract, we make sure4480 // that this case returns nullptr as well.4481 auto PreventSelfSimplify = [V](Value *Simplified) {4482 return Simplified != V ? Simplified : nullptr;4483 };4484 4485 return PreventSelfSimplify(4486 ::simplifyInstructionWithOperands(I, NewOps, Q, MaxRecurse));4487 }4488 4489 // If all operands are constant after substituting Op for RepOp then we can4490 // constant fold the instruction.4491 SmallVector<Constant *, 8> ConstOps;4492 for (Value *NewOp : NewOps) {4493 if (Constant *ConstOp = dyn_cast<Constant>(NewOp))4494 ConstOps.push_back(ConstOp);4495 else4496 return nullptr;4497 }4498 4499 // Consider:4500 // %cmp = icmp eq i32 %x, 21474836474501 // %add = add nsw i32 %x, 14502 // %sel = select i1 %cmp, i32 -2147483648, i32 %add4503 //4504 // We can't replace %sel with %add unless we strip away the flags (which4505 // will be done in InstCombine).4506 // TODO: This may be unsound, because it only catches some forms of4507 // refinement.4508 if (!AllowRefinement) {4509 if (canCreatePoison(cast<Operator>(I), !DropFlags)) {4510 // abs cannot create poison if the value is known to never be int_min.4511 if (auto *II = dyn_cast<IntrinsicInst>(I);4512 II && II->getIntrinsicID() == Intrinsic::abs) {4513 if (!ConstOps[0]->isNotMinSignedValue())4514 return nullptr;4515 } else4516 return nullptr;4517 }4518 Constant *Res = ConstantFoldInstOperands(I, ConstOps, Q.DL, Q.TLI,4519 /*AllowNonDeterministic=*/false);4520 if (DropFlags && Res && I->hasPoisonGeneratingAnnotations())4521 DropFlags->push_back(I);4522 return Res;4523 }4524 4525 return ConstantFoldInstOperands(I, ConstOps, Q.DL, Q.TLI,4526 /*AllowNonDeterministic=*/false);4527}4528 4529static Value *simplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,4530 const SimplifyQuery &Q,4531 bool AllowRefinement,4532 SmallVectorImpl<Instruction *> *DropFlags,4533 unsigned MaxRecurse) {4534 return simplifyWithOpsReplaced(V, {{Op, RepOp}}, Q, AllowRefinement,4535 DropFlags, MaxRecurse);4536}4537 4538Value *llvm::simplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,4539 const SimplifyQuery &Q,4540 bool AllowRefinement,4541 SmallVectorImpl<Instruction *> *DropFlags) {4542 // If refinement is disabled, also disable undef simplifications (which are4543 // always refinements) in SimplifyQuery.4544 if (!AllowRefinement)4545 return ::simplifyWithOpReplaced(V, Op, RepOp, Q.getWithoutUndef(),4546 AllowRefinement, DropFlags, RecursionLimit);4547 return ::simplifyWithOpReplaced(V, Op, RepOp, Q, AllowRefinement, DropFlags,4548 RecursionLimit);4549}4550 4551/// Try to simplify a select instruction when its condition operand is an4552/// integer comparison where one operand of the compare is a constant.4553static Value *simplifySelectBitTest(Value *TrueVal, Value *FalseVal, Value *X,4554 const APInt *Y, bool TrueWhenUnset) {4555 const APInt *C;4556 4557 // (X & Y) == 0 ? X & ~Y : X --> X4558 // (X & Y) != 0 ? X & ~Y : X --> X & ~Y4559 if (FalseVal == X && match(TrueVal, m_And(m_Specific(X), m_APInt(C))) &&4560 *Y == ~*C)4561 return TrueWhenUnset ? FalseVal : TrueVal;4562 4563 // (X & Y) == 0 ? X : X & ~Y --> X & ~Y4564 // (X & Y) != 0 ? X : X & ~Y --> X4565 if (TrueVal == X && match(FalseVal, m_And(m_Specific(X), m_APInt(C))) &&4566 *Y == ~*C)4567 return TrueWhenUnset ? FalseVal : TrueVal;4568 4569 if (Y->isPowerOf2()) {4570 // (X & Y) == 0 ? X | Y : X --> X | Y4571 // (X & Y) != 0 ? X | Y : X --> X4572 if (FalseVal == X && match(TrueVal, m_Or(m_Specific(X), m_APInt(C))) &&4573 *Y == *C) {4574 // We can't return the or if it has the disjoint flag.4575 if (TrueWhenUnset && cast<PossiblyDisjointInst>(TrueVal)->isDisjoint())4576 return nullptr;4577 return TrueWhenUnset ? TrueVal : FalseVal;4578 }4579 4580 // (X & Y) == 0 ? X : X | Y --> X4581 // (X & Y) != 0 ? X : X | Y --> X | Y4582 if (TrueVal == X && match(FalseVal, m_Or(m_Specific(X), m_APInt(C))) &&4583 *Y == *C) {4584 // We can't return the or if it has the disjoint flag.4585 if (!TrueWhenUnset && cast<PossiblyDisjointInst>(FalseVal)->isDisjoint())4586 return nullptr;4587 return TrueWhenUnset ? TrueVal : FalseVal;4588 }4589 }4590 4591 return nullptr;4592}4593 4594static Value *simplifyCmpSelOfMaxMin(Value *CmpLHS, Value *CmpRHS,4595 CmpPredicate Pred, Value *TVal,4596 Value *FVal) {4597 // Canonicalize common cmp+sel operand as CmpLHS.4598 if (CmpRHS == TVal || CmpRHS == FVal) {4599 std::swap(CmpLHS, CmpRHS);4600 Pred = ICmpInst::getSwappedPredicate(Pred);4601 }4602 4603 // Canonicalize common cmp+sel operand as TVal.4604 if (CmpLHS == FVal) {4605 std::swap(TVal, FVal);4606 Pred = ICmpInst::getInversePredicate(Pred);4607 }4608 4609 // A vector select may be shuffling together elements that are equivalent4610 // based on the max/min/select relationship.4611 Value *X = CmpLHS, *Y = CmpRHS;4612 bool PeekedThroughSelectShuffle = false;4613 auto *Shuf = dyn_cast<ShuffleVectorInst>(FVal);4614 if (Shuf && Shuf->isSelect()) {4615 if (Shuf->getOperand(0) == Y)4616 FVal = Shuf->getOperand(1);4617 else if (Shuf->getOperand(1) == Y)4618 FVal = Shuf->getOperand(0);4619 else4620 return nullptr;4621 PeekedThroughSelectShuffle = true;4622 }4623 4624 // (X pred Y) ? X : max/min(X, Y)4625 auto *MMI = dyn_cast<MinMaxIntrinsic>(FVal);4626 if (!MMI || TVal != X ||4627 !match(FVal, m_c_MaxOrMin(m_Specific(X), m_Specific(Y))))4628 return nullptr;4629 4630 // (X > Y) ? X : max(X, Y) --> max(X, Y)4631 // (X >= Y) ? X : max(X, Y) --> max(X, Y)4632 // (X < Y) ? X : min(X, Y) --> min(X, Y)4633 // (X <= Y) ? X : min(X, Y) --> min(X, Y)4634 //4635 // The equivalence allows a vector select (shuffle) of max/min and Y. Ex:4636 // (X > Y) ? X : (Z ? max(X, Y) : Y)4637 // If Z is true, this reduces as above, and if Z is false:4638 // (X > Y) ? X : Y --> max(X, Y)4639 ICmpInst::Predicate MMPred = MMI->getPredicate();4640 if (MMPred == CmpInst::getStrictPredicate(Pred))4641 return MMI;4642 4643 // Other transforms are not valid with a shuffle.4644 if (PeekedThroughSelectShuffle)4645 return nullptr;4646 4647 // (X == Y) ? X : max/min(X, Y) --> max/min(X, Y)4648 if (Pred == CmpInst::ICMP_EQ)4649 return MMI;4650 4651 // (X != Y) ? X : max/min(X, Y) --> X4652 if (Pred == CmpInst::ICMP_NE)4653 return X;4654 4655 // (X < Y) ? X : max(X, Y) --> X4656 // (X <= Y) ? X : max(X, Y) --> X4657 // (X > Y) ? X : min(X, Y) --> X4658 // (X >= Y) ? X : min(X, Y) --> X4659 ICmpInst::Predicate InvPred = CmpInst::getInversePredicate(Pred);4660 if (MMPred == CmpInst::getStrictPredicate(InvPred))4661 return X;4662 4663 return nullptr;4664}4665 4666/// An alternative way to test if a bit is set or not.4667/// uses e.g. sgt/slt or trunc instead of eq/ne.4668static Value *simplifySelectWithBitTest(Value *CondVal, Value *TrueVal,4669 Value *FalseVal) {4670 if (auto Res = decomposeBitTest(CondVal))4671 return simplifySelectBitTest(TrueVal, FalseVal, Res->X, &Res->Mask,4672 Res->Pred == ICmpInst::ICMP_EQ);4673 4674 return nullptr;4675}4676 4677/// Try to simplify a select instruction when its condition operand is an4678/// integer equality or floating-point equivalence comparison.4679static Value *simplifySelectWithEquivalence(4680 ArrayRef<std::pair<Value *, Value *>> Replacements, Value *TrueVal,4681 Value *FalseVal, const SimplifyQuery &Q, unsigned MaxRecurse) {4682 Value *SimplifiedFalseVal =4683 simplifyWithOpsReplaced(FalseVal, Replacements, Q.getWithoutUndef(),4684 /* AllowRefinement */ false,4685 /* DropFlags */ nullptr, MaxRecurse);4686 if (!SimplifiedFalseVal)4687 SimplifiedFalseVal = FalseVal;4688 4689 Value *SimplifiedTrueVal =4690 simplifyWithOpsReplaced(TrueVal, Replacements, Q,4691 /* AllowRefinement */ true,4692 /* DropFlags */ nullptr, MaxRecurse);4693 if (!SimplifiedTrueVal)4694 SimplifiedTrueVal = TrueVal;4695 4696 if (SimplifiedFalseVal == SimplifiedTrueVal)4697 return FalseVal;4698 4699 return nullptr;4700}4701 4702/// Try to simplify a select instruction when its condition operand is an4703/// integer comparison.4704static Value *simplifySelectWithICmpCond(Value *CondVal, Value *TrueVal,4705 Value *FalseVal,4706 const SimplifyQuery &Q,4707 unsigned MaxRecurse) {4708 CmpPredicate Pred;4709 Value *CmpLHS, *CmpRHS;4710 if (!match(CondVal, m_ICmp(Pred, m_Value(CmpLHS), m_Value(CmpRHS))))4711 return nullptr;4712 4713 if (Value *V = simplifyCmpSelOfMaxMin(CmpLHS, CmpRHS, Pred, TrueVal, FalseVal))4714 return V;4715 4716 // Canonicalize ne to eq predicate.4717 if (Pred == ICmpInst::ICMP_NE) {4718 Pred = ICmpInst::ICMP_EQ;4719 std::swap(TrueVal, FalseVal);4720 }4721 4722 // Check for integer min/max with a limit constant:4723 // X > MIN_INT ? X : MIN_INT --> X4724 // X < MAX_INT ? X : MAX_INT --> X4725 if (TrueVal->getType()->isIntOrIntVectorTy()) {4726 Value *X, *Y;4727 SelectPatternFlavor SPF =4728 matchDecomposedSelectPattern(cast<ICmpInst>(CondVal), TrueVal, FalseVal,4729 X, Y)4730 .Flavor;4731 if (SelectPatternResult::isMinOrMax(SPF) && Pred == getMinMaxPred(SPF)) {4732 APInt LimitC = getMinMaxLimit(getInverseMinMaxFlavor(SPF),4733 X->getType()->getScalarSizeInBits());4734 if (match(Y, m_SpecificInt(LimitC)))4735 return X;4736 }4737 }4738 4739 if (Pred == ICmpInst::ICMP_EQ && match(CmpRHS, m_Zero())) {4740 Value *X;4741 const APInt *Y;4742 if (match(CmpLHS, m_And(m_Value(X), m_APInt(Y))))4743 if (Value *V = simplifySelectBitTest(TrueVal, FalseVal, X, Y,4744 /*TrueWhenUnset=*/true))4745 return V;4746 4747 // Test for a bogus zero-shift-guard-op around funnel-shift or rotate.4748 Value *ShAmt;4749 auto isFsh = m_CombineOr(m_FShl(m_Value(X), m_Value(), m_Value(ShAmt)),4750 m_FShr(m_Value(), m_Value(X), m_Value(ShAmt)));4751 // (ShAmt == 0) ? fshl(X, *, ShAmt) : X --> X4752 // (ShAmt == 0) ? fshr(*, X, ShAmt) : X --> X4753 if (match(TrueVal, isFsh) && FalseVal == X && CmpLHS == ShAmt)4754 return X;4755 4756 // Test for a zero-shift-guard-op around rotates. These are used to4757 // avoid UB from oversized shifts in raw IR rotate patterns, but the4758 // intrinsics do not have that problem.4759 // We do not allow this transform for the general funnel shift case because4760 // that would not preserve the poison safety of the original code.4761 auto isRotate =4762 m_CombineOr(m_FShl(m_Value(X), m_Deferred(X), m_Value(ShAmt)),4763 m_FShr(m_Value(X), m_Deferred(X), m_Value(ShAmt)));4764 // (ShAmt == 0) ? X : fshl(X, X, ShAmt) --> fshl(X, X, ShAmt)4765 // (ShAmt == 0) ? X : fshr(X, X, ShAmt) --> fshr(X, X, ShAmt)4766 if (match(FalseVal, isRotate) && TrueVal == X && CmpLHS == ShAmt &&4767 Pred == ICmpInst::ICMP_EQ)4768 return FalseVal;4769 4770 // X == 0 ? abs(X) : -abs(X) --> -abs(X)4771 // X == 0 ? -abs(X) : abs(X) --> abs(X)4772 if (match(TrueVal, m_Intrinsic<Intrinsic::abs>(m_Specific(CmpLHS))) &&4773 match(FalseVal, m_Neg(m_Intrinsic<Intrinsic::abs>(m_Specific(CmpLHS)))))4774 return FalseVal;4775 if (match(TrueVal,4776 m_Neg(m_Intrinsic<Intrinsic::abs>(m_Specific(CmpLHS)))) &&4777 match(FalseVal, m_Intrinsic<Intrinsic::abs>(m_Specific(CmpLHS))))4778 return FalseVal;4779 }4780 4781 // If we have a scalar equality comparison, then we know the value in one of4782 // the arms of the select. See if substituting this value into the arm and4783 // simplifying the result yields the same value as the other arm.4784 if (Pred == ICmpInst::ICMP_EQ) {4785 if (CmpLHS->getType()->isIntOrIntVectorTy() ||4786 canReplacePointersIfEqual(CmpLHS, CmpRHS, Q.DL))4787 if (Value *V = simplifySelectWithEquivalence({{CmpLHS, CmpRHS}}, TrueVal,4788 FalseVal, Q, MaxRecurse))4789 return V;4790 if (CmpLHS->getType()->isIntOrIntVectorTy() ||4791 canReplacePointersIfEqual(CmpRHS, CmpLHS, Q.DL))4792 if (Value *V = simplifySelectWithEquivalence({{CmpRHS, CmpLHS}}, TrueVal,4793 FalseVal, Q, MaxRecurse))4794 return V;4795 4796 Value *X;4797 Value *Y;4798 // select((X | Y) == 0 ? X : 0) --> 0 (commuted 2 ways)4799 if (match(CmpLHS, m_Or(m_Value(X), m_Value(Y))) &&4800 match(CmpRHS, m_Zero())) {4801 // (X | Y) == 0 implies X == 0 and Y == 0.4802 if (Value *V = simplifySelectWithEquivalence(4803 {{X, CmpRHS}, {Y, CmpRHS}}, TrueVal, FalseVal, Q, MaxRecurse))4804 return V;4805 }4806 4807 // select((X & Y) == -1 ? X : -1) --> -1 (commuted 2 ways)4808 if (match(CmpLHS, m_And(m_Value(X), m_Value(Y))) &&4809 match(CmpRHS, m_AllOnes())) {4810 // (X & Y) == -1 implies X == -1 and Y == -1.4811 if (Value *V = simplifySelectWithEquivalence(4812 {{X, CmpRHS}, {Y, CmpRHS}}, TrueVal, FalseVal, Q, MaxRecurse))4813 return V;4814 }4815 }4816 4817 return nullptr;4818}4819 4820/// Try to simplify a select instruction when its condition operand is a4821/// floating-point comparison.4822static Value *simplifySelectWithFCmp(Value *Cond, Value *T, Value *F,4823 const SimplifyQuery &Q,4824 unsigned MaxRecurse) {4825 CmpPredicate Pred;4826 Value *CmpLHS, *CmpRHS;4827 if (!match(Cond, m_FCmp(Pred, m_Value(CmpLHS), m_Value(CmpRHS))))4828 return nullptr;4829 FCmpInst *I = cast<FCmpInst>(Cond);4830 4831 bool IsEquiv = I->isEquivalence();4832 if (I->isEquivalence(/*Invert=*/true)) {4833 std::swap(T, F);4834 Pred = FCmpInst::getInversePredicate(Pred);4835 IsEquiv = true;4836 }4837 4838 // This transforms is safe if at least one operand is known to not be zero.4839 // Otherwise, the select can change the sign of a zero operand.4840 if (IsEquiv) {4841 if (Value *V = simplifySelectWithEquivalence({{CmpLHS, CmpRHS}}, T, F, Q,4842 MaxRecurse))4843 return V;4844 if (Value *V = simplifySelectWithEquivalence({{CmpRHS, CmpLHS}}, T, F, Q,4845 MaxRecurse))4846 return V;4847 }4848 4849 // Canonicalize CmpLHS to be T, and CmpRHS to be F, if they're swapped.4850 if (CmpLHS == F && CmpRHS == T)4851 std::swap(CmpLHS, CmpRHS);4852 4853 if (CmpLHS != T || CmpRHS != F)4854 return nullptr;4855 4856 // This transform is also safe if we do not have (do not care about) -0.0.4857 if (Q.CxtI && isa<FPMathOperator>(Q.CxtI) && Q.CxtI->hasNoSignedZeros()) {4858 // (T == F) ? T : F --> F4859 if (Pred == FCmpInst::FCMP_OEQ)4860 return F;4861 4862 // (T != F) ? T : F --> T4863 if (Pred == FCmpInst::FCMP_UNE)4864 return T;4865 }4866 4867 return nullptr;4868}4869 4870/// Look for the following pattern and simplify %to_fold to %identicalPhi.4871/// Here %phi, %to_fold and %phi.next perform the same functionality as4872/// %identicalPhi and hence the select instruction %to_fold can be folded4873/// into %identicalPhi.4874///4875/// BB1:4876/// %identicalPhi = phi [ X, %BB0 ], [ %identicalPhi.next, %BB1 ]4877/// %phi = phi [ X, %BB0 ], [ %phi.next, %BB1 ]4878/// ...4879/// %identicalPhi.next = select %cmp, %val, %identicalPhi4880/// (or select %cmp, %identicalPhi, %val)4881/// %to_fold = select %cmp2, %identicalPhi, %phi4882/// %phi.next = select %cmp, %val, %to_fold4883/// (or select %cmp, %to_fold, %val)4884///4885/// Prove that %phi and %identicalPhi are the same by induction:4886///4887/// Base case: Both %phi and %identicalPhi are equal on entry to the loop.4888/// Inductive case:4889/// Suppose %phi and %identicalPhi are equal at iteration i.4890/// We look at their values at iteration i+1 which are %phi.next and4891/// %identicalPhi.next. They would have become different only when %cmp is4892/// false and the corresponding values %to_fold and %identicalPhi differ4893/// (similar reason for the other "or" case in the bracket).4894///4895/// The only condition when %to_fold and %identicalPh could differ is when %cmp24896/// is false and %to_fold is %phi, which contradicts our inductive hypothesis4897/// that %phi and %identicalPhi are equal. Thus %phi and %identicalPhi are4898/// always equal at iteration i+1.4899bool isSelectWithIdenticalPHI(PHINode &PN, PHINode &IdenticalPN) {4900 if (PN.getParent() != IdenticalPN.getParent())4901 return false;4902 if (PN.getNumIncomingValues() != 2)4903 return false;4904 4905 // Check that only the backedge incoming value is different.4906 unsigned DiffVals = 0;4907 BasicBlock *DiffValBB = nullptr;4908 for (unsigned i = 0; i < 2; i++) {4909 BasicBlock *PredBB = PN.getIncomingBlock(i);4910 if (PN.getIncomingValue(i) !=4911 IdenticalPN.getIncomingValueForBlock(PredBB)) {4912 DiffVals++;4913 DiffValBB = PredBB;4914 }4915 }4916 if (DiffVals != 1)4917 return false;4918 // Now check that the backedge incoming values are two select4919 // instructions with the same condition. Either their true4920 // values are the same, or their false values are the same.4921 auto *SI = dyn_cast<SelectInst>(PN.getIncomingValueForBlock(DiffValBB));4922 auto *IdenticalSI =4923 dyn_cast<SelectInst>(IdenticalPN.getIncomingValueForBlock(DiffValBB));4924 if (!SI || !IdenticalSI)4925 return false;4926 if (SI->getCondition() != IdenticalSI->getCondition())4927 return false;4928 4929 SelectInst *SIOtherVal = nullptr;4930 Value *IdenticalSIOtherVal = nullptr;4931 if (SI->getTrueValue() == IdenticalSI->getTrueValue()) {4932 SIOtherVal = dyn_cast<SelectInst>(SI->getFalseValue());4933 IdenticalSIOtherVal = IdenticalSI->getFalseValue();4934 } else if (SI->getFalseValue() == IdenticalSI->getFalseValue()) {4935 SIOtherVal = dyn_cast<SelectInst>(SI->getTrueValue());4936 IdenticalSIOtherVal = IdenticalSI->getTrueValue();4937 } else {4938 return false;4939 }4940 4941 // Now check that the other values in select, i.e., %to_fold and4942 // %identicalPhi, are essentially the same value.4943 if (!SIOtherVal || IdenticalSIOtherVal != &IdenticalPN)4944 return false;4945 if (!(SIOtherVal->getTrueValue() == &IdenticalPN &&4946 SIOtherVal->getFalseValue() == &PN) &&4947 !(SIOtherVal->getTrueValue() == &PN &&4948 SIOtherVal->getFalseValue() == &IdenticalPN))4949 return false;4950 return true;4951}4952 4953/// Given operands for a SelectInst, see if we can fold the result.4954/// If not, this returns null.4955static Value *simplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal,4956 const SimplifyQuery &Q, unsigned MaxRecurse) {4957 if (auto *CondC = dyn_cast<Constant>(Cond)) {4958 if (auto *TrueC = dyn_cast<Constant>(TrueVal))4959 if (auto *FalseC = dyn_cast<Constant>(FalseVal))4960 if (Constant *C = ConstantFoldSelectInstruction(CondC, TrueC, FalseC))4961 return C;4962 4963 // select poison, X, Y -> poison4964 if (isa<PoisonValue>(CondC))4965 return PoisonValue::get(TrueVal->getType());4966 4967 // select undef, X, Y -> X or Y4968 if (Q.isUndefValue(CondC))4969 return isa<Constant>(FalseVal) ? FalseVal : TrueVal;4970 4971 // select true, X, Y --> X4972 // select false, X, Y --> Y4973 // For vectors, allow undef/poison elements in the condition to match the4974 // defined elements, so we can eliminate the select.4975 if (match(CondC, m_One()))4976 return TrueVal;4977 if (match(CondC, m_Zero()))4978 return FalseVal;4979 }4980 4981 assert(Cond->getType()->isIntOrIntVectorTy(1) &&4982 "Select must have bool or bool vector condition");4983 assert(TrueVal->getType() == FalseVal->getType() &&4984 "Select must have same types for true/false ops");4985 4986 if (Cond->getType() == TrueVal->getType()) {4987 // select i1 Cond, i1 true, i1 false --> i1 Cond4988 if (match(TrueVal, m_One()) && match(FalseVal, m_ZeroInt()))4989 return Cond;4990 4991 // (X && Y) ? X : Y --> Y (commuted 2 ways)4992 if (match(Cond, m_c_LogicalAnd(m_Specific(TrueVal), m_Specific(FalseVal))))4993 return FalseVal;4994 4995 // (X || Y) ? X : Y --> X (commuted 2 ways)4996 if (match(Cond, m_c_LogicalOr(m_Specific(TrueVal), m_Specific(FalseVal))))4997 return TrueVal;4998 4999 // (X || Y) ? false : X --> false (commuted 2 ways)5000 if (match(Cond, m_c_LogicalOr(m_Specific(FalseVal), m_Value())) &&5001 match(TrueVal, m_ZeroInt()))5002 return ConstantInt::getFalse(Cond->getType());5003 5004 // Match patterns that end in logical-and.5005 if (match(FalseVal, m_ZeroInt())) {5006 // !(X || Y) && X --> false (commuted 2 ways)5007 if (match(Cond, m_Not(m_c_LogicalOr(m_Specific(TrueVal), m_Value()))))5008 return ConstantInt::getFalse(Cond->getType());5009 // X && !(X || Y) --> false (commuted 2 ways)5010 if (match(TrueVal, m_Not(m_c_LogicalOr(m_Specific(Cond), m_Value()))))5011 return ConstantInt::getFalse(Cond->getType());5012 5013 // (X || Y) && Y --> Y (commuted 2 ways)5014 if (match(Cond, m_c_LogicalOr(m_Specific(TrueVal), m_Value())))5015 return TrueVal;5016 // Y && (X || Y) --> Y (commuted 2 ways)5017 if (match(TrueVal, m_c_LogicalOr(m_Specific(Cond), m_Value())))5018 return Cond;5019 5020 // (X || Y) && (X || !Y) --> X (commuted 8 ways)5021 Value *X, *Y;5022 if (match(Cond, m_c_LogicalOr(m_Value(X), m_Not(m_Value(Y)))) &&5023 match(TrueVal, m_c_LogicalOr(m_Specific(X), m_Specific(Y))))5024 return X;5025 if (match(TrueVal, m_c_LogicalOr(m_Value(X), m_Not(m_Value(Y)))) &&5026 match(Cond, m_c_LogicalOr(m_Specific(X), m_Specific(Y))))5027 return X;5028 }5029 5030 // Match patterns that end in logical-or.5031 if (match(TrueVal, m_One())) {5032 // !(X && Y) || X --> true (commuted 2 ways)5033 if (match(Cond, m_Not(m_c_LogicalAnd(m_Specific(FalseVal), m_Value()))))5034 return ConstantInt::getTrue(Cond->getType());5035 // X || !(X && Y) --> true (commuted 2 ways)5036 if (match(FalseVal, m_Not(m_c_LogicalAnd(m_Specific(Cond), m_Value()))))5037 return ConstantInt::getTrue(Cond->getType());5038 5039 // (X && Y) || Y --> Y (commuted 2 ways)5040 if (match(Cond, m_c_LogicalAnd(m_Specific(FalseVal), m_Value())))5041 return FalseVal;5042 // Y || (X && Y) --> Y (commuted 2 ways)5043 if (match(FalseVal, m_c_LogicalAnd(m_Specific(Cond), m_Value())))5044 return Cond;5045 }5046 }5047 5048 // select ?, X, X -> X5049 if (TrueVal == FalseVal)5050 return TrueVal;5051 5052 if (Cond == TrueVal) {5053 // select i1 X, i1 X, i1 false --> X (logical-and)5054 if (match(FalseVal, m_ZeroInt()))5055 return Cond;5056 // select i1 X, i1 X, i1 true --> true5057 if (match(FalseVal, m_One()))5058 return ConstantInt::getTrue(Cond->getType());5059 }5060 if (Cond == FalseVal) {5061 // select i1 X, i1 true, i1 X --> X (logical-or)5062 if (match(TrueVal, m_One()))5063 return Cond;5064 // select i1 X, i1 false, i1 X --> false5065 if (match(TrueVal, m_ZeroInt()))5066 return ConstantInt::getFalse(Cond->getType());5067 }5068 5069 // If the true or false value is poison, we can fold to the other value.5070 // If the true or false value is undef, we can fold to the other value as5071 // long as the other value isn't poison.5072 // select ?, poison, X -> X5073 // select ?, undef, X -> X5074 if (isa<PoisonValue>(TrueVal) ||5075 (Q.isUndefValue(TrueVal) && impliesPoison(FalseVal, Cond)))5076 return FalseVal;5077 // select ?, X, poison -> X5078 // select ?, X, undef -> X5079 if (isa<PoisonValue>(FalseVal) ||5080 (Q.isUndefValue(FalseVal) && impliesPoison(TrueVal, Cond)))5081 return TrueVal;5082 5083 // Deal with partial undef vector constants: select ?, VecC, VecC' --> VecC''5084 Constant *TrueC, *FalseC;5085 if (isa<FixedVectorType>(TrueVal->getType()) &&5086 match(TrueVal, m_Constant(TrueC)) &&5087 match(FalseVal, m_Constant(FalseC))) {5088 unsigned NumElts =5089 cast<FixedVectorType>(TrueC->getType())->getNumElements();5090 SmallVector<Constant *, 16> NewC;5091 for (unsigned i = 0; i != NumElts; ++i) {5092 // Bail out on incomplete vector constants.5093 Constant *TEltC = TrueC->getAggregateElement(i);5094 Constant *FEltC = FalseC->getAggregateElement(i);5095 if (!TEltC || !FEltC)5096 break;5097 5098 // If the elements match (undef or not), that value is the result. If only5099 // one element is undef, choose the defined element as the safe result.5100 if (TEltC == FEltC)5101 NewC.push_back(TEltC);5102 else if (isa<PoisonValue>(TEltC) ||5103 (Q.isUndefValue(TEltC) && isGuaranteedNotToBePoison(FEltC)))5104 NewC.push_back(FEltC);5105 else if (isa<PoisonValue>(FEltC) ||5106 (Q.isUndefValue(FEltC) && isGuaranteedNotToBePoison(TEltC)))5107 NewC.push_back(TEltC);5108 else5109 break;5110 }5111 if (NewC.size() == NumElts)5112 return ConstantVector::get(NewC);5113 }5114 5115 if (Value *V =5116 simplifySelectWithICmpCond(Cond, TrueVal, FalseVal, Q, MaxRecurse))5117 return V;5118 5119 if (Value *V = simplifySelectWithBitTest(Cond, TrueVal, FalseVal))5120 return V;5121 5122 if (Value *V = simplifySelectWithFCmp(Cond, TrueVal, FalseVal, Q, MaxRecurse))5123 return V;5124 5125 std::optional<bool> Imp = isImpliedByDomCondition(Cond, Q.CxtI, Q.DL);5126 if (Imp)5127 return *Imp ? TrueVal : FalseVal;5128 // Look for same PHIs in the true and false values.5129 if (auto *TruePHI = dyn_cast<PHINode>(TrueVal))5130 if (auto *FalsePHI = dyn_cast<PHINode>(FalseVal)) {5131 if (isSelectWithIdenticalPHI(*TruePHI, *FalsePHI))5132 return FalseVal;5133 if (isSelectWithIdenticalPHI(*FalsePHI, *TruePHI))5134 return TrueVal;5135 }5136 return nullptr;5137}5138 5139Value *llvm::simplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal,5140 const SimplifyQuery &Q) {5141 return ::simplifySelectInst(Cond, TrueVal, FalseVal, Q, RecursionLimit);5142}5143 5144/// Given operands for an GetElementPtrInst, see if we can fold the result.5145/// If not, this returns null.5146static Value *simplifyGEPInst(Type *SrcTy, Value *Ptr,5147 ArrayRef<Value *> Indices, GEPNoWrapFlags NW,5148 const SimplifyQuery &Q, unsigned) {5149 // The type of the GEP pointer operand.5150 unsigned AS =5151 cast<PointerType>(Ptr->getType()->getScalarType())->getAddressSpace();5152 5153 // getelementptr P -> P.5154 if (Indices.empty())5155 return Ptr;5156 5157 // Compute the (pointer) type returned by the GEP instruction.5158 Type *LastType = GetElementPtrInst::getIndexedType(SrcTy, Indices);5159 Type *GEPTy = Ptr->getType();5160 if (!GEPTy->isVectorTy()) {5161 for (Value *Op : Indices) {5162 // If one of the operands is a vector, the result type is a vector of5163 // pointers. All vector operands must have the same number of elements.5164 if (VectorType *VT = dyn_cast<VectorType>(Op->getType())) {5165 GEPTy = VectorType::get(GEPTy, VT->getElementCount());5166 break;5167 }5168 }5169 }5170 5171 // All-zero GEP is a no-op, unless it performs a vector splat.5172 if (Ptr->getType() == GEPTy && all_of(Indices, match_fn(m_Zero())))5173 return Ptr;5174 5175 // getelementptr poison, idx -> poison5176 // getelementptr baseptr, poison -> poison5177 if (isa<PoisonValue>(Ptr) || any_of(Indices, IsaPred<PoisonValue>))5178 return PoisonValue::get(GEPTy);5179 5180 // getelementptr undef, idx -> undef5181 if (Q.isUndefValue(Ptr))5182 return UndefValue::get(GEPTy);5183 5184 bool IsScalableVec =5185 SrcTy->isScalableTy() || any_of(Indices, [](const Value *V) {5186 return isa<ScalableVectorType>(V->getType());5187 });5188 5189 if (Indices.size() == 1) {5190 Type *Ty = SrcTy;5191 if (!IsScalableVec && Ty->isSized()) {5192 Value *P;5193 uint64_t C;5194 uint64_t TyAllocSize = Q.DL.getTypeAllocSize(Ty);5195 // getelementptr P, N -> P if P points to a type of zero size.5196 if (TyAllocSize == 0 && Ptr->getType() == GEPTy)5197 return Ptr;5198 5199 // The following transforms are only safe if the ptrtoint cast5200 // doesn't truncate the address of the pointers. The non-address bits5201 // must be the same, as the underlying objects are the same.5202 if (Indices[0]->getType()->getScalarSizeInBits() >=5203 Q.DL.getAddressSizeInBits(AS)) {5204 auto CanSimplify = [GEPTy, &P, Ptr]() -> bool {5205 return P->getType() == GEPTy &&5206 getUnderlyingObject(P) == getUnderlyingObject(Ptr);5207 };5208 // getelementptr V, (sub P, V) -> P if P points to a type of size 1.5209 if (TyAllocSize == 1 &&5210 match(Indices[0], m_Sub(m_PtrToIntOrAddr(m_Value(P)),5211 m_PtrToIntOrAddr(m_Specific(Ptr)))) &&5212 CanSimplify())5213 return P;5214 5215 // getelementptr V, (ashr (sub P, V), C) -> P if P points to a type of5216 // size 1 << C.5217 if (match(Indices[0], m_AShr(m_Sub(m_PtrToIntOrAddr(m_Value(P)),5218 m_PtrToIntOrAddr(m_Specific(Ptr))),5219 m_ConstantInt(C))) &&5220 TyAllocSize == 1ULL << C && CanSimplify())5221 return P;5222 5223 // getelementptr V, (sdiv (sub P, V), C) -> P if P points to a type of5224 // size C.5225 if (match(Indices[0], m_SDiv(m_Sub(m_PtrToIntOrAddr(m_Value(P)),5226 m_PtrToIntOrAddr(m_Specific(Ptr))),5227 m_SpecificInt(TyAllocSize))) &&5228 CanSimplify())5229 return P;5230 }5231 }5232 }5233 5234 if (!IsScalableVec && Q.DL.getTypeAllocSize(LastType) == 1 &&5235 all_of(Indices.drop_back(1), match_fn(m_Zero()))) {5236 unsigned IdxWidth =5237 Q.DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace());5238 if (Q.DL.getTypeSizeInBits(Indices.back()->getType()) == IdxWidth) {5239 APInt BasePtrOffset(IdxWidth, 0);5240 Value *StrippedBasePtr =5241 Ptr->stripAndAccumulateInBoundsConstantOffsets(Q.DL, BasePtrOffset);5242 5243 // Avoid creating inttoptr of zero here: While LLVMs treatment of5244 // inttoptr is generally conservative, this particular case is folded to5245 // a null pointer, which will have incorrect provenance.5246 5247 // gep (gep V, C), (sub 0, V) -> C5248 if (match(Indices.back(),5249 m_Neg(m_PtrToInt(m_Specific(StrippedBasePtr)))) &&5250 !BasePtrOffset.isZero()) {5251 auto *CI = ConstantInt::get(GEPTy->getContext(), BasePtrOffset);5252 return ConstantExpr::getIntToPtr(CI, GEPTy);5253 }5254 // gep (gep V, C), (xor V, -1) -> C-15255 if (match(Indices.back(),5256 m_Xor(m_PtrToInt(m_Specific(StrippedBasePtr)), m_AllOnes())) &&5257 !BasePtrOffset.isOne()) {5258 auto *CI = ConstantInt::get(GEPTy->getContext(), BasePtrOffset - 1);5259 return ConstantExpr::getIntToPtr(CI, GEPTy);5260 }5261 }5262 }5263 5264 // Check to see if this is constant foldable.5265 if (!isa<Constant>(Ptr) || !all_of(Indices, IsaPred<Constant>))5266 return nullptr;5267 5268 if (!ConstantExpr::isSupportedGetElementPtr(SrcTy))5269 return ConstantFoldGetElementPtr(SrcTy, cast<Constant>(Ptr), std::nullopt,5270 Indices);5271 5272 auto *CE =5273 ConstantExpr::getGetElementPtr(SrcTy, cast<Constant>(Ptr), Indices, NW);5274 return ConstantFoldConstant(CE, Q.DL);5275}5276 5277Value *llvm::simplifyGEPInst(Type *SrcTy, Value *Ptr, ArrayRef<Value *> Indices,5278 GEPNoWrapFlags NW, const SimplifyQuery &Q) {5279 return ::simplifyGEPInst(SrcTy, Ptr, Indices, NW, Q, RecursionLimit);5280}5281 5282/// Given operands for an InsertValueInst, see if we can fold the result.5283/// If not, this returns null.5284static Value *simplifyInsertValueInst(Value *Agg, Value *Val,5285 ArrayRef<unsigned> Idxs,5286 const SimplifyQuery &Q, unsigned) {5287 if (Constant *CAgg = dyn_cast<Constant>(Agg))5288 if (Constant *CVal = dyn_cast<Constant>(Val))5289 return ConstantFoldInsertValueInstruction(CAgg, CVal, Idxs);5290 5291 // insertvalue x, poison, n -> x5292 // insertvalue x, undef, n -> x if x cannot be poison5293 if (isa<PoisonValue>(Val) ||5294 (Q.isUndefValue(Val) && isGuaranteedNotToBePoison(Agg)))5295 return Agg;5296 5297 // insertvalue x, (extractvalue y, n), n5298 if (ExtractValueInst *EV = dyn_cast<ExtractValueInst>(Val))5299 if (EV->getAggregateOperand()->getType() == Agg->getType() &&5300 EV->getIndices() == Idxs) {5301 // insertvalue poison, (extractvalue y, n), n -> y5302 // insertvalue undef, (extractvalue y, n), n -> y if y cannot be poison5303 if (isa<PoisonValue>(Agg) ||5304 (Q.isUndefValue(Agg) &&5305 isGuaranteedNotToBePoison(EV->getAggregateOperand())))5306 return EV->getAggregateOperand();5307 5308 // insertvalue y, (extractvalue y, n), n -> y5309 if (Agg == EV->getAggregateOperand())5310 return Agg;5311 }5312 5313 return nullptr;5314}5315 5316Value *llvm::simplifyInsertValueInst(Value *Agg, Value *Val,5317 ArrayRef<unsigned> Idxs,5318 const SimplifyQuery &Q) {5319 return ::simplifyInsertValueInst(Agg, Val, Idxs, Q, RecursionLimit);5320}5321 5322Value *llvm::simplifyInsertElementInst(Value *Vec, Value *Val, Value *Idx,5323 const SimplifyQuery &Q) {5324 // Try to constant fold.5325 auto *VecC = dyn_cast<Constant>(Vec);5326 auto *ValC = dyn_cast<Constant>(Val);5327 auto *IdxC = dyn_cast<Constant>(Idx);5328 if (VecC && ValC && IdxC)5329 return ConstantExpr::getInsertElement(VecC, ValC, IdxC);5330 5331 // For fixed-length vector, fold into poison if index is out of bounds.5332 if (auto *CI = dyn_cast<ConstantInt>(Idx)) {5333 if (isa<FixedVectorType>(Vec->getType()) &&5334 CI->uge(cast<FixedVectorType>(Vec->getType())->getNumElements()))5335 return PoisonValue::get(Vec->getType());5336 }5337 5338 // If index is undef, it might be out of bounds (see above case)5339 if (Q.isUndefValue(Idx))5340 return PoisonValue::get(Vec->getType());5341 5342 // If the scalar is poison, or it is undef and there is no risk of5343 // propagating poison from the vector value, simplify to the vector value.5344 if (isa<PoisonValue>(Val) ||5345 (Q.isUndefValue(Val) && isGuaranteedNotToBePoison(Vec)))5346 return Vec;5347 5348 // Inserting the splatted value into a constant splat does nothing.5349 if (VecC && ValC && VecC->getSplatValue() == ValC)5350 return Vec;5351 5352 // If we are extracting a value from a vector, then inserting it into the same5353 // place, that's the input vector:5354 // insertelt Vec, (extractelt Vec, Idx), Idx --> Vec5355 if (match(Val, m_ExtractElt(m_Specific(Vec), m_Specific(Idx))))5356 return Vec;5357 5358 return nullptr;5359}5360 5361/// Given operands for an ExtractValueInst, see if we can fold the result.5362/// If not, this returns null.5363static Value *simplifyExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs,5364 const SimplifyQuery &, unsigned) {5365 if (auto *CAgg = dyn_cast<Constant>(Agg))5366 return ConstantFoldExtractValueInstruction(CAgg, Idxs);5367 5368 // extractvalue x, (insertvalue y, elt, n), n -> elt5369 unsigned NumIdxs = Idxs.size();5370 for (auto *IVI = dyn_cast<InsertValueInst>(Agg); IVI != nullptr;5371 IVI = dyn_cast<InsertValueInst>(IVI->getAggregateOperand())) {5372 ArrayRef<unsigned> InsertValueIdxs = IVI->getIndices();5373 unsigned NumInsertValueIdxs = InsertValueIdxs.size();5374 unsigned NumCommonIdxs = std::min(NumInsertValueIdxs, NumIdxs);5375 if (InsertValueIdxs.slice(0, NumCommonIdxs) ==5376 Idxs.slice(0, NumCommonIdxs)) {5377 if (NumIdxs == NumInsertValueIdxs)5378 return IVI->getInsertedValueOperand();5379 break;5380 }5381 }5382 5383 // Simplify umul_with_overflow where one operand is 1.5384 Value *V;5385 if (Idxs.size() == 1 &&5386 (match(Agg,5387 m_Intrinsic<Intrinsic::umul_with_overflow>(m_Value(V), m_One())) ||5388 match(Agg, m_Intrinsic<Intrinsic::umul_with_overflow>(m_One(),5389 m_Value(V))))) {5390 if (Idxs[0] == 0)5391 return V;5392 assert(Idxs[0] == 1 && "invalid index");5393 return getFalse(CmpInst::makeCmpResultType(V->getType()));5394 }5395 5396 return nullptr;5397}5398 5399Value *llvm::simplifyExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs,5400 const SimplifyQuery &Q) {5401 return ::simplifyExtractValueInst(Agg, Idxs, Q, RecursionLimit);5402}5403 5404/// Given operands for an ExtractElementInst, see if we can fold the result.5405/// If not, this returns null.5406static Value *simplifyExtractElementInst(Value *Vec, Value *Idx,5407 const SimplifyQuery &Q, unsigned) {5408 auto *VecVTy = cast<VectorType>(Vec->getType());5409 if (auto *CVec = dyn_cast<Constant>(Vec)) {5410 if (auto *CIdx = dyn_cast<Constant>(Idx))5411 return ConstantExpr::getExtractElement(CVec, CIdx);5412 5413 if (Q.isUndefValue(Vec))5414 return UndefValue::get(VecVTy->getElementType());5415 }5416 5417 // An undef extract index can be arbitrarily chosen to be an out-of-range5418 // index value, which would result in the instruction being poison.5419 if (Q.isUndefValue(Idx))5420 return PoisonValue::get(VecVTy->getElementType());5421 5422 // If extracting a specified index from the vector, see if we can recursively5423 // find a previously computed scalar that was inserted into the vector.5424 if (auto *IdxC = dyn_cast<ConstantInt>(Idx)) {5425 // For fixed-length vector, fold into undef if index is out of bounds.5426 unsigned MinNumElts = VecVTy->getElementCount().getKnownMinValue();5427 if (isa<FixedVectorType>(VecVTy) && IdxC->getValue().uge(MinNumElts))5428 return PoisonValue::get(VecVTy->getElementType());5429 // Handle case where an element is extracted from a splat.5430 if (IdxC->getValue().ult(MinNumElts))5431 if (auto *Splat = getSplatValue(Vec))5432 return Splat;5433 if (Value *Elt = findScalarElement(Vec, IdxC->getZExtValue()))5434 return Elt;5435 } else {5436 // extractelt x, (insertelt y, elt, n), n -> elt5437 // If the possibly-variable indices are trivially known to be equal5438 // (because they are the same operand) then use the value that was5439 // inserted directly.5440 auto *IE = dyn_cast<InsertElementInst>(Vec);5441 if (IE && IE->getOperand(2) == Idx)5442 return IE->getOperand(1);5443 5444 // The index is not relevant if our vector is a splat.5445 if (Value *Splat = getSplatValue(Vec))5446 return Splat;5447 }5448 return nullptr;5449}5450 5451Value *llvm::simplifyExtractElementInst(Value *Vec, Value *Idx,5452 const SimplifyQuery &Q) {5453 return ::simplifyExtractElementInst(Vec, Idx, Q, RecursionLimit);5454}5455 5456/// See if we can fold the given phi. If not, returns null.5457static Value *simplifyPHINode(PHINode *PN, ArrayRef<Value *> IncomingValues,5458 const SimplifyQuery &Q) {5459 // WARNING: no matter how worthwhile it may seem, we can not perform PHI CSE5460 // here, because the PHI we may succeed simplifying to was not5461 // def-reachable from the original PHI!5462 5463 // If all of the PHI's incoming values are the same then replace the PHI node5464 // with the common value.5465 Value *CommonValue = nullptr;5466 bool HasPoisonInput = false;5467 bool HasUndefInput = false;5468 for (Value *Incoming : IncomingValues) {5469 // If the incoming value is the phi node itself, it can safely be skipped.5470 if (Incoming == PN)5471 continue;5472 if (isa<PoisonValue>(Incoming)) {5473 HasPoisonInput = true;5474 continue;5475 }5476 if (Q.isUndefValue(Incoming)) {5477 // Remember that we saw an undef value, but otherwise ignore them.5478 HasUndefInput = true;5479 continue;5480 }5481 if (CommonValue && Incoming != CommonValue)5482 return nullptr; // Not the same, bail out.5483 CommonValue = Incoming;5484 }5485 5486 // If CommonValue is null then all of the incoming values were either undef,5487 // poison or equal to the phi node itself.5488 if (!CommonValue)5489 return HasUndefInput ? UndefValue::get(PN->getType())5490 : PoisonValue::get(PN->getType());5491 5492 if (HasPoisonInput || HasUndefInput) {5493 // If we have a PHI node like phi(X, undef, X), where X is defined by some5494 // instruction, we cannot return X as the result of the PHI node unless it5495 // dominates the PHI block.5496 if (!valueDominatesPHI(CommonValue, PN, Q.DT))5497 return nullptr;5498 5499 // Make sure we do not replace an undef value with poison.5500 if (HasUndefInput &&5501 !isGuaranteedNotToBePoison(CommonValue, Q.AC, Q.CxtI, Q.DT))5502 return nullptr;5503 return CommonValue;5504 }5505 5506 return CommonValue;5507}5508 5509static Value *simplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty,5510 const SimplifyQuery &Q, unsigned MaxRecurse) {5511 if (auto *C = dyn_cast<Constant>(Op))5512 return ConstantFoldCastOperand(CastOpc, C, Ty, Q.DL);5513 5514 if (auto *CI = dyn_cast<CastInst>(Op)) {5515 auto *Src = CI->getOperand(0);5516 Type *SrcTy = Src->getType();5517 Type *MidTy = CI->getType();5518 Type *DstTy = Ty;5519 if (Src->getType() == Ty) {5520 auto FirstOp = CI->getOpcode();5521 auto SecondOp = static_cast<Instruction::CastOps>(CastOpc);5522 if (CastInst::isEliminableCastPair(FirstOp, SecondOp, SrcTy, MidTy, DstTy,5523 &Q.DL) == Instruction::BitCast)5524 return Src;5525 }5526 }5527 5528 // bitcast x -> x5529 if (CastOpc == Instruction::BitCast)5530 if (Op->getType() == Ty)5531 return Op;5532 5533 // ptrtoint (ptradd (Ptr, X - ptrtoint(Ptr))) -> X5534 Value *Ptr, *X;5535 if ((CastOpc == Instruction::PtrToInt || CastOpc == Instruction::PtrToAddr) &&5536 match(Op,5537 m_PtrAdd(m_Value(Ptr),5538 m_Sub(m_Value(X), m_PtrToIntOrAddr(m_Deferred(Ptr))))) &&5539 X->getType() == Ty && Ty == Q.DL.getIndexType(Ptr->getType()))5540 return X;5541 5542 return nullptr;5543}5544 5545Value *llvm::simplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty,5546 const SimplifyQuery &Q) {5547 return ::simplifyCastInst(CastOpc, Op, Ty, Q, RecursionLimit);5548}5549 5550/// For the given destination element of a shuffle, peek through shuffles to5551/// match a root vector source operand that contains that element in the same5552/// vector lane (ie, the same mask index), so we can eliminate the shuffle(s).5553static Value *foldIdentityShuffles(int DestElt, Value *Op0, Value *Op1,5554 int MaskVal, Value *RootVec,5555 unsigned MaxRecurse) {5556 if (!MaxRecurse--)5557 return nullptr;5558 5559 // Bail out if any mask value is undefined. That kind of shuffle may be5560 // simplified further based on demanded bits or other folds.5561 if (MaskVal == -1)5562 return nullptr;5563 5564 // The mask value chooses which source operand we need to look at next.5565 int InVecNumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();5566 int RootElt = MaskVal;5567 Value *SourceOp = Op0;5568 if (MaskVal >= InVecNumElts) {5569 RootElt = MaskVal - InVecNumElts;5570 SourceOp = Op1;5571 }5572 5573 // If the source operand is a shuffle itself, look through it to find the5574 // matching root vector.5575 if (auto *SourceShuf = dyn_cast<ShuffleVectorInst>(SourceOp)) {5576 return foldIdentityShuffles(5577 DestElt, SourceShuf->getOperand(0), SourceShuf->getOperand(1),5578 SourceShuf->getMaskValue(RootElt), RootVec, MaxRecurse);5579 }5580 5581 // The source operand is not a shuffle. Initialize the root vector value for5582 // this shuffle if that has not been done yet.5583 if (!RootVec)5584 RootVec = SourceOp;5585 5586 // Give up as soon as a source operand does not match the existing root value.5587 if (RootVec != SourceOp)5588 return nullptr;5589 5590 // The element must be coming from the same lane in the source vector5591 // (although it may have crossed lanes in intermediate shuffles).5592 if (RootElt != DestElt)5593 return nullptr;5594 5595 return RootVec;5596}5597 5598static Value *simplifyShuffleVectorInst(Value *Op0, Value *Op1,5599 ArrayRef<int> Mask, Type *RetTy,5600 const SimplifyQuery &Q,5601 unsigned MaxRecurse) {5602 if (all_of(Mask, [](int Elem) { return Elem == PoisonMaskElem; }))5603 return PoisonValue::get(RetTy);5604 5605 auto *InVecTy = cast<VectorType>(Op0->getType());5606 unsigned MaskNumElts = Mask.size();5607 ElementCount InVecEltCount = InVecTy->getElementCount();5608 5609 bool Scalable = InVecEltCount.isScalable();5610 5611 SmallVector<int, 32> Indices;5612 Indices.assign(Mask.begin(), Mask.end());5613 5614 // Canonicalization: If mask does not select elements from an input vector,5615 // replace that input vector with poison.5616 if (!Scalable) {5617 bool MaskSelects0 = false, MaskSelects1 = false;5618 unsigned InVecNumElts = InVecEltCount.getKnownMinValue();5619 for (unsigned i = 0; i != MaskNumElts; ++i) {5620 if (Indices[i] == -1)5621 continue;5622 if ((unsigned)Indices[i] < InVecNumElts)5623 MaskSelects0 = true;5624 else5625 MaskSelects1 = true;5626 }5627 if (!MaskSelects0)5628 Op0 = PoisonValue::get(InVecTy);5629 if (!MaskSelects1)5630 Op1 = PoisonValue::get(InVecTy);5631 }5632 5633 auto *Op0Const = dyn_cast<Constant>(Op0);5634 auto *Op1Const = dyn_cast<Constant>(Op1);5635 5636 // If all operands are constant, constant fold the shuffle. This5637 // transformation depends on the value of the mask which is not known at5638 // compile time for scalable vectors5639 if (Op0Const && Op1Const)5640 return ConstantExpr::getShuffleVector(Op0Const, Op1Const, Mask);5641 5642 // Canonicalization: if only one input vector is constant, it shall be the5643 // second one. This transformation depends on the value of the mask which5644 // is not known at compile time for scalable vectors5645 if (!Scalable && Op0Const && !Op1Const) {5646 std::swap(Op0, Op1);5647 ShuffleVectorInst::commuteShuffleMask(Indices,5648 InVecEltCount.getKnownMinValue());5649 }5650 5651 // A splat of an inserted scalar constant becomes a vector constant:5652 // shuf (inselt ?, C, IndexC), undef, <IndexC, IndexC...> --> <C, C...>5653 // NOTE: We may have commuted above, so analyze the updated Indices, not the5654 // original mask constant.5655 // NOTE: This transformation depends on the value of the mask which is not5656 // known at compile time for scalable vectors5657 Constant *C;5658 ConstantInt *IndexC;5659 if (!Scalable && match(Op0, m_InsertElt(m_Value(), m_Constant(C),5660 m_ConstantInt(IndexC)))) {5661 // Match a splat shuffle mask of the insert index allowing undef elements.5662 int InsertIndex = IndexC->getZExtValue();5663 if (all_of(Indices, [InsertIndex](int MaskElt) {5664 return MaskElt == InsertIndex || MaskElt == -1;5665 })) {5666 assert(isa<UndefValue>(Op1) && "Expected undef operand 1 for splat");5667 5668 // Shuffle mask poisons become poison constant result elements.5669 SmallVector<Constant *, 16> VecC(MaskNumElts, C);5670 for (unsigned i = 0; i != MaskNumElts; ++i)5671 if (Indices[i] == -1)5672 VecC[i] = PoisonValue::get(C->getType());5673 return ConstantVector::get(VecC);5674 }5675 }5676 5677 // A shuffle of a splat is always the splat itself. Legal if the shuffle's5678 // value type is same as the input vectors' type.5679 if (auto *OpShuf = dyn_cast<ShuffleVectorInst>(Op0))5680 if (Q.isUndefValue(Op1) && RetTy == InVecTy &&5681 all_equal(OpShuf->getShuffleMask()))5682 return Op0;5683 5684 // All remaining transformation depend on the value of the mask, which is5685 // not known at compile time for scalable vectors.5686 if (Scalable)5687 return nullptr;5688 5689 // Don't fold a shuffle with undef mask elements. This may get folded in a5690 // better way using demanded bits or other analysis.5691 // TODO: Should we allow this?5692 if (is_contained(Indices, -1))5693 return nullptr;5694 5695 // Check if every element of this shuffle can be mapped back to the5696 // corresponding element of a single root vector. If so, we don't need this5697 // shuffle. This handles simple identity shuffles as well as chains of5698 // shuffles that may widen/narrow and/or move elements across lanes and back.5699 Value *RootVec = nullptr;5700 for (unsigned i = 0; i != MaskNumElts; ++i) {5701 // Note that recursion is limited for each vector element, so if any element5702 // exceeds the limit, this will fail to simplify.5703 RootVec =5704 foldIdentityShuffles(i, Op0, Op1, Indices[i], RootVec, MaxRecurse);5705 5706 // We can't replace a widening/narrowing shuffle with one of its operands.5707 if (!RootVec || RootVec->getType() != RetTy)5708 return nullptr;5709 }5710 return RootVec;5711}5712 5713/// Given operands for a ShuffleVectorInst, fold the result or return null.5714Value *llvm::simplifyShuffleVectorInst(Value *Op0, Value *Op1,5715 ArrayRef<int> Mask, Type *RetTy,5716 const SimplifyQuery &Q) {5717 return ::simplifyShuffleVectorInst(Op0, Op1, Mask, RetTy, Q, RecursionLimit);5718}5719 5720static Constant *foldConstant(Instruction::UnaryOps Opcode, Value *&Op,5721 const SimplifyQuery &Q) {5722 if (auto *C = dyn_cast<Constant>(Op))5723 return ConstantFoldUnaryOpOperand(Opcode, C, Q.DL);5724 return nullptr;5725}5726 5727/// Given the operand for an FNeg, see if we can fold the result. If not, this5728/// returns null.5729static Value *simplifyFNegInst(Value *Op, FastMathFlags FMF,5730 const SimplifyQuery &Q, unsigned MaxRecurse) {5731 if (Constant *C = foldConstant(Instruction::FNeg, Op, Q))5732 return C;5733 5734 Value *X;5735 // fneg (fneg X) ==> X5736 if (match(Op, m_FNeg(m_Value(X))))5737 return X;5738 5739 return nullptr;5740}5741 5742Value *llvm::simplifyFNegInst(Value *Op, FastMathFlags FMF,5743 const SimplifyQuery &Q) {5744 return ::simplifyFNegInst(Op, FMF, Q, RecursionLimit);5745}5746 5747/// Try to propagate existing NaN values when possible. If not, replace the5748/// constant or elements in the constant with a canonical NaN.5749static Constant *propagateNaN(Constant *In) {5750 Type *Ty = In->getType();5751 if (auto *VecTy = dyn_cast<FixedVectorType>(Ty)) {5752 unsigned NumElts = VecTy->getNumElements();5753 SmallVector<Constant *, 32> NewC(NumElts);5754 for (unsigned i = 0; i != NumElts; ++i) {5755 Constant *EltC = In->getAggregateElement(i);5756 // Poison elements propagate. NaN propagates except signaling is quieted.5757 // Replace unknown or undef elements with canonical NaN.5758 if (EltC && isa<PoisonValue>(EltC))5759 NewC[i] = EltC;5760 else if (EltC && EltC->isNaN())5761 NewC[i] = ConstantFP::get(5762 EltC->getType(), cast<ConstantFP>(EltC)->getValue().makeQuiet());5763 else5764 NewC[i] = ConstantFP::getNaN(VecTy->getElementType());5765 }5766 return ConstantVector::get(NewC);5767 }5768 5769 // If it is not a fixed vector, but not a simple NaN either, return a5770 // canonical NaN.5771 if (!In->isNaN())5772 return ConstantFP::getNaN(Ty);5773 5774 // If we known this is a NaN, and it's scalable vector, we must have a splat5775 // on our hands. Grab that before splatting a QNaN constant.5776 if (isa<ScalableVectorType>(Ty)) {5777 auto *Splat = In->getSplatValue();5778 assert(Splat && Splat->isNaN() &&5779 "Found a scalable-vector NaN but not a splat");5780 In = Splat;5781 }5782 5783 // Propagate an existing QNaN constant. If it is an SNaN, make it quiet, but5784 // preserve the sign/payload.5785 return ConstantFP::get(Ty, cast<ConstantFP>(In)->getValue().makeQuiet());5786}5787 5788/// Perform folds that are common to any floating-point operation. This implies5789/// transforms based on poison/undef/NaN because the operation itself makes no5790/// difference to the result.5791static Constant *simplifyFPOp(ArrayRef<Value *> Ops, FastMathFlags FMF,5792 const SimplifyQuery &Q,5793 fp::ExceptionBehavior ExBehavior,5794 RoundingMode Rounding) {5795 // Poison is independent of anything else. It always propagates from an5796 // operand to a math result.5797 if (any_of(Ops, IsaPred<PoisonValue>))5798 return PoisonValue::get(Ops[0]->getType());5799 5800 for (Value *V : Ops) {5801 bool IsNan = match(V, m_NaN());5802 bool IsInf = match(V, m_Inf());5803 bool IsUndef = Q.isUndefValue(V);5804 5805 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand5806 // (an undef operand can be chosen to be Nan/Inf), then the result of5807 // this operation is poison.5808 if (FMF.noNaNs() && (IsNan || IsUndef))5809 return PoisonValue::get(V->getType());5810 if (FMF.noInfs() && (IsInf || IsUndef))5811 return PoisonValue::get(V->getType());5812 5813 if (isDefaultFPEnvironment(ExBehavior, Rounding)) {5814 // Undef does not propagate because undef means that all bits can take on5815 // any value. If this is undef * NaN for example, then the result values5816 // (at least the exponent bits) are limited. Assume the undef is a5817 // canonical NaN and propagate that.5818 if (IsUndef)5819 return ConstantFP::getNaN(V->getType());5820 if (IsNan)5821 return propagateNaN(cast<Constant>(V));5822 } else if (ExBehavior != fp::ebStrict) {5823 if (IsNan)5824 return propagateNaN(cast<Constant>(V));5825 }5826 }5827 return nullptr;5828}5829 5830/// Given operands for an FAdd, see if we can fold the result. If not, this5831/// returns null.5832static Value *5833simplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF,5834 const SimplifyQuery &Q, unsigned MaxRecurse,5835 fp::ExceptionBehavior ExBehavior = fp::ebIgnore,5836 RoundingMode Rounding = RoundingMode::NearestTiesToEven) {5837 if (isDefaultFPEnvironment(ExBehavior, Rounding))5838 if (Constant *C = foldOrCommuteConstant(Instruction::FAdd, Op0, Op1, Q))5839 return C;5840 5841 if (Constant *C = simplifyFPOp({Op0, Op1}, FMF, Q, ExBehavior, Rounding))5842 return C;5843 5844 // fadd X, -0 ==> X5845 // With strict/constrained FP, we have these possible edge cases that do5846 // not simplify to Op0:5847 // fadd SNaN, -0.0 --> QNaN5848 // fadd +0.0, -0.0 --> -0.0 (but only with round toward negative)5849 if (canIgnoreSNaN(ExBehavior, FMF) &&5850 (!canRoundingModeBe(Rounding, RoundingMode::TowardNegative) ||5851 FMF.noSignedZeros()))5852 if (match(Op1, m_NegZeroFP()))5853 return Op0;5854 5855 // fadd X, 0 ==> X, when we know X is not -05856 if (canIgnoreSNaN(ExBehavior, FMF))5857 if (match(Op1, m_PosZeroFP()) &&5858 (FMF.noSignedZeros() || cannotBeNegativeZero(Op0, Q)))5859 return Op0;5860 5861 if (!isDefaultFPEnvironment(ExBehavior, Rounding))5862 return nullptr;5863 5864 if (FMF.noNaNs()) {5865 // With nnan: X + {+/-}Inf --> {+/-}Inf5866 if (match(Op1, m_Inf()))5867 return Op1;5868 5869 // With nnan: -X + X --> 0.0 (and commuted variant)5870 // We don't have to explicitly exclude infinities (ninf): INF + -INF == NaN.5871 // Negative zeros are allowed because we always end up with positive zero:5872 // X = -0.0: (-0.0 - (-0.0)) + (-0.0) == ( 0.0) + (-0.0) == 0.05873 // X = -0.0: ( 0.0 - (-0.0)) + (-0.0) == ( 0.0) + (-0.0) == 0.05874 // X = 0.0: (-0.0 - ( 0.0)) + ( 0.0) == (-0.0) + ( 0.0) == 0.05875 // X = 0.0: ( 0.0 - ( 0.0)) + ( 0.0) == ( 0.0) + ( 0.0) == 0.05876 if (match(Op0, m_FSub(m_AnyZeroFP(), m_Specific(Op1))) ||5877 match(Op1, m_FSub(m_AnyZeroFP(), m_Specific(Op0))))5878 return ConstantFP::getZero(Op0->getType());5879 5880 if (match(Op0, m_FNeg(m_Specific(Op1))) ||5881 match(Op1, m_FNeg(m_Specific(Op0))))5882 return ConstantFP::getZero(Op0->getType());5883 }5884 5885 // (X - Y) + Y --> X5886 // Y + (X - Y) --> X5887 Value *X;5888 if (FMF.noSignedZeros() && FMF.allowReassoc() &&5889 (match(Op0, m_FSub(m_Value(X), m_Specific(Op1))) ||5890 match(Op1, m_FSub(m_Value(X), m_Specific(Op0)))))5891 return X;5892 5893 return nullptr;5894}5895 5896/// Given operands for an FSub, see if we can fold the result. If not, this5897/// returns null.5898static Value *5899simplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF,5900 const SimplifyQuery &Q, unsigned MaxRecurse,5901 fp::ExceptionBehavior ExBehavior = fp::ebIgnore,5902 RoundingMode Rounding = RoundingMode::NearestTiesToEven) {5903 if (isDefaultFPEnvironment(ExBehavior, Rounding))5904 if (Constant *C = foldOrCommuteConstant(Instruction::FSub, Op0, Op1, Q))5905 return C;5906 5907 if (Constant *C = simplifyFPOp({Op0, Op1}, FMF, Q, ExBehavior, Rounding))5908 return C;5909 5910 // fsub X, +0 ==> X5911 if (canIgnoreSNaN(ExBehavior, FMF) &&5912 (!canRoundingModeBe(Rounding, RoundingMode::TowardNegative) ||5913 FMF.noSignedZeros()))5914 if (match(Op1, m_PosZeroFP()))5915 return Op0;5916 5917 // fsub X, -0 ==> X, when we know X is not -05918 if (canIgnoreSNaN(ExBehavior, FMF))5919 if (match(Op1, m_NegZeroFP()) &&5920 (FMF.noSignedZeros() || cannotBeNegativeZero(Op0, Q)))5921 return Op0;5922 5923 // fsub -0.0, (fsub -0.0, X) ==> X5924 // fsub -0.0, (fneg X) ==> X5925 Value *X;5926 if (canIgnoreSNaN(ExBehavior, FMF))5927 if (match(Op0, m_NegZeroFP()) && match(Op1, m_FNeg(m_Value(X))))5928 return X;5929 5930 // fsub 0.0, (fsub 0.0, X) ==> X if signed zeros are ignored.5931 // fsub 0.0, (fneg X) ==> X if signed zeros are ignored.5932 if (canIgnoreSNaN(ExBehavior, FMF))5933 if (FMF.noSignedZeros() && match(Op0, m_AnyZeroFP()) &&5934 (match(Op1, m_FSub(m_AnyZeroFP(), m_Value(X))) ||5935 match(Op1, m_FNeg(m_Value(X)))))5936 return X;5937 5938 if (!isDefaultFPEnvironment(ExBehavior, Rounding))5939 return nullptr;5940 5941 if (FMF.noNaNs()) {5942 // fsub nnan x, x ==> 0.05943 if (Op0 == Op1)5944 return Constant::getNullValue(Op0->getType());5945 5946 // With nnan: {+/-}Inf - X --> {+/-}Inf5947 if (match(Op0, m_Inf()))5948 return Op0;5949 5950 // With nnan: X - {+/-}Inf --> {-/+}Inf5951 if (match(Op1, m_Inf()))5952 return foldConstant(Instruction::FNeg, Op1, Q);5953 }5954 5955 // Y - (Y - X) --> X5956 // (X + Y) - Y --> X5957 if (FMF.noSignedZeros() && FMF.allowReassoc() &&5958 (match(Op1, m_FSub(m_Specific(Op0), m_Value(X))) ||5959 match(Op0, m_c_FAdd(m_Specific(Op1), m_Value(X)))))5960 return X;5961 5962 return nullptr;5963}5964 5965static Value *simplifyFMAFMul(Value *Op0, Value *Op1, FastMathFlags FMF,5966 const SimplifyQuery &Q, unsigned MaxRecurse,5967 fp::ExceptionBehavior ExBehavior,5968 RoundingMode Rounding) {5969 if (Constant *C = simplifyFPOp({Op0, Op1}, FMF, Q, ExBehavior, Rounding))5970 return C;5971 5972 if (!isDefaultFPEnvironment(ExBehavior, Rounding))5973 return nullptr;5974 5975 // Canonicalize special constants as operand 1.5976 if (match(Op0, m_FPOne()) || match(Op0, m_AnyZeroFP()))5977 std::swap(Op0, Op1);5978 5979 // X * 1.0 --> X5980 if (match(Op1, m_FPOne()))5981 return Op0;5982 5983 if (match(Op1, m_AnyZeroFP())) {5984 // X * 0.0 --> 0.0 (with nnan and nsz)5985 if (FMF.noNaNs() && FMF.noSignedZeros())5986 return ConstantFP::getZero(Op0->getType());5987 5988 KnownFPClass Known = computeKnownFPClass(Op0, FMF, fcInf | fcNan, Q);5989 if (Known.isKnownNever(fcInf | fcNan)) {5990 // if nsz is set, return 0.05991 if (FMF.noSignedZeros())5992 return ConstantFP::getZero(Op0->getType());5993 // +normal number * (-)0.0 --> (-)0.05994 if (Known.SignBit == false)5995 return Op1;5996 // -normal number * (-)0.0 --> -(-)0.05997 if (Known.SignBit == true)5998 return foldConstant(Instruction::FNeg, Op1, Q);5999 }6000 }6001 6002 // sqrt(X) * sqrt(X) --> X, if we can:6003 // 1. Remove the intermediate rounding (reassociate).6004 // 2. Ignore non-zero negative numbers because sqrt would produce NAN.6005 // 3. Ignore -0.0 because sqrt(-0.0) == -0.0, but -0.0 * -0.0 == 0.0.6006 Value *X;6007 if (Op0 == Op1 && match(Op0, m_Sqrt(m_Value(X))) && FMF.allowReassoc() &&6008 FMF.noNaNs() && FMF.noSignedZeros())6009 return X;6010 6011 return nullptr;6012}6013 6014/// Given the operands for an FMul, see if we can fold the result6015static Value *6016simplifyFMulInst(Value *Op0, Value *Op1, FastMathFlags FMF,6017 const SimplifyQuery &Q, unsigned MaxRecurse,6018 fp::ExceptionBehavior ExBehavior = fp::ebIgnore,6019 RoundingMode Rounding = RoundingMode::NearestTiesToEven) {6020 if (isDefaultFPEnvironment(ExBehavior, Rounding))6021 if (Constant *C = foldOrCommuteConstant(Instruction::FMul, Op0, Op1, Q))6022 return C;6023 6024 // Now apply simplifications that do not require rounding.6025 return simplifyFMAFMul(Op0, Op1, FMF, Q, MaxRecurse, ExBehavior, Rounding);6026}6027 6028Value *llvm::simplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF,6029 const SimplifyQuery &Q,6030 fp::ExceptionBehavior ExBehavior,6031 RoundingMode Rounding) {6032 return ::simplifyFAddInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior,6033 Rounding);6034}6035 6036Value *llvm::simplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF,6037 const SimplifyQuery &Q,6038 fp::ExceptionBehavior ExBehavior,6039 RoundingMode Rounding) {6040 return ::simplifyFSubInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior,6041 Rounding);6042}6043 6044Value *llvm::simplifyFMulInst(Value *Op0, Value *Op1, FastMathFlags FMF,6045 const SimplifyQuery &Q,6046 fp::ExceptionBehavior ExBehavior,6047 RoundingMode Rounding) {6048 return ::simplifyFMulInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior,6049 Rounding);6050}6051 6052Value *llvm::simplifyFMAFMul(Value *Op0, Value *Op1, FastMathFlags FMF,6053 const SimplifyQuery &Q,6054 fp::ExceptionBehavior ExBehavior,6055 RoundingMode Rounding) {6056 return ::simplifyFMAFMul(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior,6057 Rounding);6058}6059 6060static Value *6061simplifyFDivInst(Value *Op0, Value *Op1, FastMathFlags FMF,6062 const SimplifyQuery &Q, unsigned,6063 fp::ExceptionBehavior ExBehavior = fp::ebIgnore,6064 RoundingMode Rounding = RoundingMode::NearestTiesToEven) {6065 if (isDefaultFPEnvironment(ExBehavior, Rounding))6066 if (Constant *C = foldOrCommuteConstant(Instruction::FDiv, Op0, Op1, Q))6067 return C;6068 6069 if (Constant *C = simplifyFPOp({Op0, Op1}, FMF, Q, ExBehavior, Rounding))6070 return C;6071 6072 if (!isDefaultFPEnvironment(ExBehavior, Rounding))6073 return nullptr;6074 6075 // X / 1.0 -> X6076 if (match(Op1, m_FPOne()))6077 return Op0;6078 6079 // 0 / X -> 06080 // Requires that NaNs are off (X could be zero) and signed zeroes are6081 // ignored (X could be positive or negative, so the output sign is unknown).6082 if (FMF.noNaNs() && FMF.noSignedZeros() && match(Op0, m_AnyZeroFP()))6083 return ConstantFP::getZero(Op0->getType());6084 6085 if (FMF.noNaNs()) {6086 // X / X -> 1.0 is legal when NaNs are ignored.6087 // We can ignore infinities because INF/INF is NaN.6088 if (Op0 == Op1)6089 return ConstantFP::get(Op0->getType(), 1.0);6090 6091 // (X * Y) / Y --> X if we can reassociate to the above form.6092 Value *X;6093 if (FMF.allowReassoc() && match(Op0, m_c_FMul(m_Value(X), m_Specific(Op1))))6094 return X;6095 6096 // -X / X -> -1.0 and6097 // X / -X -> -1.0 are legal when NaNs are ignored.6098 // We can ignore signed zeros because +-0.0/+-0.0 is NaN and ignored.6099 if (match(Op0, m_FNegNSZ(m_Specific(Op1))) ||6100 match(Op1, m_FNegNSZ(m_Specific(Op0))))6101 return ConstantFP::get(Op0->getType(), -1.0);6102 6103 // nnan ninf X / [-]0.0 -> poison6104 if (FMF.noInfs() && match(Op1, m_AnyZeroFP()))6105 return PoisonValue::get(Op1->getType());6106 }6107 6108 return nullptr;6109}6110 6111Value *llvm::simplifyFDivInst(Value *Op0, Value *Op1, FastMathFlags FMF,6112 const SimplifyQuery &Q,6113 fp::ExceptionBehavior ExBehavior,6114 RoundingMode Rounding) {6115 return ::simplifyFDivInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior,6116 Rounding);6117}6118 6119static Value *6120simplifyFRemInst(Value *Op0, Value *Op1, FastMathFlags FMF,6121 const SimplifyQuery &Q, unsigned,6122 fp::ExceptionBehavior ExBehavior = fp::ebIgnore,6123 RoundingMode Rounding = RoundingMode::NearestTiesToEven) {6124 if (isDefaultFPEnvironment(ExBehavior, Rounding))6125 if (Constant *C = foldOrCommuteConstant(Instruction::FRem, Op0, Op1, Q))6126 return C;6127 6128 if (Constant *C = simplifyFPOp({Op0, Op1}, FMF, Q, ExBehavior, Rounding))6129 return C;6130 6131 if (!isDefaultFPEnvironment(ExBehavior, Rounding))6132 return nullptr;6133 6134 // Unlike fdiv, the result of frem always matches the sign of the dividend.6135 // The constant match may include undef elements in a vector, so return a full6136 // zero constant as the result.6137 if (FMF.noNaNs()) {6138 // +0 % X -> 06139 if (match(Op0, m_PosZeroFP()))6140 return ConstantFP::getZero(Op0->getType());6141 // -0 % X -> -06142 if (match(Op0, m_NegZeroFP()))6143 return ConstantFP::getNegativeZero(Op0->getType());6144 }6145 6146 return nullptr;6147}6148 6149Value *llvm::simplifyFRemInst(Value *Op0, Value *Op1, FastMathFlags FMF,6150 const SimplifyQuery &Q,6151 fp::ExceptionBehavior ExBehavior,6152 RoundingMode Rounding) {6153 return ::simplifyFRemInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior,6154 Rounding);6155}6156 6157//=== Helper functions for higher up the class hierarchy.6158 6159/// Given the operand for a UnaryOperator, see if we can fold the result.6160/// If not, this returns null.6161static Value *simplifyUnOp(unsigned Opcode, Value *Op, const SimplifyQuery &Q,6162 unsigned MaxRecurse) {6163 switch (Opcode) {6164 case Instruction::FNeg:6165 return simplifyFNegInst(Op, FastMathFlags(), Q, MaxRecurse);6166 default:6167 llvm_unreachable("Unexpected opcode");6168 }6169}6170 6171/// Given the operand for a UnaryOperator, see if we can fold the result.6172/// If not, this returns null.6173/// Try to use FastMathFlags when folding the result.6174static Value *simplifyFPUnOp(unsigned Opcode, Value *Op,6175 const FastMathFlags &FMF, const SimplifyQuery &Q,6176 unsigned MaxRecurse) {6177 switch (Opcode) {6178 case Instruction::FNeg:6179 return simplifyFNegInst(Op, FMF, Q, MaxRecurse);6180 default:6181 return simplifyUnOp(Opcode, Op, Q, MaxRecurse);6182 }6183}6184 6185Value *llvm::simplifyUnOp(unsigned Opcode, Value *Op, const SimplifyQuery &Q) {6186 return ::simplifyUnOp(Opcode, Op, Q, RecursionLimit);6187}6188 6189Value *llvm::simplifyUnOp(unsigned Opcode, Value *Op, FastMathFlags FMF,6190 const SimplifyQuery &Q) {6191 return ::simplifyFPUnOp(Opcode, Op, FMF, Q, RecursionLimit);6192}6193 6194/// Given operands for a BinaryOperator, see if we can fold the result.6195/// If not, this returns null.6196static Value *simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS,6197 const SimplifyQuery &Q, unsigned MaxRecurse) {6198 switch (Opcode) {6199 case Instruction::Add:6200 return simplifyAddInst(LHS, RHS, /* IsNSW */ false, /* IsNUW */ false, Q,6201 MaxRecurse);6202 case Instruction::Sub:6203 return simplifySubInst(LHS, RHS, /* IsNSW */ false, /* IsNUW */ false, Q,6204 MaxRecurse);6205 case Instruction::Mul:6206 return simplifyMulInst(LHS, RHS, /* IsNSW */ false, /* IsNUW */ false, Q,6207 MaxRecurse);6208 case Instruction::SDiv:6209 return simplifySDivInst(LHS, RHS, /* IsExact */ false, Q, MaxRecurse);6210 case Instruction::UDiv:6211 return simplifyUDivInst(LHS, RHS, /* IsExact */ false, Q, MaxRecurse);6212 case Instruction::SRem:6213 return simplifySRemInst(LHS, RHS, Q, MaxRecurse);6214 case Instruction::URem:6215 return simplifyURemInst(LHS, RHS, Q, MaxRecurse);6216 case Instruction::Shl:6217 return simplifyShlInst(LHS, RHS, /* IsNSW */ false, /* IsNUW */ false, Q,6218 MaxRecurse);6219 case Instruction::LShr:6220 return simplifyLShrInst(LHS, RHS, /* IsExact */ false, Q, MaxRecurse);6221 case Instruction::AShr:6222 return simplifyAShrInst(LHS, RHS, /* IsExact */ false, Q, MaxRecurse);6223 case Instruction::And:6224 return simplifyAndInst(LHS, RHS, Q, MaxRecurse);6225 case Instruction::Or:6226 return simplifyOrInst(LHS, RHS, Q, MaxRecurse);6227 case Instruction::Xor:6228 return simplifyXorInst(LHS, RHS, Q, MaxRecurse);6229 case Instruction::FAdd:6230 return simplifyFAddInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse);6231 case Instruction::FSub:6232 return simplifyFSubInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse);6233 case Instruction::FMul:6234 return simplifyFMulInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse);6235 case Instruction::FDiv:6236 return simplifyFDivInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse);6237 case Instruction::FRem:6238 return simplifyFRemInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse);6239 default:6240 llvm_unreachable("Unexpected opcode");6241 }6242}6243 6244/// Given operands for a BinaryOperator, see if we can fold the result.6245/// If not, this returns null.6246/// Try to use FastMathFlags when folding the result.6247static Value *simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS,6248 const FastMathFlags &FMF, const SimplifyQuery &Q,6249 unsigned MaxRecurse) {6250 switch (Opcode) {6251 case Instruction::FAdd:6252 return simplifyFAddInst(LHS, RHS, FMF, Q, MaxRecurse);6253 case Instruction::FSub:6254 return simplifyFSubInst(LHS, RHS, FMF, Q, MaxRecurse);6255 case Instruction::FMul:6256 return simplifyFMulInst(LHS, RHS, FMF, Q, MaxRecurse);6257 case Instruction::FDiv:6258 return simplifyFDivInst(LHS, RHS, FMF, Q, MaxRecurse);6259 default:6260 return simplifyBinOp(Opcode, LHS, RHS, Q, MaxRecurse);6261 }6262}6263 6264Value *llvm::simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS,6265 const SimplifyQuery &Q) {6266 return ::simplifyBinOp(Opcode, LHS, RHS, Q, RecursionLimit);6267}6268 6269Value *llvm::simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS,6270 FastMathFlags FMF, const SimplifyQuery &Q) {6271 return ::simplifyBinOp(Opcode, LHS, RHS, FMF, Q, RecursionLimit);6272}6273 6274/// Given operands for a CmpInst, see if we can fold the result.6275static Value *simplifyCmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS,6276 const SimplifyQuery &Q, unsigned MaxRecurse) {6277 if (CmpInst::isIntPredicate(Predicate))6278 return simplifyICmpInst(Predicate, LHS, RHS, Q, MaxRecurse);6279 return simplifyFCmpInst(Predicate, LHS, RHS, FastMathFlags(), Q, MaxRecurse);6280}6281 6282Value *llvm::simplifyCmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS,6283 const SimplifyQuery &Q) {6284 return ::simplifyCmpInst(Predicate, LHS, RHS, Q, RecursionLimit);6285}6286 6287static bool isIdempotent(Intrinsic::ID ID) {6288 switch (ID) {6289 default:6290 return false;6291 6292 // Unary idempotent: f(f(x)) = f(x)6293 case Intrinsic::fabs:6294 case Intrinsic::floor:6295 case Intrinsic::ceil:6296 case Intrinsic::trunc:6297 case Intrinsic::rint:6298 case Intrinsic::nearbyint:6299 case Intrinsic::round:6300 case Intrinsic::roundeven:6301 case Intrinsic::canonicalize:6302 case Intrinsic::arithmetic_fence:6303 return true;6304 }6305}6306 6307/// Return true if the intrinsic rounds a floating-point value to an integral6308/// floating-point value (not an integer type).6309static bool removesFPFraction(Intrinsic::ID ID) {6310 switch (ID) {6311 default:6312 return false;6313 6314 case Intrinsic::floor:6315 case Intrinsic::ceil:6316 case Intrinsic::trunc:6317 case Intrinsic::rint:6318 case Intrinsic::nearbyint:6319 case Intrinsic::round:6320 case Intrinsic::roundeven:6321 return true;6322 }6323}6324 6325static Value *simplifyRelativeLoad(Constant *Ptr, Constant *Offset,6326 const DataLayout &DL) {6327 GlobalValue *PtrSym;6328 APInt PtrOffset;6329 if (!IsConstantOffsetFromGlobal(Ptr, PtrSym, PtrOffset, DL))6330 return nullptr;6331 6332 Type *Int32Ty = Type::getInt32Ty(Ptr->getContext());6333 6334 auto *OffsetConstInt = dyn_cast<ConstantInt>(Offset);6335 if (!OffsetConstInt || OffsetConstInt->getBitWidth() > 64)6336 return nullptr;6337 6338 APInt OffsetInt = OffsetConstInt->getValue().sextOrTrunc(6339 DL.getIndexTypeSizeInBits(Ptr->getType()));6340 if (OffsetInt.srem(4) != 0)6341 return nullptr;6342 6343 Constant *Loaded =6344 ConstantFoldLoadFromConstPtr(Ptr, Int32Ty, std::move(OffsetInt), DL);6345 if (!Loaded)6346 return nullptr;6347 6348 auto *LoadedCE = dyn_cast<ConstantExpr>(Loaded);6349 if (!LoadedCE)6350 return nullptr;6351 6352 if (LoadedCE->getOpcode() == Instruction::Trunc) {6353 LoadedCE = dyn_cast<ConstantExpr>(LoadedCE->getOperand(0));6354 if (!LoadedCE)6355 return nullptr;6356 }6357 6358 if (LoadedCE->getOpcode() != Instruction::Sub)6359 return nullptr;6360 6361 auto *LoadedLHS = dyn_cast<ConstantExpr>(LoadedCE->getOperand(0));6362 if (!LoadedLHS || LoadedLHS->getOpcode() != Instruction::PtrToInt)6363 return nullptr;6364 auto *LoadedLHSPtr = LoadedLHS->getOperand(0);6365 6366 Constant *LoadedRHS = LoadedCE->getOperand(1);6367 GlobalValue *LoadedRHSSym;6368 APInt LoadedRHSOffset;6369 if (!IsConstantOffsetFromGlobal(LoadedRHS, LoadedRHSSym, LoadedRHSOffset,6370 DL) ||6371 PtrSym != LoadedRHSSym || PtrOffset != LoadedRHSOffset)6372 return nullptr;6373 6374 return LoadedLHSPtr;6375}6376 6377// TODO: Need to pass in FastMathFlags6378static Value *simplifyLdexp(Value *Op0, Value *Op1, const SimplifyQuery &Q,6379 bool IsStrict) {6380 // ldexp(poison, x) -> poison6381 // ldexp(x, poison) -> poison6382 if (isa<PoisonValue>(Op0) || isa<PoisonValue>(Op1))6383 return Op0;6384 6385 // ldexp(undef, x) -> nan6386 if (Q.isUndefValue(Op0))6387 return ConstantFP::getNaN(Op0->getType());6388 6389 if (!IsStrict) {6390 // TODO: Could insert a canonicalize for strict6391 6392 // ldexp(x, undef) -> x6393 if (Q.isUndefValue(Op1))6394 return Op0;6395 }6396 6397 const APFloat *C = nullptr;6398 match(Op0, PatternMatch::m_APFloat(C));6399 6400 // These cases should be safe, even with strictfp.6401 // ldexp(0.0, x) -> 0.06402 // ldexp(-0.0, x) -> -0.06403 // ldexp(inf, x) -> inf6404 // ldexp(-inf, x) -> -inf6405 if (C && (C->isZero() || C->isInfinity()))6406 return Op0;6407 6408 // These are canonicalization dropping, could do it if we knew how we could6409 // ignore denormal flushes and target handling of nan payload bits.6410 if (IsStrict)6411 return nullptr;6412 6413 // TODO: Could quiet this with strictfp if the exception mode isn't strict.6414 if (C && C->isNaN())6415 return ConstantFP::get(Op0->getType(), C->makeQuiet());6416 6417 // ldexp(x, 0) -> x6418 6419 // TODO: Could fold this if we know the exception mode isn't6420 // strict, we know the denormal mode and other target modes.6421 if (match(Op1, PatternMatch::m_ZeroInt()))6422 return Op0;6423 6424 return nullptr;6425}6426 6427static Value *simplifyUnaryIntrinsic(Function *F, Value *Op0,6428 const SimplifyQuery &Q,6429 const CallBase *Call) {6430 // Idempotent functions return the same result when called repeatedly.6431 Intrinsic::ID IID = F->getIntrinsicID();6432 if (isIdempotent(IID))6433 if (auto *II = dyn_cast<IntrinsicInst>(Op0))6434 if (II->getIntrinsicID() == IID)6435 return II;6436 6437 if (removesFPFraction(IID)) {6438 // Converting from int or calling a rounding function always results in a6439 // finite integral number or infinity. For those inputs, rounding functions6440 // always return the same value, so the (2nd) rounding is eliminated. Ex:6441 // floor (sitofp x) -> sitofp x6442 // round (ceil x) -> ceil x6443 auto *II = dyn_cast<IntrinsicInst>(Op0);6444 if ((II && removesFPFraction(II->getIntrinsicID())) ||6445 match(Op0, m_SIToFP(m_Value())) || match(Op0, m_UIToFP(m_Value())))6446 return Op0;6447 }6448 6449 Value *X;6450 switch (IID) {6451 case Intrinsic::fabs:6452 if (computeKnownFPSignBit(Op0, Q) == false)6453 return Op0;6454 break;6455 case Intrinsic::bswap:6456 // bswap(bswap(x)) -> x6457 if (match(Op0, m_BSwap(m_Value(X))))6458 return X;6459 break;6460 case Intrinsic::bitreverse:6461 // bitreverse(bitreverse(x)) -> x6462 if (match(Op0, m_BitReverse(m_Value(X))))6463 return X;6464 break;6465 case Intrinsic::ctpop: {6466 // ctpop(X) -> 1 iff X is non-zero power of 2.6467 if (isKnownToBeAPowerOfTwo(Op0, Q.DL, /*OrZero*/ false, Q.AC, Q.CxtI, Q.DT))6468 return ConstantInt::get(Op0->getType(), 1);6469 // If everything but the lowest bit is zero, that bit is the pop-count. Ex:6470 // ctpop(and X, 1) --> and X, 16471 unsigned BitWidth = Op0->getType()->getScalarSizeInBits();6472 if (MaskedValueIsZero(Op0, APInt::getHighBitsSet(BitWidth, BitWidth - 1),6473 Q))6474 return Op0;6475 break;6476 }6477 case Intrinsic::exp:6478 // exp(log(x)) -> x6479 if (Call->hasAllowReassoc() &&6480 match(Op0, m_Intrinsic<Intrinsic::log>(m_Value(X))))6481 return X;6482 break;6483 case Intrinsic::exp2:6484 // exp2(log2(x)) -> x6485 if (Call->hasAllowReassoc() &&6486 match(Op0, m_Intrinsic<Intrinsic::log2>(m_Value(X))))6487 return X;6488 break;6489 case Intrinsic::exp10:6490 // exp10(log10(x)) -> x6491 if (Call->hasAllowReassoc() &&6492 match(Op0, m_Intrinsic<Intrinsic::log10>(m_Value(X))))6493 return X;6494 break;6495 case Intrinsic::log:6496 // log(exp(x)) -> x6497 if (Call->hasAllowReassoc() &&6498 match(Op0, m_Intrinsic<Intrinsic::exp>(m_Value(X))))6499 return X;6500 break;6501 case Intrinsic::log2:6502 // log2(exp2(x)) -> x6503 if (Call->hasAllowReassoc() &&6504 (match(Op0, m_Intrinsic<Intrinsic::exp2>(m_Value(X))) ||6505 match(Op0,6506 m_Intrinsic<Intrinsic::pow>(m_SpecificFP(2.0), m_Value(X)))))6507 return X;6508 break;6509 case Intrinsic::log10:6510 // log10(pow(10.0, x)) -> x6511 // log10(exp10(x)) -> x6512 if (Call->hasAllowReassoc() &&6513 (match(Op0, m_Intrinsic<Intrinsic::exp10>(m_Value(X))) ||6514 match(Op0,6515 m_Intrinsic<Intrinsic::pow>(m_SpecificFP(10.0), m_Value(X)))))6516 return X;6517 break;6518 case Intrinsic::vector_reverse:6519 // vector.reverse(vector.reverse(x)) -> x6520 if (match(Op0, m_VecReverse(m_Value(X))))6521 return X;6522 // vector.reverse(splat(X)) -> splat(X)6523 if (isSplatValue(Op0))6524 return Op0;6525 break;6526 default:6527 break;6528 }6529 6530 return nullptr;6531}6532 6533/// Given a min/max intrinsic, see if it can be removed based on having an6534/// operand that is another min/max intrinsic with shared operand(s). The caller6535/// is expected to swap the operand arguments to handle commutation.6536static Value *foldMinMaxSharedOp(Intrinsic::ID IID, Value *Op0, Value *Op1) {6537 Value *X, *Y;6538 if (!match(Op0, m_MaxOrMin(m_Value(X), m_Value(Y))))6539 return nullptr;6540 6541 auto *MM0 = dyn_cast<IntrinsicInst>(Op0);6542 if (!MM0)6543 return nullptr;6544 Intrinsic::ID IID0 = MM0->getIntrinsicID();6545 6546 if (Op1 == X || Op1 == Y ||6547 match(Op1, m_c_MaxOrMin(m_Specific(X), m_Specific(Y)))) {6548 // max (max X, Y), X --> max X, Y6549 if (IID0 == IID)6550 return MM0;6551 // max (min X, Y), X --> X6552 if (IID0 == getInverseMinMaxIntrinsic(IID))6553 return Op1;6554 }6555 return nullptr;6556}6557 6558/// Given a min/max intrinsic, see if it can be removed based on having an6559/// operand that is another min/max intrinsic with shared operand(s). The caller6560/// is expected to swap the operand arguments to handle commutation.6561static Value *foldMinimumMaximumSharedOp(Intrinsic::ID IID, Value *Op0,6562 Value *Op1) {6563 assert((IID == Intrinsic::maxnum || IID == Intrinsic::minnum ||6564 IID == Intrinsic::maximum || IID == Intrinsic::minimum ||6565 IID == Intrinsic::maximumnum || IID == Intrinsic::minimumnum) &&6566 "Unsupported intrinsic");6567 6568 auto *M0 = dyn_cast<IntrinsicInst>(Op0);6569 // If Op0 is not the same intrinsic as IID, do not process.6570 // This is a difference with integer min/max handling. We do not process the6571 // case like max(min(X,Y),min(X,Y)) => min(X,Y). But it can be handled by GVN.6572 if (!M0 || M0->getIntrinsicID() != IID)6573 return nullptr;6574 Value *X0 = M0->getOperand(0);6575 Value *Y0 = M0->getOperand(1);6576 // Simple case, m(m(X,Y), X) => m(X, Y)6577 // m(m(X,Y), Y) => m(X, Y)6578 // For minimum/maximum, X is NaN => m(NaN, Y) == NaN and m(NaN, NaN) == NaN.6579 // For minimum/maximum, Y is NaN => m(X, NaN) == NaN and m(NaN, NaN) == NaN.6580 // For minnum/maxnum, X is NaN => m(NaN, Y) == Y and m(Y, Y) == Y.6581 // For minnum/maxnum, Y is NaN => m(X, NaN) == X and m(X, NaN) == X.6582 if (X0 == Op1 || Y0 == Op1)6583 return M0;6584 6585 auto *M1 = dyn_cast<IntrinsicInst>(Op1);6586 if (!M1)6587 return nullptr;6588 Value *X1 = M1->getOperand(0);6589 Value *Y1 = M1->getOperand(1);6590 Intrinsic::ID IID1 = M1->getIntrinsicID();6591 // we have a case m(m(X,Y),m'(X,Y)) taking into account m' is commutative.6592 // if m' is m or inversion of m => m(m(X,Y),m'(X,Y)) == m(X,Y).6593 // For minimum/maximum, X is NaN => m(NaN,Y) == m'(NaN, Y) == NaN.6594 // For minimum/maximum, Y is NaN => m(X,NaN) == m'(X, NaN) == NaN.6595 // For minnum/maxnum, X is NaN => m(NaN,Y) == m'(NaN, Y) == Y.6596 // For minnum/maxnum, Y is NaN => m(X,NaN) == m'(X, NaN) == X.6597 if ((X0 == X1 && Y0 == Y1) || (X0 == Y1 && Y0 == X1))6598 if (IID1 == IID || getInverseMinMaxIntrinsic(IID1) == IID)6599 return M0;6600 6601 return nullptr;6602}6603 6604enum class MinMaxOptResult {6605 CannotOptimize = 0,6606 UseNewConstVal = 1,6607 UseOtherVal = 2,6608 // For undef/poison, we can choose to either propgate undef/poison or6609 // use the LHS value depending on what will allow more optimization.6610 UseEither = 36611};6612// Get the optimized value for a min/max instruction with a single constant6613// input (either undef or scalar constantFP). The result may indicate to6614// use the non-const LHS value, use a new constant value instead (with NaNs6615// quieted), or to choose either option in the case of undef/poison.6616static MinMaxOptResult OptimizeConstMinMax(const Constant *RHSConst,6617 const Intrinsic::ID IID,6618 const CallBase *Call,6619 Constant **OutNewConstVal) {6620 assert(OutNewConstVal != nullptr);6621 6622 bool PropagateNaN = IID == Intrinsic::minimum || IID == Intrinsic::maximum;6623 bool PropagateSNaN = IID == Intrinsic::minnum || IID == Intrinsic::maxnum;6624 bool IsMin = IID == Intrinsic::minimum || IID == Intrinsic::minnum ||6625 IID == Intrinsic::minimumnum;6626 6627 // min/max(x, poison) -> either x or poison6628 if (isa<UndefValue>(RHSConst)) {6629 *OutNewConstVal = const_cast<Constant *>(RHSConst);6630 return MinMaxOptResult::UseEither;6631 }6632 6633 const ConstantFP *CFP = dyn_cast<ConstantFP>(RHSConst);6634 if (!CFP)6635 return MinMaxOptResult::CannotOptimize;6636 APFloat CAPF = CFP->getValueAPF();6637 6638 // minnum(x, qnan) -> x6639 // maxnum(x, qnan) -> x6640 // minnum(x, snan) -> qnan6641 // maxnum(x, snan) -> qnan6642 // minimum(X, nan) -> qnan6643 // maximum(X, nan) -> qnan6644 // minimumnum(X, nan) -> x6645 // maximumnum(X, nan) -> x6646 if (CAPF.isNaN()) {6647 if (PropagateNaN || (PropagateSNaN && CAPF.isSignaling())) {6648 *OutNewConstVal = ConstantFP::get(CFP->getType(), CAPF.makeQuiet());6649 return MinMaxOptResult::UseNewConstVal;6650 }6651 return MinMaxOptResult::UseOtherVal;6652 }6653 6654 if (CAPF.isInfinity() || (Call && Call->hasNoInfs() && CAPF.isLargest())) {6655 // minnum(X, -inf) -> -inf (ignoring sNaN -> qNaN propagation)6656 // maxnum(X, +inf) -> +inf (ignoring sNaN -> qNaN propagation)6657 // minimum(X, -inf) -> -inf if nnan6658 // maximum(X, +inf) -> +inf if nnan6659 // minimumnum(X, -inf) -> -inf6660 // maximumnum(X, +inf) -> +inf6661 if (CAPF.isNegative() == IsMin &&6662 (!PropagateNaN || (Call && Call->hasNoNaNs()))) {6663 *OutNewConstVal = const_cast<Constant *>(RHSConst);6664 return MinMaxOptResult::UseNewConstVal;6665 }6666 6667 // minnum(X, +inf) -> X if nnan6668 // maxnum(X, -inf) -> X if nnan6669 // minimum(X, +inf) -> X (ignoring quieting of sNaNs)6670 // maximum(X, -inf) -> X (ignoring quieting of sNaNs)6671 // minimumnum(X, +inf) -> X if nnan6672 // maximumnum(X, -inf) -> X if nnan6673 if (CAPF.isNegative() != IsMin &&6674 (PropagateNaN || (Call && Call->hasNoNaNs())))6675 return MinMaxOptResult::UseOtherVal;6676 }6677 return MinMaxOptResult::CannotOptimize;6678}6679 6680static Value *simplifySVEIntReduction(Intrinsic::ID IID, Type *ReturnType,6681 Value *Op0, Value *Op1) {6682 Constant *C0 = dyn_cast<Constant>(Op0);6683 Constant *C1 = dyn_cast<Constant>(Op1);6684 unsigned Width = ReturnType->getPrimitiveSizeInBits();6685 6686 // All false predicate or reduction of neutral values ==> neutral result.6687 switch (IID) {6688 case Intrinsic::aarch64_sve_eorv:6689 case Intrinsic::aarch64_sve_orv:6690 case Intrinsic::aarch64_sve_saddv:6691 case Intrinsic::aarch64_sve_uaddv:6692 case Intrinsic::aarch64_sve_umaxv:6693 if ((C0 && C0->isNullValue()) || (C1 && C1->isNullValue()))6694 return ConstantInt::get(ReturnType, 0);6695 break;6696 case Intrinsic::aarch64_sve_andv:6697 case Intrinsic::aarch64_sve_uminv:6698 if ((C0 && C0->isNullValue()) || (C1 && C1->isAllOnesValue()))6699 return ConstantInt::get(ReturnType, APInt::getMaxValue(Width));6700 break;6701 case Intrinsic::aarch64_sve_smaxv:6702 if ((C0 && C0->isNullValue()) || (C1 && C1->isMinSignedValue()))6703 return ConstantInt::get(ReturnType, APInt::getSignedMinValue(Width));6704 break;6705 case Intrinsic::aarch64_sve_sminv:6706 if ((C0 && C0->isNullValue()) || (C1 && C1->isMaxSignedValue()))6707 return ConstantInt::get(ReturnType, APInt::getSignedMaxValue(Width));6708 break;6709 }6710 6711 switch (IID) {6712 case Intrinsic::aarch64_sve_andv:6713 case Intrinsic::aarch64_sve_orv:6714 case Intrinsic::aarch64_sve_smaxv:6715 case Intrinsic::aarch64_sve_sminv:6716 case Intrinsic::aarch64_sve_umaxv:6717 case Intrinsic::aarch64_sve_uminv:6718 // sve_reduce_##(all, splat(X)) ==> X6719 if (C0 && C0->isAllOnesValue()) {6720 if (Value *SplatVal = getSplatValue(Op1)) {6721 assert(SplatVal->getType() == ReturnType && "Unexpected result type!");6722 return SplatVal;6723 }6724 }6725 break;6726 case Intrinsic::aarch64_sve_eorv:6727 // sve_reduce_xor(all, splat(X)) ==> 06728 if (C0 && C0->isAllOnesValue())6729 return ConstantInt::get(ReturnType, 0);6730 break;6731 }6732 6733 return nullptr;6734}6735 6736Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType,6737 Value *Op0, Value *Op1,6738 const SimplifyQuery &Q,6739 const CallBase *Call) {6740 unsigned BitWidth = ReturnType->getScalarSizeInBits();6741 switch (IID) {6742 case Intrinsic::get_active_lane_mask: {6743 if (match(Op1, m_Zero()))6744 return ConstantInt::getFalse(ReturnType);6745 6746 const Function *F = Call->getFunction();6747 auto *ScalableTy = dyn_cast<ScalableVectorType>(ReturnType);6748 Attribute Attr = F->getFnAttribute(Attribute::VScaleRange);6749 if (ScalableTy && Attr.isValid()) {6750 std::optional<unsigned> VScaleMax = Attr.getVScaleRangeMax();6751 if (!VScaleMax)6752 break;6753 uint64_t MaxPossibleMaskElements =6754 (uint64_t)ScalableTy->getMinNumElements() * (*VScaleMax);6755 6756 const APInt *Op1Val;6757 if (match(Op0, m_Zero()) && match(Op1, m_APInt(Op1Val)) &&6758 Op1Val->uge(MaxPossibleMaskElements))6759 return ConstantInt::getAllOnesValue(ReturnType);6760 }6761 break;6762 }6763 case Intrinsic::abs:6764 // abs(abs(x)) -> abs(x). We don't need to worry about the nsw arg here.6765 // It is always ok to pick the earlier abs. We'll just lose nsw if its only6766 // on the outer abs.6767 if (match(Op0, m_Intrinsic<Intrinsic::abs>(m_Value(), m_Value())))6768 return Op0;6769 break;6770 6771 case Intrinsic::cttz: {6772 Value *X;6773 if (match(Op0, m_Shl(m_One(), m_Value(X))))6774 return X;6775 break;6776 }6777 case Intrinsic::ctlz: {6778 Value *X;6779 if (match(Op0, m_LShr(m_Negative(), m_Value(X))))6780 return X;6781 if (match(Op0, m_AShr(m_Negative(), m_Value())))6782 return Constant::getNullValue(ReturnType);6783 break;6784 }6785 case Intrinsic::ptrmask: {6786 // NOTE: We can't apply this simplifications based on the value of Op16787 // because we need to preserve provenance.6788 if (Q.isUndefValue(Op0) || match(Op0, m_Zero()))6789 return Constant::getNullValue(Op0->getType());6790 6791 assert(Op1->getType()->getScalarSizeInBits() ==6792 Q.DL.getIndexTypeSizeInBits(Op0->getType()) &&6793 "Invalid mask width");6794 // If index-width (mask size) is less than pointer-size then mask is6795 // 1-extended.6796 if (match(Op1, m_PtrToIntOrAddr(m_Specific(Op0))))6797 return Op0;6798 6799 // NOTE: We may have attributes associated with the return value of the6800 // llvm.ptrmask intrinsic that will be lost when we just return the6801 // operand. We should try to preserve them.6802 if (match(Op1, m_AllOnes()) || Q.isUndefValue(Op1))6803 return Op0;6804 6805 Constant *C;6806 if (match(Op1, m_ImmConstant(C))) {6807 KnownBits PtrKnown = computeKnownBits(Op0, Q);6808 // See if we only masking off bits we know are already zero due to6809 // alignment.6810 APInt IrrelevantPtrBits =6811 PtrKnown.Zero.zextOrTrunc(C->getType()->getScalarSizeInBits());6812 C = ConstantFoldBinaryOpOperands(6813 Instruction::Or, C, ConstantInt::get(C->getType(), IrrelevantPtrBits),6814 Q.DL);6815 if (C != nullptr && C->isAllOnesValue())6816 return Op0;6817 }6818 break;6819 }6820 case Intrinsic::smax:6821 case Intrinsic::smin:6822 case Intrinsic::umax:6823 case Intrinsic::umin: {6824 // If the arguments are the same, this is a no-op.6825 if (Op0 == Op1)6826 return Op0;6827 6828 // Canonicalize immediate constant operand as Op1.6829 if (match(Op0, m_ImmConstant()))6830 std::swap(Op0, Op1);6831 6832 // Assume undef is the limit value.6833 if (Q.isUndefValue(Op1))6834 return ConstantInt::get(6835 ReturnType, MinMaxIntrinsic::getSaturationPoint(IID, BitWidth));6836 6837 const APInt *C;6838 if (match(Op1, m_APIntAllowPoison(C))) {6839 // Clamp to limit value. For example:6840 // umax(i8 %x, i8 255) --> 2556841 if (*C == MinMaxIntrinsic::getSaturationPoint(IID, BitWidth))6842 return ConstantInt::get(ReturnType, *C);6843 6844 // If the constant op is the opposite of the limit value, the other must6845 // be larger/smaller or equal. For example:6846 // umin(i8 %x, i8 255) --> %x6847 if (*C == MinMaxIntrinsic::getSaturationPoint(6848 getInverseMinMaxIntrinsic(IID), BitWidth))6849 return Op0;6850 6851 // Remove nested call if constant operands allow it. Example:6852 // max (max X, 7), 5 -> max X, 76853 auto *MinMax0 = dyn_cast<IntrinsicInst>(Op0);6854 if (MinMax0 && MinMax0->getIntrinsicID() == IID) {6855 // TODO: loosen undef/splat restrictions for vector constants.6856 Value *M00 = MinMax0->getOperand(0), *M01 = MinMax0->getOperand(1);6857 const APInt *InnerC;6858 if ((match(M00, m_APInt(InnerC)) || match(M01, m_APInt(InnerC))) &&6859 ICmpInst::compare(*InnerC, *C,6860 ICmpInst::getNonStrictPredicate(6861 MinMaxIntrinsic::getPredicate(IID))))6862 return Op0;6863 }6864 }6865 6866 if (Value *V = foldMinMaxSharedOp(IID, Op0, Op1))6867 return V;6868 if (Value *V = foldMinMaxSharedOp(IID, Op1, Op0))6869 return V;6870 6871 ICmpInst::Predicate Pred =6872 ICmpInst::getNonStrictPredicate(MinMaxIntrinsic::getPredicate(IID));6873 if (isICmpTrue(Pred, Op0, Op1, Q.getWithoutUndef(), RecursionLimit))6874 return Op0;6875 if (isICmpTrue(Pred, Op1, Op0, Q.getWithoutUndef(), RecursionLimit))6876 return Op1;6877 6878 break;6879 }6880 case Intrinsic::scmp:6881 case Intrinsic::ucmp: {6882 // Fold to a constant if the relationship between operands can be6883 // established with certainty6884 if (isICmpTrue(CmpInst::ICMP_EQ, Op0, Op1, Q, RecursionLimit))6885 return Constant::getNullValue(ReturnType);6886 6887 ICmpInst::Predicate PredGT =6888 IID == Intrinsic::scmp ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;6889 if (isICmpTrue(PredGT, Op0, Op1, Q, RecursionLimit))6890 return ConstantInt::get(ReturnType, 1);6891 6892 ICmpInst::Predicate PredLT =6893 IID == Intrinsic::scmp ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;6894 if (isICmpTrue(PredLT, Op0, Op1, Q, RecursionLimit))6895 return ConstantInt::getSigned(ReturnType, -1);6896 6897 break;6898 }6899 case Intrinsic::usub_with_overflow:6900 case Intrinsic::ssub_with_overflow:6901 // X - X -> { 0, false }6902 // X - undef -> { 0, false }6903 // undef - X -> { 0, false }6904 if (Op0 == Op1 || Q.isUndefValue(Op0) || Q.isUndefValue(Op1))6905 return Constant::getNullValue(ReturnType);6906 break;6907 case Intrinsic::uadd_with_overflow:6908 case Intrinsic::sadd_with_overflow:6909 // X + undef -> { -1, false }6910 // undef + x -> { -1, false }6911 if (Q.isUndefValue(Op0) || Q.isUndefValue(Op1)) {6912 return ConstantStruct::get(6913 cast<StructType>(ReturnType),6914 {Constant::getAllOnesValue(ReturnType->getStructElementType(0)),6915 Constant::getNullValue(ReturnType->getStructElementType(1))});6916 }6917 break;6918 case Intrinsic::umul_with_overflow:6919 case Intrinsic::smul_with_overflow:6920 // 0 * X -> { 0, false }6921 // X * 0 -> { 0, false }6922 if (match(Op0, m_Zero()) || match(Op1, m_Zero()))6923 return Constant::getNullValue(ReturnType);6924 // undef * X -> { 0, false }6925 // X * undef -> { 0, false }6926 if (Q.isUndefValue(Op0) || Q.isUndefValue(Op1))6927 return Constant::getNullValue(ReturnType);6928 break;6929 case Intrinsic::uadd_sat:6930 // sat(MAX + X) -> MAX6931 // sat(X + MAX) -> MAX6932 if (match(Op0, m_AllOnes()) || match(Op1, m_AllOnes()))6933 return Constant::getAllOnesValue(ReturnType);6934 [[fallthrough]];6935 case Intrinsic::sadd_sat:6936 // sat(X + undef) -> -16937 // sat(undef + X) -> -16938 // For unsigned: Assume undef is MAX, thus we saturate to MAX (-1).6939 // For signed: Assume undef is ~X, in which case X + ~X = -1.6940 if (Q.isUndefValue(Op0) || Q.isUndefValue(Op1))6941 return Constant::getAllOnesValue(ReturnType);6942 6943 // X + 0 -> X6944 if (match(Op1, m_Zero()))6945 return Op0;6946 // 0 + X -> X6947 if (match(Op0, m_Zero()))6948 return Op1;6949 break;6950 case Intrinsic::usub_sat:6951 // sat(0 - X) -> 0, sat(X - MAX) -> 06952 if (match(Op0, m_Zero()) || match(Op1, m_AllOnes()))6953 return Constant::getNullValue(ReturnType);6954 [[fallthrough]];6955 case Intrinsic::ssub_sat:6956 // X - X -> 0, X - undef -> 0, undef - X -> 06957 if (Op0 == Op1 || Q.isUndefValue(Op0) || Q.isUndefValue(Op1))6958 return Constant::getNullValue(ReturnType);6959 // X - 0 -> X6960 if (match(Op1, m_Zero()))6961 return Op0;6962 break;6963 case Intrinsic::load_relative:6964 if (auto *C0 = dyn_cast<Constant>(Op0))6965 if (auto *C1 = dyn_cast<Constant>(Op1))6966 return simplifyRelativeLoad(C0, C1, Q.DL);6967 break;6968 case Intrinsic::powi:6969 if (auto *Power = dyn_cast<ConstantInt>(Op1)) {6970 // powi(x, 0) -> 1.06971 if (Power->isZero())6972 return ConstantFP::get(Op0->getType(), 1.0);6973 // powi(x, 1) -> x6974 if (Power->isOne())6975 return Op0;6976 }6977 break;6978 case Intrinsic::ldexp:6979 return simplifyLdexp(Op0, Op1, Q, false);6980 case Intrinsic::copysign:6981 // copysign X, X --> X6982 if (Op0 == Op1)6983 return Op0;6984 // copysign -X, X --> X6985 // copysign X, -X --> -X6986 if (match(Op0, m_FNeg(m_Specific(Op1))) ||6987 match(Op1, m_FNeg(m_Specific(Op0))))6988 return Op1;6989 break;6990 case Intrinsic::is_fpclass: {6991 uint64_t Mask = cast<ConstantInt>(Op1)->getZExtValue();6992 // If all tests are made, it doesn't matter what the value is.6993 if ((Mask & fcAllFlags) == fcAllFlags)6994 return ConstantInt::get(ReturnType, true);6995 if ((Mask & fcAllFlags) == 0)6996 return ConstantInt::get(ReturnType, false);6997 if (Q.isUndefValue(Op0))6998 return UndefValue::get(ReturnType);6999 break;7000 }7001 case Intrinsic::maxnum:7002 case Intrinsic::minnum:7003 case Intrinsic::maximum:7004 case Intrinsic::minimum:7005 case Intrinsic::maximumnum:7006 case Intrinsic::minimumnum: {7007 // In several cases here, we deviate from exact IEEE 754 semantics7008 // to enable optimizations (as allowed by the LLVM IR spec).7009 //7010 // For instance, we may return one of the arguments unmodified instead of7011 // inserting an llvm.canonicalize to transform input sNaNs into qNaNs,7012 // or may assume all NaN inputs are qNaNs.7013 7014 // If the arguments are the same, this is a no-op (ignoring NaN quieting)7015 if (Op0 == Op1)7016 return Op0;7017 7018 // Canonicalize constant operand as Op1.7019 if (isa<Constant>(Op0))7020 std::swap(Op0, Op1);7021 7022 if (Constant *C = dyn_cast<Constant>(Op1)) {7023 MinMaxOptResult OptResult = MinMaxOptResult::CannotOptimize;7024 Constant *NewConst = nullptr;7025 7026 if (VectorType *VTy = dyn_cast<VectorType>(C->getType())) {7027 ElementCount ElemCount = VTy->getElementCount();7028 7029 if (Constant *SplatVal = C->getSplatValue()) {7030 // Handle splat vectors (including scalable vectors)7031 OptResult = OptimizeConstMinMax(SplatVal, IID, Call, &NewConst);7032 if (OptResult == MinMaxOptResult::UseNewConstVal)7033 NewConst = ConstantVector::getSplat(ElemCount, NewConst);7034 7035 } else if (ElemCount.isFixed()) {7036 // Storage to build up new const return value (with NaNs quieted)7037 SmallVector<Constant *, 16> NewC(ElemCount.getFixedValue());7038 7039 // Check elementwise whether we can optimize to either a constant7040 // value or return the LHS value. We cannot mix and match LHS +7041 // constant elements, as this would require inserting a new7042 // VectorShuffle instruction, which is not allowed in simplifyBinOp.7043 OptResult = MinMaxOptResult::UseEither;7044 for (unsigned i = 0; i != ElemCount.getFixedValue(); ++i) {7045 auto *Elt = C->getAggregateElement(i);7046 if (!Elt) {7047 OptResult = MinMaxOptResult::CannotOptimize;7048 break;7049 }7050 auto ElemResult = OptimizeConstMinMax(Elt, IID, Call, &NewConst);7051 if (ElemResult == MinMaxOptResult::CannotOptimize ||7052 (ElemResult != OptResult &&7053 OptResult != MinMaxOptResult::UseEither &&7054 ElemResult != MinMaxOptResult::UseEither)) {7055 OptResult = MinMaxOptResult::CannotOptimize;7056 break;7057 }7058 NewC[i] = NewConst;7059 if (ElemResult != MinMaxOptResult::UseEither)7060 OptResult = ElemResult;7061 }7062 if (OptResult == MinMaxOptResult::UseNewConstVal)7063 NewConst = ConstantVector::get(NewC);7064 }7065 } else {7066 // Handle scalar inputs7067 OptResult = OptimizeConstMinMax(C, IID, Call, &NewConst);7068 }7069 7070 if (OptResult == MinMaxOptResult::UseOtherVal ||7071 OptResult == MinMaxOptResult::UseEither)7072 return Op0; // Return the other arg (ignoring NaN quieting)7073 else if (OptResult == MinMaxOptResult::UseNewConstVal)7074 return NewConst;7075 }7076 7077 // Min/max of the same operation with common operand:7078 // m(m(X, Y)), X --> m(X, Y) (4 commuted variants)7079 if (Value *V = foldMinimumMaximumSharedOp(IID, Op0, Op1))7080 return V;7081 if (Value *V = foldMinimumMaximumSharedOp(IID, Op1, Op0))7082 return V;7083 7084 break;7085 }7086 case Intrinsic::vector_extract: {7087 // (extract_vector (insert_vector _, X, 0), 0) -> X7088 unsigned IdxN = cast<ConstantInt>(Op1)->getZExtValue();7089 Value *X = nullptr;7090 if (match(Op0, m_Intrinsic<Intrinsic::vector_insert>(m_Value(), m_Value(X),7091 m_Zero())) &&7092 IdxN == 0 && X->getType() == ReturnType)7093 return X;7094 7095 break;7096 }7097 7098 case Intrinsic::aarch64_sve_andv:7099 case Intrinsic::aarch64_sve_eorv:7100 case Intrinsic::aarch64_sve_orv:7101 case Intrinsic::aarch64_sve_saddv:7102 case Intrinsic::aarch64_sve_smaxv:7103 case Intrinsic::aarch64_sve_sminv:7104 case Intrinsic::aarch64_sve_uaddv:7105 case Intrinsic::aarch64_sve_umaxv:7106 case Intrinsic::aarch64_sve_uminv:7107 return simplifySVEIntReduction(IID, ReturnType, Op0, Op1);7108 default:7109 break;7110 }7111 7112 return nullptr;7113}7114 7115static Value *simplifyIntrinsic(CallBase *Call, Value *Callee,7116 ArrayRef<Value *> Args,7117 const SimplifyQuery &Q) {7118 // Operand bundles should not be in Args.7119 assert(Call->arg_size() == Args.size());7120 unsigned NumOperands = Args.size();7121 Function *F = cast<Function>(Callee);7122 Intrinsic::ID IID = F->getIntrinsicID();7123 7124 if (IID != Intrinsic::not_intrinsic && intrinsicPropagatesPoison(IID) &&7125 any_of(Args, IsaPred<PoisonValue>))7126 return PoisonValue::get(F->getReturnType());7127 // Most of the intrinsics with no operands have some kind of side effect.7128 // Don't simplify.7129 if (!NumOperands) {7130 switch (IID) {7131 case Intrinsic::vscale: {7132 Type *RetTy = F->getReturnType();7133 ConstantRange CR = getVScaleRange(Call->getFunction(), 64);7134 if (const APInt *C = CR.getSingleElement())7135 return ConstantInt::get(RetTy, C->getZExtValue());7136 return nullptr;7137 }7138 default:7139 return nullptr;7140 }7141 }7142 7143 if (NumOperands == 1)7144 return simplifyUnaryIntrinsic(F, Args[0], Q, Call);7145 7146 if (NumOperands == 2)7147 return simplifyBinaryIntrinsic(IID, F->getReturnType(), Args[0], Args[1], Q,7148 Call);7149 7150 // Handle intrinsics with 3 or more arguments.7151 switch (IID) {7152 case Intrinsic::masked_load:7153 case Intrinsic::masked_gather: {7154 Value *MaskArg = Args[1];7155 Value *PassthruArg = Args[2];7156 // If the mask is all zeros or undef, the "passthru" argument is the result.7157 if (maskIsAllZeroOrUndef(MaskArg))7158 return PassthruArg;7159 return nullptr;7160 }7161 case Intrinsic::fshl:7162 case Intrinsic::fshr: {7163 Value *Op0 = Args[0], *Op1 = Args[1], *ShAmtArg = Args[2];7164 7165 // If both operands are undef, the result is undef.7166 if (Q.isUndefValue(Op0) && Q.isUndefValue(Op1))7167 return UndefValue::get(F->getReturnType());7168 7169 // If shift amount is undef, assume it is zero.7170 if (Q.isUndefValue(ShAmtArg))7171 return Args[IID == Intrinsic::fshl ? 0 : 1];7172 7173 const APInt *ShAmtC;7174 if (match(ShAmtArg, m_APInt(ShAmtC))) {7175 // If there's effectively no shift, return the 1st arg or 2nd arg.7176 APInt BitWidth = APInt(ShAmtC->getBitWidth(), ShAmtC->getBitWidth());7177 if (ShAmtC->urem(BitWidth).isZero())7178 return Args[IID == Intrinsic::fshl ? 0 : 1];7179 }7180 7181 // Rotating zero by anything is zero.7182 if (match(Op0, m_Zero()) && match(Op1, m_Zero()))7183 return ConstantInt::getNullValue(F->getReturnType());7184 7185 // Rotating -1 by anything is -1.7186 if (match(Op0, m_AllOnes()) && match(Op1, m_AllOnes()))7187 return ConstantInt::getAllOnesValue(F->getReturnType());7188 7189 return nullptr;7190 }7191 case Intrinsic::experimental_constrained_fma: {7192 auto *FPI = cast<ConstrainedFPIntrinsic>(Call);7193 if (Value *V = simplifyFPOp(Args, {}, Q, *FPI->getExceptionBehavior(),7194 *FPI->getRoundingMode()))7195 return V;7196 return nullptr;7197 }7198 case Intrinsic::fma:7199 case Intrinsic::fmuladd: {7200 if (Value *V = simplifyFPOp(Args, {}, Q, fp::ebIgnore,7201 RoundingMode::NearestTiesToEven))7202 return V;7203 return nullptr;7204 }7205 case Intrinsic::smul_fix:7206 case Intrinsic::smul_fix_sat: {7207 Value *Op0 = Args[0];7208 Value *Op1 = Args[1];7209 Value *Op2 = Args[2];7210 Type *ReturnType = F->getReturnType();7211 7212 // Canonicalize constant operand as Op1 (ConstantFolding handles the case7213 // when both Op0 and Op1 are constant so we do not care about that special7214 // case here).7215 if (isa<Constant>(Op0))7216 std::swap(Op0, Op1);7217 7218 // X * 0 -> 07219 if (match(Op1, m_Zero()))7220 return Constant::getNullValue(ReturnType);7221 7222 // X * undef -> 07223 if (Q.isUndefValue(Op1))7224 return Constant::getNullValue(ReturnType);7225 7226 // X * (1 << Scale) -> X7227 APInt ScaledOne =7228 APInt::getOneBitSet(ReturnType->getScalarSizeInBits(),7229 cast<ConstantInt>(Op2)->getZExtValue());7230 if (ScaledOne.isNonNegative() && match(Op1, m_SpecificInt(ScaledOne)))7231 return Op0;7232 7233 return nullptr;7234 }7235 case Intrinsic::vector_insert: {7236 Value *Vec = Args[0];7237 Value *SubVec = Args[1];7238 Value *Idx = Args[2];7239 Type *ReturnType = F->getReturnType();7240 7241 // (insert_vector Y, (extract_vector X, 0), 0) -> X7242 // where: Y is X, or Y is undef7243 unsigned IdxN = cast<ConstantInt>(Idx)->getZExtValue();7244 Value *X = nullptr;7245 if (match(SubVec,7246 m_Intrinsic<Intrinsic::vector_extract>(m_Value(X), m_Zero())) &&7247 (Q.isUndefValue(Vec) || Vec == X) && IdxN == 0 &&7248 X->getType() == ReturnType)7249 return X;7250 7251 return nullptr;7252 }7253 case Intrinsic::experimental_constrained_fadd: {7254 auto *FPI = cast<ConstrainedFPIntrinsic>(Call);7255 return simplifyFAddInst(Args[0], Args[1], FPI->getFastMathFlags(), Q,7256 *FPI->getExceptionBehavior(),7257 *FPI->getRoundingMode());7258 }7259 case Intrinsic::experimental_constrained_fsub: {7260 auto *FPI = cast<ConstrainedFPIntrinsic>(Call);7261 return simplifyFSubInst(Args[0], Args[1], FPI->getFastMathFlags(), Q,7262 *FPI->getExceptionBehavior(),7263 *FPI->getRoundingMode());7264 }7265 case Intrinsic::experimental_constrained_fmul: {7266 auto *FPI = cast<ConstrainedFPIntrinsic>(Call);7267 return simplifyFMulInst(Args[0], Args[1], FPI->getFastMathFlags(), Q,7268 *FPI->getExceptionBehavior(),7269 *FPI->getRoundingMode());7270 }7271 case Intrinsic::experimental_constrained_fdiv: {7272 auto *FPI = cast<ConstrainedFPIntrinsic>(Call);7273 return simplifyFDivInst(Args[0], Args[1], FPI->getFastMathFlags(), Q,7274 *FPI->getExceptionBehavior(),7275 *FPI->getRoundingMode());7276 }7277 case Intrinsic::experimental_constrained_frem: {7278 auto *FPI = cast<ConstrainedFPIntrinsic>(Call);7279 return simplifyFRemInst(Args[0], Args[1], FPI->getFastMathFlags(), Q,7280 *FPI->getExceptionBehavior(),7281 *FPI->getRoundingMode());7282 }7283 case Intrinsic::experimental_constrained_ldexp:7284 return simplifyLdexp(Args[0], Args[1], Q, true);7285 case Intrinsic::experimental_gc_relocate: {7286 GCRelocateInst &GCR = *cast<GCRelocateInst>(Call);7287 Value *DerivedPtr = GCR.getDerivedPtr();7288 Value *BasePtr = GCR.getBasePtr();7289 7290 // Undef is undef, even after relocation.7291 if (isa<UndefValue>(DerivedPtr) || isa<UndefValue>(BasePtr)) {7292 return UndefValue::get(GCR.getType());7293 }7294 7295 if (auto *PT = dyn_cast<PointerType>(GCR.getType())) {7296 // For now, the assumption is that the relocation of null will be null7297 // for most any collector. If this ever changes, a corresponding hook7298 // should be added to GCStrategy and this code should check it first.7299 if (isa<ConstantPointerNull>(DerivedPtr)) {7300 // Use null-pointer of gc_relocate's type to replace it.7301 return ConstantPointerNull::get(PT);7302 }7303 }7304 return nullptr;7305 }7306 case Intrinsic::experimental_vp_reverse: {7307 Value *Vec = Call->getArgOperand(0);7308 Value *Mask = Call->getArgOperand(1);7309 Value *EVL = Call->getArgOperand(2);7310 7311 Value *X;7312 // vp.reverse(vp.reverse(X)) == X (with all ones mask and matching EVL)7313 if (match(Mask, m_AllOnes()) &&7314 match(Vec, m_Intrinsic<Intrinsic::experimental_vp_reverse>(7315 m_Value(X), m_AllOnes(), m_Specific(EVL))))7316 return X;7317 7318 // vp.reverse(splat(X)) -> splat(X) (regardless of mask and EVL)7319 if (isSplatValue(Vec))7320 return Vec;7321 return nullptr;7322 }7323 default:7324 return nullptr;7325 }7326}7327 7328static Value *tryConstantFoldCall(CallBase *Call, Value *Callee,7329 ArrayRef<Value *> Args,7330 const SimplifyQuery &Q) {7331 auto *F = dyn_cast<Function>(Callee);7332 if (!F || !canConstantFoldCallTo(Call, F))7333 return nullptr;7334 7335 SmallVector<Constant *, 4> ConstantArgs;7336 ConstantArgs.reserve(Args.size());7337 for (Value *Arg : Args) {7338 Constant *C = dyn_cast<Constant>(Arg);7339 if (!C) {7340 if (isa<MetadataAsValue>(Arg))7341 continue;7342 return nullptr;7343 }7344 ConstantArgs.push_back(C);7345 }7346 7347 return ConstantFoldCall(Call, F, ConstantArgs, Q.TLI);7348}7349 7350Value *llvm::simplifyCall(CallBase *Call, Value *Callee, ArrayRef<Value *> Args,7351 const SimplifyQuery &Q) {7352 // Args should not contain operand bundle operands.7353 assert(Call->arg_size() == Args.size());7354 7355 // musttail calls can only be simplified if they are also DCEd.7356 // As we can't guarantee this here, don't simplify them.7357 if (Call->isMustTailCall())7358 return nullptr;7359 7360 // call undef -> poison7361 // call null -> poison7362 if (isa<UndefValue>(Callee) || isa<ConstantPointerNull>(Callee))7363 return PoisonValue::get(Call->getType());7364 7365 if (Value *V = tryConstantFoldCall(Call, Callee, Args, Q))7366 return V;7367 7368 auto *F = dyn_cast<Function>(Callee);7369 if (F && F->isIntrinsic())7370 if (Value *Ret = simplifyIntrinsic(Call, Callee, Args, Q))7371 return Ret;7372 7373 return nullptr;7374}7375 7376Value *llvm::simplifyConstrainedFPCall(CallBase *Call, const SimplifyQuery &Q) {7377 assert(isa<ConstrainedFPIntrinsic>(Call));7378 SmallVector<Value *, 4> Args(Call->args());7379 if (Value *V = tryConstantFoldCall(Call, Call->getCalledOperand(), Args, Q))7380 return V;7381 if (Value *Ret = simplifyIntrinsic(Call, Call->getCalledOperand(), Args, Q))7382 return Ret;7383 return nullptr;7384}7385 7386/// Given operands for a Freeze, see if we can fold the result.7387static Value *simplifyFreezeInst(Value *Op0, const SimplifyQuery &Q) {7388 // Use a utility function defined in ValueTracking.7389 if (llvm::isGuaranteedNotToBeUndefOrPoison(Op0, Q.AC, Q.CxtI, Q.DT))7390 return Op0;7391 // We have room for improvement.7392 return nullptr;7393}7394 7395Value *llvm::simplifyFreezeInst(Value *Op0, const SimplifyQuery &Q) {7396 return ::simplifyFreezeInst(Op0, Q);7397}7398 7399Value *llvm::simplifyLoadInst(LoadInst *LI, Value *PtrOp,7400 const SimplifyQuery &Q) {7401 if (LI->isVolatile())7402 return nullptr;7403 7404 if (auto *PtrOpC = dyn_cast<Constant>(PtrOp))7405 return ConstantFoldLoadFromConstPtr(PtrOpC, LI->getType(), Q.DL);7406 7407 // We can only fold the load if it is from a constant global with definitive7408 // initializer. Skip expensive logic if this is not the case.7409 auto *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(PtrOp));7410 if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())7411 return nullptr;7412 7413 // If GlobalVariable's initializer is uniform, then return the constant7414 // regardless of its offset.7415 if (Constant *C = ConstantFoldLoadFromUniformValue(GV->getInitializer(),7416 LI->getType(), Q.DL))7417 return C;7418 7419 // Try to convert operand into a constant by stripping offsets while looking7420 // through invariant.group intrinsics.7421 APInt Offset(Q.DL.getIndexTypeSizeInBits(PtrOp->getType()), 0);7422 PtrOp = PtrOp->stripAndAccumulateConstantOffsets(7423 Q.DL, Offset, /* AllowNonInbounts */ true,7424 /* AllowInvariantGroup */ true);7425 if (PtrOp == GV) {7426 // Index size may have changed due to address space casts.7427 Offset = Offset.sextOrTrunc(Q.DL.getIndexTypeSizeInBits(PtrOp->getType()));7428 return ConstantFoldLoadFromConstPtr(GV, LI->getType(), std::move(Offset),7429 Q.DL);7430 }7431 7432 return nullptr;7433}7434 7435/// See if we can compute a simplified version of this instruction.7436/// If not, this returns null.7437 7438static Value *simplifyInstructionWithOperands(Instruction *I,7439 ArrayRef<Value *> NewOps,7440 const SimplifyQuery &SQ,7441 unsigned MaxRecurse) {7442 assert(I->getFunction() && "instruction should be inserted in a function");7443 assert((!SQ.CxtI || SQ.CxtI->getFunction() == I->getFunction()) &&7444 "context instruction should be in the same function");7445 7446 const SimplifyQuery Q = SQ.CxtI ? SQ : SQ.getWithInstruction(I);7447 7448 switch (I->getOpcode()) {7449 default:7450 if (all_of(NewOps, IsaPred<Constant>)) {7451 SmallVector<Constant *, 8> NewConstOps(NewOps.size());7452 transform(NewOps, NewConstOps.begin(),7453 [](Value *V) { return cast<Constant>(V); });7454 return ConstantFoldInstOperands(I, NewConstOps, Q.DL, Q.TLI);7455 }7456 return nullptr;7457 case Instruction::FNeg:7458 return simplifyFNegInst(NewOps[0], I->getFastMathFlags(), Q, MaxRecurse);7459 case Instruction::FAdd:7460 return simplifyFAddInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q,7461 MaxRecurse);7462 case Instruction::Add:7463 return simplifyAddInst(7464 NewOps[0], NewOps[1], Q.IIQ.hasNoSignedWrap(cast<BinaryOperator>(I)),7465 Q.IIQ.hasNoUnsignedWrap(cast<BinaryOperator>(I)), Q, MaxRecurse);7466 case Instruction::FSub:7467 return simplifyFSubInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q,7468 MaxRecurse);7469 case Instruction::Sub:7470 return simplifySubInst(7471 NewOps[0], NewOps[1], Q.IIQ.hasNoSignedWrap(cast<BinaryOperator>(I)),7472 Q.IIQ.hasNoUnsignedWrap(cast<BinaryOperator>(I)), Q, MaxRecurse);7473 case Instruction::FMul:7474 return simplifyFMulInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q,7475 MaxRecurse);7476 case Instruction::Mul:7477 return simplifyMulInst(7478 NewOps[0], NewOps[1], Q.IIQ.hasNoSignedWrap(cast<BinaryOperator>(I)),7479 Q.IIQ.hasNoUnsignedWrap(cast<BinaryOperator>(I)), Q, MaxRecurse);7480 case Instruction::SDiv:7481 return simplifySDivInst(NewOps[0], NewOps[1],7482 Q.IIQ.isExact(cast<BinaryOperator>(I)), Q,7483 MaxRecurse);7484 case Instruction::UDiv:7485 return simplifyUDivInst(NewOps[0], NewOps[1],7486 Q.IIQ.isExact(cast<BinaryOperator>(I)), Q,7487 MaxRecurse);7488 case Instruction::FDiv:7489 return simplifyFDivInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q,7490 MaxRecurse);7491 case Instruction::SRem:7492 return simplifySRemInst(NewOps[0], NewOps[1], Q, MaxRecurse);7493 case Instruction::URem:7494 return simplifyURemInst(NewOps[0], NewOps[1], Q, MaxRecurse);7495 case Instruction::FRem:7496 return simplifyFRemInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q,7497 MaxRecurse);7498 case Instruction::Shl:7499 return simplifyShlInst(7500 NewOps[0], NewOps[1], Q.IIQ.hasNoSignedWrap(cast<BinaryOperator>(I)),7501 Q.IIQ.hasNoUnsignedWrap(cast<BinaryOperator>(I)), Q, MaxRecurse);7502 case Instruction::LShr:7503 return simplifyLShrInst(NewOps[0], NewOps[1],7504 Q.IIQ.isExact(cast<BinaryOperator>(I)), Q,7505 MaxRecurse);7506 case Instruction::AShr:7507 return simplifyAShrInst(NewOps[0], NewOps[1],7508 Q.IIQ.isExact(cast<BinaryOperator>(I)), Q,7509 MaxRecurse);7510 case Instruction::And:7511 return simplifyAndInst(NewOps[0], NewOps[1], Q, MaxRecurse);7512 case Instruction::Or:7513 return simplifyOrInst(NewOps[0], NewOps[1], Q, MaxRecurse);7514 case Instruction::Xor:7515 return simplifyXorInst(NewOps[0], NewOps[1], Q, MaxRecurse);7516 case Instruction::ICmp:7517 return simplifyICmpInst(cast<ICmpInst>(I)->getCmpPredicate(), NewOps[0],7518 NewOps[1], Q, MaxRecurse);7519 case Instruction::FCmp:7520 return simplifyFCmpInst(cast<FCmpInst>(I)->getPredicate(), NewOps[0],7521 NewOps[1], I->getFastMathFlags(), Q, MaxRecurse);7522 case Instruction::Select:7523 return simplifySelectInst(NewOps[0], NewOps[1], NewOps[2], Q, MaxRecurse);7524 case Instruction::GetElementPtr: {7525 auto *GEPI = cast<GetElementPtrInst>(I);7526 return simplifyGEPInst(GEPI->getSourceElementType(), NewOps[0],7527 ArrayRef(NewOps).slice(1), GEPI->getNoWrapFlags(), Q,7528 MaxRecurse);7529 }7530 case Instruction::InsertValue: {7531 InsertValueInst *IV = cast<InsertValueInst>(I);7532 return simplifyInsertValueInst(NewOps[0], NewOps[1], IV->getIndices(), Q,7533 MaxRecurse);7534 }7535 case Instruction::InsertElement:7536 return simplifyInsertElementInst(NewOps[0], NewOps[1], NewOps[2], Q);7537 case Instruction::ExtractValue: {7538 auto *EVI = cast<ExtractValueInst>(I);7539 return simplifyExtractValueInst(NewOps[0], EVI->getIndices(), Q,7540 MaxRecurse);7541 }7542 case Instruction::ExtractElement:7543 return simplifyExtractElementInst(NewOps[0], NewOps[1], Q, MaxRecurse);7544 case Instruction::ShuffleVector: {7545 auto *SVI = cast<ShuffleVectorInst>(I);7546 return simplifyShuffleVectorInst(NewOps[0], NewOps[1],7547 SVI->getShuffleMask(), SVI->getType(), Q,7548 MaxRecurse);7549 }7550 case Instruction::PHI:7551 return simplifyPHINode(cast<PHINode>(I), NewOps, Q);7552 case Instruction::Call:7553 return simplifyCall(7554 cast<CallInst>(I), NewOps.back(),7555 NewOps.drop_back(1 + cast<CallInst>(I)->getNumTotalBundleOperands()), Q);7556 case Instruction::Freeze:7557 return llvm::simplifyFreezeInst(NewOps[0], Q);7558#define HANDLE_CAST_INST(num, opc, clas) case Instruction::opc:7559#include "llvm/IR/Instruction.def"7560#undef HANDLE_CAST_INST7561 return simplifyCastInst(I->getOpcode(), NewOps[0], I->getType(), Q,7562 MaxRecurse);7563 case Instruction::Alloca:7564 // No simplifications for Alloca and it can't be constant folded.7565 return nullptr;7566 case Instruction::Load:7567 return simplifyLoadInst(cast<LoadInst>(I), NewOps[0], Q);7568 }7569}7570 7571Value *llvm::simplifyInstructionWithOperands(Instruction *I,7572 ArrayRef<Value *> NewOps,7573 const SimplifyQuery &SQ) {7574 assert(NewOps.size() == I->getNumOperands() &&7575 "Number of operands should match the instruction!");7576 return ::simplifyInstructionWithOperands(I, NewOps, SQ, RecursionLimit);7577}7578 7579Value *llvm::simplifyInstruction(Instruction *I, const SimplifyQuery &SQ) {7580 SmallVector<Value *, 8> Ops(I->operands());7581 Value *Result = ::simplifyInstructionWithOperands(I, Ops, SQ, RecursionLimit);7582 7583 /// If called on unreachable code, the instruction may simplify to itself.7584 /// Make life easier for users by detecting that case here, and returning a7585 /// safe value instead.7586 return Result == I ? PoisonValue::get(I->getType()) : Result;7587}7588 7589/// Implementation of recursive simplification through an instruction's7590/// uses.7591///7592/// This is the common implementation of the recursive simplification routines.7593/// If we have a pre-simplified value in 'SimpleV', that is forcibly used to7594/// replace the instruction 'I'. Otherwise, we simply add 'I' to the list of7595/// instructions to process and attempt to simplify it using7596/// InstructionSimplify. Recursively visited users which could not be7597/// simplified themselves are to the optional UnsimplifiedUsers set for7598/// further processing by the caller.7599///7600/// This routine returns 'true' only when *it* simplifies something. The passed7601/// in simplified value does not count toward this.7602static bool replaceAndRecursivelySimplifyImpl(7603 Instruction *I, Value *SimpleV, const TargetLibraryInfo *TLI,7604 const DominatorTree *DT, AssumptionCache *AC,7605 SmallSetVector<Instruction *, 8> *UnsimplifiedUsers = nullptr) {7606 bool Simplified = false;7607 SmallSetVector<Instruction *, 8> Worklist;7608 const DataLayout &DL = I->getDataLayout();7609 7610 // If we have an explicit value to collapse to, do that round of the7611 // simplification loop by hand initially.7612 if (SimpleV) {7613 for (User *U : I->users())7614 if (U != I)7615 Worklist.insert(cast<Instruction>(U));7616 7617 // Replace the instruction with its simplified value.7618 I->replaceAllUsesWith(SimpleV);7619 7620 if (!I->isEHPad() && !I->isTerminator() && !I->mayHaveSideEffects())7621 I->eraseFromParent();7622 } else {7623 Worklist.insert(I);7624 }7625 7626 // Note that we must test the size on each iteration, the worklist can grow.7627 for (unsigned Idx = 0; Idx != Worklist.size(); ++Idx) {7628 I = Worklist[Idx];7629 7630 // See if this instruction simplifies.7631 SimpleV = simplifyInstruction(I, {DL, TLI, DT, AC});7632 if (!SimpleV) {7633 if (UnsimplifiedUsers)7634 UnsimplifiedUsers->insert(I);7635 continue;7636 }7637 7638 Simplified = true;7639 7640 // Stash away all the uses of the old instruction so we can check them for7641 // recursive simplifications after a RAUW. This is cheaper than checking all7642 // uses of To on the recursive step in most cases.7643 for (User *U : I->users())7644 Worklist.insert(cast<Instruction>(U));7645 7646 // Replace the instruction with its simplified value.7647 I->replaceAllUsesWith(SimpleV);7648 7649 if (!I->isEHPad() && !I->isTerminator() && !I->mayHaveSideEffects())7650 I->eraseFromParent();7651 }7652 return Simplified;7653}7654 7655bool llvm::replaceAndRecursivelySimplify(7656 Instruction *I, Value *SimpleV, const TargetLibraryInfo *TLI,7657 const DominatorTree *DT, AssumptionCache *AC,7658 SmallSetVector<Instruction *, 8> *UnsimplifiedUsers) {7659 assert(I != SimpleV && "replaceAndRecursivelySimplify(X,X) is not valid!");7660 assert(SimpleV && "Must provide a simplified value.");7661 return replaceAndRecursivelySimplifyImpl(I, SimpleV, TLI, DT, AC,7662 UnsimplifiedUsers);7663}7664 7665namespace llvm {7666const SimplifyQuery getBestSimplifyQuery(Pass &P, Function &F) {7667 auto *DTWP = P.getAnalysisIfAvailable<DominatorTreeWrapperPass>();7668 auto *DT = DTWP ? &DTWP->getDomTree() : nullptr;7669 auto *TLIWP = P.getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();7670 auto *TLI = TLIWP ? &TLIWP->getTLI(F) : nullptr;7671 auto *ACWP = P.getAnalysisIfAvailable<AssumptionCacheTracker>();7672 auto *AC = ACWP ? &ACWP->getAssumptionCache(F) : nullptr;7673 return {F.getDataLayout(), TLI, DT, AC};7674}7675 7676const SimplifyQuery getBestSimplifyQuery(LoopStandardAnalysisResults &AR,7677 const DataLayout &DL) {7678 return {DL, &AR.TLI, &AR.DT, &AR.AC};7679}7680 7681template <class T, class... TArgs>7682const SimplifyQuery getBestSimplifyQuery(AnalysisManager<T, TArgs...> &AM,7683 Function &F) {7684 auto *DT = AM.template getCachedResult<DominatorTreeAnalysis>(F);7685 auto *TLI = AM.template getCachedResult<TargetLibraryAnalysis>(F);7686 auto *AC = AM.template getCachedResult<AssumptionAnalysis>(F);7687 return {F.getDataLayout(), TLI, DT, AC};7688}7689template const SimplifyQuery getBestSimplifyQuery(AnalysisManager<Function> &,7690 Function &);7691 7692bool SimplifyQuery::isUndefValue(Value *V) const {7693 if (!CanUseUndef)7694 return false;7695 7696 return match(V, m_Undef());7697}7698 7699} // namespace llvm7700 7701void InstSimplifyFolder::anchor() {}7702