4588 lines · cpp
1//===- Instructions.cpp - Implement the LLVM instructions -----------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements all of the non-inline methods for the LLVM instruction10// classes.11//12//===----------------------------------------------------------------------===//13 14#include "llvm/IR/Instructions.h"15#include "LLVMContextImpl.h"16#include "llvm/ADT/SmallBitVector.h"17#include "llvm/ADT/SmallVector.h"18#include "llvm/ADT/Twine.h"19#include "llvm/IR/Attributes.h"20#include "llvm/IR/BasicBlock.h"21#include "llvm/IR/Constant.h"22#include "llvm/IR/ConstantRange.h"23#include "llvm/IR/Constants.h"24#include "llvm/IR/DataLayout.h"25#include "llvm/IR/DerivedTypes.h"26#include "llvm/IR/Function.h"27#include "llvm/IR/InstrTypes.h"28#include "llvm/IR/Instruction.h"29#include "llvm/IR/Intrinsics.h"30#include "llvm/IR/LLVMContext.h"31#include "llvm/IR/MDBuilder.h"32#include "llvm/IR/Metadata.h"33#include "llvm/IR/Module.h"34#include "llvm/IR/Operator.h"35#include "llvm/IR/PatternMatch.h"36#include "llvm/IR/ProfDataUtils.h"37#include "llvm/IR/Type.h"38#include "llvm/IR/Value.h"39#include "llvm/Support/AtomicOrdering.h"40#include "llvm/Support/Casting.h"41#include "llvm/Support/CheckedArithmetic.h"42#include "llvm/Support/Compiler.h"43#include "llvm/Support/ErrorHandling.h"44#include "llvm/Support/KnownBits.h"45#include "llvm/Support/MathExtras.h"46#include "llvm/Support/ModRef.h"47#include "llvm/Support/TypeSize.h"48#include <algorithm>49#include <cassert>50#include <cstdint>51#include <optional>52#include <vector>53 54using namespace llvm;55 56static cl::opt<bool> DisableI2pP2iOpt(57 "disable-i2p-p2i-opt", cl::init(false),58 cl::desc("Disables inttoptr/ptrtoint roundtrip optimization"));59 60//===----------------------------------------------------------------------===//61// AllocaInst Class62//===----------------------------------------------------------------------===//63 64std::optional<TypeSize>65AllocaInst::getAllocationSize(const DataLayout &DL) const {66 TypeSize Size = DL.getTypeAllocSize(getAllocatedType());67 if (isArrayAllocation()) {68 auto *C = dyn_cast<ConstantInt>(getArraySize());69 if (!C)70 return std::nullopt;71 assert(!Size.isScalable() && "Array elements cannot have a scalable size");72 auto CheckedProd =73 checkedMulUnsigned(Size.getKnownMinValue(), C->getZExtValue());74 if (!CheckedProd)75 return std::nullopt;76 return TypeSize::getFixed(*CheckedProd);77 }78 return Size;79}80 81std::optional<TypeSize>82AllocaInst::getAllocationSizeInBits(const DataLayout &DL) const {83 std::optional<TypeSize> Size = getAllocationSize(DL);84 if (!Size)85 return std::nullopt;86 auto CheckedProd = checkedMulUnsigned(Size->getKnownMinValue(),87 static_cast<TypeSize::ScalarTy>(8));88 if (!CheckedProd)89 return std::nullopt;90 return TypeSize::get(*CheckedProd, Size->isScalable());91}92 93//===----------------------------------------------------------------------===//94// SelectInst Class95//===----------------------------------------------------------------------===//96 97/// areInvalidOperands - Return a string if the specified operands are invalid98/// for a select operation, otherwise return null.99const char *SelectInst::areInvalidOperands(Value *Op0, Value *Op1, Value *Op2) {100 if (Op1->getType() != Op2->getType())101 return "both values to select must have same type";102 103 if (Op1->getType()->isTokenTy())104 return "select values cannot have token type";105 106 if (VectorType *VT = dyn_cast<VectorType>(Op0->getType())) {107 // Vector select.108 if (VT->getElementType() != Type::getInt1Ty(Op0->getContext()))109 return "vector select condition element type must be i1";110 VectorType *ET = dyn_cast<VectorType>(Op1->getType());111 if (!ET)112 return "selected values for vector select must be vectors";113 if (ET->getElementCount() != VT->getElementCount())114 return "vector select requires selected vectors to have "115 "the same vector length as select condition";116 } else if (Op0->getType() != Type::getInt1Ty(Op0->getContext())) {117 return "select condition must be i1 or <n x i1>";118 }119 return nullptr;120}121 122//===----------------------------------------------------------------------===//123// PHINode Class124//===----------------------------------------------------------------------===//125 126PHINode::PHINode(const PHINode &PN)127 : Instruction(PN.getType(), Instruction::PHI, AllocMarker),128 ReservedSpace(PN.getNumOperands()) {129 NumUserOperands = PN.getNumOperands();130 allocHungoffUses(PN.getNumOperands());131 std::copy(PN.op_begin(), PN.op_end(), op_begin());132 copyIncomingBlocks(make_range(PN.block_begin(), PN.block_end()));133 SubclassOptionalData = PN.SubclassOptionalData;134}135 136// removeIncomingValue - Remove an incoming value. This is useful if a137// predecessor basic block is deleted.138Value *PHINode::removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty) {139 Value *Removed = getIncomingValue(Idx);140 141 // Move everything after this operand down.142 //143 // FIXME: we could just swap with the end of the list, then erase. However,144 // clients might not expect this to happen. The code as it is thrashes the145 // use/def lists, which is kinda lame.146 std::copy(op_begin() + Idx + 1, op_end(), op_begin() + Idx);147 copyIncomingBlocks(drop_begin(blocks(), Idx + 1), Idx);148 149 // Nuke the last value.150 Op<-1>().set(nullptr);151 setNumHungOffUseOperands(getNumOperands() - 1);152 153 // If the PHI node is dead, because it has zero entries, nuke it now.154 if (getNumOperands() == 0 && DeletePHIIfEmpty) {155 // If anyone is using this PHI, make them use a dummy value instead...156 replaceAllUsesWith(PoisonValue::get(getType()));157 eraseFromParent();158 }159 return Removed;160}161 162void PHINode::removeIncomingValueIf(function_ref<bool(unsigned)> Predicate,163 bool DeletePHIIfEmpty) {164 SmallDenseSet<unsigned> RemoveIndices;165 for (unsigned Idx = 0; Idx < getNumIncomingValues(); ++Idx)166 if (Predicate(Idx))167 RemoveIndices.insert(Idx);168 169 if (RemoveIndices.empty())170 return;171 172 // Remove operands.173 auto NewOpEnd = remove_if(operands(), [&](Use &U) {174 return RemoveIndices.contains(U.getOperandNo());175 });176 for (Use &U : make_range(NewOpEnd, op_end()))177 U.set(nullptr);178 179 // Remove incoming blocks.180 (void)std::remove_if(const_cast<block_iterator>(block_begin()),181 const_cast<block_iterator>(block_end()), [&](BasicBlock *&BB) {182 return RemoveIndices.contains(&BB - block_begin());183 });184 185 setNumHungOffUseOperands(getNumOperands() - RemoveIndices.size());186 187 // If the PHI node is dead, because it has zero entries, nuke it now.188 if (getNumOperands() == 0 && DeletePHIIfEmpty) {189 // If anyone is using this PHI, make them use a dummy value instead...190 replaceAllUsesWith(PoisonValue::get(getType()));191 eraseFromParent();192 }193}194 195/// growOperands - grow operands - This grows the operand list in response196/// to a push_back style of operation. This grows the number of ops by 1.5197/// times.198///199void PHINode::growOperands() {200 unsigned e = getNumOperands();201 unsigned NumOps = e + e / 2;202 if (NumOps < 2) NumOps = 2; // 2 op PHI nodes are VERY common.203 204 ReservedSpace = NumOps;205 growHungoffUses(ReservedSpace, /* IsPhi */ true);206}207 208/// hasConstantValue - If the specified PHI node always merges together the same209/// value, return the value, otherwise return null.210Value *PHINode::hasConstantValue() const {211 // Exploit the fact that phi nodes always have at least one entry.212 Value *ConstantValue = getIncomingValue(0);213 for (unsigned i = 1, e = getNumIncomingValues(); i != e; ++i)214 if (getIncomingValue(i) != ConstantValue && getIncomingValue(i) != this) {215 if (ConstantValue != this)216 return nullptr; // Incoming values not all the same.217 // The case where the first value is this PHI.218 ConstantValue = getIncomingValue(i);219 }220 if (ConstantValue == this)221 return PoisonValue::get(getType());222 return ConstantValue;223}224 225/// hasConstantOrUndefValue - Whether the specified PHI node always merges226/// together the same value, assuming that undefs result in the same value as227/// non-undefs.228/// Unlike \ref hasConstantValue, this does not return a value because the229/// unique non-undef incoming value need not dominate the PHI node.230bool PHINode::hasConstantOrUndefValue() const {231 Value *ConstantValue = nullptr;232 for (unsigned i = 0, e = getNumIncomingValues(); i != e; ++i) {233 Value *Incoming = getIncomingValue(i);234 if (Incoming != this && !isa<UndefValue>(Incoming)) {235 if (ConstantValue && ConstantValue != Incoming)236 return false;237 ConstantValue = Incoming;238 }239 }240 return true;241}242 243//===----------------------------------------------------------------------===//244// LandingPadInst Implementation245//===----------------------------------------------------------------------===//246 247LandingPadInst::LandingPadInst(Type *RetTy, unsigned NumReservedValues,248 const Twine &NameStr,249 InsertPosition InsertBefore)250 : Instruction(RetTy, Instruction::LandingPad, AllocMarker, InsertBefore) {251 init(NumReservedValues, NameStr);252}253 254LandingPadInst::LandingPadInst(const LandingPadInst &LP)255 : Instruction(LP.getType(), Instruction::LandingPad, AllocMarker),256 ReservedSpace(LP.getNumOperands()) {257 NumUserOperands = LP.getNumOperands();258 allocHungoffUses(LP.getNumOperands());259 Use *OL = getOperandList();260 const Use *InOL = LP.getOperandList();261 for (unsigned I = 0, E = ReservedSpace; I != E; ++I)262 OL[I] = InOL[I];263 264 setCleanup(LP.isCleanup());265}266 267LandingPadInst *LandingPadInst::Create(Type *RetTy, unsigned NumReservedClauses,268 const Twine &NameStr,269 InsertPosition InsertBefore) {270 return new LandingPadInst(RetTy, NumReservedClauses, NameStr, InsertBefore);271}272 273void LandingPadInst::init(unsigned NumReservedValues, const Twine &NameStr) {274 ReservedSpace = NumReservedValues;275 setNumHungOffUseOperands(0);276 allocHungoffUses(ReservedSpace);277 setName(NameStr);278 setCleanup(false);279}280 281/// growOperands - grow operands - This grows the operand list in response to a282/// push_back style of operation. This grows the number of ops by 2 times.283void LandingPadInst::growOperands(unsigned Size) {284 unsigned e = getNumOperands();285 if (ReservedSpace >= e + Size) return;286 ReservedSpace = (std::max(e, 1U) + Size / 2) * 2;287 growHungoffUses(ReservedSpace);288}289 290void LandingPadInst::addClause(Constant *Val) {291 unsigned OpNo = getNumOperands();292 growOperands(1);293 assert(OpNo < ReservedSpace && "Growing didn't work!");294 setNumHungOffUseOperands(getNumOperands() + 1);295 getOperandList()[OpNo] = Val;296}297 298//===----------------------------------------------------------------------===//299// CallBase Implementation300//===----------------------------------------------------------------------===//301 302CallBase *CallBase::Create(CallBase *CB, ArrayRef<OperandBundleDef> Bundles,303 InsertPosition InsertPt) {304 switch (CB->getOpcode()) {305 case Instruction::Call:306 return CallInst::Create(cast<CallInst>(CB), Bundles, InsertPt);307 case Instruction::Invoke:308 return InvokeInst::Create(cast<InvokeInst>(CB), Bundles, InsertPt);309 case Instruction::CallBr:310 return CallBrInst::Create(cast<CallBrInst>(CB), Bundles, InsertPt);311 default:312 llvm_unreachable("Unknown CallBase sub-class!");313 }314}315 316CallBase *CallBase::Create(CallBase *CI, OperandBundleDef OpB,317 InsertPosition InsertPt) {318 SmallVector<OperandBundleDef, 2> OpDefs;319 for (unsigned i = 0, e = CI->getNumOperandBundles(); i < e; ++i) {320 auto ChildOB = CI->getOperandBundleAt(i);321 if (ChildOB.getTagName() != OpB.getTag())322 OpDefs.emplace_back(ChildOB);323 }324 OpDefs.emplace_back(OpB);325 return CallBase::Create(CI, OpDefs, InsertPt);326}327 328Function *CallBase::getCaller() { return getParent()->getParent(); }329 330unsigned CallBase::getNumSubclassExtraOperandsDynamic() const {331 assert(getOpcode() == Instruction::CallBr && "Unexpected opcode!");332 return cast<CallBrInst>(this)->getNumIndirectDests() + 1;333}334 335bool CallBase::isIndirectCall() const {336 const Value *V = getCalledOperand();337 if (isa<Function>(V) || isa<Constant>(V))338 return false;339 return !isInlineAsm();340}341 342/// Tests if this call site must be tail call optimized. Only a CallInst can343/// be tail call optimized.344bool CallBase::isMustTailCall() const {345 if (auto *CI = dyn_cast<CallInst>(this))346 return CI->isMustTailCall();347 return false;348}349 350/// Tests if this call site is marked as a tail call.351bool CallBase::isTailCall() const {352 if (auto *CI = dyn_cast<CallInst>(this))353 return CI->isTailCall();354 return false;355}356 357Intrinsic::ID CallBase::getIntrinsicID() const {358 if (auto *F = dyn_cast_or_null<Function>(getCalledOperand()))359 return F->getIntrinsicID();360 return Intrinsic::not_intrinsic;361}362 363FPClassTest CallBase::getRetNoFPClass() const {364 FPClassTest Mask = Attrs.getRetNoFPClass();365 366 if (const Function *F = getCalledFunction())367 Mask |= F->getAttributes().getRetNoFPClass();368 return Mask;369}370 371FPClassTest CallBase::getParamNoFPClass(unsigned i) const {372 FPClassTest Mask = Attrs.getParamNoFPClass(i);373 374 if (const Function *F = getCalledFunction())375 Mask |= F->getAttributes().getParamNoFPClass(i);376 return Mask;377}378 379std::optional<ConstantRange> CallBase::getRange() const {380 Attribute CallAttr = Attrs.getRetAttr(Attribute::Range);381 Attribute FnAttr;382 if (const Function *F = getCalledFunction())383 FnAttr = F->getRetAttribute(Attribute::Range);384 385 if (CallAttr.isValid() && FnAttr.isValid())386 return CallAttr.getRange().intersectWith(FnAttr.getRange());387 if (CallAttr.isValid())388 return CallAttr.getRange();389 if (FnAttr.isValid())390 return FnAttr.getRange();391 return std::nullopt;392}393 394bool CallBase::isReturnNonNull() const {395 if (hasRetAttr(Attribute::NonNull))396 return true;397 398 if (getRetDereferenceableBytes() > 0 &&399 !NullPointerIsDefined(getCaller(), getType()->getPointerAddressSpace()))400 return true;401 402 return false;403}404 405Value *CallBase::getArgOperandWithAttribute(Attribute::AttrKind Kind) const {406 unsigned Index;407 408 if (Attrs.hasAttrSomewhere(Kind, &Index))409 return getArgOperand(Index - AttributeList::FirstArgIndex);410 if (const Function *F = getCalledFunction())411 if (F->getAttributes().hasAttrSomewhere(Kind, &Index))412 return getArgOperand(Index - AttributeList::FirstArgIndex);413 414 return nullptr;415}416 417/// Determine whether the argument or parameter has the given attribute.418bool CallBase::paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const {419 assert(ArgNo < arg_size() && "Param index out of bounds!");420 421 if (Attrs.hasParamAttr(ArgNo, Kind))422 return true;423 424 const Function *F = getCalledFunction();425 if (!F)426 return false;427 428 if (!F->getAttributes().hasParamAttr(ArgNo, Kind))429 return false;430 431 // Take into account mod/ref by operand bundles.432 switch (Kind) {433 case Attribute::ReadNone:434 return !hasReadingOperandBundles() && !hasClobberingOperandBundles();435 case Attribute::ReadOnly:436 return !hasClobberingOperandBundles();437 case Attribute::WriteOnly:438 return !hasReadingOperandBundles();439 default:440 return true;441 }442}443 444bool CallBase::paramHasNonNullAttr(unsigned ArgNo,445 bool AllowUndefOrPoison) const {446 assert(getArgOperand(ArgNo)->getType()->isPointerTy() &&447 "Argument must be a pointer");448 if (paramHasAttr(ArgNo, Attribute::NonNull) &&449 (AllowUndefOrPoison || paramHasAttr(ArgNo, Attribute::NoUndef)))450 return true;451 452 if (paramHasAttr(ArgNo, Attribute::Dereferenceable) &&453 !NullPointerIsDefined(454 getCaller(),455 getArgOperand(ArgNo)->getType()->getPointerAddressSpace()))456 return true;457 458 return false;459}460 461bool CallBase::hasFnAttrOnCalledFunction(Attribute::AttrKind Kind) const {462 if (auto *F = dyn_cast<Function>(getCalledOperand()))463 return F->getAttributes().hasFnAttr(Kind);464 465 return false;466}467 468bool CallBase::hasFnAttrOnCalledFunction(StringRef Kind) const {469 if (auto *F = dyn_cast<Function>(getCalledOperand()))470 return F->getAttributes().hasFnAttr(Kind);471 472 return false;473}474 475template <typename AK>476Attribute CallBase::getFnAttrOnCalledFunction(AK Kind) const {477 if constexpr (std::is_same_v<AK, Attribute::AttrKind>) {478 // getMemoryEffects() correctly combines memory effects from the call-site,479 // operand bundles and function.480 assert(Kind != Attribute::Memory && "Use getMemoryEffects() instead");481 }482 483 if (auto *F = dyn_cast<Function>(getCalledOperand()))484 return F->getAttributes().getFnAttr(Kind);485 486 return Attribute();487}488 489template LLVM_ABI Attribute490CallBase::getFnAttrOnCalledFunction(Attribute::AttrKind Kind) const;491template LLVM_ABI Attribute492CallBase::getFnAttrOnCalledFunction(StringRef Kind) const;493 494template <typename AK>495Attribute CallBase::getParamAttrOnCalledFunction(unsigned ArgNo,496 AK Kind) const {497 Value *V = getCalledOperand();498 499 if (auto *F = dyn_cast<Function>(V))500 return F->getAttributes().getParamAttr(ArgNo, Kind);501 502 return Attribute();503}504template LLVM_ABI Attribute CallBase::getParamAttrOnCalledFunction(505 unsigned ArgNo, Attribute::AttrKind Kind) const;506template LLVM_ABI Attribute507CallBase::getParamAttrOnCalledFunction(unsigned ArgNo, StringRef Kind) const;508 509void CallBase::getOperandBundlesAsDefs(510 SmallVectorImpl<OperandBundleDef> &Defs) const {511 for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i)512 Defs.emplace_back(getOperandBundleAt(i));513}514 515CallBase::op_iterator516CallBase::populateBundleOperandInfos(ArrayRef<OperandBundleDef> Bundles,517 const unsigned BeginIndex) {518 auto It = op_begin() + BeginIndex;519 for (auto &B : Bundles)520 It = std::copy(B.input_begin(), B.input_end(), It);521 522 auto *ContextImpl = getContext().pImpl;523 auto BI = Bundles.begin();524 unsigned CurrentIndex = BeginIndex;525 526 for (auto &BOI : bundle_op_infos()) {527 assert(BI != Bundles.end() && "Incorrect allocation?");528 529 BOI.Tag = ContextImpl->getOrInsertBundleTag(BI->getTag());530 BOI.Begin = CurrentIndex;531 BOI.End = CurrentIndex + BI->input_size();532 CurrentIndex = BOI.End;533 BI++;534 }535 536 assert(BI == Bundles.end() && "Incorrect allocation?");537 538 return It;539}540 541CallBase::BundleOpInfo &CallBase::getBundleOpInfoForOperand(unsigned OpIdx) {542 /// When there isn't many bundles, we do a simple linear search.543 /// Else fallback to a binary-search that use the fact that bundles usually544 /// have similar number of argument to get faster convergence.545 if (bundle_op_info_end() - bundle_op_info_begin() < 8) {546 for (auto &BOI : bundle_op_infos())547 if (BOI.Begin <= OpIdx && OpIdx < BOI.End)548 return BOI;549 550 llvm_unreachable("Did not find operand bundle for operand!");551 }552 553 assert(OpIdx >= arg_size() && "the Idx is not in the operand bundles");554 assert(bundle_op_info_end() - bundle_op_info_begin() > 0 &&555 OpIdx < std::prev(bundle_op_info_end())->End &&556 "The Idx isn't in the operand bundle");557 558 /// We need a decimal number below and to prevent using floating point numbers559 /// we use an intergal value multiplied by this constant.560 constexpr unsigned NumberScaling = 1024;561 562 bundle_op_iterator Begin = bundle_op_info_begin();563 bundle_op_iterator End = bundle_op_info_end();564 bundle_op_iterator Current = Begin;565 566 while (Begin != End) {567 unsigned ScaledOperandPerBundle =568 NumberScaling * (std::prev(End)->End - Begin->Begin) / (End - Begin);569 Current = Begin + (((OpIdx - Begin->Begin) * NumberScaling) /570 ScaledOperandPerBundle);571 if (Current >= End)572 Current = std::prev(End);573 assert(Current < End && Current >= Begin &&574 "the operand bundle doesn't cover every value in the range");575 if (OpIdx >= Current->Begin && OpIdx < Current->End)576 break;577 if (OpIdx >= Current->End)578 Begin = Current + 1;579 else580 End = Current;581 }582 583 assert(OpIdx >= Current->Begin && OpIdx < Current->End &&584 "the operand bundle doesn't cover every value in the range");585 return *Current;586}587 588CallBase *CallBase::addOperandBundle(CallBase *CB, uint32_t ID,589 OperandBundleDef OB,590 InsertPosition InsertPt) {591 if (CB->getOperandBundle(ID))592 return CB;593 594 SmallVector<OperandBundleDef, 1> Bundles;595 CB->getOperandBundlesAsDefs(Bundles);596 Bundles.push_back(OB);597 return Create(CB, Bundles, InsertPt);598}599 600CallBase *CallBase::removeOperandBundle(CallBase *CB, uint32_t ID,601 InsertPosition InsertPt) {602 SmallVector<OperandBundleDef, 1> Bundles;603 bool CreateNew = false;604 605 for (unsigned I = 0, E = CB->getNumOperandBundles(); I != E; ++I) {606 auto Bundle = CB->getOperandBundleAt(I);607 if (Bundle.getTagID() == ID) {608 CreateNew = true;609 continue;610 }611 Bundles.emplace_back(Bundle);612 }613 614 return CreateNew ? Create(CB, Bundles, InsertPt) : CB;615}616 617bool CallBase::hasReadingOperandBundles() const {618 // Implementation note: this is a conservative implementation of operand619 // bundle semantics, where *any* non-assume operand bundle (other than620 // ptrauth) forces a callsite to be at least readonly.621 return hasOperandBundlesOtherThan({LLVMContext::OB_ptrauth,622 LLVMContext::OB_kcfi,623 LLVMContext::OB_convergencectrl,624 LLVMContext::OB_deactivation_symbol}) &&625 getIntrinsicID() != Intrinsic::assume;626}627 628bool CallBase::hasClobberingOperandBundles() const {629 return hasOperandBundlesOtherThan(630 {LLVMContext::OB_deopt, LLVMContext::OB_funclet,631 LLVMContext::OB_ptrauth, LLVMContext::OB_kcfi,632 LLVMContext::OB_convergencectrl,633 LLVMContext::OB_deactivation_symbol}) &&634 getIntrinsicID() != Intrinsic::assume;635}636 637MemoryEffects CallBase::getMemoryEffects() const {638 MemoryEffects ME = getAttributes().getMemoryEffects();639 if (auto *Fn = dyn_cast<Function>(getCalledOperand())) {640 MemoryEffects FnME = Fn->getMemoryEffects();641 if (hasOperandBundles()) {642 // TODO: Add a method to get memory effects for operand bundles instead.643 if (hasReadingOperandBundles())644 FnME |= MemoryEffects::readOnly();645 if (hasClobberingOperandBundles())646 FnME |= MemoryEffects::writeOnly();647 }648 if (isVolatile()) {649 // Volatile operations also access inaccessible memory.650 FnME |= MemoryEffects::inaccessibleMemOnly();651 }652 ME &= FnME;653 }654 return ME;655}656void CallBase::setMemoryEffects(MemoryEffects ME) {657 addFnAttr(Attribute::getWithMemoryEffects(getContext(), ME));658}659 660/// Determine if the function does not access memory.661bool CallBase::doesNotAccessMemory() const {662 return getMemoryEffects().doesNotAccessMemory();663}664void CallBase::setDoesNotAccessMemory() {665 setMemoryEffects(MemoryEffects::none());666}667 668/// Determine if the function does not access or only reads memory.669bool CallBase::onlyReadsMemory() const {670 return getMemoryEffects().onlyReadsMemory();671}672void CallBase::setOnlyReadsMemory() {673 setMemoryEffects(getMemoryEffects() & MemoryEffects::readOnly());674}675 676/// Determine if the function does not access or only writes memory.677bool CallBase::onlyWritesMemory() const {678 return getMemoryEffects().onlyWritesMemory();679}680void CallBase::setOnlyWritesMemory() {681 setMemoryEffects(getMemoryEffects() & MemoryEffects::writeOnly());682}683 684/// Determine if the call can access memmory only using pointers based685/// on its arguments.686bool CallBase::onlyAccessesArgMemory() const {687 return getMemoryEffects().onlyAccessesArgPointees();688}689void CallBase::setOnlyAccessesArgMemory() {690 setMemoryEffects(getMemoryEffects() & MemoryEffects::argMemOnly());691}692 693/// Determine if the function may only access memory that is694/// inaccessible from the IR.695bool CallBase::onlyAccessesInaccessibleMemory() const {696 return getMemoryEffects().onlyAccessesInaccessibleMem();697}698void CallBase::setOnlyAccessesInaccessibleMemory() {699 setMemoryEffects(getMemoryEffects() & MemoryEffects::inaccessibleMemOnly());700}701 702/// Determine if the function may only access memory that is703/// either inaccessible from the IR or pointed to by its arguments.704bool CallBase::onlyAccessesInaccessibleMemOrArgMem() const {705 return getMemoryEffects().onlyAccessesInaccessibleOrArgMem();706}707void CallBase::setOnlyAccessesInaccessibleMemOrArgMem() {708 setMemoryEffects(getMemoryEffects() &709 MemoryEffects::inaccessibleOrArgMemOnly());710}711 712CaptureInfo CallBase::getCaptureInfo(unsigned OpNo) const {713 if (OpNo < arg_size()) {714 // If the argument is passed byval, the callee does not have access to the715 // original pointer and thus cannot capture it.716 if (isByValArgument(OpNo))717 return CaptureInfo::none();718 719 CaptureInfo CI = getParamAttributes(OpNo).getCaptureInfo();720 if (auto *Fn = dyn_cast<Function>(getCalledOperand()))721 CI &= Fn->getAttributes().getParamAttrs(OpNo).getCaptureInfo();722 return CI;723 }724 725 // Bundles on assumes are captures(none).726 if (getIntrinsicID() == Intrinsic::assume)727 return CaptureInfo::none();728 729 // deopt operand bundles are captures(none)730 auto &BOI = getBundleOpInfoForOperand(OpNo);731 auto OBU = operandBundleFromBundleOpInfo(BOI);732 return OBU.isDeoptOperandBundle() ? CaptureInfo::none() : CaptureInfo::all();733}734 735bool CallBase::hasArgumentWithAdditionalReturnCaptureComponents() const {736 for (unsigned I = 0, E = arg_size(); I < E; ++I) {737 if (!getArgOperand(I)->getType()->isPointerTy())738 continue;739 740 CaptureInfo CI = getParamAttributes(I).getCaptureInfo();741 if (auto *Fn = dyn_cast<Function>(getCalledOperand()))742 CI &= Fn->getAttributes().getParamAttrs(I).getCaptureInfo();743 if (capturesAnything(CI.getRetComponents() & ~CI.getOtherComponents()))744 return true;745 }746 return false;747}748 749//===----------------------------------------------------------------------===//750// CallInst Implementation751//===----------------------------------------------------------------------===//752 753void CallInst::init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,754 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr) {755 this->FTy = FTy;756 assert(getNumOperands() == Args.size() + CountBundleInputs(Bundles) + 1 &&757 "NumOperands not set up?");758 759#ifndef NDEBUG760 assert((Args.size() == FTy->getNumParams() ||761 (FTy->isVarArg() && Args.size() > FTy->getNumParams())) &&762 "Calling a function with bad signature!");763 764 for (unsigned i = 0; i != Args.size(); ++i)765 assert((i >= FTy->getNumParams() ||766 FTy->getParamType(i) == Args[i]->getType()) &&767 "Calling a function with a bad signature!");768#endif769 770 // Set operands in order of their index to match use-list-order771 // prediction.772 llvm::copy(Args, op_begin());773 setCalledOperand(Func);774 775 auto It = populateBundleOperandInfos(Bundles, Args.size());776 (void)It;777 assert(It + 1 == op_end() && "Should add up!");778 779 setName(NameStr);780}781 782void CallInst::init(FunctionType *FTy, Value *Func, const Twine &NameStr) {783 this->FTy = FTy;784 assert(getNumOperands() == 1 && "NumOperands not set up?");785 setCalledOperand(Func);786 787 assert(FTy->getNumParams() == 0 && "Calling a function with bad signature");788 789 setName(NameStr);790}791 792CallInst::CallInst(FunctionType *Ty, Value *Func, const Twine &Name,793 AllocInfo AllocInfo, InsertPosition InsertBefore)794 : CallBase(Ty->getReturnType(), Instruction::Call, AllocInfo,795 InsertBefore) {796 init(Ty, Func, Name);797}798 799CallInst::CallInst(const CallInst &CI, AllocInfo AllocInfo)800 : CallBase(CI.Attrs, CI.FTy, CI.getType(), Instruction::Call, AllocInfo) {801 assert(getNumOperands() == CI.getNumOperands() &&802 "Wrong number of operands allocated");803 setTailCallKind(CI.getTailCallKind());804 setCallingConv(CI.getCallingConv());805 806 std::copy(CI.op_begin(), CI.op_end(), op_begin());807 std::copy(CI.bundle_op_info_begin(), CI.bundle_op_info_end(),808 bundle_op_info_begin());809 SubclassOptionalData = CI.SubclassOptionalData;810}811 812CallInst *CallInst::Create(CallInst *CI, ArrayRef<OperandBundleDef> OpB,813 InsertPosition InsertPt) {814 std::vector<Value *> Args(CI->arg_begin(), CI->arg_end());815 816 auto *NewCI = CallInst::Create(CI->getFunctionType(), CI->getCalledOperand(),817 Args, OpB, CI->getName(), InsertPt);818 NewCI->setTailCallKind(CI->getTailCallKind());819 NewCI->setCallingConv(CI->getCallingConv());820 NewCI->SubclassOptionalData = CI->SubclassOptionalData;821 NewCI->setAttributes(CI->getAttributes());822 NewCI->setDebugLoc(CI->getDebugLoc());823 return NewCI;824}825 826// Update profile weight for call instruction by scaling it using the ratio827// of S/T. The meaning of "branch_weights" meta data for call instruction is828// transfered to represent call count.829void CallInst::updateProfWeight(uint64_t S, uint64_t T) {830 if (T == 0) {831 LLVM_DEBUG(dbgs() << "Attempting to update profile weights will result in "832 "div by 0. Ignoring. Likely the function "833 << getParent()->getParent()->getName()834 << " has 0 entry count, and contains call instructions "835 "with non-zero prof info.");836 return;837 }838 scaleProfData(*this, S, T);839}840 841//===----------------------------------------------------------------------===//842// InvokeInst Implementation843//===----------------------------------------------------------------------===//844 845void InvokeInst::init(FunctionType *FTy, Value *Fn, BasicBlock *IfNormal,846 BasicBlock *IfException, ArrayRef<Value *> Args,847 ArrayRef<OperandBundleDef> Bundles,848 const Twine &NameStr) {849 this->FTy = FTy;850 851 assert(getNumOperands() ==852 ComputeNumOperands(Args.size(), CountBundleInputs(Bundles)) &&853 "NumOperands not set up?");854 855#ifndef NDEBUG856 assert(((Args.size() == FTy->getNumParams()) ||857 (FTy->isVarArg() && Args.size() > FTy->getNumParams())) &&858 "Invoking a function with bad signature");859 860 for (unsigned i = 0, e = Args.size(); i != e; i++)861 assert((i >= FTy->getNumParams() ||862 FTy->getParamType(i) == Args[i]->getType()) &&863 "Invoking a function with a bad signature!");864#endif865 866 // Set operands in order of their index to match use-list-order867 // prediction.868 llvm::copy(Args, op_begin());869 setNormalDest(IfNormal);870 setUnwindDest(IfException);871 setCalledOperand(Fn);872 873 auto It = populateBundleOperandInfos(Bundles, Args.size());874 (void)It;875 assert(It + 3 == op_end() && "Should add up!");876 877 setName(NameStr);878}879 880InvokeInst::InvokeInst(const InvokeInst &II, AllocInfo AllocInfo)881 : CallBase(II.Attrs, II.FTy, II.getType(), Instruction::Invoke, AllocInfo) {882 assert(getNumOperands() == II.getNumOperands() &&883 "Wrong number of operands allocated");884 setCallingConv(II.getCallingConv());885 std::copy(II.op_begin(), II.op_end(), op_begin());886 std::copy(II.bundle_op_info_begin(), II.bundle_op_info_end(),887 bundle_op_info_begin());888 SubclassOptionalData = II.SubclassOptionalData;889}890 891InvokeInst *InvokeInst::Create(InvokeInst *II, ArrayRef<OperandBundleDef> OpB,892 InsertPosition InsertPt) {893 std::vector<Value *> Args(II->arg_begin(), II->arg_end());894 895 auto *NewII = InvokeInst::Create(896 II->getFunctionType(), II->getCalledOperand(), II->getNormalDest(),897 II->getUnwindDest(), Args, OpB, II->getName(), InsertPt);898 NewII->setCallingConv(II->getCallingConv());899 NewII->SubclassOptionalData = II->SubclassOptionalData;900 NewII->setAttributes(II->getAttributes());901 NewII->setDebugLoc(II->getDebugLoc());902 return NewII;903}904 905LandingPadInst *InvokeInst::getLandingPadInst() const {906 return cast<LandingPadInst>(getUnwindDest()->getFirstNonPHIIt());907}908 909void InvokeInst::updateProfWeight(uint64_t S, uint64_t T) {910 if (T == 0) {911 LLVM_DEBUG(dbgs() << "Attempting to update profile weights will result in "912 "div by 0. Ignoring. Likely the function "913 << getParent()->getParent()->getName()914 << " has 0 entry count, and contains call instructions "915 "with non-zero prof info.");916 return;917 }918 scaleProfData(*this, S, T);919}920 921//===----------------------------------------------------------------------===//922// CallBrInst Implementation923//===----------------------------------------------------------------------===//924 925void CallBrInst::init(FunctionType *FTy, Value *Fn, BasicBlock *Fallthrough,926 ArrayRef<BasicBlock *> IndirectDests,927 ArrayRef<Value *> Args,928 ArrayRef<OperandBundleDef> Bundles,929 const Twine &NameStr) {930 this->FTy = FTy;931 932 assert(getNumOperands() == ComputeNumOperands(Args.size(),933 IndirectDests.size(),934 CountBundleInputs(Bundles)) &&935 "NumOperands not set up?");936 937#ifndef NDEBUG938 assert(((Args.size() == FTy->getNumParams()) ||939 (FTy->isVarArg() && Args.size() > FTy->getNumParams())) &&940 "Calling a function with bad signature");941 942 for (unsigned i = 0, e = Args.size(); i != e; i++)943 assert((i >= FTy->getNumParams() ||944 FTy->getParamType(i) == Args[i]->getType()) &&945 "Calling a function with a bad signature!");946#endif947 948 // Set operands in order of their index to match use-list-order949 // prediction.950 llvm::copy(Args, op_begin());951 NumIndirectDests = IndirectDests.size();952 setDefaultDest(Fallthrough);953 for (unsigned i = 0; i != NumIndirectDests; ++i)954 setIndirectDest(i, IndirectDests[i]);955 setCalledOperand(Fn);956 957 auto It = populateBundleOperandInfos(Bundles, Args.size());958 (void)It;959 assert(It + 2 + IndirectDests.size() == op_end() && "Should add up!");960 961 setName(NameStr);962}963 964CallBrInst::CallBrInst(const CallBrInst &CBI, AllocInfo AllocInfo)965 : CallBase(CBI.Attrs, CBI.FTy, CBI.getType(), Instruction::CallBr,966 AllocInfo) {967 assert(getNumOperands() == CBI.getNumOperands() &&968 "Wrong number of operands allocated");969 setCallingConv(CBI.getCallingConv());970 std::copy(CBI.op_begin(), CBI.op_end(), op_begin());971 std::copy(CBI.bundle_op_info_begin(), CBI.bundle_op_info_end(),972 bundle_op_info_begin());973 SubclassOptionalData = CBI.SubclassOptionalData;974 NumIndirectDests = CBI.NumIndirectDests;975}976 977CallBrInst *CallBrInst::Create(CallBrInst *CBI, ArrayRef<OperandBundleDef> OpB,978 InsertPosition InsertPt) {979 std::vector<Value *> Args(CBI->arg_begin(), CBI->arg_end());980 981 auto *NewCBI = CallBrInst::Create(982 CBI->getFunctionType(), CBI->getCalledOperand(), CBI->getDefaultDest(),983 CBI->getIndirectDests(), Args, OpB, CBI->getName(), InsertPt);984 NewCBI->setCallingConv(CBI->getCallingConv());985 NewCBI->SubclassOptionalData = CBI->SubclassOptionalData;986 NewCBI->setAttributes(CBI->getAttributes());987 NewCBI->setDebugLoc(CBI->getDebugLoc());988 NewCBI->NumIndirectDests = CBI->NumIndirectDests;989 return NewCBI;990}991 992//===----------------------------------------------------------------------===//993// ReturnInst Implementation994//===----------------------------------------------------------------------===//995 996ReturnInst::ReturnInst(const ReturnInst &RI, AllocInfo AllocInfo)997 : Instruction(Type::getVoidTy(RI.getContext()), Instruction::Ret,998 AllocInfo) {999 assert(getNumOperands() == RI.getNumOperands() &&1000 "Wrong number of operands allocated");1001 if (RI.getNumOperands())1002 Op<0>() = RI.Op<0>();1003 SubclassOptionalData = RI.SubclassOptionalData;1004}1005 1006ReturnInst::ReturnInst(LLVMContext &C, Value *retVal, AllocInfo AllocInfo,1007 InsertPosition InsertBefore)1008 : Instruction(Type::getVoidTy(C), Instruction::Ret, AllocInfo,1009 InsertBefore) {1010 if (retVal)1011 Op<0>() = retVal;1012}1013 1014//===----------------------------------------------------------------------===//1015// ResumeInst Implementation1016//===----------------------------------------------------------------------===//1017 1018ResumeInst::ResumeInst(const ResumeInst &RI)1019 : Instruction(Type::getVoidTy(RI.getContext()), Instruction::Resume,1020 AllocMarker) {1021 Op<0>() = RI.Op<0>();1022}1023 1024ResumeInst::ResumeInst(Value *Exn, InsertPosition InsertBefore)1025 : Instruction(Type::getVoidTy(Exn->getContext()), Instruction::Resume,1026 AllocMarker, InsertBefore) {1027 Op<0>() = Exn;1028}1029 1030//===----------------------------------------------------------------------===//1031// CleanupReturnInst Implementation1032//===----------------------------------------------------------------------===//1033 1034CleanupReturnInst::CleanupReturnInst(const CleanupReturnInst &CRI,1035 AllocInfo AllocInfo)1036 : Instruction(CRI.getType(), Instruction::CleanupRet, AllocInfo) {1037 assert(getNumOperands() == CRI.getNumOperands() &&1038 "Wrong number of operands allocated");1039 setSubclassData<Instruction::OpaqueField>(1040 CRI.getSubclassData<Instruction::OpaqueField>());1041 Op<0>() = CRI.Op<0>();1042 if (CRI.hasUnwindDest())1043 Op<1>() = CRI.Op<1>();1044}1045 1046void CleanupReturnInst::init(Value *CleanupPad, BasicBlock *UnwindBB) {1047 if (UnwindBB)1048 setSubclassData<UnwindDestField>(true);1049 1050 Op<0>() = CleanupPad;1051 if (UnwindBB)1052 Op<1>() = UnwindBB;1053}1054 1055CleanupReturnInst::CleanupReturnInst(Value *CleanupPad, BasicBlock *UnwindBB,1056 AllocInfo AllocInfo,1057 InsertPosition InsertBefore)1058 : Instruction(Type::getVoidTy(CleanupPad->getContext()),1059 Instruction::CleanupRet, AllocInfo, InsertBefore) {1060 init(CleanupPad, UnwindBB);1061}1062 1063//===----------------------------------------------------------------------===//1064// CatchReturnInst Implementation1065//===----------------------------------------------------------------------===//1066void CatchReturnInst::init(Value *CatchPad, BasicBlock *BB) {1067 Op<0>() = CatchPad;1068 Op<1>() = BB;1069}1070 1071CatchReturnInst::CatchReturnInst(const CatchReturnInst &CRI)1072 : Instruction(Type::getVoidTy(CRI.getContext()), Instruction::CatchRet,1073 AllocMarker) {1074 Op<0>() = CRI.Op<0>();1075 Op<1>() = CRI.Op<1>();1076}1077 1078CatchReturnInst::CatchReturnInst(Value *CatchPad, BasicBlock *BB,1079 InsertPosition InsertBefore)1080 : Instruction(Type::getVoidTy(BB->getContext()), Instruction::CatchRet,1081 AllocMarker, InsertBefore) {1082 init(CatchPad, BB);1083}1084 1085//===----------------------------------------------------------------------===//1086// CatchSwitchInst Implementation1087//===----------------------------------------------------------------------===//1088 1089CatchSwitchInst::CatchSwitchInst(Value *ParentPad, BasicBlock *UnwindDest,1090 unsigned NumReservedValues,1091 const Twine &NameStr,1092 InsertPosition InsertBefore)1093 : Instruction(ParentPad->getType(), Instruction::CatchSwitch, AllocMarker,1094 InsertBefore) {1095 if (UnwindDest)1096 ++NumReservedValues;1097 init(ParentPad, UnwindDest, NumReservedValues + 1);1098 setName(NameStr);1099}1100 1101CatchSwitchInst::CatchSwitchInst(const CatchSwitchInst &CSI)1102 : Instruction(CSI.getType(), Instruction::CatchSwitch, AllocMarker) {1103 NumUserOperands = CSI.NumUserOperands;1104 init(CSI.getParentPad(), CSI.getUnwindDest(), CSI.getNumOperands());1105 setNumHungOffUseOperands(ReservedSpace);1106 Use *OL = getOperandList();1107 const Use *InOL = CSI.getOperandList();1108 for (unsigned I = 1, E = ReservedSpace; I != E; ++I)1109 OL[I] = InOL[I];1110}1111 1112void CatchSwitchInst::init(Value *ParentPad, BasicBlock *UnwindDest,1113 unsigned NumReservedValues) {1114 assert(ParentPad && NumReservedValues);1115 1116 ReservedSpace = NumReservedValues;1117 setNumHungOffUseOperands(UnwindDest ? 2 : 1);1118 allocHungoffUses(ReservedSpace);1119 1120 Op<0>() = ParentPad;1121 if (UnwindDest) {1122 setSubclassData<UnwindDestField>(true);1123 setUnwindDest(UnwindDest);1124 }1125}1126 1127/// growOperands - grow operands - This grows the operand list in response to a1128/// push_back style of operation. This grows the number of ops by 2 times.1129void CatchSwitchInst::growOperands(unsigned Size) {1130 unsigned NumOperands = getNumOperands();1131 assert(NumOperands >= 1);1132 if (ReservedSpace >= NumOperands + Size)1133 return;1134 ReservedSpace = (NumOperands + Size / 2) * 2;1135 growHungoffUses(ReservedSpace);1136}1137 1138void CatchSwitchInst::addHandler(BasicBlock *Handler) {1139 unsigned OpNo = getNumOperands();1140 growOperands(1);1141 assert(OpNo < ReservedSpace && "Growing didn't work!");1142 setNumHungOffUseOperands(getNumOperands() + 1);1143 getOperandList()[OpNo] = Handler;1144}1145 1146void CatchSwitchInst::removeHandler(handler_iterator HI) {1147 // Move all subsequent handlers up one.1148 Use *EndDst = op_end() - 1;1149 for (Use *CurDst = HI.getCurrent(); CurDst != EndDst; ++CurDst)1150 *CurDst = *(CurDst + 1);1151 // Null out the last handler use.1152 *EndDst = nullptr;1153 1154 setNumHungOffUseOperands(getNumOperands() - 1);1155}1156 1157//===----------------------------------------------------------------------===//1158// FuncletPadInst Implementation1159//===----------------------------------------------------------------------===//1160void FuncletPadInst::init(Value *ParentPad, ArrayRef<Value *> Args,1161 const Twine &NameStr) {1162 assert(getNumOperands() == 1 + Args.size() && "NumOperands not set up?");1163 llvm::copy(Args, op_begin());1164 setParentPad(ParentPad);1165 setName(NameStr);1166}1167 1168FuncletPadInst::FuncletPadInst(const FuncletPadInst &FPI, AllocInfo AllocInfo)1169 : Instruction(FPI.getType(), FPI.getOpcode(), AllocInfo) {1170 assert(getNumOperands() == FPI.getNumOperands() &&1171 "Wrong number of operands allocated");1172 std::copy(FPI.op_begin(), FPI.op_end(), op_begin());1173 setParentPad(FPI.getParentPad());1174}1175 1176FuncletPadInst::FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad,1177 ArrayRef<Value *> Args, AllocInfo AllocInfo,1178 const Twine &NameStr,1179 InsertPosition InsertBefore)1180 : Instruction(ParentPad->getType(), Op, AllocInfo, InsertBefore) {1181 init(ParentPad, Args, NameStr);1182}1183 1184//===----------------------------------------------------------------------===//1185// UnreachableInst Implementation1186//===----------------------------------------------------------------------===//1187 1188UnreachableInst::UnreachableInst(LLVMContext &Context,1189 InsertPosition InsertBefore)1190 : Instruction(Type::getVoidTy(Context), Instruction::Unreachable,1191 AllocMarker, InsertBefore) {}1192 1193//===----------------------------------------------------------------------===//1194// BranchInst Implementation1195//===----------------------------------------------------------------------===//1196 1197void BranchInst::AssertOK() {1198 if (isConditional())1199 assert(getCondition()->getType()->isIntegerTy(1) &&1200 "May only branch on boolean predicates!");1201}1202 1203BranchInst::BranchInst(BasicBlock *IfTrue, AllocInfo AllocInfo,1204 InsertPosition InsertBefore)1205 : Instruction(Type::getVoidTy(IfTrue->getContext()), Instruction::Br,1206 AllocInfo, InsertBefore) {1207 assert(IfTrue && "Branch destination may not be null!");1208 Op<-1>() = IfTrue;1209}1210 1211BranchInst::BranchInst(BasicBlock *IfTrue, BasicBlock *IfFalse, Value *Cond,1212 AllocInfo AllocInfo, InsertPosition InsertBefore)1213 : Instruction(Type::getVoidTy(IfTrue->getContext()), Instruction::Br,1214 AllocInfo, InsertBefore) {1215 // Assign in order of operand index to make use-list order predictable.1216 Op<-3>() = Cond;1217 Op<-2>() = IfFalse;1218 Op<-1>() = IfTrue;1219#ifndef NDEBUG1220 AssertOK();1221#endif1222}1223 1224BranchInst::BranchInst(const BranchInst &BI, AllocInfo AllocInfo)1225 : Instruction(Type::getVoidTy(BI.getContext()), Instruction::Br,1226 AllocInfo) {1227 assert(getNumOperands() == BI.getNumOperands() &&1228 "Wrong number of operands allocated");1229 // Assign in order of operand index to make use-list order predictable.1230 if (BI.getNumOperands() != 1) {1231 assert(BI.getNumOperands() == 3 && "BR can have 1 or 3 operands!");1232 Op<-3>() = BI.Op<-3>();1233 Op<-2>() = BI.Op<-2>();1234 }1235 Op<-1>() = BI.Op<-1>();1236 SubclassOptionalData = BI.SubclassOptionalData;1237}1238 1239void BranchInst::swapSuccessors() {1240 assert(isConditional() &&1241 "Cannot swap successors of an unconditional branch");1242 Op<-1>().swap(Op<-2>());1243 1244 // Update profile metadata if present and it matches our structural1245 // expectations.1246 swapProfMetadata();1247}1248 1249//===----------------------------------------------------------------------===//1250// AllocaInst Implementation1251//===----------------------------------------------------------------------===//1252 1253static Value *getAISize(LLVMContext &Context, Value *Amt) {1254 if (!Amt)1255 Amt = ConstantInt::get(Type::getInt32Ty(Context), 1);1256 else {1257 assert(!isa<BasicBlock>(Amt) &&1258 "Passed basic block into allocation size parameter! Use other ctor");1259 assert(Amt->getType()->isIntegerTy() &&1260 "Allocation array size is not an integer!");1261 }1262 return Amt;1263}1264 1265static Align computeAllocaDefaultAlign(Type *Ty, InsertPosition Pos) {1266 assert(Pos.isValid() &&1267 "Insertion position cannot be null when alignment not provided!");1268 BasicBlock *BB = Pos.getBasicBlock();1269 assert(BB->getParent() &&1270 "BB must be in a Function when alignment not provided!");1271 const DataLayout &DL = BB->getDataLayout();1272 return DL.getPrefTypeAlign(Ty);1273}1274 1275AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, const Twine &Name,1276 InsertPosition InsertBefore)1277 : AllocaInst(Ty, AddrSpace, /*ArraySize=*/nullptr, Name, InsertBefore) {}1278 1279AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,1280 const Twine &Name, InsertPosition InsertBefore)1281 : AllocaInst(Ty, AddrSpace, ArraySize,1282 computeAllocaDefaultAlign(Ty, InsertBefore), Name,1283 InsertBefore) {}1284 1285AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,1286 Align Align, const Twine &Name,1287 InsertPosition InsertBefore)1288 : UnaryInstruction(PointerType::get(Ty->getContext(), AddrSpace), Alloca,1289 getAISize(Ty->getContext(), ArraySize), InsertBefore),1290 AllocatedType(Ty) {1291 setAlignment(Align);1292 assert(!Ty->isVoidTy() && "Cannot allocate void!");1293 setName(Name);1294}1295 1296bool AllocaInst::isArrayAllocation() const {1297 if (ConstantInt *CI = dyn_cast<ConstantInt>(getOperand(0)))1298 return !CI->isOne();1299 return true;1300}1301 1302/// isStaticAlloca - Return true if this alloca is in the entry block of the1303/// function and is a constant size. If so, the code generator will fold it1304/// into the prolog/epilog code, so it is basically free.1305bool AllocaInst::isStaticAlloca() const {1306 // Must be constant size.1307 if (!isa<ConstantInt>(getArraySize())) return false;1308 1309 // Must be in the entry block.1310 const BasicBlock *Parent = getParent();1311 return Parent->isEntryBlock() && !isUsedWithInAlloca();1312}1313 1314//===----------------------------------------------------------------------===//1315// LoadInst Implementation1316//===----------------------------------------------------------------------===//1317 1318void LoadInst::AssertOK() {1319 assert(getOperand(0)->getType()->isPointerTy() &&1320 "Ptr must have pointer type.");1321}1322 1323static Align computeLoadStoreDefaultAlign(Type *Ty, InsertPosition Pos) {1324 assert(Pos.isValid() &&1325 "Insertion position cannot be null when alignment not provided!");1326 BasicBlock *BB = Pos.getBasicBlock();1327 assert(BB->getParent() &&1328 "BB must be in a Function when alignment not provided!");1329 const DataLayout &DL = BB->getDataLayout();1330 return DL.getABITypeAlign(Ty);1331}1332 1333LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name,1334 InsertPosition InsertBef)1335 : LoadInst(Ty, Ptr, Name, /*isVolatile=*/false, InsertBef) {}1336 1337LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,1338 InsertPosition InsertBef)1339 : LoadInst(Ty, Ptr, Name, isVolatile,1340 computeLoadStoreDefaultAlign(Ty, InsertBef), InsertBef) {}1341 1342LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,1343 Align Align, InsertPosition InsertBef)1344 : LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic,1345 SyncScope::System, InsertBef) {}1346 1347LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,1348 Align Align, AtomicOrdering Order, SyncScope::ID SSID,1349 InsertPosition InsertBef)1350 : UnaryInstruction(Ty, Load, Ptr, InsertBef) {1351 setVolatile(isVolatile);1352 setAlignment(Align);1353 setAtomic(Order, SSID);1354 AssertOK();1355 setName(Name);1356}1357 1358//===----------------------------------------------------------------------===//1359// StoreInst Implementation1360//===----------------------------------------------------------------------===//1361 1362void StoreInst::AssertOK() {1363 assert(getOperand(0) && getOperand(1) && "Both operands must be non-null!");1364 assert(getOperand(1)->getType()->isPointerTy() &&1365 "Ptr must have pointer type!");1366}1367 1368StoreInst::StoreInst(Value *val, Value *addr, InsertPosition InsertBefore)1369 : StoreInst(val, addr, /*isVolatile=*/false, InsertBefore) {}1370 1371StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile,1372 InsertPosition InsertBefore)1373 : StoreInst(val, addr, isVolatile,1374 computeLoadStoreDefaultAlign(val->getType(), InsertBefore),1375 InsertBefore) {}1376 1377StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, Align Align,1378 InsertPosition InsertBefore)1379 : StoreInst(val, addr, isVolatile, Align, AtomicOrdering::NotAtomic,1380 SyncScope::System, InsertBefore) {}1381 1382StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, Align Align,1383 AtomicOrdering Order, SyncScope::ID SSID,1384 InsertPosition InsertBefore)1385 : Instruction(Type::getVoidTy(val->getContext()), Store, AllocMarker,1386 InsertBefore) {1387 Op<0>() = val;1388 Op<1>() = addr;1389 setVolatile(isVolatile);1390 setAlignment(Align);1391 setAtomic(Order, SSID);1392 AssertOK();1393}1394 1395//===----------------------------------------------------------------------===//1396// AtomicCmpXchgInst Implementation1397//===----------------------------------------------------------------------===//1398 1399void AtomicCmpXchgInst::Init(Value *Ptr, Value *Cmp, Value *NewVal,1400 Align Alignment, AtomicOrdering SuccessOrdering,1401 AtomicOrdering FailureOrdering,1402 SyncScope::ID SSID) {1403 Op<0>() = Ptr;1404 Op<1>() = Cmp;1405 Op<2>() = NewVal;1406 setSuccessOrdering(SuccessOrdering);1407 setFailureOrdering(FailureOrdering);1408 setSyncScopeID(SSID);1409 setAlignment(Alignment);1410 1411 assert(getOperand(0) && getOperand(1) && getOperand(2) &&1412 "All operands must be non-null!");1413 assert(getOperand(0)->getType()->isPointerTy() &&1414 "Ptr must have pointer type!");1415 assert(getOperand(1)->getType() == getOperand(2)->getType() &&1416 "Cmp type and NewVal type must be same!");1417}1418 1419AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,1420 Align Alignment,1421 AtomicOrdering SuccessOrdering,1422 AtomicOrdering FailureOrdering,1423 SyncScope::ID SSID,1424 InsertPosition InsertBefore)1425 : Instruction(1426 StructType::get(Cmp->getType(), Type::getInt1Ty(Cmp->getContext())),1427 AtomicCmpXchg, AllocMarker, InsertBefore) {1428 Init(Ptr, Cmp, NewVal, Alignment, SuccessOrdering, FailureOrdering, SSID);1429}1430 1431//===----------------------------------------------------------------------===//1432// AtomicRMWInst Implementation1433//===----------------------------------------------------------------------===//1434 1435void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val,1436 Align Alignment, AtomicOrdering Ordering,1437 SyncScope::ID SSID) {1438 assert(Ordering != AtomicOrdering::NotAtomic &&1439 "atomicrmw instructions can only be atomic.");1440 assert(Ordering != AtomicOrdering::Unordered &&1441 "atomicrmw instructions cannot be unordered.");1442 Op<0>() = Ptr;1443 Op<1>() = Val;1444 setOperation(Operation);1445 setOrdering(Ordering);1446 setSyncScopeID(SSID);1447 setAlignment(Alignment);1448 1449 assert(getOperand(0) && getOperand(1) && "All operands must be non-null!");1450 assert(getOperand(0)->getType()->isPointerTy() &&1451 "Ptr must have pointer type!");1452 assert(Ordering != AtomicOrdering::NotAtomic &&1453 "AtomicRMW instructions must be atomic!");1454}1455 1456AtomicRMWInst::AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,1457 Align Alignment, AtomicOrdering Ordering,1458 SyncScope::ID SSID, InsertPosition InsertBefore)1459 : Instruction(Val->getType(), AtomicRMW, AllocMarker, InsertBefore) {1460 Init(Operation, Ptr, Val, Alignment, Ordering, SSID);1461}1462 1463StringRef AtomicRMWInst::getOperationName(BinOp Op) {1464 switch (Op) {1465 case AtomicRMWInst::Xchg:1466 return "xchg";1467 case AtomicRMWInst::Add:1468 return "add";1469 case AtomicRMWInst::Sub:1470 return "sub";1471 case AtomicRMWInst::And:1472 return "and";1473 case AtomicRMWInst::Nand:1474 return "nand";1475 case AtomicRMWInst::Or:1476 return "or";1477 case AtomicRMWInst::Xor:1478 return "xor";1479 case AtomicRMWInst::Max:1480 return "max";1481 case AtomicRMWInst::Min:1482 return "min";1483 case AtomicRMWInst::UMax:1484 return "umax";1485 case AtomicRMWInst::UMin:1486 return "umin";1487 case AtomicRMWInst::FAdd:1488 return "fadd";1489 case AtomicRMWInst::FSub:1490 return "fsub";1491 case AtomicRMWInst::FMax:1492 return "fmax";1493 case AtomicRMWInst::FMin:1494 return "fmin";1495 case AtomicRMWInst::FMaximum:1496 return "fmaximum";1497 case AtomicRMWInst::FMinimum:1498 return "fminimum";1499 case AtomicRMWInst::UIncWrap:1500 return "uinc_wrap";1501 case AtomicRMWInst::UDecWrap:1502 return "udec_wrap";1503 case AtomicRMWInst::USubCond:1504 return "usub_cond";1505 case AtomicRMWInst::USubSat:1506 return "usub_sat";1507 case AtomicRMWInst::BAD_BINOP:1508 return "<invalid operation>";1509 }1510 1511 llvm_unreachable("invalid atomicrmw operation");1512}1513 1514//===----------------------------------------------------------------------===//1515// FenceInst Implementation1516//===----------------------------------------------------------------------===//1517 1518FenceInst::FenceInst(LLVMContext &C, AtomicOrdering Ordering,1519 SyncScope::ID SSID, InsertPosition InsertBefore)1520 : Instruction(Type::getVoidTy(C), Fence, AllocMarker, InsertBefore) {1521 setOrdering(Ordering);1522 setSyncScopeID(SSID);1523}1524 1525//===----------------------------------------------------------------------===//1526// GetElementPtrInst Implementation1527//===----------------------------------------------------------------------===//1528 1529void GetElementPtrInst::init(Value *Ptr, ArrayRef<Value *> IdxList,1530 const Twine &Name) {1531 assert(getNumOperands() == 1 + IdxList.size() &&1532 "NumOperands not initialized?");1533 Op<0>() = Ptr;1534 llvm::copy(IdxList, op_begin() + 1);1535 setName(Name);1536}1537 1538GetElementPtrInst::GetElementPtrInst(const GetElementPtrInst &GEPI,1539 AllocInfo AllocInfo)1540 : Instruction(GEPI.getType(), GetElementPtr, AllocInfo),1541 SourceElementType(GEPI.SourceElementType),1542 ResultElementType(GEPI.ResultElementType) {1543 assert(getNumOperands() == GEPI.getNumOperands() &&1544 "Wrong number of operands allocated");1545 std::copy(GEPI.op_begin(), GEPI.op_end(), op_begin());1546 SubclassOptionalData = GEPI.SubclassOptionalData;1547}1548 1549Type *GetElementPtrInst::getTypeAtIndex(Type *Ty, Value *Idx) {1550 if (auto *Struct = dyn_cast<StructType>(Ty)) {1551 if (!Struct->indexValid(Idx))1552 return nullptr;1553 return Struct->getTypeAtIndex(Idx);1554 }1555 if (!Idx->getType()->isIntOrIntVectorTy())1556 return nullptr;1557 if (auto *Array = dyn_cast<ArrayType>(Ty))1558 return Array->getElementType();1559 if (auto *Vector = dyn_cast<VectorType>(Ty))1560 return Vector->getElementType();1561 return nullptr;1562}1563 1564Type *GetElementPtrInst::getTypeAtIndex(Type *Ty, uint64_t Idx) {1565 if (auto *Struct = dyn_cast<StructType>(Ty)) {1566 if (Idx >= Struct->getNumElements())1567 return nullptr;1568 return Struct->getElementType(Idx);1569 }1570 if (auto *Array = dyn_cast<ArrayType>(Ty))1571 return Array->getElementType();1572 if (auto *Vector = dyn_cast<VectorType>(Ty))1573 return Vector->getElementType();1574 return nullptr;1575}1576 1577template <typename IndexTy>1578static Type *getIndexedTypeInternal(Type *Ty, ArrayRef<IndexTy> IdxList) {1579 if (IdxList.empty())1580 return Ty;1581 for (IndexTy V : IdxList.slice(1)) {1582 Ty = GetElementPtrInst::getTypeAtIndex(Ty, V);1583 if (!Ty)1584 return Ty;1585 }1586 return Ty;1587}1588 1589Type *GetElementPtrInst::getIndexedType(Type *Ty, ArrayRef<Value *> IdxList) {1590 return getIndexedTypeInternal(Ty, IdxList);1591}1592 1593Type *GetElementPtrInst::getIndexedType(Type *Ty,1594 ArrayRef<Constant *> IdxList) {1595 return getIndexedTypeInternal(Ty, IdxList);1596}1597 1598Type *GetElementPtrInst::getIndexedType(Type *Ty, ArrayRef<uint64_t> IdxList) {1599 return getIndexedTypeInternal(Ty, IdxList);1600}1601 1602/// hasAllZeroIndices - Return true if all of the indices of this GEP are1603/// zeros. If so, the result pointer and the first operand have the same1604/// value, just potentially different types.1605bool GetElementPtrInst::hasAllZeroIndices() const {1606 for (unsigned i = 1, e = getNumOperands(); i != e; ++i) {1607 if (ConstantInt *CI = dyn_cast<ConstantInt>(getOperand(i))) {1608 if (!CI->isZero()) return false;1609 } else {1610 return false;1611 }1612 }1613 return true;1614}1615 1616/// hasAllConstantIndices - Return true if all of the indices of this GEP are1617/// constant integers. If so, the result pointer and the first operand have1618/// a constant offset between them.1619bool GetElementPtrInst::hasAllConstantIndices() const {1620 for (unsigned i = 1, e = getNumOperands(); i != e; ++i) {1621 if (!isa<ConstantInt>(getOperand(i)))1622 return false;1623 }1624 return true;1625}1626 1627void GetElementPtrInst::setNoWrapFlags(GEPNoWrapFlags NW) {1628 SubclassOptionalData = NW.getRaw();1629}1630 1631void GetElementPtrInst::setIsInBounds(bool B) {1632 GEPNoWrapFlags NW = cast<GEPOperator>(this)->getNoWrapFlags();1633 if (B)1634 NW |= GEPNoWrapFlags::inBounds();1635 else1636 NW = NW.withoutInBounds();1637 setNoWrapFlags(NW);1638}1639 1640GEPNoWrapFlags GetElementPtrInst::getNoWrapFlags() const {1641 return cast<GEPOperator>(this)->getNoWrapFlags();1642}1643 1644bool GetElementPtrInst::isInBounds() const {1645 return cast<GEPOperator>(this)->isInBounds();1646}1647 1648bool GetElementPtrInst::hasNoUnsignedSignedWrap() const {1649 return cast<GEPOperator>(this)->hasNoUnsignedSignedWrap();1650}1651 1652bool GetElementPtrInst::hasNoUnsignedWrap() const {1653 return cast<GEPOperator>(this)->hasNoUnsignedWrap();1654}1655 1656bool GetElementPtrInst::accumulateConstantOffset(const DataLayout &DL,1657 APInt &Offset) const {1658 // Delegate to the generic GEPOperator implementation.1659 return cast<GEPOperator>(this)->accumulateConstantOffset(DL, Offset);1660}1661 1662bool GetElementPtrInst::collectOffset(1663 const DataLayout &DL, unsigned BitWidth,1664 SmallMapVector<Value *, APInt, 4> &VariableOffsets,1665 APInt &ConstantOffset) const {1666 // Delegate to the generic GEPOperator implementation.1667 return cast<GEPOperator>(this)->collectOffset(DL, BitWidth, VariableOffsets,1668 ConstantOffset);1669}1670 1671//===----------------------------------------------------------------------===//1672// ExtractElementInst Implementation1673//===----------------------------------------------------------------------===//1674 1675ExtractElementInst::ExtractElementInst(Value *Val, Value *Index,1676 const Twine &Name,1677 InsertPosition InsertBef)1678 : Instruction(cast<VectorType>(Val->getType())->getElementType(),1679 ExtractElement, AllocMarker, InsertBef) {1680 assert(isValidOperands(Val, Index) &&1681 "Invalid extractelement instruction operands!");1682 Op<0>() = Val;1683 Op<1>() = Index;1684 setName(Name);1685}1686 1687bool ExtractElementInst::isValidOperands(const Value *Val, const Value *Index) {1688 if (!Val->getType()->isVectorTy() || !Index->getType()->isIntegerTy())1689 return false;1690 return true;1691}1692 1693//===----------------------------------------------------------------------===//1694// InsertElementInst Implementation1695//===----------------------------------------------------------------------===//1696 1697InsertElementInst::InsertElementInst(Value *Vec, Value *Elt, Value *Index,1698 const Twine &Name,1699 InsertPosition InsertBef)1700 : Instruction(Vec->getType(), InsertElement, AllocMarker, InsertBef) {1701 assert(isValidOperands(Vec, Elt, Index) &&1702 "Invalid insertelement instruction operands!");1703 Op<0>() = Vec;1704 Op<1>() = Elt;1705 Op<2>() = Index;1706 setName(Name);1707}1708 1709bool InsertElementInst::isValidOperands(const Value *Vec, const Value *Elt,1710 const Value *Index) {1711 if (!Vec->getType()->isVectorTy())1712 return false; // First operand of insertelement must be vector type.1713 1714 if (Elt->getType() != cast<VectorType>(Vec->getType())->getElementType())1715 return false;// Second operand of insertelement must be vector element type.1716 1717 if (!Index->getType()->isIntegerTy())1718 return false; // Third operand of insertelement must be i32.1719 return true;1720}1721 1722//===----------------------------------------------------------------------===//1723// ShuffleVectorInst Implementation1724//===----------------------------------------------------------------------===//1725 1726static Value *createPlaceholderForShuffleVector(Value *V) {1727 assert(V && "Cannot create placeholder of nullptr V");1728 return PoisonValue::get(V->getType());1729}1730 1731ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *Mask, const Twine &Name,1732 InsertPosition InsertBefore)1733 : ShuffleVectorInst(V1, createPlaceholderForShuffleVector(V1), Mask, Name,1734 InsertBefore) {}1735 1736ShuffleVectorInst::ShuffleVectorInst(Value *V1, ArrayRef<int> Mask,1737 const Twine &Name,1738 InsertPosition InsertBefore)1739 : ShuffleVectorInst(V1, createPlaceholderForShuffleVector(V1), Mask, Name,1740 InsertBefore) {}1741 1742ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,1743 const Twine &Name,1744 InsertPosition InsertBefore)1745 : Instruction(1746 VectorType::get(cast<VectorType>(V1->getType())->getElementType(),1747 cast<VectorType>(Mask->getType())->getElementCount()),1748 ShuffleVector, AllocMarker, InsertBefore) {1749 assert(isValidOperands(V1, V2, Mask) &&1750 "Invalid shuffle vector instruction operands!");1751 1752 Op<0>() = V1;1753 Op<1>() = V2;1754 SmallVector<int, 16> MaskArr;1755 getShuffleMask(cast<Constant>(Mask), MaskArr);1756 setShuffleMask(MaskArr);1757 setName(Name);1758}1759 1760ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,1761 const Twine &Name,1762 InsertPosition InsertBefore)1763 : Instruction(1764 VectorType::get(cast<VectorType>(V1->getType())->getElementType(),1765 Mask.size(), isa<ScalableVectorType>(V1->getType())),1766 ShuffleVector, AllocMarker, InsertBefore) {1767 assert(isValidOperands(V1, V2, Mask) &&1768 "Invalid shuffle vector instruction operands!");1769 Op<0>() = V1;1770 Op<1>() = V2;1771 setShuffleMask(Mask);1772 setName(Name);1773}1774 1775void ShuffleVectorInst::commute() {1776 int NumOpElts = cast<FixedVectorType>(Op<0>()->getType())->getNumElements();1777 int NumMaskElts = ShuffleMask.size();1778 SmallVector<int, 16> NewMask(NumMaskElts);1779 for (int i = 0; i != NumMaskElts; ++i) {1780 int MaskElt = getMaskValue(i);1781 if (MaskElt == PoisonMaskElem) {1782 NewMask[i] = PoisonMaskElem;1783 continue;1784 }1785 assert(MaskElt >= 0 && MaskElt < 2 * NumOpElts && "Out-of-range mask");1786 MaskElt = (MaskElt < NumOpElts) ? MaskElt + NumOpElts : MaskElt - NumOpElts;1787 NewMask[i] = MaskElt;1788 }1789 setShuffleMask(NewMask);1790 Op<0>().swap(Op<1>());1791}1792 1793bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,1794 ArrayRef<int> Mask) {1795 // V1 and V2 must be vectors of the same type.1796 if (!isa<VectorType>(V1->getType()) || V1->getType() != V2->getType())1797 return false;1798 1799 // Make sure the mask elements make sense.1800 int V1Size =1801 cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();1802 for (int Elem : Mask)1803 if (Elem != PoisonMaskElem && Elem >= V1Size * 2)1804 return false;1805 1806 if (isa<ScalableVectorType>(V1->getType()))1807 if ((Mask[0] != 0 && Mask[0] != PoisonMaskElem) || !all_equal(Mask))1808 return false;1809 1810 return true;1811}1812 1813bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,1814 const Value *Mask) {1815 // V1 and V2 must be vectors of the same type.1816 if (!V1->getType()->isVectorTy() || V1->getType() != V2->getType())1817 return false;1818 1819 // Mask must be vector of i32, and must be the same kind of vector as the1820 // input vectors1821 auto *MaskTy = dyn_cast<VectorType>(Mask->getType());1822 if (!MaskTy || !MaskTy->getElementType()->isIntegerTy(32) ||1823 isa<ScalableVectorType>(MaskTy) != isa<ScalableVectorType>(V1->getType()))1824 return false;1825 1826 // Check to see if Mask is valid.1827 if (isa<UndefValue>(Mask) || isa<ConstantAggregateZero>(Mask))1828 return true;1829 1830 // NOTE: Through vector ConstantInt we have the potential to support more1831 // than just zero splat masks but that requires a LangRef change.1832 if (isa<ScalableVectorType>(MaskTy))1833 return false;1834 1835 unsigned V1Size = cast<FixedVectorType>(V1->getType())->getNumElements();1836 1837 if (const auto *CI = dyn_cast<ConstantInt>(Mask))1838 return !CI->uge(V1Size * 2);1839 1840 if (const auto *MV = dyn_cast<ConstantVector>(Mask)) {1841 for (Value *Op : MV->operands()) {1842 if (auto *CI = dyn_cast<ConstantInt>(Op)) {1843 if (CI->uge(V1Size*2))1844 return false;1845 } else if (!isa<UndefValue>(Op)) {1846 return false;1847 }1848 }1849 return true;1850 }1851 1852 if (const auto *CDS = dyn_cast<ConstantDataSequential>(Mask)) {1853 for (unsigned i = 0, e = cast<FixedVectorType>(MaskTy)->getNumElements();1854 i != e; ++i)1855 if (CDS->getElementAsInteger(i) >= V1Size*2)1856 return false;1857 return true;1858 }1859 1860 return false;1861}1862 1863void ShuffleVectorInst::getShuffleMask(const Constant *Mask,1864 SmallVectorImpl<int> &Result) {1865 ElementCount EC = cast<VectorType>(Mask->getType())->getElementCount();1866 1867 if (isa<ConstantAggregateZero>(Mask) || isa<UndefValue>(Mask)) {1868 int MaskVal = isa<UndefValue>(Mask) ? -1 : 0;1869 Result.append(EC.getKnownMinValue(), MaskVal);1870 return;1871 }1872 1873 assert(!EC.isScalable() &&1874 "Scalable vector shuffle mask must be undef or zeroinitializer");1875 1876 unsigned NumElts = EC.getFixedValue();1877 1878 Result.reserve(NumElts);1879 1880 if (auto *CDS = dyn_cast<ConstantDataSequential>(Mask)) {1881 for (unsigned i = 0; i != NumElts; ++i)1882 Result.push_back(CDS->getElementAsInteger(i));1883 return;1884 }1885 for (unsigned i = 0; i != NumElts; ++i) {1886 Constant *C = Mask->getAggregateElement(i);1887 Result.push_back(isa<UndefValue>(C) ? -1 :1888 cast<ConstantInt>(C)->getZExtValue());1889 }1890}1891 1892void ShuffleVectorInst::setShuffleMask(ArrayRef<int> Mask) {1893 ShuffleMask.assign(Mask.begin(), Mask.end());1894 ShuffleMaskForBitcode = convertShuffleMaskForBitcode(Mask, getType());1895}1896 1897Constant *ShuffleVectorInst::convertShuffleMaskForBitcode(ArrayRef<int> Mask,1898 Type *ResultTy) {1899 Type *Int32Ty = Type::getInt32Ty(ResultTy->getContext());1900 if (isa<ScalableVectorType>(ResultTy)) {1901 assert(all_equal(Mask) && "Unexpected shuffle");1902 Type *VecTy = VectorType::get(Int32Ty, Mask.size(), true);1903 if (Mask[0] == 0)1904 return Constant::getNullValue(VecTy);1905 return PoisonValue::get(VecTy);1906 }1907 SmallVector<Constant *, 16> MaskConst;1908 for (int Elem : Mask) {1909 if (Elem == PoisonMaskElem)1910 MaskConst.push_back(PoisonValue::get(Int32Ty));1911 else1912 MaskConst.push_back(ConstantInt::get(Int32Ty, Elem));1913 }1914 return ConstantVector::get(MaskConst);1915}1916 1917static bool isSingleSourceMaskImpl(ArrayRef<int> Mask, int NumOpElts) {1918 assert(!Mask.empty() && "Shuffle mask must contain elements");1919 bool UsesLHS = false;1920 bool UsesRHS = false;1921 for (int I : Mask) {1922 if (I == -1)1923 continue;1924 assert(I >= 0 && I < (NumOpElts * 2) &&1925 "Out-of-bounds shuffle mask element");1926 UsesLHS |= (I < NumOpElts);1927 UsesRHS |= (I >= NumOpElts);1928 if (UsesLHS && UsesRHS)1929 return false;1930 }1931 // Allow for degenerate case: completely undef mask means neither source is used.1932 return UsesLHS || UsesRHS;1933}1934 1935bool ShuffleVectorInst::isSingleSourceMask(ArrayRef<int> Mask, int NumSrcElts) {1936 // We don't have vector operand size information, so assume operands are the1937 // same size as the mask.1938 return isSingleSourceMaskImpl(Mask, NumSrcElts);1939}1940 1941static bool isIdentityMaskImpl(ArrayRef<int> Mask, int NumOpElts) {1942 if (!isSingleSourceMaskImpl(Mask, NumOpElts))1943 return false;1944 for (int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {1945 if (Mask[i] == -1)1946 continue;1947 if (Mask[i] != i && Mask[i] != (NumOpElts + i))1948 return false;1949 }1950 return true;1951}1952 1953bool ShuffleVectorInst::isIdentityMask(ArrayRef<int> Mask, int NumSrcElts) {1954 if (Mask.size() != static_cast<unsigned>(NumSrcElts))1955 return false;1956 // We don't have vector operand size information, so assume operands are the1957 // same size as the mask.1958 return isIdentityMaskImpl(Mask, NumSrcElts);1959}1960 1961bool ShuffleVectorInst::isReverseMask(ArrayRef<int> Mask, int NumSrcElts) {1962 if (Mask.size() != static_cast<unsigned>(NumSrcElts))1963 return false;1964 if (!isSingleSourceMask(Mask, NumSrcElts))1965 return false;1966 1967 // The number of elements in the mask must be at least 2.1968 if (NumSrcElts < 2)1969 return false;1970 1971 for (int I = 0, E = Mask.size(); I < E; ++I) {1972 if (Mask[I] == -1)1973 continue;1974 if (Mask[I] != (NumSrcElts - 1 - I) &&1975 Mask[I] != (NumSrcElts + NumSrcElts - 1 - I))1976 return false;1977 }1978 return true;1979}1980 1981bool ShuffleVectorInst::isZeroEltSplatMask(ArrayRef<int> Mask, int NumSrcElts) {1982 if (Mask.size() != static_cast<unsigned>(NumSrcElts))1983 return false;1984 if (!isSingleSourceMask(Mask, NumSrcElts))1985 return false;1986 for (int I = 0, E = Mask.size(); I < E; ++I) {1987 if (Mask[I] == -1)1988 continue;1989 if (Mask[I] != 0 && Mask[I] != NumSrcElts)1990 return false;1991 }1992 return true;1993}1994 1995bool ShuffleVectorInst::isSelectMask(ArrayRef<int> Mask, int NumSrcElts) {1996 if (Mask.size() != static_cast<unsigned>(NumSrcElts))1997 return false;1998 // Select is differentiated from identity. It requires using both sources.1999 if (isSingleSourceMask(Mask, NumSrcElts))2000 return false;2001 for (int I = 0, E = Mask.size(); I < E; ++I) {2002 if (Mask[I] == -1)2003 continue;2004 if (Mask[I] != I && Mask[I] != (NumSrcElts + I))2005 return false;2006 }2007 return true;2008}2009 2010bool ShuffleVectorInst::isTransposeMask(ArrayRef<int> Mask, int NumSrcElts) {2011 // Example masks that will return true:2012 // v1 = <a, b, c, d>2013 // v2 = <e, f, g, h>2014 // trn1 = shufflevector v1, v2 <0, 4, 2, 6> = <a, e, c, g>2015 // trn2 = shufflevector v1, v2 <1, 5, 3, 7> = <b, f, d, h>2016 2017 if (Mask.size() != static_cast<unsigned>(NumSrcElts))2018 return false;2019 // 1. The number of elements in the mask must be a power-of-2 and at least 2.2020 int Sz = Mask.size();2021 if (Sz < 2 || !isPowerOf2_32(Sz))2022 return false;2023 2024 // 2. The first element of the mask must be either a 0 or a 1.2025 if (Mask[0] != 0 && Mask[0] != 1)2026 return false;2027 2028 // 3. The difference between the first 2 elements must be equal to the2029 // number of elements in the mask.2030 if ((Mask[1] - Mask[0]) != NumSrcElts)2031 return false;2032 2033 // 4. The difference between consecutive even-numbered and odd-numbered2034 // elements must be equal to 2.2035 for (int I = 2; I < Sz; ++I) {2036 int MaskEltVal = Mask[I];2037 if (MaskEltVal == -1)2038 return false;2039 int MaskEltPrevVal = Mask[I - 2];2040 if (MaskEltVal - MaskEltPrevVal != 2)2041 return false;2042 }2043 return true;2044}2045 2046bool ShuffleVectorInst::isSpliceMask(ArrayRef<int> Mask, int NumSrcElts,2047 int &Index) {2048 if (Mask.size() != static_cast<unsigned>(NumSrcElts))2049 return false;2050 // Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4>2051 int StartIndex = -1;2052 for (int I = 0, E = Mask.size(); I != E; ++I) {2053 int MaskEltVal = Mask[I];2054 if (MaskEltVal == -1)2055 continue;2056 2057 if (StartIndex == -1) {2058 // Don't support a StartIndex that begins in the second input, or if the2059 // first non-undef index would access below the StartIndex.2060 if (MaskEltVal < I || NumSrcElts <= (MaskEltVal - I))2061 return false;2062 2063 StartIndex = MaskEltVal - I;2064 continue;2065 }2066 2067 // Splice is sequential starting from StartIndex.2068 if (MaskEltVal != (StartIndex + I))2069 return false;2070 }2071 2072 if (StartIndex == -1)2073 return false;2074 2075 // NOTE: This accepts StartIndex == 0 (COPY).2076 Index = StartIndex;2077 return true;2078}2079 2080bool ShuffleVectorInst::isExtractSubvectorMask(ArrayRef<int> Mask,2081 int NumSrcElts, int &Index) {2082 // Must extract from a single source.2083 if (!isSingleSourceMaskImpl(Mask, NumSrcElts))2084 return false;2085 2086 // Must be smaller (else this is an Identity shuffle).2087 if (NumSrcElts <= (int)Mask.size())2088 return false;2089 2090 // Find start of extraction, accounting that we may start with an UNDEF.2091 int SubIndex = -1;2092 for (int i = 0, e = Mask.size(); i != e; ++i) {2093 int M = Mask[i];2094 if (M < 0)2095 continue;2096 int Offset = (M % NumSrcElts) - i;2097 if (0 <= SubIndex && SubIndex != Offset)2098 return false;2099 SubIndex = Offset;2100 }2101 2102 if (0 <= SubIndex && SubIndex + (int)Mask.size() <= NumSrcElts) {2103 Index = SubIndex;2104 return true;2105 }2106 return false;2107}2108 2109bool ShuffleVectorInst::isInsertSubvectorMask(ArrayRef<int> Mask,2110 int NumSrcElts, int &NumSubElts,2111 int &Index) {2112 int NumMaskElts = Mask.size();2113 2114 // Don't try to match if we're shuffling to a smaller size.2115 if (NumMaskElts < NumSrcElts)2116 return false;2117 2118 // TODO: We don't recognize self-insertion/widening.2119 if (isSingleSourceMaskImpl(Mask, NumSrcElts))2120 return false;2121 2122 // Determine which mask elements are attributed to which source.2123 APInt UndefElts = APInt::getZero(NumMaskElts);2124 APInt Src0Elts = APInt::getZero(NumMaskElts);2125 APInt Src1Elts = APInt::getZero(NumMaskElts);2126 bool Src0Identity = true;2127 bool Src1Identity = true;2128 2129 for (int i = 0; i != NumMaskElts; ++i) {2130 int M = Mask[i];2131 if (M < 0) {2132 UndefElts.setBit(i);2133 continue;2134 }2135 if (M < NumSrcElts) {2136 Src0Elts.setBit(i);2137 Src0Identity &= (M == i);2138 continue;2139 }2140 Src1Elts.setBit(i);2141 Src1Identity &= (M == (i + NumSrcElts));2142 }2143 assert((Src0Elts | Src1Elts | UndefElts).isAllOnes() &&2144 "unknown shuffle elements");2145 assert(!Src0Elts.isZero() && !Src1Elts.isZero() &&2146 "2-source shuffle not found");2147 2148 // Determine lo/hi span ranges.2149 // TODO: How should we handle undefs at the start of subvector insertions?2150 int Src0Lo = Src0Elts.countr_zero();2151 int Src1Lo = Src1Elts.countr_zero();2152 int Src0Hi = NumMaskElts - Src0Elts.countl_zero();2153 int Src1Hi = NumMaskElts - Src1Elts.countl_zero();2154 2155 // If src0 is in place, see if the src1 elements is inplace within its own2156 // span.2157 if (Src0Identity) {2158 int NumSub1Elts = Src1Hi - Src1Lo;2159 ArrayRef<int> Sub1Mask = Mask.slice(Src1Lo, NumSub1Elts);2160 if (isIdentityMaskImpl(Sub1Mask, NumSrcElts)) {2161 NumSubElts = NumSub1Elts;2162 Index = Src1Lo;2163 return true;2164 }2165 }2166 2167 // If src1 is in place, see if the src0 elements is inplace within its own2168 // span.2169 if (Src1Identity) {2170 int NumSub0Elts = Src0Hi - Src0Lo;2171 ArrayRef<int> Sub0Mask = Mask.slice(Src0Lo, NumSub0Elts);2172 if (isIdentityMaskImpl(Sub0Mask, NumSrcElts)) {2173 NumSubElts = NumSub0Elts;2174 Index = Src0Lo;2175 return true;2176 }2177 }2178 2179 return false;2180}2181 2182bool ShuffleVectorInst::isIdentityWithPadding() const {2183 // FIXME: Not currently possible to express a shuffle mask for a scalable2184 // vector for this case.2185 if (isa<ScalableVectorType>(getType()))2186 return false;2187 2188 int NumOpElts = cast<FixedVectorType>(Op<0>()->getType())->getNumElements();2189 int NumMaskElts = cast<FixedVectorType>(getType())->getNumElements();2190 if (NumMaskElts <= NumOpElts)2191 return false;2192 2193 // The first part of the mask must choose elements from exactly 1 source op.2194 ArrayRef<int> Mask = getShuffleMask();2195 if (!isIdentityMaskImpl(Mask, NumOpElts))2196 return false;2197 2198 // All extending must be with undef elements.2199 for (int i = NumOpElts; i < NumMaskElts; ++i)2200 if (Mask[i] != -1)2201 return false;2202 2203 return true;2204}2205 2206bool ShuffleVectorInst::isIdentityWithExtract() const {2207 // FIXME: Not currently possible to express a shuffle mask for a scalable2208 // vector for this case.2209 if (isa<ScalableVectorType>(getType()))2210 return false;2211 2212 int NumOpElts = cast<FixedVectorType>(Op<0>()->getType())->getNumElements();2213 int NumMaskElts = cast<FixedVectorType>(getType())->getNumElements();2214 if (NumMaskElts >= NumOpElts)2215 return false;2216 2217 return isIdentityMaskImpl(getShuffleMask(), NumOpElts);2218}2219 2220bool ShuffleVectorInst::isConcat() const {2221 // Vector concatenation is differentiated from identity with padding.2222 if (isa<UndefValue>(Op<0>()) || isa<UndefValue>(Op<1>()))2223 return false;2224 2225 // FIXME: Not currently possible to express a shuffle mask for a scalable2226 // vector for this case.2227 if (isa<ScalableVectorType>(getType()))2228 return false;2229 2230 int NumOpElts = cast<FixedVectorType>(Op<0>()->getType())->getNumElements();2231 int NumMaskElts = cast<FixedVectorType>(getType())->getNumElements();2232 if (NumMaskElts != NumOpElts * 2)2233 return false;2234 2235 // Use the mask length rather than the operands' vector lengths here. We2236 // already know that the shuffle returns a vector twice as long as the inputs,2237 // and neither of the inputs are undef vectors. If the mask picks consecutive2238 // elements from both inputs, then this is a concatenation of the inputs.2239 return isIdentityMaskImpl(getShuffleMask(), NumMaskElts);2240}2241 2242static bool isReplicationMaskWithParams(ArrayRef<int> Mask,2243 int ReplicationFactor, int VF) {2244 assert(Mask.size() == (unsigned)ReplicationFactor * VF &&2245 "Unexpected mask size.");2246 2247 for (int CurrElt : seq(VF)) {2248 ArrayRef<int> CurrSubMask = Mask.take_front(ReplicationFactor);2249 assert(CurrSubMask.size() == (unsigned)ReplicationFactor &&2250 "Run out of mask?");2251 Mask = Mask.drop_front(ReplicationFactor);2252 if (!all_of(CurrSubMask, [CurrElt](int MaskElt) {2253 return MaskElt == PoisonMaskElem || MaskElt == CurrElt;2254 }))2255 return false;2256 }2257 assert(Mask.empty() && "Did not consume the whole mask?");2258 2259 return true;2260}2261 2262bool ShuffleVectorInst::isReplicationMask(ArrayRef<int> Mask,2263 int &ReplicationFactor, int &VF) {2264 // undef-less case is trivial.2265 if (!llvm::is_contained(Mask, PoisonMaskElem)) {2266 ReplicationFactor =2267 Mask.take_while([](int MaskElt) { return MaskElt == 0; }).size();2268 if (ReplicationFactor == 0 || Mask.size() % ReplicationFactor != 0)2269 return false;2270 VF = Mask.size() / ReplicationFactor;2271 return isReplicationMaskWithParams(Mask, ReplicationFactor, VF);2272 }2273 2274 // However, if the mask contains undef's, we have to enumerate possible tuples2275 // and pick one. There are bounds on replication factor: [1, mask size]2276 // (where RF=1 is an identity shuffle, RF=mask size is a broadcast shuffle)2277 // Additionally, mask size is a replication factor multiplied by vector size,2278 // which further significantly reduces the search space.2279 2280 // Before doing that, let's perform basic correctness checking first.2281 int Largest = -1;2282 for (int MaskElt : Mask) {2283 if (MaskElt == PoisonMaskElem)2284 continue;2285 // Elements must be in non-decreasing order.2286 if (MaskElt < Largest)2287 return false;2288 Largest = std::max(Largest, MaskElt);2289 }2290 2291 // Prefer larger replication factor if all else equal.2292 for (int PossibleReplicationFactor :2293 reverse(seq_inclusive<unsigned>(1, Mask.size()))) {2294 if (Mask.size() % PossibleReplicationFactor != 0)2295 continue;2296 int PossibleVF = Mask.size() / PossibleReplicationFactor;2297 if (!isReplicationMaskWithParams(Mask, PossibleReplicationFactor,2298 PossibleVF))2299 continue;2300 ReplicationFactor = PossibleReplicationFactor;2301 VF = PossibleVF;2302 return true;2303 }2304 2305 return false;2306}2307 2308bool ShuffleVectorInst::isReplicationMask(int &ReplicationFactor,2309 int &VF) const {2310 // Not possible to express a shuffle mask for a scalable vector for this2311 // case.2312 if (isa<ScalableVectorType>(getType()))2313 return false;2314 2315 VF = cast<FixedVectorType>(Op<0>()->getType())->getNumElements();2316 if (ShuffleMask.size() % VF != 0)2317 return false;2318 ReplicationFactor = ShuffleMask.size() / VF;2319 2320 return isReplicationMaskWithParams(ShuffleMask, ReplicationFactor, VF);2321}2322 2323bool ShuffleVectorInst::isOneUseSingleSourceMask(ArrayRef<int> Mask, int VF) {2324 if (VF <= 0 || Mask.size() < static_cast<unsigned>(VF) ||2325 Mask.size() % VF != 0)2326 return false;2327 for (unsigned K = 0, Sz = Mask.size(); K < Sz; K += VF) {2328 ArrayRef<int> SubMask = Mask.slice(K, VF);2329 if (all_of(SubMask, [](int Idx) { return Idx == PoisonMaskElem; }))2330 continue;2331 SmallBitVector Used(VF, false);2332 for (int Idx : SubMask) {2333 if (Idx != PoisonMaskElem && Idx < VF)2334 Used.set(Idx);2335 }2336 if (!Used.all())2337 return false;2338 }2339 return true;2340}2341 2342/// Return true if this shuffle mask is a replication mask.2343bool ShuffleVectorInst::isOneUseSingleSourceMask(int VF) const {2344 // Not possible to express a shuffle mask for a scalable vector for this2345 // case.2346 if (isa<ScalableVectorType>(getType()))2347 return false;2348 if (!isSingleSourceMask(ShuffleMask, VF))2349 return false;2350 2351 return isOneUseSingleSourceMask(ShuffleMask, VF);2352}2353 2354bool ShuffleVectorInst::isInterleave(unsigned Factor) {2355 FixedVectorType *OpTy = dyn_cast<FixedVectorType>(getOperand(0)->getType());2356 // shuffle_vector can only interleave fixed length vectors - for scalable2357 // vectors, see the @llvm.vector.interleave2 intrinsic2358 if (!OpTy)2359 return false;2360 unsigned OpNumElts = OpTy->getNumElements();2361 2362 return isInterleaveMask(ShuffleMask, Factor, OpNumElts * 2);2363}2364 2365bool ShuffleVectorInst::isInterleaveMask(2366 ArrayRef<int> Mask, unsigned Factor, unsigned NumInputElts,2367 SmallVectorImpl<unsigned> &StartIndexes) {2368 unsigned NumElts = Mask.size();2369 if (NumElts % Factor)2370 return false;2371 2372 unsigned LaneLen = NumElts / Factor;2373 if (!isPowerOf2_32(LaneLen))2374 return false;2375 2376 StartIndexes.resize(Factor);2377 2378 // Check whether each element matches the general interleaved rule.2379 // Ignore undef elements, as long as the defined elements match the rule.2380 // Outer loop processes all factors (x, y, z in the above example)2381 unsigned I = 0, J;2382 for (; I < Factor; I++) {2383 unsigned SavedLaneValue;2384 unsigned SavedNoUndefs = 0;2385 2386 // Inner loop processes consecutive accesses (x, x+1... in the example)2387 for (J = 0; J < LaneLen - 1; J++) {2388 // Lane computes x's position in the Mask2389 unsigned Lane = J * Factor + I;2390 unsigned NextLane = Lane + Factor;2391 int LaneValue = Mask[Lane];2392 int NextLaneValue = Mask[NextLane];2393 2394 // If both are defined, values must be sequential2395 if (LaneValue >= 0 && NextLaneValue >= 0 &&2396 LaneValue + 1 != NextLaneValue)2397 break;2398 2399 // If the next value is undef, save the current one as reference2400 if (LaneValue >= 0 && NextLaneValue < 0) {2401 SavedLaneValue = LaneValue;2402 SavedNoUndefs = 1;2403 }2404 2405 // Undefs are allowed, but defined elements must still be consecutive:2406 // i.e.: x,..., undef,..., x + 2,..., undef,..., undef,..., x + 5, ....2407 // Verify this by storing the last non-undef followed by an undef2408 // Check that following non-undef masks are incremented with the2409 // corresponding distance.2410 if (SavedNoUndefs > 0 && LaneValue < 0) {2411 SavedNoUndefs++;2412 if (NextLaneValue >= 0 &&2413 SavedLaneValue + SavedNoUndefs != (unsigned)NextLaneValue)2414 break;2415 }2416 }2417 2418 if (J < LaneLen - 1)2419 return false;2420 2421 int StartMask = 0;2422 if (Mask[I] >= 0) {2423 // Check that the start of the I range (J=0) is greater than 02424 StartMask = Mask[I];2425 } else if (Mask[(LaneLen - 1) * Factor + I] >= 0) {2426 // StartMask defined by the last value in lane2427 StartMask = Mask[(LaneLen - 1) * Factor + I] - J;2428 } else if (SavedNoUndefs > 0) {2429 // StartMask defined by some non-zero value in the j loop2430 StartMask = SavedLaneValue - (LaneLen - 1 - SavedNoUndefs);2431 }2432 // else StartMask remains set to 0, i.e. all elements are undefs2433 2434 if (StartMask < 0)2435 return false;2436 // We must stay within the vectors; This case can happen with undefs.2437 if (StartMask + LaneLen > NumInputElts)2438 return false;2439 2440 StartIndexes[I] = StartMask;2441 }2442 2443 return true;2444}2445 2446/// Check if the mask is a DE-interleave mask of the given factor2447/// \p Factor like:2448/// <Index, Index+Factor, ..., Index+(NumElts-1)*Factor>2449bool ShuffleVectorInst::isDeInterleaveMaskOfFactor(ArrayRef<int> Mask,2450 unsigned Factor,2451 unsigned &Index) {2452 // Check all potential start indices from 0 to (Factor - 1).2453 for (unsigned Idx = 0; Idx < Factor; Idx++) {2454 unsigned I = 0;2455 2456 // Check that elements are in ascending order by Factor. Ignore undef2457 // elements.2458 for (; I < Mask.size(); I++)2459 if (Mask[I] >= 0 && static_cast<unsigned>(Mask[I]) != Idx + I * Factor)2460 break;2461 2462 if (I == Mask.size()) {2463 Index = Idx;2464 return true;2465 }2466 }2467 2468 return false;2469}2470 2471/// Try to lower a vector shuffle as a bit rotation.2472///2473/// Look for a repeated rotation pattern in each sub group.2474/// Returns an element-wise left bit rotation amount or -1 if failed.2475static int matchShuffleAsBitRotate(ArrayRef<int> Mask, int NumSubElts) {2476 int NumElts = Mask.size();2477 assert((NumElts % NumSubElts) == 0 && "Illegal shuffle mask");2478 2479 int RotateAmt = -1;2480 for (int i = 0; i != NumElts; i += NumSubElts) {2481 for (int j = 0; j != NumSubElts; ++j) {2482 int M = Mask[i + j];2483 if (M < 0)2484 continue;2485 if (M < i || M >= i + NumSubElts)2486 return -1;2487 int Offset = (NumSubElts - (M - (i + j))) % NumSubElts;2488 if (0 <= RotateAmt && Offset != RotateAmt)2489 return -1;2490 RotateAmt = Offset;2491 }2492 }2493 return RotateAmt;2494}2495 2496bool ShuffleVectorInst::isBitRotateMask(2497 ArrayRef<int> Mask, unsigned EltSizeInBits, unsigned MinSubElts,2498 unsigned MaxSubElts, unsigned &NumSubElts, unsigned &RotateAmt) {2499 for (NumSubElts = MinSubElts; NumSubElts <= MaxSubElts; NumSubElts *= 2) {2500 int EltRotateAmt = matchShuffleAsBitRotate(Mask, NumSubElts);2501 if (EltRotateAmt < 0)2502 continue;2503 RotateAmt = EltRotateAmt * EltSizeInBits;2504 return true;2505 }2506 2507 return false;2508}2509 2510//===----------------------------------------------------------------------===//2511// InsertValueInst Class2512//===----------------------------------------------------------------------===//2513 2514void InsertValueInst::init(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,2515 const Twine &Name) {2516 assert(getNumOperands() == 2 && "NumOperands not initialized?");2517 2518 // There's no fundamental reason why we require at least one index2519 // (other than weirdness with &*IdxBegin being invalid; see2520 // getelementptr's init routine for example). But there's no2521 // present need to support it.2522 assert(!Idxs.empty() && "InsertValueInst must have at least one index");2523 2524 assert(ExtractValueInst::getIndexedType(Agg->getType(), Idxs) ==2525 Val->getType() && "Inserted value must match indexed type!");2526 Op<0>() = Agg;2527 Op<1>() = Val;2528 2529 Indices.append(Idxs.begin(), Idxs.end());2530 setName(Name);2531}2532 2533InsertValueInst::InsertValueInst(const InsertValueInst &IVI)2534 : Instruction(IVI.getType(), InsertValue, AllocMarker),2535 Indices(IVI.Indices) {2536 Op<0>() = IVI.getOperand(0);2537 Op<1>() = IVI.getOperand(1);2538 SubclassOptionalData = IVI.SubclassOptionalData;2539}2540 2541//===----------------------------------------------------------------------===//2542// ExtractValueInst Class2543//===----------------------------------------------------------------------===//2544 2545void ExtractValueInst::init(ArrayRef<unsigned> Idxs, const Twine &Name) {2546 assert(getNumOperands() == 1 && "NumOperands not initialized?");2547 2548 // There's no fundamental reason why we require at least one index.2549 // But there's no present need to support it.2550 assert(!Idxs.empty() && "ExtractValueInst must have at least one index");2551 2552 Indices.append(Idxs.begin(), Idxs.end());2553 setName(Name);2554}2555 2556ExtractValueInst::ExtractValueInst(const ExtractValueInst &EVI)2557 : UnaryInstruction(EVI.getType(), ExtractValue, EVI.getOperand(0),2558 (BasicBlock *)nullptr),2559 Indices(EVI.Indices) {2560 SubclassOptionalData = EVI.SubclassOptionalData;2561}2562 2563// getIndexedType - Returns the type of the element that would be extracted2564// with an extractvalue instruction with the specified parameters.2565//2566// A null type is returned if the indices are invalid for the specified2567// pointer type.2568//2569Type *ExtractValueInst::getIndexedType(Type *Agg,2570 ArrayRef<unsigned> Idxs) {2571 for (unsigned Index : Idxs) {2572 // We can't use CompositeType::indexValid(Index) here.2573 // indexValid() always returns true for arrays because getelementptr allows2574 // out-of-bounds indices. Since we don't allow those for extractvalue and2575 // insertvalue we need to check array indexing manually.2576 // Since the only other types we can index into are struct types it's just2577 // as easy to check those manually as well.2578 if (ArrayType *AT = dyn_cast<ArrayType>(Agg)) {2579 if (Index >= AT->getNumElements())2580 return nullptr;2581 Agg = AT->getElementType();2582 } else if (StructType *ST = dyn_cast<StructType>(Agg)) {2583 if (Index >= ST->getNumElements())2584 return nullptr;2585 Agg = ST->getElementType(Index);2586 } else {2587 // Not a valid type to index into.2588 return nullptr;2589 }2590 }2591 return Agg;2592}2593 2594//===----------------------------------------------------------------------===//2595// UnaryOperator Class2596//===----------------------------------------------------------------------===//2597 2598UnaryOperator::UnaryOperator(UnaryOps iType, Value *S, Type *Ty,2599 const Twine &Name, InsertPosition InsertBefore)2600 : UnaryInstruction(Ty, iType, S, InsertBefore) {2601 Op<0>() = S;2602 setName(Name);2603 AssertOK();2604}2605 2606UnaryOperator *UnaryOperator::Create(UnaryOps Op, Value *S, const Twine &Name,2607 InsertPosition InsertBefore) {2608 return new UnaryOperator(Op, S, S->getType(), Name, InsertBefore);2609}2610 2611void UnaryOperator::AssertOK() {2612 Value *LHS = getOperand(0);2613 (void)LHS; // Silence warnings.2614#ifndef NDEBUG2615 switch (getOpcode()) {2616 case FNeg:2617 assert(getType() == LHS->getType() &&2618 "Unary operation should return same type as operand!");2619 assert(getType()->isFPOrFPVectorTy() &&2620 "Tried to create a floating-point operation on a "2621 "non-floating-point type!");2622 break;2623 default: llvm_unreachable("Invalid opcode provided");2624 }2625#endif2626}2627 2628//===----------------------------------------------------------------------===//2629// BinaryOperator Class2630//===----------------------------------------------------------------------===//2631 2632BinaryOperator::BinaryOperator(BinaryOps iType, Value *S1, Value *S2, Type *Ty,2633 const Twine &Name, InsertPosition InsertBefore)2634 : Instruction(Ty, iType, AllocMarker, InsertBefore) {2635 Op<0>() = S1;2636 Op<1>() = S2;2637 setName(Name);2638 AssertOK();2639}2640 2641void BinaryOperator::AssertOK() {2642 Value *LHS = getOperand(0), *RHS = getOperand(1);2643 (void)LHS; (void)RHS; // Silence warnings.2644 assert(LHS->getType() == RHS->getType() &&2645 "Binary operator operand types must match!");2646#ifndef NDEBUG2647 switch (getOpcode()) {2648 case Add: case Sub:2649 case Mul:2650 assert(getType() == LHS->getType() &&2651 "Arithmetic operation should return same type as operands!");2652 assert(getType()->isIntOrIntVectorTy() &&2653 "Tried to create an integer operation on a non-integer type!");2654 break;2655 case FAdd: case FSub:2656 case FMul:2657 assert(getType() == LHS->getType() &&2658 "Arithmetic operation should return same type as operands!");2659 assert(getType()->isFPOrFPVectorTy() &&2660 "Tried to create a floating-point operation on a "2661 "non-floating-point type!");2662 break;2663 case UDiv:2664 case SDiv:2665 assert(getType() == LHS->getType() &&2666 "Arithmetic operation should return same type as operands!");2667 assert(getType()->isIntOrIntVectorTy() &&2668 "Incorrect operand type (not integer) for S/UDIV");2669 break;2670 case FDiv:2671 assert(getType() == LHS->getType() &&2672 "Arithmetic operation should return same type as operands!");2673 assert(getType()->isFPOrFPVectorTy() &&2674 "Incorrect operand type (not floating point) for FDIV");2675 break;2676 case URem:2677 case SRem:2678 assert(getType() == LHS->getType() &&2679 "Arithmetic operation should return same type as operands!");2680 assert(getType()->isIntOrIntVectorTy() &&2681 "Incorrect operand type (not integer) for S/UREM");2682 break;2683 case FRem:2684 assert(getType() == LHS->getType() &&2685 "Arithmetic operation should return same type as operands!");2686 assert(getType()->isFPOrFPVectorTy() &&2687 "Incorrect operand type (not floating point) for FREM");2688 break;2689 case Shl:2690 case LShr:2691 case AShr:2692 assert(getType() == LHS->getType() &&2693 "Shift operation should return same type as operands!");2694 assert(getType()->isIntOrIntVectorTy() &&2695 "Tried to create a shift operation on a non-integral type!");2696 break;2697 case And: case Or:2698 case Xor:2699 assert(getType() == LHS->getType() &&2700 "Logical operation should return same type as operands!");2701 assert(getType()->isIntOrIntVectorTy() &&2702 "Tried to create a logical operation on a non-integral type!");2703 break;2704 default: llvm_unreachable("Invalid opcode provided");2705 }2706#endif2707}2708 2709BinaryOperator *BinaryOperator::Create(BinaryOps Op, Value *S1, Value *S2,2710 const Twine &Name,2711 InsertPosition InsertBefore) {2712 assert(S1->getType() == S2->getType() &&2713 "Cannot create binary operator with two operands of differing type!");2714 return new BinaryOperator(Op, S1, S2, S1->getType(), Name, InsertBefore);2715}2716 2717BinaryOperator *BinaryOperator::CreateNeg(Value *Op, const Twine &Name,2718 InsertPosition InsertBefore) {2719 Value *Zero = ConstantInt::get(Op->getType(), 0);2720 return new BinaryOperator(Instruction::Sub, Zero, Op, Op->getType(), Name,2721 InsertBefore);2722}2723 2724BinaryOperator *BinaryOperator::CreateNSWNeg(Value *Op, const Twine &Name,2725 InsertPosition InsertBefore) {2726 Value *Zero = ConstantInt::get(Op->getType(), 0);2727 return BinaryOperator::CreateNSWSub(Zero, Op, Name, InsertBefore);2728}2729 2730BinaryOperator *BinaryOperator::CreateNot(Value *Op, const Twine &Name,2731 InsertPosition InsertBefore) {2732 Constant *C = Constant::getAllOnesValue(Op->getType());2733 return new BinaryOperator(Instruction::Xor, Op, C,2734 Op->getType(), Name, InsertBefore);2735}2736 2737// Exchange the two operands to this instruction. This instruction is safe to2738// use on any binary instruction and does not modify the semantics of the2739// instruction.2740bool BinaryOperator::swapOperands() {2741 if (!isCommutative())2742 return true; // Can't commute operands2743 Op<0>().swap(Op<1>());2744 return false;2745}2746 2747//===----------------------------------------------------------------------===//2748// FPMathOperator Class2749//===----------------------------------------------------------------------===//2750 2751float FPMathOperator::getFPAccuracy() const {2752 const MDNode *MD =2753 cast<Instruction>(this)->getMetadata(LLVMContext::MD_fpmath);2754 if (!MD)2755 return 0.0;2756 ConstantFP *Accuracy = mdconst::extract<ConstantFP>(MD->getOperand(0));2757 return Accuracy->getValueAPF().convertToFloat();2758}2759 2760//===----------------------------------------------------------------------===//2761// CastInst Class2762//===----------------------------------------------------------------------===//2763 2764// Just determine if this cast only deals with integral->integral conversion.2765bool CastInst::isIntegerCast() const {2766 switch (getOpcode()) {2767 default: return false;2768 case Instruction::ZExt:2769 case Instruction::SExt:2770 case Instruction::Trunc:2771 return true;2772 case Instruction::BitCast:2773 return getOperand(0)->getType()->isIntegerTy() &&2774 getType()->isIntegerTy();2775 }2776}2777 2778/// This function determines if the CastInst does not require any bits to be2779/// changed in order to effect the cast. Essentially, it identifies cases where2780/// no code gen is necessary for the cast, hence the name no-op cast. For2781/// example, the following are all no-op casts:2782/// # bitcast i32* %x to i8*2783/// # bitcast <2 x i32> %x to <4 x i16>2784/// # ptrtoint i32* %x to i32 ; on 32-bit plaforms only2785/// Determine if the described cast is a no-op.2786bool CastInst::isNoopCast(Instruction::CastOps Opcode,2787 Type *SrcTy,2788 Type *DestTy,2789 const DataLayout &DL) {2790 assert(castIsValid(Opcode, SrcTy, DestTy) && "method precondition");2791 switch (Opcode) {2792 default: llvm_unreachable("Invalid CastOp");2793 case Instruction::Trunc:2794 case Instruction::ZExt:2795 case Instruction::SExt:2796 case Instruction::FPTrunc:2797 case Instruction::FPExt:2798 case Instruction::UIToFP:2799 case Instruction::SIToFP:2800 case Instruction::FPToUI:2801 case Instruction::FPToSI:2802 case Instruction::AddrSpaceCast:2803 // TODO: Target informations may give a more accurate answer here.2804 return false;2805 case Instruction::BitCast:2806 return true; // BitCast never modifies bits.2807 case Instruction::PtrToAddr:2808 case Instruction::PtrToInt:2809 return DL.getIntPtrType(SrcTy)->getScalarSizeInBits() ==2810 DestTy->getScalarSizeInBits();2811 case Instruction::IntToPtr:2812 return DL.getIntPtrType(DestTy)->getScalarSizeInBits() ==2813 SrcTy->getScalarSizeInBits();2814 }2815}2816 2817bool CastInst::isNoopCast(const DataLayout &DL) const {2818 return isNoopCast(getOpcode(), getOperand(0)->getType(), getType(), DL);2819}2820 2821/// This function determines if a pair of casts can be eliminated and what2822/// opcode should be used in the elimination. This assumes that there are two2823/// instructions like this:2824/// * %F = firstOpcode SrcTy %x to MidTy2825/// * %S = secondOpcode MidTy %F to DstTy2826/// The function returns a resultOpcode so these two casts can be replaced with:2827/// * %Replacement = resultOpcode %SrcTy %x to DstTy2828/// If no such cast is permitted, the function returns 0.2829unsigned CastInst::isEliminableCastPair(Instruction::CastOps firstOp,2830 Instruction::CastOps secondOp,2831 Type *SrcTy, Type *MidTy, Type *DstTy,2832 const DataLayout *DL) {2833 // Define the 144 possibilities for these two cast instructions. The values2834 // in this matrix determine what to do in a given situation and select the2835 // case in the switch below. The rows correspond to firstOp, the columns2836 // correspond to secondOp. In looking at the table below, keep in mind2837 // the following cast properties:2838 //2839 // Size Compare Source Destination2840 // Operator Src ? Size Type Sign Type Sign2841 // -------- ------------ ------------------- ---------------------2842 // TRUNC > Integer Any Integral Any2843 // ZEXT < Integral Unsigned Integer Any2844 // SEXT < Integral Signed Integer Any2845 // FPTOUI n/a FloatPt n/a Integral Unsigned2846 // FPTOSI n/a FloatPt n/a Integral Signed2847 // UITOFP n/a Integral Unsigned FloatPt n/a2848 // SITOFP n/a Integral Signed FloatPt n/a2849 // FPTRUNC > FloatPt n/a FloatPt n/a2850 // FPEXT < FloatPt n/a FloatPt n/a2851 // PTRTOINT n/a Pointer n/a Integral Unsigned2852 // PTRTOADDR n/a Pointer n/a Integral Unsigned2853 // INTTOPTR n/a Integral Unsigned Pointer n/a2854 // BITCAST = FirstClass n/a FirstClass n/a2855 // ADDRSPCST n/a Pointer n/a Pointer n/a2856 //2857 // NOTE: some transforms are safe, but we consider them to be non-profitable.2858 // For example, we could merge "fptoui double to i32" + "zext i32 to i64",2859 // into "fptoui double to i64", but this loses information about the range2860 // of the produced value (we no longer know the top-part is all zeros).2861 // Further this conversion is often much more expensive for typical hardware,2862 // and causes issues when building libgcc. We disallow fptosi+sext for the2863 // same reason.2864 const unsigned numCastOps =2865 Instruction::CastOpsEnd - Instruction::CastOpsBegin;2866 // clang-format off2867 static const uint8_t CastResults[numCastOps][numCastOps] = {2868 // T F F U S F F P P I B A -+2869 // R Z S P P I I T P 2 2 N T S |2870 // U E E 2 2 2 2 R E I A T C C +- secondOp2871 // N X X U S F F N X N D 2 V V |2872 // C T T I I P P C T T R P T T -+2873 { 1, 0, 0,99,99, 0, 0,99,99,99,99, 0, 3, 0}, // Trunc -+2874 { 8, 1, 9,99,99, 2,17,99,99,99,99, 2, 3, 0}, // ZExt |2875 { 8, 0, 1,99,99, 0, 2,99,99,99,99, 0, 3, 0}, // SExt |2876 { 0, 0, 0,99,99, 0, 0,99,99,99,99, 0, 3, 0}, // FPToUI |2877 { 0, 0, 0,99,99, 0, 0,99,99,99,99, 0, 3, 0}, // FPToSI |2878 { 99,99,99, 0, 0,99,99, 0, 0,99,99,99, 4, 0}, // UIToFP +- firstOp2879 { 99,99,99, 0, 0,99,99, 0, 0,99,99,99, 4, 0}, // SIToFP |2880 { 99,99,99, 0, 0,99,99, 0, 0,99,99,99, 4, 0}, // FPTrunc |2881 { 99,99,99, 2, 2,99,99, 8, 2,99,99,99, 4, 0}, // FPExt |2882 { 1, 0, 0,99,99, 0, 0,99,99,99,99, 7, 3, 0}, // PtrToInt |2883 { 0, 0, 0,99,99, 0, 0,99,99,99,99, 0, 3, 0}, // PtrToAddr |2884 { 99,99,99,99,99,99,99,99,99,11,11,99,15, 0}, // IntToPtr |2885 { 5, 5, 5, 0, 0, 5, 5, 0, 0,16,16, 5, 1,14}, // BitCast |2886 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,13,12}, // AddrSpaceCast -+2887 };2888 // clang-format on2889 2890 // TODO: This logic could be encoded into the table above and handled in the2891 // switch below.2892 // If either of the casts are a bitcast from scalar to vector, disallow the2893 // merging. However, any pair of bitcasts are allowed.2894 bool IsFirstBitcast = (firstOp == Instruction::BitCast);2895 bool IsSecondBitcast = (secondOp == Instruction::BitCast);2896 bool AreBothBitcasts = IsFirstBitcast && IsSecondBitcast;2897 2898 // Check if any of the casts convert scalars <-> vectors.2899 if ((IsFirstBitcast && isa<VectorType>(SrcTy) != isa<VectorType>(MidTy)) ||2900 (IsSecondBitcast && isa<VectorType>(MidTy) != isa<VectorType>(DstTy)))2901 if (!AreBothBitcasts)2902 return 0;2903 2904 int ElimCase = CastResults[firstOp-Instruction::CastOpsBegin]2905 [secondOp-Instruction::CastOpsBegin];2906 switch (ElimCase) {2907 case 0:2908 // Categorically disallowed.2909 return 0;2910 case 1:2911 // Allowed, use first cast's opcode.2912 return firstOp;2913 case 2:2914 // Allowed, use second cast's opcode.2915 return secondOp;2916 case 3:2917 // No-op cast in second op implies firstOp as long as the DestTy2918 // is integer and we are not converting between a vector and a2919 // non-vector type.2920 if (!SrcTy->isVectorTy() && DstTy->isIntegerTy())2921 return firstOp;2922 return 0;2923 case 4:2924 // No-op cast in second op implies firstOp as long as the DestTy2925 // matches MidTy.2926 if (DstTy == MidTy)2927 return firstOp;2928 return 0;2929 case 5:2930 // No-op cast in first op implies secondOp as long as the SrcTy2931 // is an integer.2932 if (SrcTy->isIntegerTy())2933 return secondOp;2934 return 0;2935 case 7: {2936 // Disable inttoptr/ptrtoint optimization if enabled.2937 if (DisableI2pP2iOpt)2938 return 0;2939 2940 // Cannot simplify if address spaces are different!2941 if (SrcTy != DstTy)2942 return 0;2943 2944 // Cannot simplify if the intermediate integer size is smaller than the2945 // pointer size.2946 unsigned MidSize = MidTy->getScalarSizeInBits();2947 if (!DL || MidSize < DL->getPointerTypeSizeInBits(SrcTy))2948 return 0;2949 2950 return Instruction::BitCast;2951 }2952 case 8: {2953 // ext, trunc -> bitcast, if the SrcTy and DstTy are the same2954 // ext, trunc -> ext, if sizeof(SrcTy) < sizeof(DstTy)2955 // ext, trunc -> trunc, if sizeof(SrcTy) > sizeof(DstTy)2956 unsigned SrcSize = SrcTy->getScalarSizeInBits();2957 unsigned DstSize = DstTy->getScalarSizeInBits();2958 if (SrcTy == DstTy)2959 return Instruction::BitCast;2960 if (SrcSize < DstSize)2961 return firstOp;2962 if (SrcSize > DstSize)2963 return secondOp;2964 return 0;2965 }2966 case 9:2967 // zext, sext -> zext, because sext can't sign extend after zext2968 return Instruction::ZExt;2969 case 11: {2970 // inttoptr, ptrtoint/ptrtoaddr -> integer cast2971 if (!DL)2972 return 0;2973 unsigned MidSize = secondOp == Instruction::PtrToAddr2974 ? DL->getAddressSizeInBits(MidTy)2975 : DL->getPointerTypeSizeInBits(MidTy);2976 unsigned SrcSize = SrcTy->getScalarSizeInBits();2977 unsigned DstSize = DstTy->getScalarSizeInBits();2978 // If the middle size is smaller than both source and destination,2979 // an additional masking operation would be required.2980 if (MidSize < SrcSize && MidSize < DstSize)2981 return 0;2982 if (DstSize < SrcSize)2983 return Instruction::Trunc;2984 if (DstSize > SrcSize)2985 return Instruction::ZExt;2986 return Instruction::BitCast;2987 }2988 case 12:2989 // addrspacecast, addrspacecast -> bitcast, if SrcAS == DstAS2990 // addrspacecast, addrspacecast -> addrspacecast, if SrcAS != DstAS2991 if (SrcTy->getPointerAddressSpace() != DstTy->getPointerAddressSpace())2992 return Instruction::AddrSpaceCast;2993 return Instruction::BitCast;2994 case 13:2995 // FIXME: this state can be merged with (1), but the following assert2996 // is useful to check the correcteness of the sequence due to semantic2997 // change of bitcast.2998 assert(2999 SrcTy->isPtrOrPtrVectorTy() &&3000 MidTy->isPtrOrPtrVectorTy() &&3001 DstTy->isPtrOrPtrVectorTy() &&3002 SrcTy->getPointerAddressSpace() != MidTy->getPointerAddressSpace() &&3003 MidTy->getPointerAddressSpace() == DstTy->getPointerAddressSpace() &&3004 "Illegal addrspacecast, bitcast sequence!");3005 // Allowed, use first cast's opcode3006 return firstOp;3007 case 14:3008 // bitcast, addrspacecast -> addrspacecast3009 return Instruction::AddrSpaceCast;3010 case 15:3011 // FIXME: this state can be merged with (1), but the following assert3012 // is useful to check the correcteness of the sequence due to semantic3013 // change of bitcast.3014 assert(3015 SrcTy->isIntOrIntVectorTy() &&3016 MidTy->isPtrOrPtrVectorTy() &&3017 DstTy->isPtrOrPtrVectorTy() &&3018 MidTy->getPointerAddressSpace() == DstTy->getPointerAddressSpace() &&3019 "Illegal inttoptr, bitcast sequence!");3020 // Allowed, use first cast's opcode3021 return firstOp;3022 case 16:3023 // FIXME: this state can be merged with (2), but the following assert3024 // is useful to check the correcteness of the sequence due to semantic3025 // change of bitcast.3026 assert(3027 SrcTy->isPtrOrPtrVectorTy() &&3028 MidTy->isPtrOrPtrVectorTy() &&3029 DstTy->isIntOrIntVectorTy() &&3030 SrcTy->getPointerAddressSpace() == MidTy->getPointerAddressSpace() &&3031 "Illegal bitcast, ptrtoint sequence!");3032 // Allowed, use second cast's opcode3033 return secondOp;3034 case 17:3035 // (sitofp (zext x)) -> (uitofp x)3036 return Instruction::UIToFP;3037 case 99:3038 // Cast combination can't happen (error in input). This is for all cases3039 // where the MidTy is not the same for the two cast instructions.3040 llvm_unreachable("Invalid Cast Combination");3041 default:3042 llvm_unreachable("Error in CastResults table!!!");3043 }3044}3045 3046CastInst *CastInst::Create(Instruction::CastOps op, Value *S, Type *Ty,3047 const Twine &Name, InsertPosition InsertBefore) {3048 assert(castIsValid(op, S, Ty) && "Invalid cast!");3049 // Construct and return the appropriate CastInst subclass3050 switch (op) {3051 case Trunc: return new TruncInst (S, Ty, Name, InsertBefore);3052 case ZExt: return new ZExtInst (S, Ty, Name, InsertBefore);3053 case SExt: return new SExtInst (S, Ty, Name, InsertBefore);3054 case FPTrunc: return new FPTruncInst (S, Ty, Name, InsertBefore);3055 case FPExt: return new FPExtInst (S, Ty, Name, InsertBefore);3056 case UIToFP: return new UIToFPInst (S, Ty, Name, InsertBefore);3057 case SIToFP: return new SIToFPInst (S, Ty, Name, InsertBefore);3058 case FPToUI: return new FPToUIInst (S, Ty, Name, InsertBefore);3059 case FPToSI: return new FPToSIInst (S, Ty, Name, InsertBefore);3060 case PtrToAddr: return new PtrToAddrInst (S, Ty, Name, InsertBefore);3061 case PtrToInt: return new PtrToIntInst (S, Ty, Name, InsertBefore);3062 case IntToPtr: return new IntToPtrInst (S, Ty, Name, InsertBefore);3063 case BitCast:3064 return new BitCastInst(S, Ty, Name, InsertBefore);3065 case AddrSpaceCast:3066 return new AddrSpaceCastInst(S, Ty, Name, InsertBefore);3067 default:3068 llvm_unreachable("Invalid opcode provided");3069 }3070}3071 3072CastInst *CastInst::CreateZExtOrBitCast(Value *S, Type *Ty, const Twine &Name,3073 InsertPosition InsertBefore) {3074 if (S->getType()->getScalarSizeInBits() == Ty->getScalarSizeInBits())3075 return Create(Instruction::BitCast, S, Ty, Name, InsertBefore);3076 return Create(Instruction::ZExt, S, Ty, Name, InsertBefore);3077}3078 3079CastInst *CastInst::CreateSExtOrBitCast(Value *S, Type *Ty, const Twine &Name,3080 InsertPosition InsertBefore) {3081 if (S->getType()->getScalarSizeInBits() == Ty->getScalarSizeInBits())3082 return Create(Instruction::BitCast, S, Ty, Name, InsertBefore);3083 return Create(Instruction::SExt, S, Ty, Name, InsertBefore);3084}3085 3086CastInst *CastInst::CreateTruncOrBitCast(Value *S, Type *Ty, const Twine &Name,3087 InsertPosition InsertBefore) {3088 if (S->getType()->getScalarSizeInBits() == Ty->getScalarSizeInBits())3089 return Create(Instruction::BitCast, S, Ty, Name, InsertBefore);3090 return Create(Instruction::Trunc, S, Ty, Name, InsertBefore);3091}3092 3093/// Create a BitCast or a PtrToInt cast instruction3094CastInst *CastInst::CreatePointerCast(Value *S, Type *Ty, const Twine &Name,3095 InsertPosition InsertBefore) {3096 assert(S->getType()->isPtrOrPtrVectorTy() && "Invalid cast");3097 assert((Ty->isIntOrIntVectorTy() || Ty->isPtrOrPtrVectorTy()) &&3098 "Invalid cast");3099 assert(Ty->isVectorTy() == S->getType()->isVectorTy() && "Invalid cast");3100 assert((!Ty->isVectorTy() ||3101 cast<VectorType>(Ty)->getElementCount() ==3102 cast<VectorType>(S->getType())->getElementCount()) &&3103 "Invalid cast");3104 3105 if (Ty->isIntOrIntVectorTy())3106 return Create(Instruction::PtrToInt, S, Ty, Name, InsertBefore);3107 3108 return CreatePointerBitCastOrAddrSpaceCast(S, Ty, Name, InsertBefore);3109}3110 3111CastInst *CastInst::CreatePointerBitCastOrAddrSpaceCast(3112 Value *S, Type *Ty, const Twine &Name, InsertPosition InsertBefore) {3113 assert(S->getType()->isPtrOrPtrVectorTy() && "Invalid cast");3114 assert(Ty->isPtrOrPtrVectorTy() && "Invalid cast");3115 3116 if (S->getType()->getPointerAddressSpace() != Ty->getPointerAddressSpace())3117 return Create(Instruction::AddrSpaceCast, S, Ty, Name, InsertBefore);3118 3119 return Create(Instruction::BitCast, S, Ty, Name, InsertBefore);3120}3121 3122CastInst *CastInst::CreateBitOrPointerCast(Value *S, Type *Ty,3123 const Twine &Name,3124 InsertPosition InsertBefore) {3125 if (S->getType()->isPointerTy() && Ty->isIntegerTy())3126 return Create(Instruction::PtrToInt, S, Ty, Name, InsertBefore);3127 if (S->getType()->isIntegerTy() && Ty->isPointerTy())3128 return Create(Instruction::IntToPtr, S, Ty, Name, InsertBefore);3129 3130 return Create(Instruction::BitCast, S, Ty, Name, InsertBefore);3131}3132 3133CastInst *CastInst::CreateIntegerCast(Value *C, Type *Ty, bool isSigned,3134 const Twine &Name,3135 InsertPosition InsertBefore) {3136 assert(C->getType()->isIntOrIntVectorTy() && Ty->isIntOrIntVectorTy() &&3137 "Invalid integer cast");3138 unsigned SrcBits = C->getType()->getScalarSizeInBits();3139 unsigned DstBits = Ty->getScalarSizeInBits();3140 Instruction::CastOps opcode =3141 (SrcBits == DstBits ? Instruction::BitCast :3142 (SrcBits > DstBits ? Instruction::Trunc :3143 (isSigned ? Instruction::SExt : Instruction::ZExt)));3144 return Create(opcode, C, Ty, Name, InsertBefore);3145}3146 3147CastInst *CastInst::CreateFPCast(Value *C, Type *Ty, const Twine &Name,3148 InsertPosition InsertBefore) {3149 assert(C->getType()->isFPOrFPVectorTy() && Ty->isFPOrFPVectorTy() &&3150 "Invalid cast");3151 unsigned SrcBits = C->getType()->getScalarSizeInBits();3152 unsigned DstBits = Ty->getScalarSizeInBits();3153 assert((C->getType() == Ty || SrcBits != DstBits) && "Invalid cast");3154 Instruction::CastOps opcode =3155 (SrcBits == DstBits ? Instruction::BitCast :3156 (SrcBits > DstBits ? Instruction::FPTrunc : Instruction::FPExt));3157 return Create(opcode, C, Ty, Name, InsertBefore);3158}3159 3160bool CastInst::isBitCastable(Type *SrcTy, Type *DestTy) {3161 if (!SrcTy->isFirstClassType() || !DestTy->isFirstClassType())3162 return false;3163 3164 if (SrcTy == DestTy)3165 return true;3166 3167 if (VectorType *SrcVecTy = dyn_cast<VectorType>(SrcTy)) {3168 if (VectorType *DestVecTy = dyn_cast<VectorType>(DestTy)) {3169 if (SrcVecTy->getElementCount() == DestVecTy->getElementCount()) {3170 // An element by element cast. Valid if casting the elements is valid.3171 SrcTy = SrcVecTy->getElementType();3172 DestTy = DestVecTy->getElementType();3173 }3174 }3175 }3176 3177 if (PointerType *DestPtrTy = dyn_cast<PointerType>(DestTy)) {3178 if (PointerType *SrcPtrTy = dyn_cast<PointerType>(SrcTy)) {3179 return SrcPtrTy->getAddressSpace() == DestPtrTy->getAddressSpace();3180 }3181 }3182 3183 TypeSize SrcBits = SrcTy->getPrimitiveSizeInBits(); // 0 for ptr3184 TypeSize DestBits = DestTy->getPrimitiveSizeInBits(); // 0 for ptr3185 3186 // Could still have vectors of pointers if the number of elements doesn't3187 // match3188 if (SrcBits.getKnownMinValue() == 0 || DestBits.getKnownMinValue() == 0)3189 return false;3190 3191 if (SrcBits != DestBits)3192 return false;3193 3194 return true;3195}3196 3197bool CastInst::isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy,3198 const DataLayout &DL) {3199 // ptrtoint and inttoptr are not allowed on non-integral pointers3200 if (auto *PtrTy = dyn_cast<PointerType>(SrcTy))3201 if (auto *IntTy = dyn_cast<IntegerType>(DestTy))3202 return (IntTy->getBitWidth() == DL.getPointerTypeSizeInBits(PtrTy) &&3203 !DL.isNonIntegralPointerType(PtrTy));3204 if (auto *PtrTy = dyn_cast<PointerType>(DestTy))3205 if (auto *IntTy = dyn_cast<IntegerType>(SrcTy))3206 return (IntTy->getBitWidth() == DL.getPointerTypeSizeInBits(PtrTy) &&3207 !DL.isNonIntegralPointerType(PtrTy));3208 3209 return isBitCastable(SrcTy, DestTy);3210}3211 3212// Provide a way to get a "cast" where the cast opcode is inferred from the3213// types and size of the operand. This, basically, is a parallel of the3214// logic in the castIsValid function below. This axiom should hold:3215// castIsValid( getCastOpcode(Val, Ty), Val, Ty)3216// should not assert in castIsValid. In other words, this produces a "correct"3217// casting opcode for the arguments passed to it.3218Instruction::CastOps3219CastInst::getCastOpcode(3220 const Value *Src, bool SrcIsSigned, Type *DestTy, bool DestIsSigned) {3221 Type *SrcTy = Src->getType();3222 3223 assert(SrcTy->isFirstClassType() && DestTy->isFirstClassType() &&3224 "Only first class types are castable!");3225 3226 if (SrcTy == DestTy)3227 return BitCast;3228 3229 // FIXME: Check address space sizes here3230 if (VectorType *SrcVecTy = dyn_cast<VectorType>(SrcTy))3231 if (VectorType *DestVecTy = dyn_cast<VectorType>(DestTy))3232 if (SrcVecTy->getElementCount() == DestVecTy->getElementCount()) {3233 // An element by element cast. Find the appropriate opcode based on the3234 // element types.3235 SrcTy = SrcVecTy->getElementType();3236 DestTy = DestVecTy->getElementType();3237 }3238 3239 // Get the bit sizes, we'll need these3240 // FIXME: This doesn't work for scalable vector types with different element3241 // counts that don't call getElementType above.3242 unsigned SrcBits =3243 SrcTy->getPrimitiveSizeInBits().getFixedValue(); // 0 for ptr3244 unsigned DestBits =3245 DestTy->getPrimitiveSizeInBits().getFixedValue(); // 0 for ptr3246 3247 // Run through the possibilities ...3248 if (DestTy->isIntegerTy()) { // Casting to integral3249 if (SrcTy->isIntegerTy()) { // Casting from integral3250 if (DestBits < SrcBits)3251 return Trunc; // int -> smaller int3252 else if (DestBits > SrcBits) { // its an extension3253 if (SrcIsSigned)3254 return SExt; // signed -> SEXT3255 else3256 return ZExt; // unsigned -> ZEXT3257 } else {3258 return BitCast; // Same size, No-op cast3259 }3260 } else if (SrcTy->isFloatingPointTy()) { // Casting from floating pt3261 if (DestIsSigned)3262 return FPToSI; // FP -> sint3263 else3264 return FPToUI; // FP -> uint3265 } else if (SrcTy->isVectorTy()) {3266 assert(DestBits == SrcBits &&3267 "Casting vector to integer of different width");3268 return BitCast; // Same size, no-op cast3269 } else {3270 assert(SrcTy->isPointerTy() &&3271 "Casting from a value that is not first-class type");3272 return PtrToInt; // ptr -> int3273 }3274 } else if (DestTy->isFloatingPointTy()) { // Casting to floating pt3275 if (SrcTy->isIntegerTy()) { // Casting from integral3276 if (SrcIsSigned)3277 return SIToFP; // sint -> FP3278 else3279 return UIToFP; // uint -> FP3280 } else if (SrcTy->isFloatingPointTy()) { // Casting from floating pt3281 if (DestBits < SrcBits) {3282 return FPTrunc; // FP -> smaller FP3283 } else if (DestBits > SrcBits) {3284 return FPExt; // FP -> larger FP3285 } else {3286 return BitCast; // same size, no-op cast3287 }3288 } else if (SrcTy->isVectorTy()) {3289 assert(DestBits == SrcBits &&3290 "Casting vector to floating point of different width");3291 return BitCast; // same size, no-op cast3292 }3293 llvm_unreachable("Casting pointer or non-first class to float");3294 } else if (DestTy->isVectorTy()) {3295 assert(DestBits == SrcBits &&3296 "Illegal cast to vector (wrong type or size)");3297 return BitCast;3298 } else if (DestTy->isPointerTy()) {3299 if (SrcTy->isPointerTy()) {3300 if (DestTy->getPointerAddressSpace() != SrcTy->getPointerAddressSpace())3301 return AddrSpaceCast;3302 return BitCast; // ptr -> ptr3303 } else if (SrcTy->isIntegerTy()) {3304 return IntToPtr; // int -> ptr3305 }3306 llvm_unreachable("Casting pointer to other than pointer or int");3307 }3308 llvm_unreachable("Casting to type that is not first-class");3309}3310 3311//===----------------------------------------------------------------------===//3312// CastInst SubClass Constructors3313//===----------------------------------------------------------------------===//3314 3315/// Check that the construction parameters for a CastInst are correct. This3316/// could be broken out into the separate constructors but it is useful to have3317/// it in one place and to eliminate the redundant code for getting the sizes3318/// of the types involved.3319bool3320CastInst::castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy) {3321 if (!SrcTy->isFirstClassType() || !DstTy->isFirstClassType() ||3322 SrcTy->isAggregateType() || DstTy->isAggregateType())3323 return false;3324 3325 // Get the size of the types in bits, and whether we are dealing3326 // with vector types, we'll need this later.3327 bool SrcIsVec = isa<VectorType>(SrcTy);3328 bool DstIsVec = isa<VectorType>(DstTy);3329 unsigned SrcScalarBitSize = SrcTy->getScalarSizeInBits();3330 unsigned DstScalarBitSize = DstTy->getScalarSizeInBits();3331 3332 // If these are vector types, get the lengths of the vectors (using zero for3333 // scalar types means that checking that vector lengths match also checks that3334 // scalars are not being converted to vectors or vectors to scalars).3335 ElementCount SrcEC = SrcIsVec ? cast<VectorType>(SrcTy)->getElementCount()3336 : ElementCount::getFixed(0);3337 ElementCount DstEC = DstIsVec ? cast<VectorType>(DstTy)->getElementCount()3338 : ElementCount::getFixed(0);3339 3340 // Switch on the opcode provided3341 switch (op) {3342 default: return false; // This is an input error3343 case Instruction::Trunc:3344 return SrcTy->isIntOrIntVectorTy() && DstTy->isIntOrIntVectorTy() &&3345 SrcEC == DstEC && SrcScalarBitSize > DstScalarBitSize;3346 case Instruction::ZExt:3347 return SrcTy->isIntOrIntVectorTy() && DstTy->isIntOrIntVectorTy() &&3348 SrcEC == DstEC && SrcScalarBitSize < DstScalarBitSize;3349 case Instruction::SExt:3350 return SrcTy->isIntOrIntVectorTy() && DstTy->isIntOrIntVectorTy() &&3351 SrcEC == DstEC && SrcScalarBitSize < DstScalarBitSize;3352 case Instruction::FPTrunc:3353 return SrcTy->isFPOrFPVectorTy() && DstTy->isFPOrFPVectorTy() &&3354 SrcEC == DstEC && SrcScalarBitSize > DstScalarBitSize;3355 case Instruction::FPExt:3356 return SrcTy->isFPOrFPVectorTy() && DstTy->isFPOrFPVectorTy() &&3357 SrcEC == DstEC && SrcScalarBitSize < DstScalarBitSize;3358 case Instruction::UIToFP:3359 case Instruction::SIToFP:3360 return SrcTy->isIntOrIntVectorTy() && DstTy->isFPOrFPVectorTy() &&3361 SrcEC == DstEC;3362 case Instruction::FPToUI:3363 case Instruction::FPToSI:3364 return SrcTy->isFPOrFPVectorTy() && DstTy->isIntOrIntVectorTy() &&3365 SrcEC == DstEC;3366 case Instruction::PtrToAddr:3367 case Instruction::PtrToInt:3368 if (SrcEC != DstEC)3369 return false;3370 return SrcTy->isPtrOrPtrVectorTy() && DstTy->isIntOrIntVectorTy();3371 case Instruction::IntToPtr:3372 if (SrcEC != DstEC)3373 return false;3374 return SrcTy->isIntOrIntVectorTy() && DstTy->isPtrOrPtrVectorTy();3375 case Instruction::BitCast: {3376 PointerType *SrcPtrTy = dyn_cast<PointerType>(SrcTy->getScalarType());3377 PointerType *DstPtrTy = dyn_cast<PointerType>(DstTy->getScalarType());3378 3379 // BitCast implies a no-op cast of type only. No bits change.3380 // However, you can't cast pointers to anything but pointers.3381 if (!SrcPtrTy != !DstPtrTy)3382 return false;3383 3384 // For non-pointer cases, the cast is okay if the source and destination bit3385 // widths are identical.3386 if (!SrcPtrTy)3387 return SrcTy->getPrimitiveSizeInBits() == DstTy->getPrimitiveSizeInBits();3388 3389 // If both are pointers then the address spaces must match.3390 if (SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace())3391 return false;3392 3393 // A vector of pointers must have the same number of elements.3394 if (SrcIsVec && DstIsVec)3395 return SrcEC == DstEC;3396 if (SrcIsVec)3397 return SrcEC == ElementCount::getFixed(1);3398 if (DstIsVec)3399 return DstEC == ElementCount::getFixed(1);3400 3401 return true;3402 }3403 case Instruction::AddrSpaceCast: {3404 PointerType *SrcPtrTy = dyn_cast<PointerType>(SrcTy->getScalarType());3405 if (!SrcPtrTy)3406 return false;3407 3408 PointerType *DstPtrTy = dyn_cast<PointerType>(DstTy->getScalarType());3409 if (!DstPtrTy)3410 return false;3411 3412 if (SrcPtrTy->getAddressSpace() == DstPtrTy->getAddressSpace())3413 return false;3414 3415 return SrcEC == DstEC;3416 }3417 }3418}3419 3420TruncInst::TruncInst(Value *S, Type *Ty, const Twine &Name,3421 InsertPosition InsertBefore)3422 : CastInst(Ty, Trunc, S, Name, InsertBefore) {3423 assert(castIsValid(getOpcode(), S, Ty) && "Illegal Trunc");3424}3425 3426ZExtInst::ZExtInst(Value *S, Type *Ty, const Twine &Name,3427 InsertPosition InsertBefore)3428 : CastInst(Ty, ZExt, S, Name, InsertBefore) {3429 assert(castIsValid(getOpcode(), S, Ty) && "Illegal ZExt");3430}3431 3432SExtInst::SExtInst(Value *S, Type *Ty, const Twine &Name,3433 InsertPosition InsertBefore)3434 : CastInst(Ty, SExt, S, Name, InsertBefore) {3435 assert(castIsValid(getOpcode(), S, Ty) && "Illegal SExt");3436}3437 3438FPTruncInst::FPTruncInst(Value *S, Type *Ty, const Twine &Name,3439 InsertPosition InsertBefore)3440 : CastInst(Ty, FPTrunc, S, Name, InsertBefore) {3441 assert(castIsValid(getOpcode(), S, Ty) && "Illegal FPTrunc");3442}3443 3444FPExtInst::FPExtInst(Value *S, Type *Ty, const Twine &Name,3445 InsertPosition InsertBefore)3446 : CastInst(Ty, FPExt, S, Name, InsertBefore) {3447 assert(castIsValid(getOpcode(), S, Ty) && "Illegal FPExt");3448}3449 3450UIToFPInst::UIToFPInst(Value *S, Type *Ty, const Twine &Name,3451 InsertPosition InsertBefore)3452 : CastInst(Ty, UIToFP, S, Name, InsertBefore) {3453 assert(castIsValid(getOpcode(), S, Ty) && "Illegal UIToFP");3454}3455 3456SIToFPInst::SIToFPInst(Value *S, Type *Ty, const Twine &Name,3457 InsertPosition InsertBefore)3458 : CastInst(Ty, SIToFP, S, Name, InsertBefore) {3459 assert(castIsValid(getOpcode(), S, Ty) && "Illegal SIToFP");3460}3461 3462FPToUIInst::FPToUIInst(Value *S, Type *Ty, const Twine &Name,3463 InsertPosition InsertBefore)3464 : CastInst(Ty, FPToUI, S, Name, InsertBefore) {3465 assert(castIsValid(getOpcode(), S, Ty) && "Illegal FPToUI");3466}3467 3468FPToSIInst::FPToSIInst(Value *S, Type *Ty, const Twine &Name,3469 InsertPosition InsertBefore)3470 : CastInst(Ty, FPToSI, S, Name, InsertBefore) {3471 assert(castIsValid(getOpcode(), S, Ty) && "Illegal FPToSI");3472}3473 3474PtrToIntInst::PtrToIntInst(Value *S, Type *Ty, const Twine &Name,3475 InsertPosition InsertBefore)3476 : CastInst(Ty, PtrToInt, S, Name, InsertBefore) {3477 assert(castIsValid(getOpcode(), S, Ty) && "Illegal PtrToInt");3478}3479 3480PtrToAddrInst::PtrToAddrInst(Value *S, Type *Ty, const Twine &Name,3481 InsertPosition InsertBefore)3482 : CastInst(Ty, PtrToAddr, S, Name, InsertBefore) {3483 assert(castIsValid(getOpcode(), S, Ty) && "Illegal PtrToAddr");3484}3485 3486IntToPtrInst::IntToPtrInst(Value *S, Type *Ty, const Twine &Name,3487 InsertPosition InsertBefore)3488 : CastInst(Ty, IntToPtr, S, Name, InsertBefore) {3489 assert(castIsValid(getOpcode(), S, Ty) && "Illegal IntToPtr");3490}3491 3492BitCastInst::BitCastInst(Value *S, Type *Ty, const Twine &Name,3493 InsertPosition InsertBefore)3494 : CastInst(Ty, BitCast, S, Name, InsertBefore) {3495 assert(castIsValid(getOpcode(), S, Ty) && "Illegal BitCast");3496}3497 3498AddrSpaceCastInst::AddrSpaceCastInst(Value *S, Type *Ty, const Twine &Name,3499 InsertPosition InsertBefore)3500 : CastInst(Ty, AddrSpaceCast, S, Name, InsertBefore) {3501 assert(castIsValid(getOpcode(), S, Ty) && "Illegal AddrSpaceCast");3502}3503 3504//===----------------------------------------------------------------------===//3505// CmpInst Classes3506//===----------------------------------------------------------------------===//3507 3508CmpInst::CmpInst(Type *ty, OtherOps op, Predicate predicate, Value *LHS,3509 Value *RHS, const Twine &Name, InsertPosition InsertBefore,3510 Instruction *FlagsSource)3511 : Instruction(ty, op, AllocMarker, InsertBefore) {3512 Op<0>() = LHS;3513 Op<1>() = RHS;3514 setPredicate(predicate);3515 setName(Name);3516 if (FlagsSource)3517 copyIRFlags(FlagsSource);3518}3519 3520CmpInst *CmpInst::Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2,3521 const Twine &Name, InsertPosition InsertBefore) {3522 if (Op == Instruction::ICmp) {3523 if (InsertBefore.isValid())3524 return new ICmpInst(InsertBefore, CmpInst::Predicate(predicate),3525 S1, S2, Name);3526 else3527 return new ICmpInst(CmpInst::Predicate(predicate),3528 S1, S2, Name);3529 }3530 3531 if (InsertBefore.isValid())3532 return new FCmpInst(InsertBefore, CmpInst::Predicate(predicate),3533 S1, S2, Name);3534 else3535 return new FCmpInst(CmpInst::Predicate(predicate),3536 S1, S2, Name);3537}3538 3539CmpInst *CmpInst::CreateWithCopiedFlags(OtherOps Op, Predicate Pred, Value *S1,3540 Value *S2,3541 const Instruction *FlagsSource,3542 const Twine &Name,3543 InsertPosition InsertBefore) {3544 CmpInst *Inst = Create(Op, Pred, S1, S2, Name, InsertBefore);3545 Inst->copyIRFlags(FlagsSource);3546 return Inst;3547}3548 3549void CmpInst::swapOperands() {3550 if (ICmpInst *IC = dyn_cast<ICmpInst>(this))3551 IC->swapOperands();3552 else3553 cast<FCmpInst>(this)->swapOperands();3554}3555 3556bool CmpInst::isCommutative() const {3557 if (const ICmpInst *IC = dyn_cast<ICmpInst>(this))3558 return IC->isCommutative();3559 return cast<FCmpInst>(this)->isCommutative();3560}3561 3562bool CmpInst::isEquality(Predicate P) {3563 if (ICmpInst::isIntPredicate(P))3564 return ICmpInst::isEquality(P);3565 if (FCmpInst::isFPPredicate(P))3566 return FCmpInst::isEquality(P);3567 llvm_unreachable("Unsupported predicate kind");3568}3569 3570// Returns true if either operand of CmpInst is a provably non-zero3571// floating-point constant.3572static bool hasNonZeroFPOperands(const CmpInst *Cmp) {3573 auto *LHS = dyn_cast<Constant>(Cmp->getOperand(0));3574 auto *RHS = dyn_cast<Constant>(Cmp->getOperand(1));3575 if (auto *Const = LHS ? LHS : RHS) {3576 using namespace llvm::PatternMatch;3577 return match(Const, m_NonZeroNotDenormalFP());3578 }3579 return false;3580}3581 3582// Floating-point equality is not an equivalence when comparing +0.0 with3583// -0.0, when comparing NaN with another value, or when flushing3584// denormals-to-zero.3585bool CmpInst::isEquivalence(bool Invert) const {3586 switch (Invert ? getInversePredicate() : getPredicate()) {3587 case CmpInst::Predicate::ICMP_EQ:3588 return true;3589 case CmpInst::Predicate::FCMP_UEQ:3590 if (!hasNoNaNs())3591 return false;3592 [[fallthrough]];3593 case CmpInst::Predicate::FCMP_OEQ:3594 return hasNonZeroFPOperands(this);3595 default:3596 return false;3597 }3598}3599 3600CmpInst::Predicate CmpInst::getInversePredicate(Predicate pred) {3601 switch (pred) {3602 default: llvm_unreachable("Unknown cmp predicate!");3603 case ICMP_EQ: return ICMP_NE;3604 case ICMP_NE: return ICMP_EQ;3605 case ICMP_UGT: return ICMP_ULE;3606 case ICMP_ULT: return ICMP_UGE;3607 case ICMP_UGE: return ICMP_ULT;3608 case ICMP_ULE: return ICMP_UGT;3609 case ICMP_SGT: return ICMP_SLE;3610 case ICMP_SLT: return ICMP_SGE;3611 case ICMP_SGE: return ICMP_SLT;3612 case ICMP_SLE: return ICMP_SGT;3613 3614 case FCMP_OEQ: return FCMP_UNE;3615 case FCMP_ONE: return FCMP_UEQ;3616 case FCMP_OGT: return FCMP_ULE;3617 case FCMP_OLT: return FCMP_UGE;3618 case FCMP_OGE: return FCMP_ULT;3619 case FCMP_OLE: return FCMP_UGT;3620 case FCMP_UEQ: return FCMP_ONE;3621 case FCMP_UNE: return FCMP_OEQ;3622 case FCMP_UGT: return FCMP_OLE;3623 case FCMP_ULT: return FCMP_OGE;3624 case FCMP_UGE: return FCMP_OLT;3625 case FCMP_ULE: return FCMP_OGT;3626 case FCMP_ORD: return FCMP_UNO;3627 case FCMP_UNO: return FCMP_ORD;3628 case FCMP_TRUE: return FCMP_FALSE;3629 case FCMP_FALSE: return FCMP_TRUE;3630 }3631}3632 3633StringRef CmpInst::getPredicateName(Predicate Pred) {3634 switch (Pred) {3635 default: return "unknown";3636 case FCmpInst::FCMP_FALSE: return "false";3637 case FCmpInst::FCMP_OEQ: return "oeq";3638 case FCmpInst::FCMP_OGT: return "ogt";3639 case FCmpInst::FCMP_OGE: return "oge";3640 case FCmpInst::FCMP_OLT: return "olt";3641 case FCmpInst::FCMP_OLE: return "ole";3642 case FCmpInst::FCMP_ONE: return "one";3643 case FCmpInst::FCMP_ORD: return "ord";3644 case FCmpInst::FCMP_UNO: return "uno";3645 case FCmpInst::FCMP_UEQ: return "ueq";3646 case FCmpInst::FCMP_UGT: return "ugt";3647 case FCmpInst::FCMP_UGE: return "uge";3648 case FCmpInst::FCMP_ULT: return "ult";3649 case FCmpInst::FCMP_ULE: return "ule";3650 case FCmpInst::FCMP_UNE: return "une";3651 case FCmpInst::FCMP_TRUE: return "true";3652 case ICmpInst::ICMP_EQ: return "eq";3653 case ICmpInst::ICMP_NE: return "ne";3654 case ICmpInst::ICMP_SGT: return "sgt";3655 case ICmpInst::ICMP_SGE: return "sge";3656 case ICmpInst::ICMP_SLT: return "slt";3657 case ICmpInst::ICMP_SLE: return "sle";3658 case ICmpInst::ICMP_UGT: return "ugt";3659 case ICmpInst::ICMP_UGE: return "uge";3660 case ICmpInst::ICMP_ULT: return "ult";3661 case ICmpInst::ICMP_ULE: return "ule";3662 }3663}3664 3665raw_ostream &llvm::operator<<(raw_ostream &OS, CmpInst::Predicate Pred) {3666 OS << CmpInst::getPredicateName(Pred);3667 return OS;3668}3669 3670ICmpInst::Predicate ICmpInst::getSignedPredicate(Predicate pred) {3671 switch (pred) {3672 default: llvm_unreachable("Unknown icmp predicate!");3673 case ICMP_EQ: case ICMP_NE:3674 case ICMP_SGT: case ICMP_SLT: case ICMP_SGE: case ICMP_SLE:3675 return pred;3676 case ICMP_UGT: return ICMP_SGT;3677 case ICMP_ULT: return ICMP_SLT;3678 case ICMP_UGE: return ICMP_SGE;3679 case ICMP_ULE: return ICMP_SLE;3680 }3681}3682 3683ICmpInst::Predicate ICmpInst::getUnsignedPredicate(Predicate pred) {3684 switch (pred) {3685 default: llvm_unreachable("Unknown icmp predicate!");3686 case ICMP_EQ: case ICMP_NE:3687 case ICMP_UGT: case ICMP_ULT: case ICMP_UGE: case ICMP_ULE:3688 return pred;3689 case ICMP_SGT: return ICMP_UGT;3690 case ICMP_SLT: return ICMP_ULT;3691 case ICMP_SGE: return ICMP_UGE;3692 case ICMP_SLE: return ICMP_ULE;3693 }3694}3695 3696CmpInst::Predicate CmpInst::getSwappedPredicate(Predicate pred) {3697 switch (pred) {3698 default: llvm_unreachable("Unknown cmp predicate!");3699 case ICMP_EQ: case ICMP_NE:3700 return pred;3701 case ICMP_SGT: return ICMP_SLT;3702 case ICMP_SLT: return ICMP_SGT;3703 case ICMP_SGE: return ICMP_SLE;3704 case ICMP_SLE: return ICMP_SGE;3705 case ICMP_UGT: return ICMP_ULT;3706 case ICMP_ULT: return ICMP_UGT;3707 case ICMP_UGE: return ICMP_ULE;3708 case ICMP_ULE: return ICMP_UGE;3709 3710 case FCMP_FALSE: case FCMP_TRUE:3711 case FCMP_OEQ: case FCMP_ONE:3712 case FCMP_UEQ: case FCMP_UNE:3713 case FCMP_ORD: case FCMP_UNO:3714 return pred;3715 case FCMP_OGT: return FCMP_OLT;3716 case FCMP_OLT: return FCMP_OGT;3717 case FCMP_OGE: return FCMP_OLE;3718 case FCMP_OLE: return FCMP_OGE;3719 case FCMP_UGT: return FCMP_ULT;3720 case FCMP_ULT: return FCMP_UGT;3721 case FCMP_UGE: return FCMP_ULE;3722 case FCMP_ULE: return FCMP_UGE;3723 }3724}3725 3726bool CmpInst::isNonStrictPredicate(Predicate pred) {3727 switch (pred) {3728 case ICMP_SGE:3729 case ICMP_SLE:3730 case ICMP_UGE:3731 case ICMP_ULE:3732 case FCMP_OGE:3733 case FCMP_OLE:3734 case FCMP_UGE:3735 case FCMP_ULE:3736 return true;3737 default:3738 return false;3739 }3740}3741 3742bool CmpInst::isStrictPredicate(Predicate pred) {3743 switch (pred) {3744 case ICMP_SGT:3745 case ICMP_SLT:3746 case ICMP_UGT:3747 case ICMP_ULT:3748 case FCMP_OGT:3749 case FCMP_OLT:3750 case FCMP_UGT:3751 case FCMP_ULT:3752 return true;3753 default:3754 return false;3755 }3756}3757 3758CmpInst::Predicate CmpInst::getStrictPredicate(Predicate pred) {3759 switch (pred) {3760 case ICMP_SGE:3761 return ICMP_SGT;3762 case ICMP_SLE:3763 return ICMP_SLT;3764 case ICMP_UGE:3765 return ICMP_UGT;3766 case ICMP_ULE:3767 return ICMP_ULT;3768 case FCMP_OGE:3769 return FCMP_OGT;3770 case FCMP_OLE:3771 return FCMP_OLT;3772 case FCMP_UGE:3773 return FCMP_UGT;3774 case FCMP_ULE:3775 return FCMP_ULT;3776 default:3777 return pred;3778 }3779}3780 3781CmpInst::Predicate CmpInst::getNonStrictPredicate(Predicate pred) {3782 switch (pred) {3783 case ICMP_SGT:3784 return ICMP_SGE;3785 case ICMP_SLT:3786 return ICMP_SLE;3787 case ICMP_UGT:3788 return ICMP_UGE;3789 case ICMP_ULT:3790 return ICMP_ULE;3791 case FCMP_OGT:3792 return FCMP_OGE;3793 case FCMP_OLT:3794 return FCMP_OLE;3795 case FCMP_UGT:3796 return FCMP_UGE;3797 case FCMP_ULT:3798 return FCMP_ULE;3799 default:3800 return pred;3801 }3802}3803 3804CmpInst::Predicate CmpInst::getFlippedStrictnessPredicate(Predicate pred) {3805 assert(CmpInst::isRelational(pred) && "Call only with relational predicate!");3806 3807 if (isStrictPredicate(pred))3808 return getNonStrictPredicate(pred);3809 if (isNonStrictPredicate(pred))3810 return getStrictPredicate(pred);3811 3812 llvm_unreachable("Unknown predicate!");3813}3814 3815bool CmpInst::isUnsigned(Predicate predicate) {3816 switch (predicate) {3817 default: return false;3818 case ICmpInst::ICMP_ULT: case ICmpInst::ICMP_ULE: case ICmpInst::ICMP_UGT:3819 case ICmpInst::ICMP_UGE: return true;3820 }3821}3822 3823bool CmpInst::isSigned(Predicate predicate) {3824 switch (predicate) {3825 default: return false;3826 case ICmpInst::ICMP_SLT: case ICmpInst::ICMP_SLE: case ICmpInst::ICMP_SGT:3827 case ICmpInst::ICMP_SGE: return true;3828 }3829}3830 3831bool ICmpInst::compare(const APInt &LHS, const APInt &RHS,3832 ICmpInst::Predicate Pred) {3833 assert(ICmpInst::isIntPredicate(Pred) && "Only for integer predicates!");3834 switch (Pred) {3835 case ICmpInst::Predicate::ICMP_EQ:3836 return LHS.eq(RHS);3837 case ICmpInst::Predicate::ICMP_NE:3838 return LHS.ne(RHS);3839 case ICmpInst::Predicate::ICMP_UGT:3840 return LHS.ugt(RHS);3841 case ICmpInst::Predicate::ICMP_UGE:3842 return LHS.uge(RHS);3843 case ICmpInst::Predicate::ICMP_ULT:3844 return LHS.ult(RHS);3845 case ICmpInst::Predicate::ICMP_ULE:3846 return LHS.ule(RHS);3847 case ICmpInst::Predicate::ICMP_SGT:3848 return LHS.sgt(RHS);3849 case ICmpInst::Predicate::ICMP_SGE:3850 return LHS.sge(RHS);3851 case ICmpInst::Predicate::ICMP_SLT:3852 return LHS.slt(RHS);3853 case ICmpInst::Predicate::ICMP_SLE:3854 return LHS.sle(RHS);3855 default:3856 llvm_unreachable("Unexpected non-integer predicate.");3857 };3858}3859 3860bool FCmpInst::compare(const APFloat &LHS, const APFloat &RHS,3861 FCmpInst::Predicate Pred) {3862 APFloat::cmpResult R = LHS.compare(RHS);3863 switch (Pred) {3864 default:3865 llvm_unreachable("Invalid FCmp Predicate");3866 case FCmpInst::FCMP_FALSE:3867 return false;3868 case FCmpInst::FCMP_TRUE:3869 return true;3870 case FCmpInst::FCMP_UNO:3871 return R == APFloat::cmpUnordered;3872 case FCmpInst::FCMP_ORD:3873 return R != APFloat::cmpUnordered;3874 case FCmpInst::FCMP_UEQ:3875 return R == APFloat::cmpUnordered || R == APFloat::cmpEqual;3876 case FCmpInst::FCMP_OEQ:3877 return R == APFloat::cmpEqual;3878 case FCmpInst::FCMP_UNE:3879 return R != APFloat::cmpEqual;3880 case FCmpInst::FCMP_ONE:3881 return R == APFloat::cmpLessThan || R == APFloat::cmpGreaterThan;3882 case FCmpInst::FCMP_ULT:3883 return R == APFloat::cmpUnordered || R == APFloat::cmpLessThan;3884 case FCmpInst::FCMP_OLT:3885 return R == APFloat::cmpLessThan;3886 case FCmpInst::FCMP_UGT:3887 return R == APFloat::cmpUnordered || R == APFloat::cmpGreaterThan;3888 case FCmpInst::FCMP_OGT:3889 return R == APFloat::cmpGreaterThan;3890 case FCmpInst::FCMP_ULE:3891 return R != APFloat::cmpGreaterThan;3892 case FCmpInst::FCMP_OLE:3893 return R == APFloat::cmpLessThan || R == APFloat::cmpEqual;3894 case FCmpInst::FCMP_UGE:3895 return R != APFloat::cmpLessThan;3896 case FCmpInst::FCMP_OGE:3897 return R == APFloat::cmpGreaterThan || R == APFloat::cmpEqual;3898 }3899}3900 3901std::optional<bool> ICmpInst::compare(const KnownBits &LHS,3902 const KnownBits &RHS,3903 ICmpInst::Predicate Pred) {3904 switch (Pred) {3905 case ICmpInst::ICMP_EQ:3906 return KnownBits::eq(LHS, RHS);3907 case ICmpInst::ICMP_NE:3908 return KnownBits::ne(LHS, RHS);3909 case ICmpInst::ICMP_UGE:3910 return KnownBits::uge(LHS, RHS);3911 case ICmpInst::ICMP_UGT:3912 return KnownBits::ugt(LHS, RHS);3913 case ICmpInst::ICMP_ULE:3914 return KnownBits::ule(LHS, RHS);3915 case ICmpInst::ICMP_ULT:3916 return KnownBits::ult(LHS, RHS);3917 case ICmpInst::ICMP_SGE:3918 return KnownBits::sge(LHS, RHS);3919 case ICmpInst::ICMP_SGT:3920 return KnownBits::sgt(LHS, RHS);3921 case ICmpInst::ICMP_SLE:3922 return KnownBits::sle(LHS, RHS);3923 case ICmpInst::ICMP_SLT:3924 return KnownBits::slt(LHS, RHS);3925 default:3926 llvm_unreachable("Unexpected non-integer predicate.");3927 }3928}3929 3930CmpInst::Predicate ICmpInst::getFlippedSignednessPredicate(Predicate pred) {3931 if (CmpInst::isEquality(pred))3932 return pred;3933 if (isSigned(pred))3934 return getUnsignedPredicate(pred);3935 if (isUnsigned(pred))3936 return getSignedPredicate(pred);3937 3938 llvm_unreachable("Unknown predicate!");3939}3940 3941bool CmpInst::isOrdered(Predicate predicate) {3942 switch (predicate) {3943 default: return false;3944 case FCmpInst::FCMP_OEQ: case FCmpInst::FCMP_ONE: case FCmpInst::FCMP_OGT:3945 case FCmpInst::FCMP_OLT: case FCmpInst::FCMP_OGE: case FCmpInst::FCMP_OLE:3946 case FCmpInst::FCMP_ORD: return true;3947 }3948}3949 3950bool CmpInst::isUnordered(Predicate predicate) {3951 switch (predicate) {3952 default: return false;3953 case FCmpInst::FCMP_UEQ: case FCmpInst::FCMP_UNE: case FCmpInst::FCMP_UGT:3954 case FCmpInst::FCMP_ULT: case FCmpInst::FCMP_UGE: case FCmpInst::FCMP_ULE:3955 case FCmpInst::FCMP_UNO: return true;3956 }3957}3958 3959bool CmpInst::isTrueWhenEqual(Predicate predicate) {3960 switch(predicate) {3961 default: return false;3962 case ICMP_EQ: case ICMP_UGE: case ICMP_ULE: case ICMP_SGE: case ICMP_SLE:3963 case FCMP_TRUE: case FCMP_UEQ: case FCMP_UGE: case FCMP_ULE: return true;3964 }3965}3966 3967bool CmpInst::isFalseWhenEqual(Predicate predicate) {3968 switch(predicate) {3969 case ICMP_NE: case ICMP_UGT: case ICMP_ULT: case ICMP_SGT: case ICMP_SLT:3970 case FCMP_FALSE: case FCMP_ONE: case FCMP_OGT: case FCMP_OLT: return true;3971 default: return false;3972 }3973}3974 3975static bool isImpliedTrueByMatchingCmp(CmpPredicate Pred1, CmpPredicate Pred2) {3976 // If the predicates match, then we know the first condition implies the3977 // second is true.3978 if (CmpPredicate::getMatching(Pred1, Pred2))3979 return true;3980 3981 if (Pred1.hasSameSign() && CmpInst::isSigned(Pred2))3982 Pred1 = ICmpInst::getFlippedSignednessPredicate(Pred1);3983 else if (Pred2.hasSameSign() && CmpInst::isSigned(Pred1))3984 Pred2 = ICmpInst::getFlippedSignednessPredicate(Pred2);3985 3986 switch (Pred1) {3987 default:3988 break;3989 case CmpInst::ICMP_EQ:3990 // A == B implies A >=u B, A <=u B, A >=s B, and A <=s B are true.3991 return Pred2 == CmpInst::ICMP_UGE || Pred2 == CmpInst::ICMP_ULE ||3992 Pred2 == CmpInst::ICMP_SGE || Pred2 == CmpInst::ICMP_SLE;3993 case CmpInst::ICMP_UGT: // A >u B implies A != B and A >=u B are true.3994 return Pred2 == CmpInst::ICMP_NE || Pred2 == CmpInst::ICMP_UGE;3995 case CmpInst::ICMP_ULT: // A <u B implies A != B and A <=u B are true.3996 return Pred2 == CmpInst::ICMP_NE || Pred2 == CmpInst::ICMP_ULE;3997 case CmpInst::ICMP_SGT: // A >s B implies A != B and A >=s B are true.3998 return Pred2 == CmpInst::ICMP_NE || Pred2 == CmpInst::ICMP_SGE;3999 case CmpInst::ICMP_SLT: // A <s B implies A != B and A <=s B are true.4000 return Pred2 == CmpInst::ICMP_NE || Pred2 == CmpInst::ICMP_SLE;4001 }4002 return false;4003}4004 4005static bool isImpliedFalseByMatchingCmp(CmpPredicate Pred1,4006 CmpPredicate Pred2) {4007 return isImpliedTrueByMatchingCmp(Pred1,4008 ICmpInst::getInverseCmpPredicate(Pred2));4009}4010 4011std::optional<bool> ICmpInst::isImpliedByMatchingCmp(CmpPredicate Pred1,4012 CmpPredicate Pred2) {4013 if (isImpliedTrueByMatchingCmp(Pred1, Pred2))4014 return true;4015 if (isImpliedFalseByMatchingCmp(Pred1, Pred2))4016 return false;4017 return std::nullopt;4018}4019 4020//===----------------------------------------------------------------------===//4021// CmpPredicate Implementation4022//===----------------------------------------------------------------------===//4023 4024std::optional<CmpPredicate> CmpPredicate::getMatching(CmpPredicate A,4025 CmpPredicate B) {4026 if (A.Pred == B.Pred)4027 return A.HasSameSign == B.HasSameSign ? A : CmpPredicate(A.Pred);4028 if (CmpInst::isFPPredicate(A) || CmpInst::isFPPredicate(B))4029 return {};4030 if (A.HasSameSign &&4031 A.Pred == ICmpInst::getFlippedSignednessPredicate(B.Pred))4032 return B.Pred;4033 if (B.HasSameSign &&4034 B.Pred == ICmpInst::getFlippedSignednessPredicate(A.Pred))4035 return A.Pred;4036 return {};4037}4038 4039CmpInst::Predicate CmpPredicate::getPreferredSignedPredicate() const {4040 return HasSameSign ? ICmpInst::getSignedPredicate(Pred) : Pred;4041}4042 4043CmpPredicate CmpPredicate::get(const CmpInst *Cmp) {4044 if (auto *ICI = dyn_cast<ICmpInst>(Cmp))4045 return ICI->getCmpPredicate();4046 return Cmp->getPredicate();4047}4048 4049CmpPredicate CmpPredicate::getSwapped(CmpPredicate P) {4050 return {CmpInst::getSwappedPredicate(P), P.hasSameSign()};4051}4052 4053CmpPredicate CmpPredicate::getSwapped(const CmpInst *Cmp) {4054 return getSwapped(get(Cmp));4055}4056 4057//===----------------------------------------------------------------------===//4058// SwitchInst Implementation4059//===----------------------------------------------------------------------===//4060 4061void SwitchInst::init(Value *Value, BasicBlock *Default, unsigned NumReserved) {4062 assert(Value && Default && NumReserved);4063 ReservedSpace = NumReserved;4064 setNumHungOffUseOperands(2);4065 allocHungoffUses(ReservedSpace);4066 4067 Op<0>() = Value;4068 Op<1>() = Default;4069}4070 4071/// SwitchInst ctor - Create a new switch instruction, specifying a value to4072/// switch on and a default destination. The number of additional cases can4073/// be specified here to make memory allocation more efficient. This4074/// constructor can also autoinsert before another instruction.4075SwitchInst::SwitchInst(Value *Value, BasicBlock *Default, unsigned NumCases,4076 InsertPosition InsertBefore)4077 : Instruction(Type::getVoidTy(Value->getContext()), Instruction::Switch,4078 AllocMarker, InsertBefore) {4079 init(Value, Default, 2+NumCases*2);4080}4081 4082SwitchInst::SwitchInst(const SwitchInst &SI)4083 : Instruction(SI.getType(), Instruction::Switch, AllocMarker) {4084 init(SI.getCondition(), SI.getDefaultDest(), SI.getNumOperands());4085 setNumHungOffUseOperands(SI.getNumOperands());4086 Use *OL = getOperandList();4087 const Use *InOL = SI.getOperandList();4088 for (unsigned i = 2, E = SI.getNumOperands(); i != E; i += 2) {4089 OL[i] = InOL[i];4090 OL[i+1] = InOL[i+1];4091 }4092 SubclassOptionalData = SI.SubclassOptionalData;4093}4094 4095/// addCase - Add an entry to the switch instruction...4096///4097void SwitchInst::addCase(ConstantInt *OnVal, BasicBlock *Dest) {4098 unsigned NewCaseIdx = getNumCases();4099 unsigned OpNo = getNumOperands();4100 if (OpNo+2 > ReservedSpace)4101 growOperands(); // Get more space!4102 // Initialize some new operands.4103 assert(OpNo+1 < ReservedSpace && "Growing didn't work!");4104 setNumHungOffUseOperands(OpNo+2);4105 CaseHandle Case(this, NewCaseIdx);4106 Case.setValue(OnVal);4107 Case.setSuccessor(Dest);4108}4109 4110/// removeCase - This method removes the specified case and its successor4111/// from the switch instruction.4112SwitchInst::CaseIt SwitchInst::removeCase(CaseIt I) {4113 unsigned idx = I->getCaseIndex();4114 4115 assert(2 + idx*2 < getNumOperands() && "Case index out of range!!!");4116 4117 unsigned NumOps = getNumOperands();4118 Use *OL = getOperandList();4119 4120 // Overwrite this case with the end of the list.4121 if (2 + (idx + 1) * 2 != NumOps) {4122 OL[2 + idx * 2] = OL[NumOps - 2];4123 OL[2 + idx * 2 + 1] = OL[NumOps - 1];4124 }4125 4126 // Nuke the last value.4127 OL[NumOps-2].set(nullptr);4128 OL[NumOps-2+1].set(nullptr);4129 setNumHungOffUseOperands(NumOps-2);4130 4131 return CaseIt(this, idx);4132}4133 4134/// growOperands - grow operands - This grows the operand list in response4135/// to a push_back style of operation. This grows the number of ops by 3 times.4136///4137void SwitchInst::growOperands() {4138 unsigned e = getNumOperands();4139 unsigned NumOps = e*3;4140 4141 ReservedSpace = NumOps;4142 growHungoffUses(ReservedSpace);4143}4144 4145void SwitchInstProfUpdateWrapper::init() {4146 MDNode *ProfileData = getBranchWeightMDNode(SI);4147 if (!ProfileData)4148 return;4149 4150 if (getNumBranchWeights(*ProfileData) != SI.getNumSuccessors()) {4151 llvm_unreachable("number of prof branch_weights metadata operands does "4152 "not correspond to number of succesors");4153 }4154 4155 SmallVector<uint32_t, 8> Weights;4156 if (!extractBranchWeights(ProfileData, Weights))4157 return;4158 this->Weights = std::move(Weights);4159}4160 4161SwitchInst::CaseIt4162SwitchInstProfUpdateWrapper::removeCase(SwitchInst::CaseIt I) {4163 if (Weights) {4164 assert(SI.getNumSuccessors() == Weights->size() &&4165 "num of prof branch_weights must accord with num of successors");4166 Changed = true;4167 // Copy the last case to the place of the removed one and shrink.4168 // This is tightly coupled with the way SwitchInst::removeCase() removes4169 // the cases in SwitchInst::removeCase(CaseIt).4170 (*Weights)[I->getCaseIndex() + 1] = Weights->back();4171 Weights->pop_back();4172 }4173 return SI.removeCase(I);4174}4175 4176void SwitchInstProfUpdateWrapper::replaceDefaultDest(SwitchInst::CaseIt I) {4177 auto *DestBlock = I->getCaseSuccessor();4178 if (Weights) {4179 auto Weight = getSuccessorWeight(I->getCaseIndex() + 1);4180 (*Weights)[0] = Weight.value();4181 }4182 4183 SI.setDefaultDest(DestBlock);4184}4185 4186void SwitchInstProfUpdateWrapper::addCase(4187 ConstantInt *OnVal, BasicBlock *Dest,4188 SwitchInstProfUpdateWrapper::CaseWeightOpt W) {4189 SI.addCase(OnVal, Dest);4190 4191 if (!Weights && W && *W) {4192 Changed = true;4193 Weights = SmallVector<uint32_t, 8>(SI.getNumSuccessors(), 0);4194 (*Weights)[SI.getNumSuccessors() - 1] = *W;4195 } else if (Weights) {4196 Changed = true;4197 Weights->push_back(W.value_or(0));4198 }4199 if (Weights)4200 assert(SI.getNumSuccessors() == Weights->size() &&4201 "num of prof branch_weights must accord with num of successors");4202}4203 4204Instruction::InstListType::iterator4205SwitchInstProfUpdateWrapper::eraseFromParent() {4206 // Instruction is erased. Mark as unchanged to not touch it in the destructor.4207 Changed = false;4208 if (Weights)4209 Weights->resize(0);4210 return SI.eraseFromParent();4211}4212 4213SwitchInstProfUpdateWrapper::CaseWeightOpt4214SwitchInstProfUpdateWrapper::getSuccessorWeight(unsigned idx) {4215 if (!Weights)4216 return std::nullopt;4217 return (*Weights)[idx];4218}4219 4220void SwitchInstProfUpdateWrapper::setSuccessorWeight(4221 unsigned idx, SwitchInstProfUpdateWrapper::CaseWeightOpt W) {4222 if (!W)4223 return;4224 4225 if (!Weights && *W)4226 Weights = SmallVector<uint32_t, 8>(SI.getNumSuccessors(), 0);4227 4228 if (Weights) {4229 auto &OldW = (*Weights)[idx];4230 if (*W != OldW) {4231 Changed = true;4232 OldW = *W;4233 }4234 }4235}4236 4237SwitchInstProfUpdateWrapper::CaseWeightOpt4238SwitchInstProfUpdateWrapper::getSuccessorWeight(const SwitchInst &SI,4239 unsigned idx) {4240 if (MDNode *ProfileData = getBranchWeightMDNode(SI))4241 if (ProfileData->getNumOperands() == SI.getNumSuccessors() + 1)4242 return mdconst::extract<ConstantInt>(ProfileData->getOperand(idx + 1))4243 ->getValue()4244 .getZExtValue();4245 4246 return std::nullopt;4247}4248 4249//===----------------------------------------------------------------------===//4250// IndirectBrInst Implementation4251//===----------------------------------------------------------------------===//4252 4253void IndirectBrInst::init(Value *Address, unsigned NumDests) {4254 assert(Address && Address->getType()->isPointerTy() &&4255 "Address of indirectbr must be a pointer");4256 ReservedSpace = 1+NumDests;4257 setNumHungOffUseOperands(1);4258 allocHungoffUses(ReservedSpace);4259 4260 Op<0>() = Address;4261}4262 4263 4264/// growOperands - grow operands - This grows the operand list in response4265/// to a push_back style of operation. This grows the number of ops by 2 times.4266///4267void IndirectBrInst::growOperands() {4268 unsigned e = getNumOperands();4269 unsigned NumOps = e*2;4270 4271 ReservedSpace = NumOps;4272 growHungoffUses(ReservedSpace);4273}4274 4275IndirectBrInst::IndirectBrInst(Value *Address, unsigned NumCases,4276 InsertPosition InsertBefore)4277 : Instruction(Type::getVoidTy(Address->getContext()),4278 Instruction::IndirectBr, AllocMarker, InsertBefore) {4279 init(Address, NumCases);4280}4281 4282IndirectBrInst::IndirectBrInst(const IndirectBrInst &IBI)4283 : Instruction(Type::getVoidTy(IBI.getContext()), Instruction::IndirectBr,4284 AllocMarker) {4285 NumUserOperands = IBI.NumUserOperands;4286 allocHungoffUses(IBI.getNumOperands());4287 Use *OL = getOperandList();4288 const Use *InOL = IBI.getOperandList();4289 for (unsigned i = 0, E = IBI.getNumOperands(); i != E; ++i)4290 OL[i] = InOL[i];4291 SubclassOptionalData = IBI.SubclassOptionalData;4292}4293 4294/// addDestination - Add a destination.4295///4296void IndirectBrInst::addDestination(BasicBlock *DestBB) {4297 unsigned OpNo = getNumOperands();4298 if (OpNo+1 > ReservedSpace)4299 growOperands(); // Get more space!4300 // Initialize some new operands.4301 assert(OpNo < ReservedSpace && "Growing didn't work!");4302 setNumHungOffUseOperands(OpNo+1);4303 getOperandList()[OpNo] = DestBB;4304}4305 4306/// removeDestination - This method removes the specified successor from the4307/// indirectbr instruction.4308void IndirectBrInst::removeDestination(unsigned idx) {4309 assert(idx < getNumOperands()-1 && "Successor index out of range!");4310 4311 unsigned NumOps = getNumOperands();4312 Use *OL = getOperandList();4313 4314 // Replace this value with the last one.4315 OL[idx+1] = OL[NumOps-1];4316 4317 // Nuke the last value.4318 OL[NumOps-1].set(nullptr);4319 setNumHungOffUseOperands(NumOps-1);4320}4321 4322//===----------------------------------------------------------------------===//4323// FreezeInst Implementation4324//===----------------------------------------------------------------------===//4325 4326FreezeInst::FreezeInst(Value *S, const Twine &Name, InsertPosition InsertBefore)4327 : UnaryInstruction(S->getType(), Freeze, S, InsertBefore) {4328 setName(Name);4329}4330 4331//===----------------------------------------------------------------------===//4332// cloneImpl() implementations4333//===----------------------------------------------------------------------===//4334 4335// Define these methods here so vtables don't get emitted into every translation4336// unit that uses these classes.4337 4338GetElementPtrInst *GetElementPtrInst::cloneImpl() const {4339 IntrusiveOperandsAllocMarker AllocMarker{getNumOperands()};4340 return new (AllocMarker) GetElementPtrInst(*this, AllocMarker);4341}4342 4343UnaryOperator *UnaryOperator::cloneImpl() const {4344 return Create(getOpcode(), Op<0>());4345}4346 4347BinaryOperator *BinaryOperator::cloneImpl() const {4348 return Create(getOpcode(), Op<0>(), Op<1>());4349}4350 4351FCmpInst *FCmpInst::cloneImpl() const {4352 return new FCmpInst(getPredicate(), Op<0>(), Op<1>());4353}4354 4355ICmpInst *ICmpInst::cloneImpl() const {4356 return new ICmpInst(getPredicate(), Op<0>(), Op<1>());4357}4358 4359ExtractValueInst *ExtractValueInst::cloneImpl() const {4360 return new ExtractValueInst(*this);4361}4362 4363InsertValueInst *InsertValueInst::cloneImpl() const {4364 return new InsertValueInst(*this);4365}4366 4367AllocaInst *AllocaInst::cloneImpl() const {4368 AllocaInst *Result = new AllocaInst(getAllocatedType(), getAddressSpace(),4369 getOperand(0), getAlign());4370 Result->setUsedWithInAlloca(isUsedWithInAlloca());4371 Result->setSwiftError(isSwiftError());4372 return Result;4373}4374 4375LoadInst *LoadInst::cloneImpl() const {4376 return new LoadInst(getType(), getOperand(0), Twine(), isVolatile(),4377 getAlign(), getOrdering(), getSyncScopeID());4378}4379 4380StoreInst *StoreInst::cloneImpl() const {4381 return new StoreInst(getOperand(0), getOperand(1), isVolatile(), getAlign(),4382 getOrdering(), getSyncScopeID());4383}4384 4385AtomicCmpXchgInst *AtomicCmpXchgInst::cloneImpl() const {4386 AtomicCmpXchgInst *Result = new AtomicCmpXchgInst(4387 getOperand(0), getOperand(1), getOperand(2), getAlign(),4388 getSuccessOrdering(), getFailureOrdering(), getSyncScopeID());4389 Result->setVolatile(isVolatile());4390 Result->setWeak(isWeak());4391 return Result;4392}4393 4394AtomicRMWInst *AtomicRMWInst::cloneImpl() const {4395 AtomicRMWInst *Result =4396 new AtomicRMWInst(getOperation(), getOperand(0), getOperand(1),4397 getAlign(), getOrdering(), getSyncScopeID());4398 Result->setVolatile(isVolatile());4399 return Result;4400}4401 4402FenceInst *FenceInst::cloneImpl() const {4403 return new FenceInst(getContext(), getOrdering(), getSyncScopeID());4404}4405 4406TruncInst *TruncInst::cloneImpl() const {4407 return new TruncInst(getOperand(0), getType());4408}4409 4410ZExtInst *ZExtInst::cloneImpl() const {4411 return new ZExtInst(getOperand(0), getType());4412}4413 4414SExtInst *SExtInst::cloneImpl() const {4415 return new SExtInst(getOperand(0), getType());4416}4417 4418FPTruncInst *FPTruncInst::cloneImpl() const {4419 return new FPTruncInst(getOperand(0), getType());4420}4421 4422FPExtInst *FPExtInst::cloneImpl() const {4423 return new FPExtInst(getOperand(0), getType());4424}4425 4426UIToFPInst *UIToFPInst::cloneImpl() const {4427 return new UIToFPInst(getOperand(0), getType());4428}4429 4430SIToFPInst *SIToFPInst::cloneImpl() const {4431 return new SIToFPInst(getOperand(0), getType());4432}4433 4434FPToUIInst *FPToUIInst::cloneImpl() const {4435 return new FPToUIInst(getOperand(0), getType());4436}4437 4438FPToSIInst *FPToSIInst::cloneImpl() const {4439 return new FPToSIInst(getOperand(0), getType());4440}4441 4442PtrToIntInst *PtrToIntInst::cloneImpl() const {4443 return new PtrToIntInst(getOperand(0), getType());4444}4445 4446PtrToAddrInst *PtrToAddrInst::cloneImpl() const {4447 return new PtrToAddrInst(getOperand(0), getType());4448}4449 4450IntToPtrInst *IntToPtrInst::cloneImpl() const {4451 return new IntToPtrInst(getOperand(0), getType());4452}4453 4454BitCastInst *BitCastInst::cloneImpl() const {4455 return new BitCastInst(getOperand(0), getType());4456}4457 4458AddrSpaceCastInst *AddrSpaceCastInst::cloneImpl() const {4459 return new AddrSpaceCastInst(getOperand(0), getType());4460}4461 4462CallInst *CallInst::cloneImpl() const {4463 if (hasOperandBundles()) {4464 IntrusiveOperandsAndDescriptorAllocMarker AllocMarker{4465 getNumOperands(),4466 getNumOperandBundles() * unsigned(sizeof(BundleOpInfo))};4467 return new (AllocMarker) CallInst(*this, AllocMarker);4468 }4469 IntrusiveOperandsAllocMarker AllocMarker{getNumOperands()};4470 return new (AllocMarker) CallInst(*this, AllocMarker);4471}4472 4473SelectInst *SelectInst::cloneImpl() const {4474 return SelectInst::Create(getOperand(0), getOperand(1), getOperand(2));4475}4476 4477VAArgInst *VAArgInst::cloneImpl() const {4478 return new VAArgInst(getOperand(0), getType());4479}4480 4481ExtractElementInst *ExtractElementInst::cloneImpl() const {4482 return ExtractElementInst::Create(getOperand(0), getOperand(1));4483}4484 4485InsertElementInst *InsertElementInst::cloneImpl() const {4486 return InsertElementInst::Create(getOperand(0), getOperand(1), getOperand(2));4487}4488 4489ShuffleVectorInst *ShuffleVectorInst::cloneImpl() const {4490 return new ShuffleVectorInst(getOperand(0), getOperand(1), getShuffleMask());4491}4492 4493PHINode *PHINode::cloneImpl() const { return new (AllocMarker) PHINode(*this); }4494 4495LandingPadInst *LandingPadInst::cloneImpl() const {4496 return new LandingPadInst(*this);4497}4498 4499ReturnInst *ReturnInst::cloneImpl() const {4500 IntrusiveOperandsAllocMarker AllocMarker{getNumOperands()};4501 return new (AllocMarker) ReturnInst(*this, AllocMarker);4502}4503 4504BranchInst *BranchInst::cloneImpl() const {4505 IntrusiveOperandsAllocMarker AllocMarker{getNumOperands()};4506 return new (AllocMarker) BranchInst(*this, AllocMarker);4507}4508 4509SwitchInst *SwitchInst::cloneImpl() const { return new SwitchInst(*this); }4510 4511IndirectBrInst *IndirectBrInst::cloneImpl() const {4512 return new IndirectBrInst(*this);4513}4514 4515InvokeInst *InvokeInst::cloneImpl() const {4516 if (hasOperandBundles()) {4517 IntrusiveOperandsAndDescriptorAllocMarker AllocMarker{4518 getNumOperands(),4519 getNumOperandBundles() * unsigned(sizeof(BundleOpInfo))};4520 return new (AllocMarker) InvokeInst(*this, AllocMarker);4521 }4522 IntrusiveOperandsAllocMarker AllocMarker{getNumOperands()};4523 return new (AllocMarker) InvokeInst(*this, AllocMarker);4524}4525 4526CallBrInst *CallBrInst::cloneImpl() const {4527 if (hasOperandBundles()) {4528 IntrusiveOperandsAndDescriptorAllocMarker AllocMarker{4529 getNumOperands(),4530 getNumOperandBundles() * unsigned(sizeof(BundleOpInfo))};4531 return new (AllocMarker) CallBrInst(*this, AllocMarker);4532 }4533 IntrusiveOperandsAllocMarker AllocMarker{getNumOperands()};4534 return new (AllocMarker) CallBrInst(*this, AllocMarker);4535}4536 4537ResumeInst *ResumeInst::cloneImpl() const {4538 return new (AllocMarker) ResumeInst(*this);4539}4540 4541CleanupReturnInst *CleanupReturnInst::cloneImpl() const {4542 IntrusiveOperandsAllocMarker AllocMarker{getNumOperands()};4543 return new (AllocMarker) CleanupReturnInst(*this, AllocMarker);4544}4545 4546CatchReturnInst *CatchReturnInst::cloneImpl() const {4547 return new (AllocMarker) CatchReturnInst(*this);4548}4549 4550CatchSwitchInst *CatchSwitchInst::cloneImpl() const {4551 return new CatchSwitchInst(*this);4552}4553 4554FuncletPadInst *FuncletPadInst::cloneImpl() const {4555 IntrusiveOperandsAllocMarker AllocMarker{getNumOperands()};4556 return new (AllocMarker) FuncletPadInst(*this, AllocMarker);4557}4558 4559UnreachableInst *UnreachableInst::cloneImpl() const {4560 LLVMContext &Context = getContext();4561 return new UnreachableInst(Context);4562}4563 4564bool UnreachableInst::shouldLowerToTrap(bool TrapUnreachable,4565 bool NoTrapAfterNoreturn) const {4566 if (!TrapUnreachable)4567 return false;4568 4569 // We may be able to ignore unreachable behind a noreturn call.4570 if (const CallInst *Call = dyn_cast_or_null<CallInst>(getPrevNode());4571 Call && Call->doesNotReturn()) {4572 if (NoTrapAfterNoreturn)4573 return false;4574 // Do not emit an additional trap instruction.4575 if (Call->isNonContinuableTrap())4576 return false;4577 }4578 4579 if (getFunction()->hasFnAttribute(Attribute::Naked))4580 return false;4581 4582 return true;4583}4584 4585FreezeInst *FreezeInst::cloneImpl() const {4586 return new FreezeInst(getOperand(0));4587}4588