12993 lines · cpp
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This implements routines for translating from LLVM IR into SelectionDAG IR.10//11//===----------------------------------------------------------------------===//12 13#include "SelectionDAGBuilder.h"14#include "SDNodeDbgValue.h"15#include "llvm/ADT/APFloat.h"16#include "llvm/ADT/APInt.h"17#include "llvm/ADT/BitVector.h"18#include "llvm/ADT/STLExtras.h"19#include "llvm/ADT/SmallPtrSet.h"20#include "llvm/ADT/StringExtras.h"21#include "llvm/ADT/StringRef.h"22#include "llvm/ADT/Twine.h"23#include "llvm/Analysis/AliasAnalysis.h"24#include "llvm/Analysis/BranchProbabilityInfo.h"25#include "llvm/Analysis/ConstantFolding.h"26#include "llvm/Analysis/Loads.h"27#include "llvm/Analysis/MemoryLocation.h"28#include "llvm/Analysis/TargetLibraryInfo.h"29#include "llvm/Analysis/TargetTransformInfo.h"30#include "llvm/Analysis/ValueTracking.h"31#include "llvm/Analysis/VectorUtils.h"32#include "llvm/CodeGen/Analysis.h"33#include "llvm/CodeGen/AssignmentTrackingAnalysis.h"34#include "llvm/CodeGen/CodeGenCommonISel.h"35#include "llvm/CodeGen/FunctionLoweringInfo.h"36#include "llvm/CodeGen/GCMetadata.h"37#include "llvm/CodeGen/ISDOpcodes.h"38#include "llvm/CodeGen/MachineBasicBlock.h"39#include "llvm/CodeGen/MachineFrameInfo.h"40#include "llvm/CodeGen/MachineFunction.h"41#include "llvm/CodeGen/MachineInstrBuilder.h"42#include "llvm/CodeGen/MachineInstrBundleIterator.h"43#include "llvm/CodeGen/MachineMemOperand.h"44#include "llvm/CodeGen/MachineModuleInfo.h"45#include "llvm/CodeGen/MachineOperand.h"46#include "llvm/CodeGen/MachineRegisterInfo.h"47#include "llvm/CodeGen/SelectionDAG.h"48#include "llvm/CodeGen/SelectionDAGNodes.h"49#include "llvm/CodeGen/SelectionDAGTargetInfo.h"50#include "llvm/CodeGen/StackMaps.h"51#include "llvm/CodeGen/SwiftErrorValueTracking.h"52#include "llvm/CodeGen/TargetFrameLowering.h"53#include "llvm/CodeGen/TargetInstrInfo.h"54#include "llvm/CodeGen/TargetOpcodes.h"55#include "llvm/CodeGen/TargetRegisterInfo.h"56#include "llvm/CodeGen/TargetSubtargetInfo.h"57#include "llvm/CodeGen/WinEHFuncInfo.h"58#include "llvm/IR/Argument.h"59#include "llvm/IR/Attributes.h"60#include "llvm/IR/BasicBlock.h"61#include "llvm/IR/CFG.h"62#include "llvm/IR/CallingConv.h"63#include "llvm/IR/Constant.h"64#include "llvm/IR/ConstantRange.h"65#include "llvm/IR/Constants.h"66#include "llvm/IR/DataLayout.h"67#include "llvm/IR/DebugInfo.h"68#include "llvm/IR/DebugInfoMetadata.h"69#include "llvm/IR/DerivedTypes.h"70#include "llvm/IR/DiagnosticInfo.h"71#include "llvm/IR/EHPersonalities.h"72#include "llvm/IR/Function.h"73#include "llvm/IR/GetElementPtrTypeIterator.h"74#include "llvm/IR/InlineAsm.h"75#include "llvm/IR/InstrTypes.h"76#include "llvm/IR/Instructions.h"77#include "llvm/IR/IntrinsicInst.h"78#include "llvm/IR/Intrinsics.h"79#include "llvm/IR/IntrinsicsAArch64.h"80#include "llvm/IR/IntrinsicsAMDGPU.h"81#include "llvm/IR/IntrinsicsWebAssembly.h"82#include "llvm/IR/LLVMContext.h"83#include "llvm/IR/MemoryModelRelaxationAnnotations.h"84#include "llvm/IR/Metadata.h"85#include "llvm/IR/Module.h"86#include "llvm/IR/Operator.h"87#include "llvm/IR/PatternMatch.h"88#include "llvm/IR/Statepoint.h"89#include "llvm/IR/Type.h"90#include "llvm/IR/User.h"91#include "llvm/IR/Value.h"92#include "llvm/MC/MCContext.h"93#include "llvm/Support/AtomicOrdering.h"94#include "llvm/Support/Casting.h"95#include "llvm/Support/CommandLine.h"96#include "llvm/Support/Compiler.h"97#include "llvm/Support/Debug.h"98#include "llvm/Support/InstructionCost.h"99#include "llvm/Support/MathExtras.h"100#include "llvm/Support/raw_ostream.h"101#include "llvm/Target/TargetMachine.h"102#include "llvm/Target/TargetOptions.h"103#include "llvm/TargetParser/Triple.h"104#include "llvm/Transforms/Utils/Local.h"105#include <cstddef>106#include <limits>107#include <optional>108#include <tuple>109 110using namespace llvm;111using namespace PatternMatch;112using namespace SwitchCG;113 114#define DEBUG_TYPE "isel"115 116/// LimitFloatPrecision - Generate low-precision inline sequences for117/// some float libcalls (6, 8 or 12 bits).118static unsigned LimitFloatPrecision;119 120static cl::opt<bool>121 InsertAssertAlign("insert-assert-align", cl::init(true),122 cl::desc("Insert the experimental `assertalign` node."),123 cl::ReallyHidden);124 125static cl::opt<unsigned, true>126 LimitFPPrecision("limit-float-precision",127 cl::desc("Generate low-precision inline sequences "128 "for some float libcalls"),129 cl::location(LimitFloatPrecision), cl::Hidden,130 cl::init(0));131 132static cl::opt<unsigned> SwitchPeelThreshold(133 "switch-peel-threshold", cl::Hidden, cl::init(66),134 cl::desc("Set the case probability threshold for peeling the case from a "135 "switch statement. A value greater than 100 will void this "136 "optimization"));137 138// Limit the width of DAG chains. This is important in general to prevent139// DAG-based analysis from blowing up. For example, alias analysis and140// load clustering may not complete in reasonable time. It is difficult to141// recognize and avoid this situation within each individual analysis, and142// future analyses are likely to have the same behavior. Limiting DAG width is143// the safe approach and will be especially important with global DAGs.144//145// MaxParallelChains default is arbitrarily high to avoid affecting146// optimization, but could be lowered to improve compile time. Any ld-ld-st-st147// sequence over this should have been converted to llvm.memcpy by the148// frontend. It is easy to induce this behavior with .ll code such as:149// %buffer = alloca [4096 x i8]150// %data = load [4096 x i8]* %argPtr151// store [4096 x i8] %data, [4096 x i8]* %buffer152static const unsigned MaxParallelChains = 64;153 154static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,155 const SDValue *Parts, unsigned NumParts,156 MVT PartVT, EVT ValueVT, const Value *V,157 SDValue InChain,158 std::optional<CallingConv::ID> CC);159 160/// getCopyFromParts - Create a value that contains the specified legal parts161/// combined into the value they represent. If the parts combine to a type162/// larger than ValueVT then AssertOp can be used to specify whether the extra163/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT164/// (ISD::AssertSext).165static SDValue166getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,167 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,168 SDValue InChain,169 std::optional<CallingConv::ID> CC = std::nullopt,170 std::optional<ISD::NodeType> AssertOp = std::nullopt) {171 // Let the target assemble the parts if it wants to172 const TargetLowering &TLI = DAG.getTargetLoweringInfo();173 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,174 PartVT, ValueVT, CC))175 return Val;176 177 if (ValueVT.isVector())178 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,179 InChain, CC);180 181 assert(NumParts > 0 && "No parts to assemble!");182 SDValue Val = Parts[0];183 184 if (NumParts > 1) {185 // Assemble the value from multiple parts.186 if (ValueVT.isInteger()) {187 unsigned PartBits = PartVT.getSizeInBits();188 unsigned ValueBits = ValueVT.getSizeInBits();189 190 // Assemble the power of 2 part.191 unsigned RoundParts = llvm::bit_floor(NumParts);192 unsigned RoundBits = PartBits * RoundParts;193 EVT RoundVT = RoundBits == ValueBits ?194 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);195 SDValue Lo, Hi;196 197 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);198 199 if (RoundParts > 2) {200 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, PartVT, HalfVT, V,201 InChain);202 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, RoundParts / 2,203 PartVT, HalfVT, V, InChain);204 } else {205 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);206 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);207 }208 209 if (DAG.getDataLayout().isBigEndian())210 std::swap(Lo, Hi);211 212 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);213 214 if (RoundParts < NumParts) {215 // Assemble the trailing non-power-of-2 part.216 unsigned OddParts = NumParts - RoundParts;217 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);218 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,219 OddVT, V, InChain, CC);220 221 // Combine the round and odd parts.222 Lo = Val;223 if (DAG.getDataLayout().isBigEndian())224 std::swap(Lo, Hi);225 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);226 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);227 Hi = DAG.getNode(228 ISD::SHL, DL, TotalVT, Hi,229 DAG.getShiftAmountConstant(Lo.getValueSizeInBits(), TotalVT, DL));230 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);231 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);232 }233 } else if (PartVT.isFloatingPoint()) {234 // FP split into multiple FP parts (for ppcf128)235 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&236 "Unexpected split");237 SDValue Lo, Hi;238 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);239 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);240 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))241 std::swap(Lo, Hi);242 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);243 } else {244 // FP split into integer parts (soft fp)245 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&246 !PartVT.isVector() && "Unexpected split");247 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());248 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V,249 InChain, CC);250 }251 }252 253 // There is now one part, held in Val. Correct it to match ValueVT.254 // PartEVT is the type of the register class that holds the value.255 // ValueVT is the type of the inline asm operation.256 EVT PartEVT = Val.getValueType();257 258 if (PartEVT == ValueVT)259 return Val;260 261 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&262 ValueVT.bitsLT(PartEVT)) {263 // For an FP value in an integer part, we need to truncate to the right264 // width first.265 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());266 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);267 }268 269 // Handle types that have the same size.270 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())271 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);272 273 // Handle types with different sizes.274 if (PartEVT.isInteger() && ValueVT.isInteger()) {275 if (ValueVT.bitsLT(PartEVT)) {276 // For a truncate, see if we have any information to277 // indicate whether the truncated bits will always be278 // zero or sign-extension.279 if (AssertOp)280 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,281 DAG.getValueType(ValueVT));282 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);283 }284 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);285 }286 287 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {288 // FP_ROUND's are always exact here.289 if (ValueVT.bitsLT(Val.getValueType())) {290 291 SDValue NoChange =292 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));293 294 if (DAG.getMachineFunction().getFunction().getAttributes().hasFnAttr(295 llvm::Attribute::StrictFP)) {296 return DAG.getNode(ISD::STRICT_FP_ROUND, DL,297 DAG.getVTList(ValueVT, MVT::Other), InChain, Val,298 NoChange);299 }300 301 return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, NoChange);302 }303 304 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);305 }306 307 // Handle MMX to a narrower integer type by bitcasting MMX to integer and308 // then truncating.309 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&310 ValueVT.bitsLT(PartEVT)) {311 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);312 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);313 }314 315 report_fatal_error("Unknown mismatch in getCopyFromParts!");316}317 318static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,319 const Twine &ErrMsg) {320 const Instruction *I = dyn_cast_or_null<Instruction>(V);321 if (!I)322 return Ctx.emitError(ErrMsg);323 324 if (const CallInst *CI = dyn_cast<CallInst>(I))325 if (CI->isInlineAsm()) {326 return Ctx.diagnose(DiagnosticInfoInlineAsm(327 *CI, ErrMsg + ", possible invalid constraint for vector type"));328 }329 330 return Ctx.emitError(I, ErrMsg);331}332 333/// getCopyFromPartsVector - Create a value that contains the specified legal334/// parts combined into the value they represent. If the parts combine to a335/// type larger than ValueVT then AssertOp can be used to specify whether the336/// extra bits are known to be zero (ISD::AssertZext) or sign extended from337/// ValueVT (ISD::AssertSext).338static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,339 const SDValue *Parts, unsigned NumParts,340 MVT PartVT, EVT ValueVT, const Value *V,341 SDValue InChain,342 std::optional<CallingConv::ID> CallConv) {343 assert(ValueVT.isVector() && "Not a vector value");344 assert(NumParts > 0 && "No parts to assemble!");345 const bool IsABIRegCopy = CallConv.has_value();346 347 const TargetLowering &TLI = DAG.getTargetLoweringInfo();348 SDValue Val = Parts[0];349 350 // Handle a multi-element vector.351 if (NumParts > 1) {352 EVT IntermediateVT;353 MVT RegisterVT;354 unsigned NumIntermediates;355 unsigned NumRegs;356 357 if (IsABIRegCopy) {358 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(359 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,360 NumIntermediates, RegisterVT);361 } else {362 NumRegs =363 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,364 NumIntermediates, RegisterVT);365 }366 367 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");368 NumParts = NumRegs; // Silence a compiler warning.369 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");370 assert(RegisterVT.getSizeInBits() ==371 Parts[0].getSimpleValueType().getSizeInBits() &&372 "Part type sizes don't match!");373 374 // Assemble the parts into intermediate operands.375 SmallVector<SDValue, 8> Ops(NumIntermediates);376 if (NumIntermediates == NumParts) {377 // If the register was not expanded, truncate or copy the value,378 // as appropriate.379 for (unsigned i = 0; i != NumParts; ++i)380 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, PartVT, IntermediateVT,381 V, InChain, CallConv);382 } else if (NumParts > 0) {383 // If the intermediate type was expanded, build the intermediate384 // operands from the parts.385 assert(NumParts % NumIntermediates == 0 &&386 "Must expand into a divisible number of parts!");387 unsigned Factor = NumParts / NumIntermediates;388 for (unsigned i = 0; i != NumIntermediates; ++i)389 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, PartVT,390 IntermediateVT, V, InChain, CallConv);391 }392 393 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the394 // intermediate operands.395 EVT BuiltVectorTy =396 IntermediateVT.isVector()397 ? EVT::getVectorVT(398 *DAG.getContext(), IntermediateVT.getScalarType(),399 IntermediateVT.getVectorElementCount() * NumParts)400 : EVT::getVectorVT(*DAG.getContext(),401 IntermediateVT.getScalarType(),402 NumIntermediates);403 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS404 : ISD::BUILD_VECTOR,405 DL, BuiltVectorTy, Ops);406 }407 408 // There is now one part, held in Val. Correct it to match ValueVT.409 EVT PartEVT = Val.getValueType();410 411 if (PartEVT == ValueVT)412 return Val;413 414 if (PartEVT.isVector()) {415 // Vector/Vector bitcast.416 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())417 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);418 419 // If the parts vector has more elements than the value vector, then we420 // have a vector widening case (e.g. <2 x float> -> <4 x float>).421 // Extract the elements we want.422 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {423 assert((PartEVT.getVectorElementCount().getKnownMinValue() >424 ValueVT.getVectorElementCount().getKnownMinValue()) &&425 (PartEVT.getVectorElementCount().isScalable() ==426 ValueVT.getVectorElementCount().isScalable()) &&427 "Cannot narrow, it would be a lossy transformation");428 PartEVT =429 EVT::getVectorVT(*DAG.getContext(), PartEVT.getVectorElementType(),430 ValueVT.getVectorElementCount());431 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,432 DAG.getVectorIdxConstant(0, DL));433 if (PartEVT == ValueVT)434 return Val;435 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())436 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);437 438 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).439 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())440 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);441 }442 443 // Promoted vector extract444 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);445 }446 447 // Trivial bitcast if the types are the same size and the destination448 // vector type is legal.449 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&450 TLI.isTypeLegal(ValueVT))451 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);452 453 if (ValueVT.getVectorNumElements() != 1) {454 // Certain ABIs require that vectors are passed as integers. For vectors455 // are the same size, this is an obvious bitcast.456 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {457 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);458 } else if (ValueVT.bitsLT(PartEVT)) {459 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();460 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);461 // Drop the extra bits.462 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);463 return DAG.getBitcast(ValueVT, Val);464 }465 466 diagnosePossiblyInvalidConstraint(467 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");468 return DAG.getUNDEF(ValueVT);469 }470 471 // Handle cases such as i8 -> <1 x i1>472 EVT ValueSVT = ValueVT.getVectorElementType();473 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {474 unsigned ValueSize = ValueSVT.getSizeInBits();475 if (ValueSize == PartEVT.getSizeInBits()) {476 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);477 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {478 // It's possible a scalar floating point type gets softened to integer and479 // then promoted to a larger integer. If PartEVT is the larger integer480 // we need to truncate it and then bitcast to the FP type.481 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");482 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);483 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);484 Val = DAG.getBitcast(ValueSVT, Val);485 } else {486 Val = ValueVT.isFloatingPoint()487 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)488 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);489 }490 }491 492 return DAG.getBuildVector(ValueVT, DL, Val);493}494 495static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,496 SDValue Val, SDValue *Parts, unsigned NumParts,497 MVT PartVT, const Value *V,498 std::optional<CallingConv::ID> CallConv);499 500/// getCopyToParts - Create a series of nodes that contain the specified value501/// split into legal parts. If the parts contain more bits than Val, then, for502/// integers, ExtendKind can be used to specify how to generate the extra bits.503static void504getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,505 unsigned NumParts, MVT PartVT, const Value *V,506 std::optional<CallingConv::ID> CallConv = std::nullopt,507 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {508 // Let the target split the parts if it wants to509 const TargetLowering &TLI = DAG.getTargetLoweringInfo();510 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,511 CallConv))512 return;513 EVT ValueVT = Val.getValueType();514 515 // Handle the vector case separately.516 if (ValueVT.isVector())517 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,518 CallConv);519 520 unsigned OrigNumParts = NumParts;521 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&522 "Copying to an illegal type!");523 524 if (NumParts == 0)525 return;526 527 assert(!ValueVT.isVector() && "Vector case handled elsewhere");528 EVT PartEVT = PartVT;529 if (PartEVT == ValueVT) {530 assert(NumParts == 1 && "No-op copy with multiple parts!");531 Parts[0] = Val;532 return;533 }534 535 unsigned PartBits = PartVT.getSizeInBits();536 if (NumParts * PartBits > ValueVT.getSizeInBits()) {537 // If the parts cover more bits than the value has, promote the value.538 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {539 assert(NumParts == 1 && "Do not know what to promote to!");540 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);541 } else {542 if (ValueVT.isFloatingPoint()) {543 // FP values need to be bitcast, then extended if they are being put544 // into a larger container.545 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());546 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);547 }548 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&549 ValueVT.isInteger() &&550 "Unknown mismatch!");551 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);552 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);553 if (PartVT == MVT::x86mmx)554 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);555 }556 } else if (PartBits == ValueVT.getSizeInBits()) {557 // Different types of the same size.558 assert(NumParts == 1 && PartEVT != ValueVT);559 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);560 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {561 // If the parts cover less bits than value has, truncate the value.562 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&563 ValueVT.isInteger() &&564 "Unknown mismatch!");565 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);566 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);567 if (PartVT == MVT::x86mmx)568 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);569 }570 571 // The value may have changed - recompute ValueVT.572 ValueVT = Val.getValueType();573 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&574 "Failed to tile the value with PartVT!");575 576 if (NumParts == 1) {577 if (PartEVT != ValueVT) {578 diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,579 "scalar-to-vector conversion failed");580 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);581 }582 583 Parts[0] = Val;584 return;585 }586 587 // Expand the value into multiple parts.588 if (NumParts & (NumParts - 1)) {589 // The number of parts is not a power of 2. Split off and copy the tail.590 assert(PartVT.isInteger() && ValueVT.isInteger() &&591 "Do not know what to expand to!");592 unsigned RoundParts = llvm::bit_floor(NumParts);593 unsigned RoundBits = RoundParts * PartBits;594 unsigned OddParts = NumParts - RoundParts;595 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,596 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));597 598 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,599 CallConv);600 601 if (DAG.getDataLayout().isBigEndian())602 // The odd parts were reversed by getCopyToParts - unreverse them.603 std::reverse(Parts + RoundParts, Parts + NumParts);604 605 NumParts = RoundParts;606 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);607 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);608 }609 610 // The number of parts is a power of 2. Repeatedly bisect the value using611 // EXTRACT_ELEMENT.612 Parts[0] = DAG.getNode(ISD::BITCAST, DL,613 EVT::getIntegerVT(*DAG.getContext(),614 ValueVT.getSizeInBits()),615 Val);616 617 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {618 for (unsigned i = 0; i < NumParts; i += StepSize) {619 unsigned ThisBits = StepSize * PartBits / 2;620 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);621 SDValue &Part0 = Parts[i];622 SDValue &Part1 = Parts[i+StepSize/2];623 624 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,625 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));626 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,627 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));628 629 if (ThisBits == PartBits && ThisVT != PartVT) {630 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);631 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);632 }633 }634 }635 636 if (DAG.getDataLayout().isBigEndian())637 std::reverse(Parts, Parts + OrigNumParts);638}639 640static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val,641 const SDLoc &DL, EVT PartVT) {642 if (!PartVT.isVector())643 return SDValue();644 645 EVT ValueVT = Val.getValueType();646 EVT PartEVT = PartVT.getVectorElementType();647 EVT ValueEVT = ValueVT.getVectorElementType();648 ElementCount PartNumElts = PartVT.getVectorElementCount();649 ElementCount ValueNumElts = ValueVT.getVectorElementCount();650 651 // We only support widening vectors with equivalent element types and652 // fixed/scalable properties. If a target needs to widen a fixed-length type653 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.654 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||655 PartNumElts.isScalable() != ValueNumElts.isScalable())656 return SDValue();657 658 // Have a try for bf16 because some targets share its ABI with fp16.659 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {660 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&661 "Cannot widen to illegal type");662 Val = DAG.getNode(ISD::BITCAST, DL,663 ValueVT.changeVectorElementType(MVT::f16), Val);664 } else if (PartEVT != ValueEVT) {665 return SDValue();666 }667 668 // Widening a scalable vector to another scalable vector is done by inserting669 // the vector into a larger undef one.670 if (PartNumElts.isScalable())671 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),672 Val, DAG.getVectorIdxConstant(0, DL));673 674 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in675 // undef elements.676 SmallVector<SDValue, 16> Ops;677 DAG.ExtractVectorElements(Val, Ops);678 SDValue EltUndef = DAG.getUNDEF(PartEVT);679 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);680 681 // FIXME: Use CONCAT for 2x -> 4x.682 return DAG.getBuildVector(PartVT, DL, Ops);683}684 685/// getCopyToPartsVector - Create a series of nodes that contain the specified686/// value split into legal parts.687static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,688 SDValue Val, SDValue *Parts, unsigned NumParts,689 MVT PartVT, const Value *V,690 std::optional<CallingConv::ID> CallConv) {691 EVT ValueVT = Val.getValueType();692 assert(ValueVT.isVector() && "Not a vector");693 const TargetLowering &TLI = DAG.getTargetLoweringInfo();694 const bool IsABIRegCopy = CallConv.has_value();695 696 if (NumParts == 1) {697 EVT PartEVT = PartVT;698 if (PartEVT == ValueVT) {699 // Nothing to do.700 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {701 // Bitconvert vector->vector case.702 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);703 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {704 Val = Widened;705 } else if (PartVT.isVector() &&706 PartEVT.getVectorElementType().bitsGE(707 ValueVT.getVectorElementType()) &&708 PartEVT.getVectorElementCount() ==709 ValueVT.getVectorElementCount()) {710 711 // Promoted vector extract712 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);713 } else if (PartEVT.isVector() &&714 PartEVT.getVectorElementType() !=715 ValueVT.getVectorElementType() &&716 TLI.getTypeAction(*DAG.getContext(), ValueVT) ==717 TargetLowering::TypeWidenVector) {718 // Combination of widening and promotion.719 EVT WidenVT =720 EVT::getVectorVT(*DAG.getContext(), ValueVT.getVectorElementType(),721 PartVT.getVectorElementCount());722 SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);723 Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);724 } else {725 // Don't extract an integer from a float vector. This can happen if the726 // FP type gets softened to integer and then promoted. The promotion727 // prevents it from being picked up by the earlier bitcast case.728 if (ValueVT.getVectorElementCount().isScalar() &&729 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {730 // If we reach this condition and PartVT is FP, this means that731 // ValueVT is also FP and both have a different size, otherwise we732 // would have bitcasted them. Producing an EXTRACT_VECTOR_ELT here733 // would be invalid since that would mean the smaller FP type has to734 // be extended to the larger one.735 if (PartVT.isFloatingPoint()) {736 Val = DAG.getBitcast(ValueVT.getScalarType(), Val);737 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);738 } else739 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,740 DAG.getVectorIdxConstant(0, DL));741 } else {742 uint64_t ValueSize = ValueVT.getFixedSizeInBits();743 assert(PartVT.getFixedSizeInBits() > ValueSize &&744 "lossy conversion of vector to scalar type");745 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);746 Val = DAG.getBitcast(IntermediateType, Val);747 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);748 }749 }750 751 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");752 Parts[0] = Val;753 return;754 }755 756 // Handle a multi-element vector.757 EVT IntermediateVT;758 MVT RegisterVT;759 unsigned NumIntermediates;760 unsigned NumRegs;761 if (IsABIRegCopy) {762 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(763 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,764 RegisterVT);765 } else {766 NumRegs =767 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,768 NumIntermediates, RegisterVT);769 }770 771 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");772 NumParts = NumRegs; // Silence a compiler warning.773 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");774 775 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&776 "Mixing scalable and fixed vectors when copying in parts");777 778 std::optional<ElementCount> DestEltCnt;779 780 if (IntermediateVT.isVector())781 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;782 else783 DestEltCnt = ElementCount::getFixed(NumIntermediates);784 785 EVT BuiltVectorTy = EVT::getVectorVT(786 *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);787 788 if (ValueVT == BuiltVectorTy) {789 // Nothing to do.790 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {791 // Bitconvert vector->vector case.792 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);793 } else {794 if (BuiltVectorTy.getVectorElementType().bitsGT(795 ValueVT.getVectorElementType())) {796 // Integer promotion.797 ValueVT = EVT::getVectorVT(*DAG.getContext(),798 BuiltVectorTy.getVectorElementType(),799 ValueVT.getVectorElementCount());800 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);801 }802 803 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {804 Val = Widened;805 }806 }807 808 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");809 810 // Split the vector into intermediate operands.811 SmallVector<SDValue, 8> Ops(NumIntermediates);812 for (unsigned i = 0; i != NumIntermediates; ++i) {813 if (IntermediateVT.isVector()) {814 // This does something sensible for scalable vectors - see the815 // definition of EXTRACT_SUBVECTOR for further details.816 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();817 Ops[i] =818 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,819 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));820 } else {821 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,822 DAG.getVectorIdxConstant(i, DL));823 }824 }825 826 // Split the intermediate operands into legal parts.827 if (NumParts == NumIntermediates) {828 // If the register was not expanded, promote or copy the value,829 // as appropriate.830 for (unsigned i = 0; i != NumParts; ++i)831 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);832 } else if (NumParts > 0) {833 // If the intermediate type was expanded, split each the value into834 // legal parts.835 assert(NumIntermediates != 0 && "division by zero");836 assert(NumParts % NumIntermediates == 0 &&837 "Must expand into a divisible number of parts!");838 unsigned Factor = NumParts / NumIntermediates;839 for (unsigned i = 0; i != NumIntermediates; ++i)840 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,841 CallConv);842 }843}844 845static void failForInvalidBundles(const CallBase &I, StringRef Name,846 ArrayRef<uint32_t> AllowedBundles) {847 if (I.hasOperandBundlesOtherThan(AllowedBundles)) {848 ListSeparator LS;849 std::string Error;850 raw_string_ostream OS(Error);851 for (unsigned i = 0, e = I.getNumOperandBundles(); i != e; ++i) {852 OperandBundleUse U = I.getOperandBundleAt(i);853 if (!is_contained(AllowedBundles, U.getTagID()))854 OS << LS << U.getTagName();855 }856 reportFatalUsageError(857 Twine("cannot lower ", Name)858 .concat(Twine(" with arbitrary operand bundles: ", Error)));859 }860}861 862RegsForValue::RegsForValue(const SmallVector<Register, 4> ®s, MVT regvt,863 EVT valuevt, std::optional<CallingConv::ID> CC)864 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),865 RegCount(1, regs.size()), CallConv(CC) {}866 867RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,868 const DataLayout &DL, Register Reg, Type *Ty,869 std::optional<CallingConv::ID> CC) {870 ComputeValueVTs(TLI, DL, Ty, ValueVTs);871 872 CallConv = CC;873 874 for (EVT ValueVT : ValueVTs) {875 unsigned NumRegs =876 isABIMangled()877 ? TLI.getNumRegistersForCallingConv(Context, *CC, ValueVT)878 : TLI.getNumRegisters(Context, ValueVT);879 MVT RegisterVT =880 isABIMangled()881 ? TLI.getRegisterTypeForCallingConv(Context, *CC, ValueVT)882 : TLI.getRegisterType(Context, ValueVT);883 for (unsigned i = 0; i != NumRegs; ++i)884 Regs.push_back(Reg + i);885 RegVTs.push_back(RegisterVT);886 RegCount.push_back(NumRegs);887 Reg = Reg.id() + NumRegs;888 }889}890 891SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,892 FunctionLoweringInfo &FuncInfo,893 const SDLoc &dl, SDValue &Chain,894 SDValue *Glue, const Value *V) const {895 // A Value with type {} or [0 x %t] needs no registers.896 if (ValueVTs.empty())897 return SDValue();898 899 const TargetLowering &TLI = DAG.getTargetLoweringInfo();900 901 // Assemble the legal parts into the final values.902 SmallVector<SDValue, 4> Values(ValueVTs.size());903 SmallVector<SDValue, 8> Parts;904 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {905 // Copy the legal parts from the registers.906 EVT ValueVT = ValueVTs[Value];907 unsigned NumRegs = RegCount[Value];908 MVT RegisterVT = isABIMangled()909 ? TLI.getRegisterTypeForCallingConv(910 *DAG.getContext(), *CallConv, RegVTs[Value])911 : RegVTs[Value];912 913 Parts.resize(NumRegs);914 for (unsigned i = 0; i != NumRegs; ++i) {915 SDValue P;916 if (!Glue) {917 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);918 } else {919 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue);920 *Glue = P.getValue(2);921 }922 923 Chain = P.getValue(1);924 Parts[i] = P;925 926 // If the source register was virtual and if we know something about it,927 // add an assert node.928 if (!Regs[Part + i].isVirtual() || !RegisterVT.isInteger())929 continue;930 931 const FunctionLoweringInfo::LiveOutInfo *LOI =932 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);933 if (!LOI)934 continue;935 936 unsigned RegSize = RegisterVT.getScalarSizeInBits();937 unsigned NumSignBits = LOI->NumSignBits;938 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();939 940 if (NumZeroBits == RegSize) {941 // The current value is a zero.942 // Explicitly express that as it would be easier for943 // optimizations to kick in.944 Parts[i] = DAG.getConstant(0, dl, RegisterVT);945 continue;946 }947 948 // FIXME: We capture more information than the dag can represent. For949 // now, just use the tightest assertzext/assertsext possible.950 bool isSExt;951 EVT FromVT(MVT::Other);952 if (NumZeroBits) {953 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);954 isSExt = false;955 } else if (NumSignBits > 1) {956 FromVT =957 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);958 isSExt = true;959 } else {960 continue;961 }962 // Add an assertion node.963 assert(FromVT != MVT::Other);964 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,965 RegisterVT, P, DAG.getValueType(FromVT));966 }967 968 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,969 RegisterVT, ValueVT, V, Chain, CallConv);970 Part += NumRegs;971 Parts.clear();972 }973 974 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);975}976 977void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,978 const SDLoc &dl, SDValue &Chain, SDValue *Glue,979 const Value *V,980 ISD::NodeType PreferredExtendType) const {981 const TargetLowering &TLI = DAG.getTargetLoweringInfo();982 ISD::NodeType ExtendKind = PreferredExtendType;983 984 // Get the list of the values's legal parts.985 unsigned NumRegs = Regs.size();986 SmallVector<SDValue, 8> Parts(NumRegs);987 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {988 unsigned NumParts = RegCount[Value];989 990 MVT RegisterVT = isABIMangled()991 ? TLI.getRegisterTypeForCallingConv(992 *DAG.getContext(), *CallConv, RegVTs[Value])993 : RegVTs[Value];994 995 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))996 ExtendKind = ISD::ZERO_EXTEND;997 998 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],999 NumParts, RegisterVT, V, CallConv, ExtendKind);1000 Part += NumParts;1001 }1002 1003 // Copy the parts into the registers.1004 SmallVector<SDValue, 8> Chains(NumRegs);1005 for (unsigned i = 0; i != NumRegs; ++i) {1006 SDValue Part;1007 if (!Glue) {1008 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);1009 } else {1010 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Glue);1011 *Glue = Part.getValue(1);1012 }1013 1014 Chains[i] = Part.getValue(0);1015 }1016 1017 if (NumRegs == 1 || Glue)1018 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is1019 // flagged to it. That is the CopyToReg nodes and the user are considered1020 // a single scheduling unit. If we create a TokenFactor and return it as1021 // chain, then the TokenFactor is both a predecessor (operand) of the1022 // user as well as a successor (the TF operands are flagged to the user).1023 // c1, f1 = CopyToReg1024 // c2, f2 = CopyToReg1025 // c3 = TokenFactor c1, c21026 // ...1027 // = op c3, ..., f21028 Chain = Chains[NumRegs-1];1029 else1030 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);1031}1032 1033void RegsForValue::AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching,1034 unsigned MatchingIdx, const SDLoc &dl,1035 SelectionDAG &DAG,1036 std::vector<SDValue> &Ops) const {1037 const TargetLowering &TLI = DAG.getTargetLoweringInfo();1038 1039 InlineAsm::Flag Flag(Code, Regs.size());1040 if (HasMatching)1041 Flag.setMatchingOp(MatchingIdx);1042 else if (!Regs.empty() && Regs.front().isVirtual()) {1043 // Put the register class of the virtual registers in the flag word. That1044 // way, later passes can recompute register class constraints for inline1045 // assembly as well as normal instructions.1046 // Don't do this for tied operands that can use the regclass information1047 // from the def.1048 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();1049 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());1050 Flag.setRegClass(RC->getID());1051 }1052 1053 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);1054 Ops.push_back(Res);1055 1056 if (Code == InlineAsm::Kind::Clobber) {1057 // Clobbers should always have a 1:1 mapping with registers, and may1058 // reference registers that have illegal (e.g. vector) types. Hence, we1059 // shouldn't try to apply any sort of splitting logic to them.1060 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&1061 "No 1:1 mapping from clobbers to regs?");1062 Register SP = TLI.getStackPointerRegisterToSaveRestore();1063 (void)SP;1064 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {1065 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));1066 assert(1067 (Regs[I] != SP ||1068 DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&1069 "If we clobbered the stack pointer, MFI should know about it.");1070 }1071 return;1072 }1073 1074 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {1075 MVT RegisterVT = RegVTs[Value];1076 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],1077 RegisterVT);1078 for (unsigned i = 0; i != NumRegs; ++i) {1079 assert(Reg < Regs.size() && "Mismatch in # registers expected");1080 Register TheReg = Regs[Reg++];1081 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));1082 }1083 }1084}1085 1086SmallVector<std::pair<Register, TypeSize>, 4>1087RegsForValue::getRegsAndSizes() const {1088 SmallVector<std::pair<Register, TypeSize>, 4> OutVec;1089 unsigned I = 0;1090 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {1091 unsigned RegCount = std::get<0>(CountAndVT);1092 MVT RegisterVT = std::get<1>(CountAndVT);1093 TypeSize RegisterSize = RegisterVT.getSizeInBits();1094 for (unsigned E = I + RegCount; I != E; ++I)1095 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));1096 }1097 return OutVec;1098}1099 1100void SelectionDAGBuilder::init(GCFunctionInfo *gfi, BatchAAResults *aa,1101 AssumptionCache *ac, const TargetLibraryInfo *li,1102 const TargetTransformInfo &TTI) {1103 BatchAA = aa;1104 AC = ac;1105 GFI = gfi;1106 LibInfo = li;1107 Context = DAG.getContext();1108 LPadToCallSiteMap.clear();1109 this->TTI = &TTI;1110 SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());1111 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(1112 *DAG.getMachineFunction().getFunction().getParent());1113}1114 1115void SelectionDAGBuilder::clear() {1116 NodeMap.clear();1117 UnusedArgNodeMap.clear();1118 PendingLoads.clear();1119 PendingExports.clear();1120 PendingConstrainedFP.clear();1121 PendingConstrainedFPStrict.clear();1122 CurInst = nullptr;1123 HasTailCall = false;1124 SDNodeOrder = LowestSDNodeOrder;1125 StatepointLowering.clear();1126}1127 1128void SelectionDAGBuilder::clearDanglingDebugInfo() {1129 DanglingDebugInfoMap.clear();1130}1131 1132// Update DAG root to include dependencies on Pending chains.1133SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {1134 SDValue Root = DAG.getRoot();1135 1136 if (Pending.empty())1137 return Root;1138 1139 // Add current root to PendingChains, unless we already indirectly1140 // depend on it.1141 if (Root.getOpcode() != ISD::EntryToken) {1142 unsigned i = 0, e = Pending.size();1143 for (; i != e; ++i) {1144 assert(Pending[i].getNode()->getNumOperands() > 1);1145 if (Pending[i].getNode()->getOperand(0) == Root)1146 break; // Don't add the root if we already indirectly depend on it.1147 }1148 1149 if (i == e)1150 Pending.push_back(Root);1151 }1152 1153 if (Pending.size() == 1)1154 Root = Pending[0];1155 else1156 Root = DAG.getTokenFactor(getCurSDLoc(), Pending);1157 1158 DAG.setRoot(Root);1159 Pending.clear();1160 return Root;1161}1162 1163SDValue SelectionDAGBuilder::getMemoryRoot() {1164 return updateRoot(PendingLoads);1165}1166 1167SDValue SelectionDAGBuilder::getFPOperationRoot(fp::ExceptionBehavior EB) {1168 // If the new exception behavior differs from that of the pending1169 // ones, chain up them and update the root.1170 switch (EB) {1171 case fp::ExceptionBehavior::ebMayTrap:1172 case fp::ExceptionBehavior::ebIgnore:1173 // Floating-point exceptions produced by such operations are not intended1174 // to be observed, so the sequence of these operations does not need to be1175 // preserved.1176 //1177 // They however must not be mixed with the instructions that have strict1178 // exception behavior. Placing an operation with 'ebIgnore' behavior between1179 // 'ebStrict' operations could distort the observed exception behavior.1180 if (!PendingConstrainedFPStrict.empty()) {1181 assert(PendingConstrainedFP.empty());1182 updateRoot(PendingConstrainedFPStrict);1183 }1184 break;1185 case fp::ExceptionBehavior::ebStrict:1186 // Floating-point exception produced by these operations may be observed, so1187 // they must be correctly chained. If trapping on FP exceptions is1188 // disabled, the exceptions can be observed only by functions that read1189 // exception flags, like 'llvm.get_fpenv' or 'fetestexcept'. It means that1190 // the order of operations is not significant between barriers.1191 //1192 // If trapping is enabled, each operation becomes an implicit observation1193 // point, so the operations must be sequenced according their original1194 // source order.1195 if (!PendingConstrainedFP.empty()) {1196 assert(PendingConstrainedFPStrict.empty());1197 updateRoot(PendingConstrainedFP);1198 }1199 // TODO: Add support for trapping-enabled scenarios.1200 }1201 return DAG.getRoot();1202}1203 1204SDValue SelectionDAGBuilder::getRoot() {1205 // Chain up all pending constrained intrinsics together with all1206 // pending loads, by simply appending them to PendingLoads and1207 // then calling getMemoryRoot().1208 PendingLoads.reserve(PendingLoads.size() +1209 PendingConstrainedFP.size() +1210 PendingConstrainedFPStrict.size());1211 PendingLoads.append(PendingConstrainedFP.begin(),1212 PendingConstrainedFP.end());1213 PendingLoads.append(PendingConstrainedFPStrict.begin(),1214 PendingConstrainedFPStrict.end());1215 PendingConstrainedFP.clear();1216 PendingConstrainedFPStrict.clear();1217 return getMemoryRoot();1218}1219 1220SDValue SelectionDAGBuilder::getControlRoot() {1221 // We need to emit pending fpexcept.strict constrained intrinsics,1222 // so append them to the PendingExports list.1223 PendingExports.append(PendingConstrainedFPStrict.begin(),1224 PendingConstrainedFPStrict.end());1225 PendingConstrainedFPStrict.clear();1226 return updateRoot(PendingExports);1227}1228 1229void SelectionDAGBuilder::handleDebugDeclare(Value *Address,1230 DILocalVariable *Variable,1231 DIExpression *Expression,1232 DebugLoc DL) {1233 assert(Variable && "Missing variable");1234 1235 // Check if address has undef value.1236 if (!Address || isa<UndefValue>(Address) ||1237 (Address->use_empty() && !isa<Argument>(Address))) {1238 LLVM_DEBUG(1239 dbgs()1240 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");1241 return;1242 }1243 1244 bool IsParameter = Variable->isParameter() || isa<Argument>(Address);1245 1246 SDValue &N = NodeMap[Address];1247 if (!N.getNode() && isa<Argument>(Address))1248 // Check unused arguments map.1249 N = UnusedArgNodeMap[Address];1250 SDDbgValue *SDV;1251 if (N.getNode()) {1252 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))1253 Address = BCI->getOperand(0);1254 // Parameters are handled specially.1255 auto *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());1256 if (IsParameter && FINode) {1257 // Byval parameter. We have a frame index at this point.1258 SDV = DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),1259 /*IsIndirect*/ true, DL, SDNodeOrder);1260 } else if (isa<Argument>(Address)) {1261 // Address is an argument, so try to emit its dbg value using1262 // virtual register info from the FuncInfo.ValueMap.1263 EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,1264 FuncArgumentDbgValueKind::Declare, N);1265 return;1266 } else {1267 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),1268 true, DL, SDNodeOrder);1269 }1270 DAG.AddDbgValue(SDV, IsParameter);1271 } else {1272 // If Address is an argument then try to emit its dbg value using1273 // virtual register info from the FuncInfo.ValueMap.1274 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,1275 FuncArgumentDbgValueKind::Declare, N)) {1276 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"1277 << " (could not emit func-arg dbg_value)\n");1278 }1279 }1280}1281 1282void SelectionDAGBuilder::visitDbgInfo(const Instruction &I) {1283 // Add SDDbgValue nodes for any var locs here. Do so before updating1284 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.1285 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {1286 // Add SDDbgValue nodes for any var locs here. Do so before updating1287 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.1288 for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);1289 It != End; ++It) {1290 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);1291 dropDanglingDebugInfo(Var, It->Expr);1292 if (It->Values.isKillLocation(It->Expr)) {1293 handleKillDebugValue(Var, It->Expr, It->DL, SDNodeOrder);1294 continue;1295 }1296 SmallVector<Value *> Values(It->Values.location_ops());1297 if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,1298 It->Values.hasArgList())) {1299 SmallVector<Value *, 4> Vals(It->Values.location_ops());1300 addDanglingDebugInfo(Vals,1301 FnVarLocs->getDILocalVariable(It->VariableID),1302 It->Expr, Vals.size() > 1, It->DL, SDNodeOrder);1303 }1304 }1305 }1306 1307 // We must skip DbgVariableRecords if they've already been processed above as1308 // we have just emitted the debug values resulting from assignment tracking1309 // analysis, making any existing DbgVariableRecords redundant (and probably1310 // less correct). We still need to process DbgLabelRecords. This does sink1311 // DbgLabelRecords to the bottom of the group of debug records. That sholdn't1312 // be important as it does so deterministcally and ordering between1313 // DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR1314 // printing).1315 bool SkipDbgVariableRecords = DAG.getFunctionVarLocs();1316 // Is there is any debug-info attached to this instruction, in the form of1317 // DbgRecord non-instruction debug-info records.1318 for (DbgRecord &DR : I.getDbgRecordRange()) {1319 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(&DR)) {1320 assert(DLR->getLabel() && "Missing label");1321 SDDbgLabel *SDV =1322 DAG.getDbgLabel(DLR->getLabel(), DLR->getDebugLoc(), SDNodeOrder);1323 DAG.AddDbgLabel(SDV);1324 continue;1325 }1326 1327 if (SkipDbgVariableRecords)1328 continue;1329 DbgVariableRecord &DVR = cast<DbgVariableRecord>(DR);1330 DILocalVariable *Variable = DVR.getVariable();1331 DIExpression *Expression = DVR.getExpression();1332 dropDanglingDebugInfo(Variable, Expression);1333 1334 if (DVR.getType() == DbgVariableRecord::LocationType::Declare) {1335 if (FuncInfo.PreprocessedDVRDeclares.contains(&DVR))1336 continue;1337 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DVR1338 << "\n");1339 handleDebugDeclare(DVR.getVariableLocationOp(0), Variable, Expression,1340 DVR.getDebugLoc());1341 continue;1342 }1343 1344 // A DbgVariableRecord with no locations is a kill location.1345 SmallVector<Value *, 4> Values(DVR.location_ops());1346 if (Values.empty()) {1347 handleKillDebugValue(Variable, Expression, DVR.getDebugLoc(),1348 SDNodeOrder);1349 continue;1350 }1351 1352 // A DbgVariableRecord with an undef or absent location is also a kill1353 // location.1354 if (llvm::any_of(Values,1355 [](Value *V) { return !V || isa<UndefValue>(V); })) {1356 handleKillDebugValue(Variable, Expression, DVR.getDebugLoc(),1357 SDNodeOrder);1358 continue;1359 }1360 1361 bool IsVariadic = DVR.hasArgList();1362 if (!handleDebugValue(Values, Variable, Expression, DVR.getDebugLoc(),1363 SDNodeOrder, IsVariadic)) {1364 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,1365 DVR.getDebugLoc(), SDNodeOrder);1366 }1367 }1368}1369 1370void SelectionDAGBuilder::visit(const Instruction &I) {1371 visitDbgInfo(I);1372 1373 // Set up outgoing PHI node register values before emitting the terminator.1374 if (I.isTerminator()) {1375 HandlePHINodesInSuccessorBlocks(I.getParent());1376 }1377 1378 ++SDNodeOrder;1379 CurInst = &I;1380 1381 // Set inserted listener only if required.1382 bool NodeInserted = false;1383 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;1384 MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);1385 MDNode *MMRA = I.getMetadata(LLVMContext::MD_mmra);1386 if (PCSectionsMD || MMRA) {1387 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(1388 DAG, [&](SDNode *) { NodeInserted = true; });1389 }1390 1391 visit(I.getOpcode(), I);1392 1393 if (!I.isTerminator() && !HasTailCall &&1394 !isa<GCStatepointInst>(I)) // statepoints handle their exports internally1395 CopyToExportRegsIfNeeded(&I);1396 1397 // Handle metadata.1398 if (PCSectionsMD || MMRA) {1399 auto It = NodeMap.find(&I);1400 if (It != NodeMap.end()) {1401 if (PCSectionsMD)1402 DAG.addPCSections(It->second.getNode(), PCSectionsMD);1403 if (MMRA)1404 DAG.addMMRAMetadata(It->second.getNode(), MMRA);1405 } else if (NodeInserted) {1406 // This should not happen; if it does, don't let it go unnoticed so we can1407 // fix it. Relevant visit*() function is probably missing a setValue().1408 errs() << "warning: loosing !pcsections and/or !mmra metadata ["1409 << I.getModule()->getName() << "]\n";1410 LLVM_DEBUG(I.dump());1411 assert(false);1412 }1413 }1414 1415 CurInst = nullptr;1416}1417 1418void SelectionDAGBuilder::visitPHI(const PHINode &) {1419 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");1420}1421 1422void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {1423 // Note: this doesn't use InstVisitor, because it has to work with1424 // ConstantExpr's in addition to instructions.1425 switch (Opcode) {1426 default: llvm_unreachable("Unknown instruction type encountered!");1427 // Build the switch statement using the Instruction.def file.1428#define HANDLE_INST(NUM, OPCODE, CLASS) \1429 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;1430#include "llvm/IR/Instruction.def"1431 }1432}1433 1434static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG,1435 DILocalVariable *Variable,1436 DebugLoc DL, unsigned Order,1437 SmallVectorImpl<Value *> &Values,1438 DIExpression *Expression) {1439 // For variadic dbg_values we will now insert poison.1440 // FIXME: We can potentially recover these!1441 SmallVector<SDDbgOperand, 2> Locs;1442 for (const Value *V : Values) {1443 auto *Poison = PoisonValue::get(V->getType());1444 Locs.push_back(SDDbgOperand::fromConst(Poison));1445 }1446 SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},1447 /*IsIndirect=*/false, DL, Order,1448 /*IsVariadic=*/true);1449 DAG.AddDbgValue(SDV, /*isParameter=*/false);1450 return true;1451}1452 1453void SelectionDAGBuilder::addDanglingDebugInfo(SmallVectorImpl<Value *> &Values,1454 DILocalVariable *Var,1455 DIExpression *Expr,1456 bool IsVariadic, DebugLoc DL,1457 unsigned Order) {1458 if (IsVariadic) {1459 handleDanglingVariadicDebugInfo(DAG, Var, DL, Order, Values, Expr);1460 return;1461 }1462 // TODO: Dangling debug info will eventually either be resolved or produce1463 // a poison DBG_VALUE. However in the resolution case, a gap may appear1464 // between the original dbg.value location and its resolved DBG_VALUE,1465 // which we should ideally fill with an extra poison DBG_VALUE.1466 assert(Values.size() == 1);1467 DanglingDebugInfoMap[Values[0]].emplace_back(Var, Expr, DL, Order);1468}1469 1470void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,1471 const DIExpression *Expr) {1472 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {1473 DIVariable *DanglingVariable = DDI.getVariable();1474 DIExpression *DanglingExpr = DDI.getExpression();1475 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {1476 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "1477 << printDDI(nullptr, DDI) << "\n");1478 return true;1479 }1480 return false;1481 };1482 1483 for (auto &DDIMI : DanglingDebugInfoMap) {1484 DanglingDebugInfoVector &DDIV = DDIMI.second;1485 1486 // If debug info is to be dropped, run it through final checks to see1487 // whether it can be salvaged.1488 for (auto &DDI : DDIV)1489 if (isMatchingDbgValue(DDI))1490 salvageUnresolvedDbgValue(DDIMI.first, DDI);1491 1492 erase_if(DDIV, isMatchingDbgValue);1493 }1494}1495 1496// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,1497// generate the debug data structures now that we've seen its definition.1498void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,1499 SDValue Val) {1500 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);1501 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())1502 return;1503 1504 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;1505 for (auto &DDI : DDIV) {1506 DebugLoc DL = DDI.getDebugLoc();1507 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();1508 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();1509 DILocalVariable *Variable = DDI.getVariable();1510 DIExpression *Expr = DDI.getExpression();1511 assert(Variable->isValidLocationForIntrinsic(DL) &&1512 "Expected inlined-at fields to agree");1513 SDDbgValue *SDV;1514 if (Val.getNode()) {1515 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a1516 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if1517 // we couldn't resolve it directly when examining the DbgValue intrinsic1518 // in the first place we should not be more successful here). Unless we1519 // have some test case that prove this to be correct we should avoid1520 // calling EmitFuncArgumentDbgValue here.1521 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,1522 FuncArgumentDbgValueKind::Value, Val)) {1523 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "1524 << printDDI(V, DDI) << "\n");1525 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());1526 // Increase the SDNodeOrder for the DbgValue here to make sure it is1527 // inserted after the definition of Val when emitting the instructions1528 // after ISel. An alternative could be to teach1529 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.1530 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()1531 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "1532 << ValSDNodeOrder << "\n");1533 SDV = getDbgValue(Val, Variable, Expr, DL,1534 std::max(DbgSDNodeOrder, ValSDNodeOrder));1535 DAG.AddDbgValue(SDV, false);1536 } else1537 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "1538 << printDDI(V, DDI)1539 << " in EmitFuncArgumentDbgValue\n");1540 } else {1541 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)1542 << "\n");1543 auto Poison = PoisonValue::get(V->getType());1544 auto SDV =1545 DAG.getConstantDbgValue(Variable, Expr, Poison, DL, DbgSDNodeOrder);1546 DAG.AddDbgValue(SDV, false);1547 }1548 }1549 DDIV.clear();1550}1551 1552void SelectionDAGBuilder::salvageUnresolvedDbgValue(const Value *V,1553 DanglingDebugInfo &DDI) {1554 // TODO: For the variadic implementation, instead of only checking the fail1555 // state of `handleDebugValue`, we need know specifically which values were1556 // invalid, so that we attempt to salvage only those values when processing1557 // a DIArgList.1558 const Value *OrigV = V;1559 DILocalVariable *Var = DDI.getVariable();1560 DIExpression *Expr = DDI.getExpression();1561 DebugLoc DL = DDI.getDebugLoc();1562 unsigned SDOrder = DDI.getSDNodeOrder();1563 1564 // Currently we consider only dbg.value intrinsics -- we tell the salvager1565 // that DW_OP_stack_value is desired.1566 bool StackValue = true;1567 1568 // Can this Value can be encoded without any further work?1569 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false))1570 return;1571 1572 // Attempt to salvage back through as many instructions as possible. Bail if1573 // a non-instruction is seen, such as a constant expression or global1574 // variable. FIXME: Further work could recover those too.1575 while (isa<Instruction>(V)) {1576 const Instruction &VAsInst = *cast<const Instruction>(V);1577 // Temporary "0", awaiting real implementation.1578 SmallVector<uint64_t, 16> Ops;1579 SmallVector<Value *, 4> AdditionalValues;1580 V = salvageDebugInfoImpl(const_cast<Instruction &>(VAsInst),1581 Expr->getNumLocationOperands(), Ops,1582 AdditionalValues);1583 // If we cannot salvage any further, and haven't yet found a suitable debug1584 // expression, bail out.1585 if (!V)1586 break;1587 1588 // TODO: If AdditionalValues isn't empty, then the salvage can only be1589 // represented with a DBG_VALUE_LIST, so we give up. When we have support1590 // here for variadic dbg_values, remove that condition.1591 if (!AdditionalValues.empty())1592 break;1593 1594 // New value and expr now represent this debuginfo.1595 Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);1596 1597 // Some kind of simplification occurred: check whether the operand of the1598 // salvaged debug expression can be encoded in this DAG.1599 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) {1600 LLVM_DEBUG(1601 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"1602 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");1603 return;1604 }1605 }1606 1607 // This was the final opportunity to salvage this debug information, and it1608 // couldn't be done. Place a poison DBG_VALUE at this location to terminate1609 // any earlier variable location.1610 assert(OrigV && "V shouldn't be null");1611 auto *Poison = PoisonValue::get(OrigV->getType());1612 auto *SDV = DAG.getConstantDbgValue(Var, Expr, Poison, DL, SDNodeOrder);1613 DAG.AddDbgValue(SDV, false);1614 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "1615 << printDDI(OrigV, DDI) << "\n");1616}1617 1618void SelectionDAGBuilder::handleKillDebugValue(DILocalVariable *Var,1619 DIExpression *Expr,1620 DebugLoc DbgLoc,1621 unsigned Order) {1622 Value *Poison = PoisonValue::get(Type::getInt1Ty(*Context));1623 DIExpression *NewExpr =1624 const_cast<DIExpression *>(DIExpression::convertToUndefExpression(Expr));1625 handleDebugValue(Poison, Var, NewExpr, DbgLoc, Order,1626 /*IsVariadic*/ false);1627}1628 1629bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,1630 DILocalVariable *Var,1631 DIExpression *Expr, DebugLoc DbgLoc,1632 unsigned Order, bool IsVariadic) {1633 if (Values.empty())1634 return true;1635 1636 // Filter EntryValue locations out early.1637 if (visitEntryValueDbgValue(Values, Var, Expr, DbgLoc))1638 return true;1639 1640 SmallVector<SDDbgOperand> LocationOps;1641 SmallVector<SDNode *> Dependencies;1642 for (const Value *V : Values) {1643 // Constant value.1644 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||1645 isa<ConstantPointerNull>(V)) {1646 LocationOps.emplace_back(SDDbgOperand::fromConst(V));1647 continue;1648 }1649 1650 // Look through IntToPtr constants.1651 if (auto *CE = dyn_cast<ConstantExpr>(V))1652 if (CE->getOpcode() == Instruction::IntToPtr) {1653 LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));1654 continue;1655 }1656 1657 // If the Value is a frame index, we can create a FrameIndex debug value1658 // without relying on the DAG at all.1659 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {1660 auto SI = FuncInfo.StaticAllocaMap.find(AI);1661 if (SI != FuncInfo.StaticAllocaMap.end()) {1662 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));1663 continue;1664 }1665 }1666 1667 // Do not use getValue() in here; we don't want to generate code at1668 // this point if it hasn't been done yet.1669 SDValue N = NodeMap[V];1670 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.1671 N = UnusedArgNodeMap[V];1672 1673 if (N.getNode()) {1674 // Only emit func arg dbg value for non-variadic dbg.values for now.1675 if (!IsVariadic &&1676 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,1677 FuncArgumentDbgValueKind::Value, N))1678 return true;1679 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {1680 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can1681 // describe stack slot locations.1682 //1683 // Consider "int x = 0; int *px = &x;". There are two kinds of1684 // interesting debug values here after optimization:1685 //1686 // dbg.value(i32* %px, !"int *px", !DIExpression()), and1687 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))1688 //1689 // Both describe the direct values of their associated variables.1690 Dependencies.push_back(N.getNode());1691 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));1692 continue;1693 }1694 LocationOps.emplace_back(1695 SDDbgOperand::fromNode(N.getNode(), N.getResNo()));1696 continue;1697 }1698 1699 const TargetLowering &TLI = DAG.getTargetLoweringInfo();1700 // Special rules apply for the first dbg.values of parameter variables in a1701 // function. Identify them by the fact they reference Argument Values, that1702 // they're parameters, and they are parameters of the current function. We1703 // need to let them dangle until they get an SDNode.1704 bool IsParamOfFunc =1705 isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();1706 if (IsParamOfFunc)1707 return false;1708 1709 // The value is not used in this block yet (or it would have an SDNode).1710 // We still want the value to appear for the user if possible -- if it has1711 // an associated VReg, we can refer to that instead.1712 auto VMI = FuncInfo.ValueMap.find(V);1713 if (VMI != FuncInfo.ValueMap.end()) {1714 Register Reg = VMI->second;1715 // If this is a PHI node, it may be split up into several MI PHI nodes1716 // (in FunctionLoweringInfo::set).1717 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,1718 V->getType(), std::nullopt);1719 if (RFV.occupiesMultipleRegs()) {1720 // FIXME: We could potentially support variadic dbg_values here.1721 if (IsVariadic)1722 return false;1723 unsigned Offset = 0;1724 unsigned BitsToDescribe = 0;1725 if (auto VarSize = Var->getSizeInBits())1726 BitsToDescribe = *VarSize;1727 if (auto Fragment = Expr->getFragmentInfo())1728 BitsToDescribe = Fragment->SizeInBits;1729 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {1730 // Bail out if all bits are described already.1731 if (Offset >= BitsToDescribe)1732 break;1733 // TODO: handle scalable vectors.1734 unsigned RegisterSize = RegAndSize.second;1735 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)1736 ? BitsToDescribe - Offset1737 : RegisterSize;1738 auto FragmentExpr = DIExpression::createFragmentExpression(1739 Expr, Offset, FragmentSize);1740 if (!FragmentExpr)1741 continue;1742 SDDbgValue *SDV = DAG.getVRegDbgValue(1743 Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, Order);1744 DAG.AddDbgValue(SDV, false);1745 Offset += RegisterSize;1746 }1747 return true;1748 }1749 // We can use simple vreg locations for variadic dbg_values as well.1750 LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));1751 continue;1752 }1753 // We failed to create a SDDbgOperand for V.1754 return false;1755 }1756 1757 // We have created a SDDbgOperand for each Value in Values.1758 assert(!LocationOps.empty());1759 SDDbgValue *SDV =1760 DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,1761 /*IsIndirect=*/false, DbgLoc, Order, IsVariadic);1762 DAG.AddDbgValue(SDV, /*isParameter=*/false);1763 return true;1764}1765 1766void SelectionDAGBuilder::resolveOrClearDbgInfo() {1767 // Try to fixup any remaining dangling debug info -- and drop it if we can't.1768 for (auto &Pair : DanglingDebugInfoMap)1769 for (auto &DDI : Pair.second)1770 salvageUnresolvedDbgValue(const_cast<Value *>(Pair.first), DDI);1771 clearDanglingDebugInfo();1772}1773 1774/// getCopyFromRegs - If there was virtual register allocated for the value V1775/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.1776SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {1777 DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(V);1778 SDValue Result;1779 1780 if (It != FuncInfo.ValueMap.end()) {1781 Register InReg = It->second;1782 1783 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),1784 DAG.getDataLayout(), InReg, Ty,1785 std::nullopt); // This is not an ABI copy.1786 SDValue Chain = DAG.getEntryNode();1787 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,1788 V);1789 resolveDanglingDebugInfo(V, Result);1790 }1791 1792 return Result;1793}1794 1795/// getValue - Return an SDValue for the given Value.1796SDValue SelectionDAGBuilder::getValue(const Value *V) {1797 // If we already have an SDValue for this value, use it. It's important1798 // to do this first, so that we don't create a CopyFromReg if we already1799 // have a regular SDValue.1800 SDValue &N = NodeMap[V];1801 if (N.getNode()) return N;1802 1803 // If there's a virtual register allocated and initialized for this1804 // value, use it.1805 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))1806 return copyFromReg;1807 1808 // Otherwise create a new SDValue and remember it.1809 SDValue Val = getValueImpl(V);1810 NodeMap[V] = Val;1811 resolveDanglingDebugInfo(V, Val);1812 return Val;1813}1814 1815/// getNonRegisterValue - Return an SDValue for the given Value, but1816/// don't look in FuncInfo.ValueMap for a virtual register.1817SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {1818 // If we already have an SDValue for this value, use it.1819 SDValue &N = NodeMap[V];1820 if (N.getNode()) {1821 if (isIntOrFPConstant(N)) {1822 // Remove the debug location from the node as the node is about to be used1823 // in a location which may differ from the original debug location. This1824 // is relevant to Constant and ConstantFP nodes because they can appear1825 // as constant expressions inside PHI nodes.1826 N->setDebugLoc(DebugLoc());1827 }1828 return N;1829 }1830 1831 // Otherwise create a new SDValue and remember it.1832 SDValue Val = getValueImpl(V);1833 NodeMap[V] = Val;1834 resolveDanglingDebugInfo(V, Val);1835 return Val;1836}1837 1838/// getValueImpl - Helper function for getValue and getNonRegisterValue.1839/// Create an SDValue for the given value.1840SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {1841 const TargetLowering &TLI = DAG.getTargetLoweringInfo();1842 1843 if (const Constant *C = dyn_cast<Constant>(V)) {1844 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);1845 1846 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C)) {1847 SDLoc DL = getCurSDLoc();1848 1849 // DAG.getConstant() may attempt to legalise the vector constant which can1850 // significantly change the combines applied to the DAG. To reduce the1851 // divergence when enabling ConstantInt based vectors we try to construct1852 // the DAG in the same way as shufflevector based splats. TODO: The1853 // divergence sometimes leads to better optimisations. Ideally we should1854 // prevent DAG.getConstant() from legalising too early but there are some1855 // degradations preventing this.1856 if (VT.isScalableVector())1857 return DAG.getNode(1858 ISD::SPLAT_VECTOR, DL, VT,1859 DAG.getConstant(CI->getValue(), DL, VT.getVectorElementType()));1860 if (VT.isFixedLengthVector())1861 return DAG.getSplatBuildVector(1862 VT, DL,1863 DAG.getConstant(CI->getValue(), DL, VT.getVectorElementType()));1864 return DAG.getConstant(*CI, DL, VT);1865 }1866 1867 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))1868 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);1869 1870 if (const ConstantPtrAuth *CPA = dyn_cast<ConstantPtrAuth>(C)) {1871 return DAG.getNode(ISD::PtrAuthGlobalAddress, getCurSDLoc(), VT,1872 getValue(CPA->getPointer()), getValue(CPA->getKey()),1873 getValue(CPA->getAddrDiscriminator()),1874 getValue(CPA->getDiscriminator()));1875 }1876 1877 if (isa<ConstantPointerNull>(C))1878 return DAG.getConstant(0, getCurSDLoc(), VT);1879 1880 if (match(C, m_VScale()))1881 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));1882 1883 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))1884 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);1885 1886 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())1887 return isa<PoisonValue>(C) ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);1888 1889 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {1890 visit(CE->getOpcode(), *CE);1891 SDValue N1 = NodeMap[V];1892 assert(N1.getNode() && "visit didn't populate the NodeMap!");1893 return N1;1894 }1895 1896 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {1897 SmallVector<SDValue, 4> Constants;1898 for (const Use &U : C->operands()) {1899 SDNode *Val = getValue(U).getNode();1900 // If the operand is an empty aggregate, there are no values.1901 if (!Val) continue;1902 // Add each leaf value from the operand to the Constants list1903 // to form a flattened list of all the values.1904 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)1905 Constants.push_back(SDValue(Val, i));1906 }1907 1908 return DAG.getMergeValues(Constants, getCurSDLoc());1909 }1910 1911 if (const ConstantDataSequential *CDS =1912 dyn_cast<ConstantDataSequential>(C)) {1913 SmallVector<SDValue, 4> Ops;1914 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i) {1915 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();1916 // Add each leaf value from the operand to the Constants list1917 // to form a flattened list of all the values.1918 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)1919 Ops.push_back(SDValue(Val, i));1920 }1921 1922 if (isa<ArrayType>(CDS->getType()))1923 return DAG.getMergeValues(Ops, getCurSDLoc());1924 return DAG.getBuildVector(VT, getCurSDLoc(), Ops);1925 }1926 1927 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {1928 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&1929 "Unknown struct or array constant!");1930 1931 SmallVector<EVT, 4> ValueVTs;1932 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);1933 unsigned NumElts = ValueVTs.size();1934 if (NumElts == 0)1935 return SDValue(); // empty struct1936 SmallVector<SDValue, 4> Constants(NumElts);1937 for (unsigned i = 0; i != NumElts; ++i) {1938 EVT EltVT = ValueVTs[i];1939 if (isa<UndefValue>(C))1940 Constants[i] = DAG.getUNDEF(EltVT);1941 else if (EltVT.isFloatingPoint())1942 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);1943 else1944 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);1945 }1946 1947 return DAG.getMergeValues(Constants, getCurSDLoc());1948 }1949 1950 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))1951 return DAG.getBlockAddress(BA, VT);1952 1953 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))1954 return getValue(Equiv->getGlobalValue());1955 1956 if (const auto *NC = dyn_cast<NoCFIValue>(C))1957 return getValue(NC->getGlobalValue());1958 1959 if (VT == MVT::aarch64svcount) {1960 assert(C->isNullValue() && "Can only zero this target type!");1961 return DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT,1962 DAG.getConstant(0, getCurSDLoc(), MVT::nxv16i1));1963 }1964 1965 if (VT.isRISCVVectorTuple()) {1966 assert(C->isNullValue() && "Can only zero this target type!");1967 return DAG.getNode(1968 ISD::BITCAST, getCurSDLoc(), VT,1969 DAG.getNode(1970 ISD::SPLAT_VECTOR, getCurSDLoc(),1971 EVT::getVectorVT(*DAG.getContext(), MVT::i8,1972 VT.getSizeInBits().getKnownMinValue() / 8, true),1973 DAG.getConstant(0, getCurSDLoc(), MVT::getIntegerVT(8))));1974 }1975 1976 VectorType *VecTy = cast<VectorType>(V->getType());1977 1978 // Now that we know the number and type of the elements, get that number of1979 // elements into the Ops array based on what kind of constant it is.1980 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {1981 SmallVector<SDValue, 16> Ops;1982 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();1983 for (unsigned i = 0; i != NumElements; ++i)1984 Ops.push_back(getValue(CV->getOperand(i)));1985 1986 return DAG.getBuildVector(VT, getCurSDLoc(), Ops);1987 }1988 1989 if (isa<ConstantAggregateZero>(C)) {1990 EVT EltVT =1991 TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());1992 1993 SDValue Op;1994 if (EltVT.isFloatingPoint())1995 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);1996 else1997 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);1998 1999 return DAG.getSplat(VT, getCurSDLoc(), Op);2000 }2001 2002 llvm_unreachable("Unknown vector constant");2003 }2004 2005 // If this is a static alloca, generate it as the frameindex instead of2006 // computation.2007 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {2008 DenseMap<const AllocaInst*, int>::iterator SI =2009 FuncInfo.StaticAllocaMap.find(AI);2010 if (SI != FuncInfo.StaticAllocaMap.end())2011 return DAG.getFrameIndex(2012 SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));2013 }2014 2015 // If this is an instruction which fast-isel has deferred, select it now.2016 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {2017 Register InReg = FuncInfo.InitializeRegForValue(Inst);2018 2019 std::optional<CallingConv::ID> CallConv;2020 auto *CB = dyn_cast<CallBase>(Inst);2021 if (CB && !CB->isInlineAsm())2022 CallConv = CB->getCallingConv();2023 2024 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,2025 Inst->getType(), CallConv);2026 SDValue Chain = DAG.getEntryNode();2027 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);2028 }2029 2030 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))2031 return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));2032 2033 if (const auto *BB = dyn_cast<BasicBlock>(V))2034 return DAG.getBasicBlock(FuncInfo.getMBB(BB));2035 2036 llvm_unreachable("Can't get register for value!");2037}2038 2039void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {2040 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());2041 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;2042 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;2043 bool IsSEH = isAsynchronousEHPersonality(Pers);2044 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;2045 if (IsSEH) {2046 // For SEH, EHCont Guard needs to know that this catchpad is a target.2047 CatchPadMBB->setIsEHContTarget(true);2048 DAG.getMachineFunction().setHasEHContTarget(true);2049 } else2050 CatchPadMBB->setIsEHScopeEntry();2051 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.2052 if (IsMSVCCXX || IsCoreCLR)2053 CatchPadMBB->setIsEHFuncletEntry();2054}2055 2056void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {2057 // Update machine-CFG edge.2058 MachineBasicBlock *TargetMBB = FuncInfo.getMBB(I.getSuccessor());2059 FuncInfo.MBB->addSuccessor(TargetMBB);2060 2061 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());2062 bool IsSEH = isAsynchronousEHPersonality(Pers);2063 if (IsSEH) {2064 // If this is not a fall-through branch or optimizations are switched off,2065 // emit the branch.2066 if (TargetMBB != NextBlock(FuncInfo.MBB) ||2067 TM.getOptLevel() == CodeGenOptLevel::None)2068 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,2069 getControlRoot(), DAG.getBasicBlock(TargetMBB)));2070 return;2071 }2072 2073 // For non-SEH, EHCont Guard needs to know that this catchret is a target.2074 TargetMBB->setIsEHContTarget(true);2075 DAG.getMachineFunction().setHasEHContTarget(true);2076 2077 // Figure out the funclet membership for the catchret's successor.2078 // This will be used by the FuncletLayout pass to determine how to order the2079 // BB's.2080 // A 'catchret' returns to the outer scope's color.2081 Value *ParentPad = I.getCatchSwitchParentPad();2082 const BasicBlock *SuccessorColor;2083 if (isa<ConstantTokenNone>(ParentPad))2084 SuccessorColor = &FuncInfo.Fn->getEntryBlock();2085 else2086 SuccessorColor = cast<Instruction>(ParentPad)->getParent();2087 assert(SuccessorColor && "No parent funclet for catchret!");2088 MachineBasicBlock *SuccessorColorMBB = FuncInfo.getMBB(SuccessorColor);2089 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");2090 2091 // Create the terminator node.2092 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,2093 getControlRoot(), DAG.getBasicBlock(TargetMBB),2094 DAG.getBasicBlock(SuccessorColorMBB));2095 DAG.setRoot(Ret);2096}2097 2098void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {2099 // Don't emit any special code for the cleanuppad instruction. It just marks2100 // the start of an EH scope/funclet.2101 FuncInfo.MBB->setIsEHScopeEntry();2102 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());2103 if (Pers != EHPersonality::Wasm_CXX) {2104 FuncInfo.MBB->setIsEHFuncletEntry();2105 FuncInfo.MBB->setIsCleanupFuncletEntry();2106 }2107}2108 2109/// When an invoke or a cleanupret unwinds to the next EH pad, there are2110/// many places it could ultimately go. In the IR, we have a single unwind2111/// destination, but in the machine CFG, we enumerate all the possible blocks.2112/// This function skips over imaginary basic blocks that hold catchswitch2113/// instructions, and finds all the "real" machine2114/// basic block destinations. As those destinations may not be successors of2115/// EHPadBB, here we also calculate the edge probability to those destinations.2116/// The passed-in Prob is the edge probability to EHPadBB.2117static void findUnwindDestinations(2118 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,2119 BranchProbability Prob,2120 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>2121 &UnwindDests) {2122 EHPersonality Personality =2123 classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());2124 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;2125 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;2126 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;2127 bool IsSEH = isAsynchronousEHPersonality(Personality);2128 2129 while (EHPadBB) {2130 BasicBlock::const_iterator Pad = EHPadBB->getFirstNonPHIIt();2131 BasicBlock *NewEHPadBB = nullptr;2132 if (isa<LandingPadInst>(Pad)) {2133 // Stop on landingpads. They are not funclets.2134 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);2135 break;2136 } else if (isa<CleanupPadInst>(Pad)) {2137 // Stop on cleanup pads. Cleanups are always funclet entries for all known2138 // personalities except Wasm. And in Wasm this becomes a catch_all(_ref),2139 // which always catches an exception.2140 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);2141 UnwindDests.back().first->setIsEHScopeEntry();2142 // In Wasm, EH scopes are not funclets2143 if (!IsWasmCXX)2144 UnwindDests.back().first->setIsEHFuncletEntry();2145 break;2146 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {2147 // Add the catchpad handlers to the possible destinations.2148 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {2149 UnwindDests.emplace_back(FuncInfo.getMBB(CatchPadBB), Prob);2150 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.2151 if (IsMSVCCXX || IsCoreCLR)2152 UnwindDests.back().first->setIsEHFuncletEntry();2153 if (!IsSEH)2154 UnwindDests.back().first->setIsEHScopeEntry();2155 }2156 NewEHPadBB = CatchSwitch->getUnwindDest();2157 } else {2158 continue;2159 }2160 2161 BranchProbabilityInfo *BPI = FuncInfo.BPI;2162 if (BPI && NewEHPadBB)2163 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);2164 EHPadBB = NewEHPadBB;2165 }2166}2167 2168void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {2169 // Update successor info.2170 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;2171 auto UnwindDest = I.getUnwindDest();2172 BranchProbabilityInfo *BPI = FuncInfo.BPI;2173 BranchProbability UnwindDestProb =2174 (BPI && UnwindDest)2175 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)2176 : BranchProbability::getZero();2177 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);2178 for (auto &UnwindDest : UnwindDests) {2179 UnwindDest.first->setIsEHPad();2180 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);2181 }2182 FuncInfo.MBB->normalizeSuccProbs();2183 2184 // Create the terminator node.2185 MachineBasicBlock *CleanupPadMBB =2186 FuncInfo.getMBB(I.getCleanupPad()->getParent());2187 SDValue Ret = DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other,2188 getControlRoot(), DAG.getBasicBlock(CleanupPadMBB));2189 DAG.setRoot(Ret);2190}2191 2192void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {2193 report_fatal_error("visitCatchSwitch not yet implemented!");2194}2195 2196void SelectionDAGBuilder::visitRet(const ReturnInst &I) {2197 const TargetLowering &TLI = DAG.getTargetLoweringInfo();2198 auto &DL = DAG.getDataLayout();2199 SDValue Chain = getControlRoot();2200 SmallVector<ISD::OutputArg, 8> Outs;2201 SmallVector<SDValue, 8> OutVals;2202 2203 // Calls to @llvm.experimental.deoptimize don't generate a return value, so2204 // lower2205 //2206 // %val = call <ty> @llvm.experimental.deoptimize()2207 // ret <ty> %val2208 //2209 // differently.2210 if (I.getParent()->getTerminatingDeoptimizeCall()) {2211 LowerDeoptimizingReturn();2212 return;2213 }2214 2215 if (!FuncInfo.CanLowerReturn) {2216 Register DemoteReg = FuncInfo.DemoteRegister;2217 2218 // Emit a store of the return value through the virtual register.2219 // Leave Outs empty so that LowerReturn won't try to load return2220 // registers the usual way.2221 MVT PtrValueVT = TLI.getPointerTy(DL, DL.getAllocaAddrSpace());2222 SDValue RetPtr =2223 DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVT);2224 SDValue RetOp = getValue(I.getOperand(0));2225 2226 SmallVector<EVT, 4> ValueVTs, MemVTs;2227 SmallVector<uint64_t, 4> Offsets;2228 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,2229 &Offsets, 0);2230 unsigned NumValues = ValueVTs.size();2231 2232 SmallVector<SDValue, 4> Chains(NumValues);2233 Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());2234 for (unsigned i = 0; i != NumValues; ++i) {2235 // An aggregate return value cannot wrap around the address space, so2236 // offsets to its parts don't wrap either.2237 SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,2238 TypeSize::getFixed(Offsets[i]));2239 2240 SDValue Val = RetOp.getValue(RetOp.getResNo() + i);2241 if (MemVTs[i] != ValueVTs[i])2242 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);2243 Chains[i] = DAG.getStore(2244 Chain, getCurSDLoc(), Val,2245 // FIXME: better loc info would be nice.2246 Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),2247 commonAlignment(BaseAlign, Offsets[i]));2248 }2249 2250 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),2251 MVT::Other, Chains);2252 } else if (I.getNumOperands() != 0) {2253 SmallVector<Type *, 4> Types;2254 ComputeValueTypes(DL, I.getOperand(0)->getType(), Types);2255 unsigned NumValues = Types.size();2256 if (NumValues) {2257 SDValue RetOp = getValue(I.getOperand(0));2258 2259 const Function *F = I.getParent()->getParent();2260 2261 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(2262 I.getOperand(0)->getType(), F->getCallingConv(),2263 /*IsVarArg*/ false, DL);2264 2265 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;2266 if (F->getAttributes().hasRetAttr(Attribute::SExt))2267 ExtendKind = ISD::SIGN_EXTEND;2268 else if (F->getAttributes().hasRetAttr(Attribute::ZExt))2269 ExtendKind = ISD::ZERO_EXTEND;2270 2271 LLVMContext &Context = F->getContext();2272 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);2273 2274 for (unsigned j = 0; j != NumValues; ++j) {2275 EVT VT = TLI.getValueType(DL, Types[j]);2276 2277 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())2278 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);2279 2280 CallingConv::ID CC = F->getCallingConv();2281 2282 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);2283 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);2284 SmallVector<SDValue, 4> Parts(NumParts);2285 getCopyToParts(DAG, getCurSDLoc(),2286 SDValue(RetOp.getNode(), RetOp.getResNo() + j),2287 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);2288 2289 // 'inreg' on function refers to return value2290 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();2291 if (RetInReg)2292 Flags.setInReg();2293 2294 if (I.getOperand(0)->getType()->isPointerTy()) {2295 Flags.setPointer();2296 Flags.setPointerAddrSpace(2297 cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());2298 }2299 2300 if (NeedsRegBlock) {2301 Flags.setInConsecutiveRegs();2302 if (j == NumValues - 1)2303 Flags.setInConsecutiveRegsLast();2304 }2305 2306 // Propagate extension type if any2307 if (ExtendKind == ISD::SIGN_EXTEND)2308 Flags.setSExt();2309 else if (ExtendKind == ISD::ZERO_EXTEND)2310 Flags.setZExt();2311 else if (F->getAttributes().hasRetAttr(Attribute::NoExt))2312 Flags.setNoExt();2313 2314 for (unsigned i = 0; i < NumParts; ++i) {2315 Outs.push_back(ISD::OutputArg(Flags,2316 Parts[i].getValueType().getSimpleVT(),2317 VT, Types[j], 0, 0));2318 OutVals.push_back(Parts[i]);2319 }2320 }2321 }2322 }2323 2324 // Push in swifterror virtual register as the last element of Outs. This makes2325 // sure swifterror virtual register will be returned in the swifterror2326 // physical register.2327 const Function *F = I.getParent()->getParent();2328 if (TLI.supportSwiftError() &&2329 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {2330 assert(SwiftError.getFunctionArg() && "Need a swift error argument");2331 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();2332 Flags.setSwiftError();2333 Outs.push_back(ISD::OutputArg(Flags, /*vt=*/TLI.getPointerTy(DL),2334 /*argvt=*/EVT(TLI.getPointerTy(DL)),2335 PointerType::getUnqual(*DAG.getContext()),2336 /*origidx=*/1, /*partOffs=*/0));2337 // Create SDNode for the swifterror virtual register.2338 OutVals.push_back(2339 DAG.getRegister(SwiftError.getOrCreateVRegUseAt(2340 &I, FuncInfo.MBB, SwiftError.getFunctionArg()),2341 EVT(TLI.getPointerTy(DL))));2342 }2343 2344 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();2345 CallingConv::ID CallConv =2346 DAG.getMachineFunction().getFunction().getCallingConv();2347 Chain = DAG.getTargetLoweringInfo().LowerReturn(2348 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);2349 2350 // Verify that the target's LowerReturn behaved as expected.2351 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&2352 "LowerReturn didn't return a valid chain!");2353 2354 // Update the DAG with the new chain value resulting from return lowering.2355 DAG.setRoot(Chain);2356}2357 2358/// CopyToExportRegsIfNeeded - If the given value has virtual registers2359/// created for it, emit nodes to copy the value into the virtual2360/// registers.2361void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {2362 // Skip empty types2363 if (V->getType()->isEmptyTy())2364 return;2365 2366 DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V);2367 if (VMI != FuncInfo.ValueMap.end()) {2368 assert((!V->use_empty() || isa<CallBrInst>(V)) &&2369 "Unused value assigned virtual registers!");2370 CopyValueToVirtualRegister(V, VMI->second);2371 }2372}2373 2374/// ExportFromCurrentBlock - If this condition isn't known to be exported from2375/// the current basic block, add it to ValueMap now so that we'll get a2376/// CopyTo/FromReg.2377void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {2378 // No need to export constants.2379 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;2380 2381 // Already exported?2382 if (FuncInfo.isExportedInst(V)) return;2383 2384 Register Reg = FuncInfo.InitializeRegForValue(V);2385 CopyValueToVirtualRegister(V, Reg);2386}2387 2388bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,2389 const BasicBlock *FromBB) {2390 // The operands of the setcc have to be in this block. We don't know2391 // how to export them from some other block.2392 if (const Instruction *VI = dyn_cast<Instruction>(V)) {2393 // Can export from current BB.2394 if (VI->getParent() == FromBB)2395 return true;2396 2397 // Is already exported, noop.2398 return FuncInfo.isExportedInst(V);2399 }2400 2401 // If this is an argument, we can export it if the BB is the entry block or2402 // if it is already exported.2403 if (isa<Argument>(V)) {2404 if (FromBB->isEntryBlock())2405 return true;2406 2407 // Otherwise, can only export this if it is already exported.2408 return FuncInfo.isExportedInst(V);2409 }2410 2411 // Otherwise, constants can always be exported.2412 return true;2413}2414 2415/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.2416BranchProbability2417SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,2418 const MachineBasicBlock *Dst) const {2419 BranchProbabilityInfo *BPI = FuncInfo.BPI;2420 const BasicBlock *SrcBB = Src->getBasicBlock();2421 const BasicBlock *DstBB = Dst->getBasicBlock();2422 if (!BPI) {2423 // If BPI is not available, set the default probability as 1 / N, where N is2424 // the number of successors.2425 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);2426 return BranchProbability(1, SuccSize);2427 }2428 return BPI->getEdgeProbability(SrcBB, DstBB);2429}2430 2431void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,2432 MachineBasicBlock *Dst,2433 BranchProbability Prob) {2434 if (!FuncInfo.BPI)2435 Src->addSuccessorWithoutProb(Dst);2436 else {2437 if (Prob.isUnknown())2438 Prob = getEdgeProbability(Src, Dst);2439 Src->addSuccessor(Dst, Prob);2440 }2441}2442 2443static bool InBlock(const Value *V, const BasicBlock *BB) {2444 if (const Instruction *I = dyn_cast<Instruction>(V))2445 return I->getParent() == BB;2446 return true;2447}2448 2449/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.2450/// This function emits a branch and is used at the leaves of an OR or an2451/// AND operator tree.2452void2453SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,2454 MachineBasicBlock *TBB,2455 MachineBasicBlock *FBB,2456 MachineBasicBlock *CurBB,2457 MachineBasicBlock *SwitchBB,2458 BranchProbability TProb,2459 BranchProbability FProb,2460 bool InvertCond) {2461 const BasicBlock *BB = CurBB->getBasicBlock();2462 2463 // If the leaf of the tree is a comparison, merge the condition into2464 // the caseblock.2465 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {2466 // The operands of the cmp have to be in this block. We don't know2467 // how to export them from some other block. If this is the first block2468 // of the sequence, no exporting is needed.2469 if (CurBB == SwitchBB ||2470 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&2471 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {2472 ISD::CondCode Condition;2473 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {2474 ICmpInst::Predicate Pred =2475 InvertCond ? IC->getInversePredicate() : IC->getPredicate();2476 Condition = getICmpCondCode(Pred);2477 } else {2478 const FCmpInst *FC = cast<FCmpInst>(Cond);2479 FCmpInst::Predicate Pred =2480 InvertCond ? FC->getInversePredicate() : FC->getPredicate();2481 Condition = getFCmpCondCode(Pred);2482 if (TM.Options.NoNaNsFPMath)2483 Condition = getFCmpCodeWithoutNaN(Condition);2484 }2485 2486 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,2487 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);2488 SL->SwitchCases.push_back(CB);2489 return;2490 }2491 }2492 2493 // Create a CaseBlock record representing this branch.2494 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;2495 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),2496 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);2497 SL->SwitchCases.push_back(CB);2498}2499 2500// Collect dependencies on V recursively. This is used for the cost analysis in2501// `shouldKeepJumpConditionsTogether`.2502static bool collectInstructionDeps(2503 SmallMapVector<const Instruction *, bool, 8> *Deps, const Value *V,2504 SmallMapVector<const Instruction *, bool, 8> *Necessary = nullptr,2505 unsigned Depth = 0) {2506 // Return false if we have an incomplete count.2507 if (Depth >= SelectionDAG::MaxRecursionDepth)2508 return false;2509 2510 auto *I = dyn_cast<Instruction>(V);2511 if (I == nullptr)2512 return true;2513 2514 if (Necessary != nullptr) {2515 // This instruction is necessary for the other side of the condition so2516 // don't count it.2517 if (Necessary->contains(I))2518 return true;2519 }2520 2521 // Already added this dep.2522 if (!Deps->try_emplace(I, false).second)2523 return true;2524 2525 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)2526 if (!collectInstructionDeps(Deps, I->getOperand(OpIdx), Necessary,2527 Depth + 1))2528 return false;2529 return true;2530}2531 2532bool SelectionDAGBuilder::shouldKeepJumpConditionsTogether(2533 const FunctionLoweringInfo &FuncInfo, const BranchInst &I,2534 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,2535 TargetLoweringBase::CondMergingParams Params) const {2536 if (I.getNumSuccessors() != 2)2537 return false;2538 2539 if (!I.isConditional())2540 return false;2541 2542 if (Params.BaseCost < 0)2543 return false;2544 2545 // Baseline cost.2546 InstructionCost CostThresh = Params.BaseCost;2547 2548 BranchProbabilityInfo *BPI = nullptr;2549 if (Params.LikelyBias || Params.UnlikelyBias)2550 BPI = FuncInfo.BPI;2551 if (BPI != nullptr) {2552 // See if we are either likely to get an early out or compute both lhs/rhs2553 // of the condition.2554 BasicBlock *IfFalse = I.getSuccessor(0);2555 BasicBlock *IfTrue = I.getSuccessor(1);2556 2557 std::optional<bool> Likely;2558 if (BPI->isEdgeHot(I.getParent(), IfTrue))2559 Likely = true;2560 else if (BPI->isEdgeHot(I.getParent(), IfFalse))2561 Likely = false;2562 2563 if (Likely) {2564 if (Opc == (*Likely ? Instruction::And : Instruction::Or))2565 // Its likely we will have to compute both lhs and rhs of condition2566 CostThresh += Params.LikelyBias;2567 else {2568 if (Params.UnlikelyBias < 0)2569 return false;2570 // Its likely we will get an early out.2571 CostThresh -= Params.UnlikelyBias;2572 }2573 }2574 }2575 2576 if (CostThresh <= 0)2577 return false;2578 2579 // Collect "all" instructions that lhs condition is dependent on.2580 // Use map for stable iteration (to avoid non-determanism of iteration of2581 // SmallPtrSet). The `bool` value is just a dummy.2582 SmallMapVector<const Instruction *, bool, 8> LhsDeps, RhsDeps;2583 collectInstructionDeps(&LhsDeps, Lhs);2584 // Collect "all" instructions that rhs condition is dependent on AND are2585 // dependencies of lhs. This gives us an estimate on which instructions we2586 // stand to save by splitting the condition.2587 if (!collectInstructionDeps(&RhsDeps, Rhs, &LhsDeps))2588 return false;2589 // Add the compare instruction itself unless its a dependency on the LHS.2590 if (const auto *RhsI = dyn_cast<Instruction>(Rhs))2591 if (!LhsDeps.contains(RhsI))2592 RhsDeps.try_emplace(RhsI, false);2593 2594 InstructionCost CostOfIncluding = 0;2595 // See if this instruction will need to computed independently of whether RHS2596 // is.2597 Value *BrCond = I.getCondition();2598 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {2599 for (const auto *U : Ins->users()) {2600 // If user is independent of RHS calculation we don't need to count it.2601 if (auto *UIns = dyn_cast<Instruction>(U))2602 if (UIns != BrCond && !RhsDeps.contains(UIns))2603 return false;2604 }2605 return true;2606 };2607 2608 // Prune instructions from RHS Deps that are dependencies of unrelated2609 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly2610 // arbitrary and just meant to cap the how much time we spend in the pruning2611 // loop. Its highly unlikely to come into affect.2612 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;2613 // Stop after a certain point. No incorrectness from including too many2614 // instructions.2615 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {2616 const Instruction *ToDrop = nullptr;2617 for (const auto &InsPair : RhsDeps) {2618 if (!ShouldCountInsn(InsPair.first)) {2619 ToDrop = InsPair.first;2620 break;2621 }2622 }2623 if (ToDrop == nullptr)2624 break;2625 RhsDeps.erase(ToDrop);2626 }2627 2628 for (const auto &InsPair : RhsDeps) {2629 // Finally accumulate latency that we can only attribute to computing the2630 // RHS condition. Use latency because we are essentially trying to calculate2631 // the cost of the dependency chain.2632 // Possible TODO: We could try to estimate ILP and make this more precise.2633 CostOfIncluding += TTI->getInstructionCost(2634 InsPair.first, TargetTransformInfo::TCK_Latency);2635 2636 if (CostOfIncluding > CostThresh)2637 return false;2638 }2639 return true;2640}2641 2642void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,2643 MachineBasicBlock *TBB,2644 MachineBasicBlock *FBB,2645 MachineBasicBlock *CurBB,2646 MachineBasicBlock *SwitchBB,2647 Instruction::BinaryOps Opc,2648 BranchProbability TProb,2649 BranchProbability FProb,2650 bool InvertCond) {2651 // Skip over not part of the tree and remember to invert op and operands at2652 // next level.2653 Value *NotCond;2654 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&2655 InBlock(NotCond, CurBB->getBasicBlock())) {2656 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,2657 !InvertCond);2658 return;2659 }2660 2661 const Instruction *BOp = dyn_cast<Instruction>(Cond);2662 const Value *BOpOp0, *BOpOp1;2663 // Compute the effective opcode for Cond, taking into account whether it needs2664 // to be inverted, e.g.2665 // and (not (or A, B)), C2666 // gets lowered as2667 // and (and (not A, not B), C)2668 Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;2669 if (BOp) {2670 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))2671 ? Instruction::And2672 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))2673 ? Instruction::Or2674 : (Instruction::BinaryOps)0);2675 if (InvertCond) {2676 if (BOpc == Instruction::And)2677 BOpc = Instruction::Or;2678 else if (BOpc == Instruction::Or)2679 BOpc = Instruction::And;2680 }2681 }2682 2683 // If this node is not part of the or/and tree, emit it as a branch.2684 // Note that all nodes in the tree should have same opcode.2685 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();2686 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||2687 !InBlock(BOpOp0, CurBB->getBasicBlock()) ||2688 !InBlock(BOpOp1, CurBB->getBasicBlock())) {2689 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,2690 TProb, FProb, InvertCond);2691 return;2692 }2693 2694 // Create TmpBB after CurBB.2695 MachineFunction::iterator BBI(CurBB);2696 MachineFunction &MF = DAG.getMachineFunction();2697 MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());2698 CurBB->getParent()->insert(++BBI, TmpBB);2699 2700 if (Opc == Instruction::Or) {2701 // Codegen X | Y as:2702 // BB1:2703 // jmp_if_X TBB2704 // jmp TmpBB2705 // TmpBB:2706 // jmp_if_Y TBB2707 // jmp FBB2708 //2709 2710 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.2711 // The requirement is that2712 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)2713 // = TrueProb for original BB.2714 // Assuming the original probabilities are A and B, one choice is to set2715 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to2716 // A/(1+B) and 2B/(1+B). This choice assumes that2717 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.2718 // Another choice is to assume TrueProb for BB1 equals to TrueProb for2719 // TmpBB, but the math is more complicated.2720 2721 auto NewTrueProb = TProb / 2;2722 auto NewFalseProb = TProb / 2 + FProb;2723 // Emit the LHS condition.2724 FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,2725 NewFalseProb, InvertCond);2726 2727 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).2728 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};2729 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());2730 // Emit the RHS condition into TmpBB.2731 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],2732 Probs[1], InvertCond);2733 } else {2734 assert(Opc == Instruction::And && "Unknown merge op!");2735 // Codegen X & Y as:2736 // BB1:2737 // jmp_if_X TmpBB2738 // jmp FBB2739 // TmpBB:2740 // jmp_if_Y TBB2741 // jmp FBB2742 //2743 // This requires creation of TmpBB after CurBB.2744 2745 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.2746 // The requirement is that2747 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)2748 // = FalseProb for original BB.2749 // Assuming the original probabilities are A and B, one choice is to set2750 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to2751 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==2752 // TrueProb for BB1 * FalseProb for TmpBB.2753 2754 auto NewTrueProb = TProb + FProb / 2;2755 auto NewFalseProb = FProb / 2;2756 // Emit the LHS condition.2757 FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,2758 NewFalseProb, InvertCond);2759 2760 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).2761 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};2762 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());2763 // Emit the RHS condition into TmpBB.2764 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],2765 Probs[1], InvertCond);2766 }2767}2768 2769/// If the set of cases should be emitted as a series of branches, return true.2770/// If we should emit this as a bunch of and/or'd together conditions, return2771/// false.2772bool2773SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {2774 if (Cases.size() != 2) return true;2775 2776 // If this is two comparisons of the same values or'd or and'd together, they2777 // will get folded into a single comparison, so don't emit two blocks.2778 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&2779 Cases[0].CmpRHS == Cases[1].CmpRHS) ||2780 (Cases[0].CmpRHS == Cases[1].CmpLHS &&2781 Cases[0].CmpLHS == Cases[1].CmpRHS)) {2782 return false;2783 }2784 2785 // Handle: (X != null) | (Y != null) --> (X|Y) != 02786 // Handle: (X == null) & (Y == null) --> (X|Y) == 02787 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&2788 Cases[0].CC == Cases[1].CC &&2789 isa<Constant>(Cases[0].CmpRHS) &&2790 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {2791 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)2792 return false;2793 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)2794 return false;2795 }2796 2797 return true;2798}2799 2800void SelectionDAGBuilder::visitBr(const BranchInst &I) {2801 MachineBasicBlock *BrMBB = FuncInfo.MBB;2802 2803 // Update machine-CFG edges.2804 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(I.getSuccessor(0));2805 2806 if (I.isUnconditional()) {2807 // Update machine-CFG edges.2808 BrMBB->addSuccessor(Succ0MBB);2809 2810 // If this is not a fall-through branch or optimizations are switched off,2811 // emit the branch.2812 if (Succ0MBB != NextBlock(BrMBB) ||2813 TM.getOptLevel() == CodeGenOptLevel::None) {2814 auto Br = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,2815 getControlRoot(), DAG.getBasicBlock(Succ0MBB));2816 setValue(&I, Br);2817 DAG.setRoot(Br);2818 }2819 2820 return;2821 }2822 2823 // If this condition is one of the special cases we handle, do special stuff2824 // now.2825 const Value *CondVal = I.getCondition();2826 MachineBasicBlock *Succ1MBB = FuncInfo.getMBB(I.getSuccessor(1));2827 2828 // If this is a series of conditions that are or'd or and'd together, emit2829 // this as a sequence of branches instead of setcc's with and/or operations.2830 // As long as jumps are not expensive (exceptions for multi-use logic ops,2831 // unpredictable branches, and vector extracts because those jumps are likely2832 // expensive for any target), this should improve performance.2833 // For example, instead of something like:2834 // cmp A, B2835 // C = seteq2836 // cmp D, E2837 // F = setle2838 // or C, F2839 // jnz foo2840 // Emit:2841 // cmp A, B2842 // je foo2843 // cmp D, E2844 // jle foo2845 bool IsUnpredictable = I.hasMetadata(LLVMContext::MD_unpredictable);2846 const Instruction *BOp = dyn_cast<Instruction>(CondVal);2847 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&2848 BOp->hasOneUse() && !IsUnpredictable) {2849 Value *Vec;2850 const Value *BOp0, *BOp1;2851 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;2852 if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))2853 Opcode = Instruction::And;2854 else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))2855 Opcode = Instruction::Or;2856 2857 if (Opcode &&2858 !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&2859 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value()))) &&2860 !shouldKeepJumpConditionsTogether(2861 FuncInfo, I, Opcode, BOp0, BOp1,2862 DAG.getTargetLoweringInfo().getJumpConditionMergingParams(2863 Opcode, BOp0, BOp1))) {2864 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,2865 getEdgeProbability(BrMBB, Succ0MBB),2866 getEdgeProbability(BrMBB, Succ1MBB),2867 /*InvertCond=*/false);2868 // If the compares in later blocks need to use values not currently2869 // exported from this block, export them now. This block should always2870 // be the first entry.2871 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");2872 2873 // Allow some cases to be rejected.2874 if (ShouldEmitAsBranches(SL->SwitchCases)) {2875 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {2876 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);2877 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);2878 }2879 2880 // Emit the branch for this block.2881 visitSwitchCase(SL->SwitchCases[0], BrMBB);2882 SL->SwitchCases.erase(SL->SwitchCases.begin());2883 return;2884 }2885 2886 // Okay, we decided not to do this, remove any inserted MBB's and clear2887 // SwitchCases.2888 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)2889 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);2890 2891 SL->SwitchCases.clear();2892 }2893 }2894 2895 // Create a CaseBlock record representing this branch.2896 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),2897 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc(),2898 BranchProbability::getUnknown(), BranchProbability::getUnknown(),2899 IsUnpredictable);2900 2901 // Use visitSwitchCase to actually insert the fast branch sequence for this2902 // cond branch.2903 visitSwitchCase(CB, BrMBB);2904}2905 2906/// visitSwitchCase - Emits the necessary code to represent a single node in2907/// the binary search tree resulting from lowering a switch instruction.2908void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,2909 MachineBasicBlock *SwitchBB) {2910 SDValue Cond;2911 SDValue CondLHS = getValue(CB.CmpLHS);2912 SDLoc dl = CB.DL;2913 2914 if (CB.CC == ISD::SETTRUE) {2915 // Branch or fall through to TrueBB.2916 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);2917 SwitchBB->normalizeSuccProbs();2918 if (CB.TrueBB != NextBlock(SwitchBB)) {2919 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),2920 DAG.getBasicBlock(CB.TrueBB)));2921 }2922 return;2923 }2924 2925 auto &TLI = DAG.getTargetLoweringInfo();2926 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());2927 2928 // Build the setcc now.2929 if (!CB.CmpMHS) {2930 // Fold "(X == true)" to X and "(X == false)" to !X to2931 // handle common cases produced by branch lowering.2932 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&2933 CB.CC == ISD::SETEQ)2934 Cond = CondLHS;2935 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&2936 CB.CC == ISD::SETEQ) {2937 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());2938 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);2939 } else {2940 SDValue CondRHS = getValue(CB.CmpRHS);2941 2942 // If a pointer's DAG type is larger than its memory type then the DAG2943 // values are zero-extended. This breaks signed comparisons so truncate2944 // back to the underlying type before doing the compare.2945 if (CondLHS.getValueType() != MemVT) {2946 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);2947 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);2948 }2949 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);2950 }2951 } else {2952 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");2953 2954 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();2955 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();2956 2957 SDValue CmpOp = getValue(CB.CmpMHS);2958 EVT VT = CmpOp.getValueType();2959 2960 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {2961 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),2962 ISD::SETLE);2963 } else {2964 SDValue SUB = DAG.getNode(ISD::SUB, dl,2965 VT, CmpOp, DAG.getConstant(Low, dl, VT));2966 Cond = DAG.getSetCC(dl, MVT::i1, SUB,2967 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);2968 }2969 }2970 2971 // Update successor info2972 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);2973 // TrueBB and FalseBB are always different unless the incoming IR is2974 // degenerate. This only happens when running llc on weird IR.2975 if (CB.TrueBB != CB.FalseBB)2976 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);2977 SwitchBB->normalizeSuccProbs();2978 2979 // If the lhs block is the next block, invert the condition so that we can2980 // fall through to the lhs instead of the rhs block.2981 if (CB.TrueBB == NextBlock(SwitchBB)) {2982 std::swap(CB.TrueBB, CB.FalseBB);2983 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());2984 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);2985 }2986 2987 SDNodeFlags Flags;2988 Flags.setUnpredictable(CB.IsUnpredictable);2989 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(),2990 Cond, DAG.getBasicBlock(CB.TrueBB), Flags);2991 2992 setValue(CurInst, BrCond);2993 2994 // Insert the false branch. Do this even if it's a fall through branch,2995 // this makes it easier to do DAG optimizations which require inverting2996 // the branch condition.2997 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,2998 DAG.getBasicBlock(CB.FalseBB));2999 3000 DAG.setRoot(BrCond);3001}3002 3003/// visitJumpTable - Emit JumpTable node in the current MBB3004void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {3005 // Emit the code for the jump table3006 assert(JT.SL && "Should set SDLoc for SelectionDAG!");3007 assert(JT.Reg && "Should lower JT Header first!");3008 EVT PTy = DAG.getTargetLoweringInfo().getJumpTableRegTy(DAG.getDataLayout());3009 SDValue Index = DAG.getCopyFromReg(getControlRoot(), *JT.SL, JT.Reg, PTy);3010 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);3011 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, *JT.SL, MVT::Other,3012 Index.getValue(1), Table, Index);3013 DAG.setRoot(BrJumpTable);3014}3015 3016/// visitJumpTableHeader - This function emits necessary code to produce index3017/// in the JumpTable from switch case.3018void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,3019 JumpTableHeader &JTH,3020 MachineBasicBlock *SwitchBB) {3021 assert(JT.SL && "Should set SDLoc for SelectionDAG!");3022 const SDLoc &dl = *JT.SL;3023 3024 // Subtract the lowest switch case value from the value being switched on.3025 SDValue SwitchOp = getValue(JTH.SValue);3026 EVT VT = SwitchOp.getValueType();3027 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,3028 DAG.getConstant(JTH.First, dl, VT));3029 3030 // The SDNode we just created, which holds the value being switched on minus3031 // the smallest case value, needs to be copied to a virtual register so it3032 // can be used as an index into the jump table in a subsequent basic block.3033 // This value may be smaller or larger than the target's pointer type, and3034 // therefore require extension or truncating.3035 const TargetLowering &TLI = DAG.getTargetLoweringInfo();3036 SwitchOp =3037 DAG.getZExtOrTrunc(Sub, dl, TLI.getJumpTableRegTy(DAG.getDataLayout()));3038 3039 Register JumpTableReg =3040 FuncInfo.CreateReg(TLI.getJumpTableRegTy(DAG.getDataLayout()));3041 SDValue CopyTo =3042 DAG.getCopyToReg(getControlRoot(), dl, JumpTableReg, SwitchOp);3043 JT.Reg = JumpTableReg;3044 3045 if (!JTH.FallthroughUnreachable) {3046 // Emit the range check for the jump table, and branch to the default block3047 // for the switch statement if the value being switched on exceeds the3048 // largest case in the switch.3049 SDValue CMP = DAG.getSetCC(3050 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),3051 Sub.getValueType()),3052 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);3053 3054 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,3055 MVT::Other, CopyTo, CMP,3056 DAG.getBasicBlock(JT.Default));3057 3058 // Avoid emitting unnecessary branches to the next block.3059 if (JT.MBB != NextBlock(SwitchBB))3060 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,3061 DAG.getBasicBlock(JT.MBB));3062 3063 DAG.setRoot(BrCond);3064 } else {3065 // Avoid emitting unnecessary branches to the next block.3066 if (JT.MBB != NextBlock(SwitchBB))3067 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,3068 DAG.getBasicBlock(JT.MBB)));3069 else3070 DAG.setRoot(CopyTo);3071 }3072}3073 3074/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global3075/// variable if there exists one.3076static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,3077 SDValue &Chain) {3078 const TargetLowering &TLI = DAG.getTargetLoweringInfo();3079 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());3080 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());3081 MachineFunction &MF = DAG.getMachineFunction();3082 Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());3083 MachineSDNode *Node =3084 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);3085 if (Global) {3086 MachinePointerInfo MPInfo(Global);3087 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |3088 MachineMemOperand::MODereferenceable;3089 MachineMemOperand *MemRef = MF.getMachineMemOperand(3090 MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));3091 DAG.setNodeMemRefs(Node, {MemRef});3092 }3093 if (PtrTy != PtrMemTy)3094 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);3095 return SDValue(Node, 0);3096}3097 3098/// Codegen a new tail for a stack protector check ParentMBB which has had its3099/// tail spliced into a stack protector check success bb.3100///3101/// For a high level explanation of how this fits into the stack protector3102/// generation see the comment on the declaration of class3103/// StackProtectorDescriptor.3104void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,3105 MachineBasicBlock *ParentBB) {3106 3107 // First create the loads to the guard/stack slot for the comparison.3108 const TargetLowering &TLI = DAG.getTargetLoweringInfo();3109 auto &DL = DAG.getDataLayout();3110 EVT PtrTy = TLI.getFrameIndexTy(DL);3111 EVT PtrMemTy = TLI.getPointerMemTy(DL, DL.getAllocaAddrSpace());3112 3113 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();3114 int FI = MFI.getStackProtectorIndex();3115 3116 SDValue Guard;3117 SDLoc dl = getCurSDLoc();3118 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);3119 const Module &M = *ParentBB->getParent()->getFunction().getParent();3120 Align Align = DL.getPrefTypeAlign(3121 PointerType::get(M.getContext(), DL.getAllocaAddrSpace()));3122 3123 // Generate code to load the content of the guard slot.3124 SDValue GuardVal = DAG.getLoad(3125 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,3126 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,3127 MachineMemOperand::MOVolatile);3128 3129 if (TLI.useStackGuardXorFP())3130 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);3131 3132 // If we're using function-based instrumentation, call the guard check3133 // function3134 if (SPD.shouldEmitFunctionBasedCheckStackProtector()) {3135 // Get the guard check function from the target and verify it exists since3136 // we're using function-based instrumentation3137 const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M);3138 assert(GuardCheckFn && "Guard check function is null");3139 3140 // The target provides a guard check function to validate the guard value.3141 // Generate a call to that function with the content of the guard slot as3142 // argument.3143 FunctionType *FnTy = GuardCheckFn->getFunctionType();3144 assert(FnTy->getNumParams() == 1 && "Invalid function signature");3145 3146 TargetLowering::ArgListTy Args;3147 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(0));3148 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))3149 Entry.IsInReg = true;3150 Args.push_back(Entry);3151 3152 TargetLowering::CallLoweringInfo CLI(DAG);3153 CLI.setDebugLoc(getCurSDLoc())3154 .setChain(DAG.getEntryNode())3155 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),3156 getValue(GuardCheckFn), std::move(Args));3157 3158 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);3159 DAG.setRoot(Result.second);3160 return;3161 }3162 3163 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.3164 // Otherwise, emit a volatile load to retrieve the stack guard value.3165 SDValue Chain = DAG.getEntryNode();3166 if (TLI.useLoadStackGuardNode(M)) {3167 Guard = getLoadStackGuard(DAG, dl, Chain);3168 } else {3169 if (const Value *IRGuard = TLI.getSDagStackGuard(M)) {3170 SDValue GuardPtr = getValue(IRGuard);3171 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,3172 MachinePointerInfo(IRGuard, 0), Align,3173 MachineMemOperand::MOVolatile);3174 } else {3175 LLVMContext &Ctx = *DAG.getContext();3176 Ctx.diagnose(DiagnosticInfoGeneric("unable to lower stackguard"));3177 Guard = DAG.getPOISON(PtrMemTy);3178 }3179 }3180 3181 // Perform the comparison via a getsetcc.3182 SDValue Cmp = DAG.getSetCC(3183 dl, TLI.getSetCCResultType(DL, *DAG.getContext(), Guard.getValueType()),3184 Guard, GuardVal, ISD::SETNE);3185 3186 // If the guard/stackslot do not equal, branch to failure MBB.3187 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,3188 MVT::Other, GuardVal.getOperand(0),3189 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));3190 // Otherwise branch to success MBB.3191 SDValue Br = DAG.getNode(ISD::BR, dl,3192 MVT::Other, BrCond,3193 DAG.getBasicBlock(SPD.getSuccessMBB()));3194 3195 DAG.setRoot(Br);3196}3197 3198/// Codegen the failure basic block for a stack protector check.3199///3200/// A failure stack protector machine basic block consists simply of a call to3201/// __stack_chk_fail().3202///3203/// For a high level explanation of how this fits into the stack protector3204/// generation see the comment on the declaration of class3205/// StackProtectorDescriptor.3206void SelectionDAGBuilder::visitSPDescriptorFailure(3207 StackProtectorDescriptor &SPD) {3208 3209 const TargetLowering &TLI = DAG.getTargetLoweringInfo();3210 MachineBasicBlock *ParentBB = SPD.getParentMBB();3211 const Module &M = *ParentBB->getParent()->getFunction().getParent();3212 SDValue Chain;3213 3214 // For -Oz builds with a guard check function, we use function-based3215 // instrumentation. Otherwise, if we have a guard check function, we call it3216 // in the failure block.3217 auto *GuardCheckFn = TLI.getSSPStackGuardCheck(M);3218 if (GuardCheckFn && !SPD.shouldEmitFunctionBasedCheckStackProtector()) {3219 // First create the loads to the guard/stack slot for the comparison.3220 auto &DL = DAG.getDataLayout();3221 EVT PtrTy = TLI.getFrameIndexTy(DL);3222 EVT PtrMemTy = TLI.getPointerMemTy(DL, DL.getAllocaAddrSpace());3223 3224 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();3225 int FI = MFI.getStackProtectorIndex();3226 3227 SDLoc dl = getCurSDLoc();3228 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);3229 Align Align = DL.getPrefTypeAlign(3230 PointerType::get(M.getContext(), DL.getAllocaAddrSpace()));3231 3232 // Generate code to load the content of the guard slot.3233 SDValue GuardVal = DAG.getLoad(3234 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,3235 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,3236 MachineMemOperand::MOVolatile);3237 3238 if (TLI.useStackGuardXorFP())3239 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);3240 3241 // The target provides a guard check function to validate the guard value.3242 // Generate a call to that function with the content of the guard slot as3243 // argument.3244 FunctionType *FnTy = GuardCheckFn->getFunctionType();3245 assert(FnTy->getNumParams() == 1 && "Invalid function signature");3246 3247 TargetLowering::ArgListTy Args;3248 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(0));3249 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))3250 Entry.IsInReg = true;3251 Args.push_back(Entry);3252 3253 TargetLowering::CallLoweringInfo CLI(DAG);3254 CLI.setDebugLoc(getCurSDLoc())3255 .setChain(DAG.getEntryNode())3256 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),3257 getValue(GuardCheckFn), std::move(Args));3258 3259 Chain = TLI.LowerCallTo(CLI).second;3260 } else {3261 TargetLowering::MakeLibCallOptions CallOptions;3262 CallOptions.setDiscardResult(true);3263 Chain = TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,3264 {}, CallOptions, getCurSDLoc())3265 .second;3266 }3267 3268 // Emit a trap instruction if we are required to do so.3269 const TargetOptions &TargetOpts = DAG.getTarget().Options;3270 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)3271 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);3272 3273 DAG.setRoot(Chain);3274}3275 3276/// visitBitTestHeader - This function emits necessary code to produce value3277/// suitable for "bit tests"3278void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,3279 MachineBasicBlock *SwitchBB) {3280 SDLoc dl = getCurSDLoc();3281 3282 // Subtract the minimum value.3283 SDValue SwitchOp = getValue(B.SValue);3284 EVT VT = SwitchOp.getValueType();3285 SDValue RangeSub =3286 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));3287 3288 // Determine the type of the test operands.3289 const TargetLowering &TLI = DAG.getTargetLoweringInfo();3290 bool UsePtrType = false;3291 if (!TLI.isTypeLegal(VT)) {3292 UsePtrType = true;3293 } else {3294 for (const BitTestCase &Case : B.Cases)3295 if (!isUIntN(VT.getSizeInBits(), Case.Mask)) {3296 // Switch table case range are encoded into series of masks.3297 // Just use pointer type, it's guaranteed to fit.3298 UsePtrType = true;3299 break;3300 }3301 }3302 SDValue Sub = RangeSub;3303 if (UsePtrType) {3304 VT = TLI.getPointerTy(DAG.getDataLayout());3305 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);3306 }3307 3308 B.RegVT = VT.getSimpleVT();3309 B.Reg = FuncInfo.CreateReg(B.RegVT);3310 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);3311 3312 MachineBasicBlock* MBB = B.Cases[0].ThisBB;3313 3314 if (!B.FallthroughUnreachable)3315 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);3316 addSuccessorWithProb(SwitchBB, MBB, B.Prob);3317 SwitchBB->normalizeSuccProbs();3318 3319 SDValue Root = CopyTo;3320 if (!B.FallthroughUnreachable) {3321 // Conditional branch to the default block.3322 SDValue RangeCmp = DAG.getSetCC(dl,3323 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),3324 RangeSub.getValueType()),3325 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),3326 ISD::SETUGT);3327 3328 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,3329 DAG.getBasicBlock(B.Default));3330 }3331 3332 // Avoid emitting unnecessary branches to the next block.3333 if (MBB != NextBlock(SwitchBB))3334 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));3335 3336 DAG.setRoot(Root);3337}3338 3339/// visitBitTestCase - this function produces one "bit test"3340void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,3341 MachineBasicBlock *NextMBB,3342 BranchProbability BranchProbToNext,3343 Register Reg, BitTestCase &B,3344 MachineBasicBlock *SwitchBB) {3345 SDLoc dl = getCurSDLoc();3346 MVT VT = BB.RegVT;3347 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);3348 SDValue Cmp;3349 unsigned PopCount = llvm::popcount(B.Mask);3350 const TargetLowering &TLI = DAG.getTargetLoweringInfo();3351 if (PopCount == 1) {3352 // Testing for a single bit; just compare the shift count with what it3353 // would need to be to shift a 1 bit in that position.3354 Cmp = DAG.getSetCC(3355 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),3356 ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),3357 ISD::SETEQ);3358 } else if (PopCount == BB.Range) {3359 // There is only one zero bit in the range, test for it directly.3360 Cmp = DAG.getSetCC(3361 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),3362 ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);3363 } else {3364 // Make desired shift3365 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,3366 DAG.getConstant(1, dl, VT), ShiftOp);3367 3368 // Emit bit tests and jumps3369 SDValue AndOp = DAG.getNode(ISD::AND, dl,3370 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));3371 Cmp = DAG.getSetCC(3372 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),3373 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);3374 }3375 3376 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.3377 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);3378 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.3379 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);3380 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is3381 // one as they are relative probabilities (and thus work more like weights),3382 // and hence we need to normalize them to let the sum of them become one.3383 SwitchBB->normalizeSuccProbs();3384 3385 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,3386 MVT::Other, getControlRoot(),3387 Cmp, DAG.getBasicBlock(B.TargetBB));3388 3389 // Avoid emitting unnecessary branches to the next block.3390 if (NextMBB != NextBlock(SwitchBB))3391 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,3392 DAG.getBasicBlock(NextMBB));3393 3394 DAG.setRoot(BrAnd);3395}3396 3397void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {3398 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;3399 3400 // Retrieve successors. Look through artificial IR level blocks like3401 // catchswitch for successors.3402 MachineBasicBlock *Return = FuncInfo.getMBB(I.getSuccessor(0));3403 const BasicBlock *EHPadBB = I.getSuccessor(1);3404 MachineBasicBlock *EHPadMBB = FuncInfo.getMBB(EHPadBB);3405 3406 // Deopt and ptrauth bundles are lowered in helper functions, and we don't3407 // have to do anything here to lower funclet bundles.3408 failForInvalidBundles(I, "invokes",3409 {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,3410 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,3411 LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,3412 LLVMContext::OB_clang_arc_attachedcall,3413 LLVMContext::OB_kcfi});3414 3415 const Value *Callee(I.getCalledOperand());3416 const Function *Fn = dyn_cast<Function>(Callee);3417 if (isa<InlineAsm>(Callee))3418 visitInlineAsm(I, EHPadBB);3419 else if (Fn && Fn->isIntrinsic()) {3420 switch (Fn->getIntrinsicID()) {3421 default:3422 llvm_unreachable("Cannot invoke this intrinsic");3423 case Intrinsic::donothing:3424 // Ignore invokes to @llvm.donothing: jump directly to the next BB.3425 case Intrinsic::seh_try_begin:3426 case Intrinsic::seh_scope_begin:3427 case Intrinsic::seh_try_end:3428 case Intrinsic::seh_scope_end:3429 if (EHPadMBB)3430 // a block referenced by EH table3431 // so dtor-funclet not removed by opts3432 EHPadMBB->setMachineBlockAddressTaken();3433 break;3434 case Intrinsic::experimental_patchpoint_void:3435 case Intrinsic::experimental_patchpoint:3436 visitPatchpoint(I, EHPadBB);3437 break;3438 case Intrinsic::experimental_gc_statepoint:3439 LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);3440 break;3441 // wasm_throw, wasm_rethrow: This is usually done in visitTargetIntrinsic,3442 // but these intrinsics are special because they can be invoked, so we3443 // manually lower it to a DAG node here.3444 case Intrinsic::wasm_throw: {3445 const TargetLowering &TLI = DAG.getTargetLoweringInfo();3446 std::array<SDValue, 4> Ops = {3447 getControlRoot(), // inchain for the terminator node3448 DAG.getTargetConstant(Intrinsic::wasm_throw, getCurSDLoc(),3449 TLI.getPointerTy(DAG.getDataLayout())),3450 getValue(I.getArgOperand(0)), // tag3451 getValue(I.getArgOperand(1)) // thrown value3452 };3453 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain3454 DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));3455 break;3456 }3457 case Intrinsic::wasm_rethrow: {3458 const TargetLowering &TLI = DAG.getTargetLoweringInfo();3459 std::array<SDValue, 2> Ops = {3460 getControlRoot(), // inchain for the terminator node3461 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),3462 TLI.getPointerTy(DAG.getDataLayout()))};3463 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain3464 DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));3465 break;3466 }3467 }3468 } else if (I.hasDeoptState()) {3469 // Currently we do not lower any intrinsic calls with deopt operand bundles.3470 // Eventually we will support lowering the @llvm.experimental.deoptimize3471 // intrinsic, and right now there are no plans to support other intrinsics3472 // with deopt state.3473 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);3474 } else if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {3475 LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), EHPadBB);3476 } else {3477 LowerCallTo(I, getValue(Callee), false, false, EHPadBB);3478 }3479 3480 // If the value of the invoke is used outside of its defining block, make it3481 // available as a virtual register.3482 // We already took care of the exported value for the statepoint instruction3483 // during call to the LowerStatepoint.3484 if (!isa<GCStatepointInst>(I)) {3485 CopyToExportRegsIfNeeded(&I);3486 }3487 3488 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;3489 BranchProbabilityInfo *BPI = FuncInfo.BPI;3490 BranchProbability EHPadBBProb =3491 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)3492 : BranchProbability::getZero();3493 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);3494 3495 // Update successor info.3496 addSuccessorWithProb(InvokeMBB, Return);3497 for (auto &UnwindDest : UnwindDests) {3498 UnwindDest.first->setIsEHPad();3499 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);3500 }3501 InvokeMBB->normalizeSuccProbs();3502 3503 // Drop into normal successor.3504 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),3505 DAG.getBasicBlock(Return)));3506}3507 3508void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {3509 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;3510 3511 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't3512 // have to do anything here to lower funclet bundles.3513 failForInvalidBundles(I, "callbrs",3514 {LLVMContext::OB_deopt, LLVMContext::OB_funclet});3515 3516 assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");3517 visitInlineAsm(I);3518 CopyToExportRegsIfNeeded(&I);3519 3520 // Retrieve successors.3521 SmallPtrSet<BasicBlock *, 8> Dests;3522 Dests.insert(I.getDefaultDest());3523 MachineBasicBlock *Return = FuncInfo.getMBB(I.getDefaultDest());3524 3525 // Update successor info.3526 addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());3527 for (BasicBlock *Dest : I.getIndirectDests()) {3528 MachineBasicBlock *Target = FuncInfo.getMBB(Dest);3529 Target->setIsInlineAsmBrIndirectTarget();3530 // If we introduce a type of asm goto statement that is permitted to use an3531 // indirect call instruction to jump to its labels, then we should add a3532 // call to Target->setMachineBlockAddressTaken() here, to mark the target3533 // block as requiring a BTI.3534 3535 Target->setLabelMustBeEmitted();3536 // Don't add duplicate machine successors.3537 if (Dests.insert(Dest).second)3538 addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());3539 }3540 CallBrMBB->normalizeSuccProbs();3541 3542 // Drop into default successor.3543 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),3544 MVT::Other, getControlRoot(),3545 DAG.getBasicBlock(Return)));3546}3547 3548void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {3549 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");3550}3551 3552void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {3553 assert(FuncInfo.MBB->isEHPad() &&3554 "Call to landingpad not in landing pad!");3555 3556 // If there aren't registers to copy the values into (e.g., during SjLj3557 // exceptions), then don't bother to create these DAG nodes.3558 const TargetLowering &TLI = DAG.getTargetLoweringInfo();3559 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();3560 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&3561 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)3562 return;3563 3564 // If landingpad's return type is token type, we don't create DAG nodes3565 // for its exception pointer and selector value. The extraction of exception3566 // pointer or selector value from token type landingpads is not currently3567 // supported.3568 if (LP.getType()->isTokenTy())3569 return;3570 3571 SmallVector<EVT, 2> ValueVTs;3572 SDLoc dl = getCurSDLoc();3573 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);3574 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");3575 3576 // Get the two live-in registers as SDValues. The physregs have already been3577 // copied into virtual registers.3578 SDValue Ops[2];3579 if (FuncInfo.ExceptionPointerVirtReg) {3580 Ops[0] = DAG.getZExtOrTrunc(3581 DAG.getCopyFromReg(DAG.getEntryNode(), dl,3582 FuncInfo.ExceptionPointerVirtReg,3583 TLI.getPointerTy(DAG.getDataLayout())),3584 dl, ValueVTs[0]);3585 } else {3586 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));3587 }3588 Ops[1] = DAG.getZExtOrTrunc(3589 DAG.getCopyFromReg(DAG.getEntryNode(), dl,3590 FuncInfo.ExceptionSelectorVirtReg,3591 TLI.getPointerTy(DAG.getDataLayout())),3592 dl, ValueVTs[1]);3593 3594 // Merge into one.3595 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,3596 DAG.getVTList(ValueVTs), Ops);3597 setValue(&LP, Res);3598}3599 3600void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,3601 MachineBasicBlock *Last) {3602 // Update JTCases.3603 for (JumpTableBlock &JTB : SL->JTCases)3604 if (JTB.first.HeaderBB == First)3605 JTB.first.HeaderBB = Last;3606 3607 // Update BitTestCases.3608 for (BitTestBlock &BTB : SL->BitTestCases)3609 if (BTB.Parent == First)3610 BTB.Parent = Last;3611}3612 3613void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {3614 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;3615 3616 // Update machine-CFG edges with unique successors.3617 SmallPtrSet<BasicBlock *, 32> Done;3618 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {3619 BasicBlock *BB = I.getSuccessor(i);3620 bool Inserted = Done.insert(BB).second;3621 if (!Inserted)3622 continue;3623 3624 MachineBasicBlock *Succ = FuncInfo.getMBB(BB);3625 addSuccessorWithProb(IndirectBrMBB, Succ);3626 }3627 IndirectBrMBB->normalizeSuccProbs();3628 3629 DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),3630 MVT::Other, getControlRoot(),3631 getValue(I.getAddress())));3632}3633 3634void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {3635 if (!I.shouldLowerToTrap(DAG.getTarget().Options.TrapUnreachable,3636 DAG.getTarget().Options.NoTrapAfterNoreturn))3637 return;3638 3639 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));3640}3641 3642void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {3643 SDNodeFlags Flags;3644 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))3645 Flags.copyFMF(*FPOp);3646 3647 SDValue Op = getValue(I.getOperand(0));3648 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),3649 Op, Flags);3650 setValue(&I, UnNodeValue);3651}3652 3653void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {3654 SDNodeFlags Flags;3655 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {3656 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());3657 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());3658 }3659 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))3660 Flags.setExact(ExactOp->isExact());3661 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(&I))3662 Flags.setDisjoint(DisjointOp->isDisjoint());3663 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))3664 Flags.copyFMF(*FPOp);3665 3666 SDValue Op1 = getValue(I.getOperand(0));3667 SDValue Op2 = getValue(I.getOperand(1));3668 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),3669 Op1, Op2, Flags);3670 setValue(&I, BinNodeValue);3671}3672 3673void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {3674 SDValue Op1 = getValue(I.getOperand(0));3675 SDValue Op2 = getValue(I.getOperand(1));3676 3677 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(3678 Op1.getValueType(), DAG.getDataLayout());3679 3680 // Coerce the shift amount to the right type if we can. This exposes the3681 // truncate or zext to optimization early.3682 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {3683 assert(ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) &&3684 "Unexpected shift type");3685 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);3686 }3687 3688 bool nuw = false;3689 bool nsw = false;3690 bool exact = false;3691 3692 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {3693 3694 if (const OverflowingBinaryOperator *OFBinOp =3695 dyn_cast<const OverflowingBinaryOperator>(&I)) {3696 nuw = OFBinOp->hasNoUnsignedWrap();3697 nsw = OFBinOp->hasNoSignedWrap();3698 }3699 if (const PossiblyExactOperator *ExactOp =3700 dyn_cast<const PossiblyExactOperator>(&I))3701 exact = ExactOp->isExact();3702 }3703 SDNodeFlags Flags;3704 Flags.setExact(exact);3705 Flags.setNoSignedWrap(nsw);3706 Flags.setNoUnsignedWrap(nuw);3707 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,3708 Flags);3709 setValue(&I, Res);3710}3711 3712void SelectionDAGBuilder::visitSDiv(const User &I) {3713 SDValue Op1 = getValue(I.getOperand(0));3714 SDValue Op2 = getValue(I.getOperand(1));3715 3716 SDNodeFlags Flags;3717 Flags.setExact(isa<PossiblyExactOperator>(&I) &&3718 cast<PossiblyExactOperator>(&I)->isExact());3719 setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,3720 Op2, Flags));3721}3722 3723void SelectionDAGBuilder::visitICmp(const ICmpInst &I) {3724 ICmpInst::Predicate predicate = I.getPredicate();3725 SDValue Op1 = getValue(I.getOperand(0));3726 SDValue Op2 = getValue(I.getOperand(1));3727 ISD::CondCode Opcode = getICmpCondCode(predicate);3728 3729 auto &TLI = DAG.getTargetLoweringInfo();3730 EVT MemVT =3731 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());3732 3733 // If a pointer's DAG type is larger than its memory type then the DAG values3734 // are zero-extended. This breaks signed comparisons so truncate back to the3735 // underlying type before doing the compare.3736 if (Op1.getValueType() != MemVT) {3737 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);3738 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);3739 }3740 3741 SDNodeFlags Flags;3742 Flags.setSameSign(I.hasSameSign());3743 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);3744 3745 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),3746 I.getType());3747 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));3748}3749 3750void SelectionDAGBuilder::visitFCmp(const FCmpInst &I) {3751 FCmpInst::Predicate predicate = I.getPredicate();3752 SDValue Op1 = getValue(I.getOperand(0));3753 SDValue Op2 = getValue(I.getOperand(1));3754 3755 ISD::CondCode Condition = getFCmpCondCode(predicate);3756 auto *FPMO = cast<FPMathOperator>(&I);3757 if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)3758 Condition = getFCmpCodeWithoutNaN(Condition);3759 3760 SDNodeFlags Flags;3761 Flags.copyFMF(*FPMO);3762 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);3763 3764 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),3765 I.getType());3766 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));3767}3768 3769// Check if the condition of the select has one use or two users that are both3770// selects with the same condition.3771static bool hasOnlySelectUsers(const Value *Cond) {3772 return llvm::all_of(Cond->users(), [](const Value *V) {3773 return isa<SelectInst>(V);3774 });3775}3776 3777void SelectionDAGBuilder::visitSelect(const User &I) {3778 SmallVector<EVT, 4> ValueVTs;3779 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),3780 ValueVTs);3781 unsigned NumValues = ValueVTs.size();3782 if (NumValues == 0) return;3783 3784 SmallVector<SDValue, 4> Values(NumValues);3785 SDValue Cond = getValue(I.getOperand(0));3786 SDValue LHSVal = getValue(I.getOperand(1));3787 SDValue RHSVal = getValue(I.getOperand(2));3788 SmallVector<SDValue, 1> BaseOps(1, Cond);3789 ISD::NodeType OpCode =3790 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;3791 3792 bool IsUnaryAbs = false;3793 bool Negate = false;3794 3795 SDNodeFlags Flags;3796 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))3797 Flags.copyFMF(*FPOp);3798 3799 Flags.setUnpredictable(3800 cast<SelectInst>(I).getMetadata(LLVMContext::MD_unpredictable));3801 3802 // Min/max matching is only viable if all output VTs are the same.3803 if (all_equal(ValueVTs)) {3804 EVT VT = ValueVTs[0];3805 LLVMContext &Ctx = *DAG.getContext();3806 auto &TLI = DAG.getTargetLoweringInfo();3807 3808 // We care about the legality of the operation after it has been type3809 // legalized.3810 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)3811 VT = TLI.getTypeToTransformTo(Ctx, VT);3812 3813 // If the vselect is legal, assume we want to leave this as a vector setcc +3814 // vselect. Otherwise, if this is going to be scalarized, we want to see if3815 // min/max is legal on the scalar type.3816 bool UseScalarMinMax = VT.isVector() &&3817 !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);3818 3819 // ValueTracking's select pattern matching does not account for -0.0,3820 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that3821 // -0.0 is less than +0.0.3822 const Value *LHS, *RHS;3823 auto SPR = matchSelectPattern(&I, LHS, RHS);3824 ISD::NodeType Opc = ISD::DELETED_NODE;3825 switch (SPR.Flavor) {3826 case SPF_UMAX: Opc = ISD::UMAX; break;3827 case SPF_UMIN: Opc = ISD::UMIN; break;3828 case SPF_SMAX: Opc = ISD::SMAX; break;3829 case SPF_SMIN: Opc = ISD::SMIN; break;3830 case SPF_FMINNUM:3831 switch (SPR.NaNBehavior) {3832 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");3833 case SPNB_RETURNS_NAN: break;3834 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;3835 case SPNB_RETURNS_ANY:3836 if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT) ||3837 (UseScalarMinMax &&3838 TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType())))3839 Opc = ISD::FMINNUM;3840 break;3841 }3842 break;3843 case SPF_FMAXNUM:3844 switch (SPR.NaNBehavior) {3845 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");3846 case SPNB_RETURNS_NAN: break;3847 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;3848 case SPNB_RETURNS_ANY:3849 if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT) ||3850 (UseScalarMinMax &&3851 TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType())))3852 Opc = ISD::FMAXNUM;3853 break;3854 }3855 break;3856 case SPF_NABS:3857 Negate = true;3858 [[fallthrough]];3859 case SPF_ABS:3860 IsUnaryAbs = true;3861 Opc = ISD::ABS;3862 break;3863 default: break;3864 }3865 3866 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&3867 (TLI.isOperationLegalOrCustom(Opc, VT) ||3868 (UseScalarMinMax &&3869 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&3870 // If the underlying comparison instruction is used by any other3871 // instruction, the consumed instructions won't be destroyed, so it is3872 // not profitable to convert to a min/max.3873 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {3874 OpCode = Opc;3875 LHSVal = getValue(LHS);3876 RHSVal = getValue(RHS);3877 BaseOps.clear();3878 }3879 3880 if (IsUnaryAbs) {3881 OpCode = Opc;3882 LHSVal = getValue(LHS);3883 BaseOps.clear();3884 }3885 }3886 3887 if (IsUnaryAbs) {3888 for (unsigned i = 0; i != NumValues; ++i) {3889 SDLoc dl = getCurSDLoc();3890 EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);3891 Values[i] =3892 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));3893 if (Negate)3894 Values[i] = DAG.getNegative(Values[i], dl, VT);3895 }3896 } else {3897 for (unsigned i = 0; i != NumValues; ++i) {3898 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());3899 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));3900 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));3901 Values[i] = DAG.getNode(3902 OpCode, getCurSDLoc(),3903 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);3904 }3905 }3906 3907 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),3908 DAG.getVTList(ValueVTs), Values));3909}3910 3911void SelectionDAGBuilder::visitTrunc(const User &I) {3912 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).3913 SDValue N = getValue(I.getOperand(0));3914 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),3915 I.getType());3916 SDNodeFlags Flags;3917 if (auto *Trunc = dyn_cast<TruncInst>(&I)) {3918 Flags.setNoSignedWrap(Trunc->hasNoSignedWrap());3919 Flags.setNoUnsignedWrap(Trunc->hasNoUnsignedWrap());3920 }3921 3922 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N, Flags));3923}3924 3925void SelectionDAGBuilder::visitZExt(const User &I) {3926 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).3927 // ZExt also can't be a cast to bool for same reason. So, nothing much to do3928 SDValue N = getValue(I.getOperand(0));3929 auto &TLI = DAG.getTargetLoweringInfo();3930 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());3931 3932 SDNodeFlags Flags;3933 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))3934 Flags.setNonNeg(PNI->hasNonNeg());3935 3936 // Eagerly use nonneg information to canonicalize towards sign_extend if3937 // that is the target's preference.3938 // TODO: Let the target do this later.3939 if (Flags.hasNonNeg() &&3940 TLI.isSExtCheaperThanZExt(N.getValueType(), DestVT)) {3941 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));3942 return;3943 }3944 3945 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N, Flags));3946}3947 3948void SelectionDAGBuilder::visitSExt(const User &I) {3949 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).3950 // SExt also can't be a cast to bool for same reason. So, nothing much to do3951 SDValue N = getValue(I.getOperand(0));3952 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),3953 I.getType());3954 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));3955}3956 3957void SelectionDAGBuilder::visitFPTrunc(const User &I) {3958 // FPTrunc is never a no-op cast, no need to check3959 SDValue N = getValue(I.getOperand(0));3960 SDLoc dl = getCurSDLoc();3961 SDNodeFlags Flags;3962 if (auto *TruncInst = dyn_cast<FPMathOperator>(&I))3963 Flags.copyFMF(*TruncInst);3964 const TargetLowering &TLI = DAG.getTargetLoweringInfo();3965 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());3966 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,3967 DAG.getTargetConstant(3968 0, dl, TLI.getPointerTy(DAG.getDataLayout())),3969 Flags));3970}3971 3972void SelectionDAGBuilder::visitFPExt(const User &I) {3973 // FPExt is never a no-op cast, no need to check3974 SDValue N = getValue(I.getOperand(0));3975 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),3976 I.getType());3977 SDNodeFlags Flags;3978 if (auto *TruncInst = dyn_cast<FPMathOperator>(&I))3979 Flags.copyFMF(*TruncInst);3980 setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N, Flags));3981}3982 3983void SelectionDAGBuilder::visitFPToUI(const User &I) {3984 // FPToUI is never a no-op cast, no need to check3985 SDValue N = getValue(I.getOperand(0));3986 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),3987 I.getType());3988 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));3989}3990 3991void SelectionDAGBuilder::visitFPToSI(const User &I) {3992 // FPToSI is never a no-op cast, no need to check3993 SDValue N = getValue(I.getOperand(0));3994 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),3995 I.getType());3996 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));3997}3998 3999void SelectionDAGBuilder::visitUIToFP(const User &I) {4000 // UIToFP is never a no-op cast, no need to check4001 SDValue N = getValue(I.getOperand(0));4002 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),4003 I.getType());4004 SDNodeFlags Flags;4005 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))4006 Flags.setNonNeg(PNI->hasNonNeg());4007 4008 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N, Flags));4009}4010 4011void SelectionDAGBuilder::visitSIToFP(const User &I) {4012 // SIToFP is never a no-op cast, no need to check4013 SDValue N = getValue(I.getOperand(0));4014 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),4015 I.getType());4016 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));4017}4018 4019void SelectionDAGBuilder::visitPtrToAddr(const User &I) {4020 SDValue N = getValue(I.getOperand(0));4021 // By definition the type of the ptrtoaddr must be equal to the address type.4022 const auto &TLI = DAG.getTargetLoweringInfo();4023 EVT AddrVT = TLI.getValueType(DAG.getDataLayout(), I.getType());4024 // The address width must be smaller or equal to the pointer representation4025 // width, so we lower ptrtoaddr as a truncate (possibly folded to a no-op).4026 N = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), AddrVT, N);4027 setValue(&I, N);4028}4029 4030void SelectionDAGBuilder::visitPtrToInt(const User &I) {4031 // What to do depends on the size of the integer and the size of the pointer.4032 // We can either truncate, zero extend, or no-op, accordingly.4033 SDValue N = getValue(I.getOperand(0));4034 auto &TLI = DAG.getTargetLoweringInfo();4035 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),4036 I.getType());4037 EVT PtrMemVT =4038 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());4039 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);4040 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);4041 setValue(&I, N);4042}4043 4044void SelectionDAGBuilder::visitIntToPtr(const User &I) {4045 // What to do depends on the size of the integer and the size of the pointer.4046 // We can either truncate, zero extend, or no-op, accordingly.4047 SDValue N = getValue(I.getOperand(0));4048 auto &TLI = DAG.getTargetLoweringInfo();4049 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());4050 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());4051 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);4052 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);4053 setValue(&I, N);4054}4055 4056void SelectionDAGBuilder::visitBitCast(const User &I) {4057 SDValue N = getValue(I.getOperand(0));4058 SDLoc dl = getCurSDLoc();4059 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),4060 I.getType());4061 4062 // BitCast assures us that source and destination are the same size so this is4063 // either a BITCAST or a no-op.4064 if (DestVT != N.getValueType())4065 setValue(&I, DAG.getNode(ISD::BITCAST, dl,4066 DestVT, N)); // convert types.4067 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()4068 // might fold any kind of constant expression to an integer constant and that4069 // is not what we are looking for. Only recognize a bitcast of a genuine4070 // constant integer as an opaque constant.4071 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))4072 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,4073 /*isOpaque*/true));4074 else4075 setValue(&I, N); // noop cast.4076}4077 4078void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {4079 const TargetLowering &TLI = DAG.getTargetLoweringInfo();4080 const Value *SV = I.getOperand(0);4081 SDValue N = getValue(SV);4082 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());4083 4084 unsigned SrcAS = SV->getType()->getPointerAddressSpace();4085 unsigned DestAS = I.getType()->getPointerAddressSpace();4086 4087 if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))4088 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);4089 4090 setValue(&I, N);4091}4092 4093void SelectionDAGBuilder::visitInsertElement(const User &I) {4094 const TargetLowering &TLI = DAG.getTargetLoweringInfo();4095 SDValue InVec = getValue(I.getOperand(0));4096 SDValue InVal = getValue(I.getOperand(1));4097 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),4098 TLI.getVectorIdxTy(DAG.getDataLayout()));4099 setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),4100 TLI.getValueType(DAG.getDataLayout(), I.getType()),4101 InVec, InVal, InIdx));4102}4103 4104void SelectionDAGBuilder::visitExtractElement(const User &I) {4105 const TargetLowering &TLI = DAG.getTargetLoweringInfo();4106 SDValue InVec = getValue(I.getOperand(0));4107 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),4108 TLI.getVectorIdxTy(DAG.getDataLayout()));4109 setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),4110 TLI.getValueType(DAG.getDataLayout(), I.getType()),4111 InVec, InIdx));4112}4113 4114void SelectionDAGBuilder::visitShuffleVector(const User &I) {4115 SDValue Src1 = getValue(I.getOperand(0));4116 SDValue Src2 = getValue(I.getOperand(1));4117 ArrayRef<int> Mask;4118 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))4119 Mask = SVI->getShuffleMask();4120 else4121 Mask = cast<ConstantExpr>(I).getShuffleMask();4122 SDLoc DL = getCurSDLoc();4123 const TargetLowering &TLI = DAG.getTargetLoweringInfo();4124 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());4125 EVT SrcVT = Src1.getValueType();4126 4127 if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&4128 VT.isScalableVector()) {4129 // Canonical splat form of first element of first input vector.4130 SDValue FirstElt =4131 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,4132 DAG.getVectorIdxConstant(0, DL));4133 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));4134 return;4135 }4136 4137 // For now, we only handle splats for scalable vectors.4138 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation4139 // for targets that support a SPLAT_VECTOR for non-scalable vector types.4140 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");4141 4142 unsigned SrcNumElts = SrcVT.getVectorNumElements();4143 unsigned MaskNumElts = Mask.size();4144 4145 if (SrcNumElts == MaskNumElts) {4146 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));4147 return;4148 }4149 4150 // Normalize the shuffle vector since mask and vector length don't match.4151 if (SrcNumElts < MaskNumElts) {4152 // Mask is longer than the source vectors. We can use concatenate vector to4153 // make the mask and vectors lengths match.4154 4155 if (MaskNumElts % SrcNumElts == 0) {4156 // Mask length is a multiple of the source vector length.4157 // Check if the shuffle is some kind of concatenation of the input4158 // vectors.4159 unsigned NumConcat = MaskNumElts / SrcNumElts;4160 bool IsConcat = true;4161 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);4162 for (unsigned i = 0; i != MaskNumElts; ++i) {4163 int Idx = Mask[i];4164 if (Idx < 0)4165 continue;4166 // Ensure the indices in each SrcVT sized piece are sequential and that4167 // the same source is used for the whole piece.4168 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||4169 (ConcatSrcs[i / SrcNumElts] >= 0 &&4170 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {4171 IsConcat = false;4172 break;4173 }4174 // Remember which source this index came from.4175 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;4176 }4177 4178 // The shuffle is concatenating multiple vectors together. Just emit4179 // a CONCAT_VECTORS operation.4180 if (IsConcat) {4181 SmallVector<SDValue, 8> ConcatOps;4182 for (auto Src : ConcatSrcs) {4183 if (Src < 0)4184 ConcatOps.push_back(DAG.getUNDEF(SrcVT));4185 else if (Src == 0)4186 ConcatOps.push_back(Src1);4187 else4188 ConcatOps.push_back(Src2);4189 }4190 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));4191 return;4192 }4193 }4194 4195 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);4196 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;4197 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),4198 PaddedMaskNumElts);4199 4200 // Pad both vectors with undefs to make them the same length as the mask.4201 SDValue UndefVal = DAG.getUNDEF(SrcVT);4202 4203 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);4204 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);4205 MOps1[0] = Src1;4206 MOps2[0] = Src2;4207 4208 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);4209 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);4210 4211 // Readjust mask for new input vector length.4212 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);4213 for (unsigned i = 0; i != MaskNumElts; ++i) {4214 int Idx = Mask[i];4215 if (Idx >= (int)SrcNumElts)4216 Idx -= SrcNumElts - PaddedMaskNumElts;4217 MappedOps[i] = Idx;4218 }4219 4220 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);4221 4222 // If the concatenated vector was padded, extract a subvector with the4223 // correct number of elements.4224 if (MaskNumElts != PaddedMaskNumElts)4225 Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,4226 DAG.getVectorIdxConstant(0, DL));4227 4228 setValue(&I, Result);4229 return;4230 }4231 4232 assert(SrcNumElts > MaskNumElts);4233 4234 // Analyze the access pattern of the vector to see if we can extract4235 // two subvectors and do the shuffle.4236 int StartIdx[2] = {-1, -1}; // StartIdx to extract from4237 bool CanExtract = true;4238 for (int Idx : Mask) {4239 unsigned Input = 0;4240 if (Idx < 0)4241 continue;4242 4243 if (Idx >= (int)SrcNumElts) {4244 Input = 1;4245 Idx -= SrcNumElts;4246 }4247 4248 // If all the indices come from the same MaskNumElts sized portion of4249 // the sources we can use extract. Also make sure the extract wouldn't4250 // extract past the end of the source.4251 int NewStartIdx = alignDown(Idx, MaskNumElts);4252 if (NewStartIdx + MaskNumElts > SrcNumElts ||4253 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))4254 CanExtract = false;4255 // Make sure we always update StartIdx as we use it to track if all4256 // elements are undef.4257 StartIdx[Input] = NewStartIdx;4258 }4259 4260 if (StartIdx[0] < 0 && StartIdx[1] < 0) {4261 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.4262 return;4263 }4264 if (CanExtract) {4265 // Extract appropriate subvector and generate a vector shuffle4266 for (unsigned Input = 0; Input < 2; ++Input) {4267 SDValue &Src = Input == 0 ? Src1 : Src2;4268 if (StartIdx[Input] < 0)4269 Src = DAG.getUNDEF(VT);4270 else {4271 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,4272 DAG.getVectorIdxConstant(StartIdx[Input], DL));4273 }4274 }4275 4276 // Calculate new mask.4277 SmallVector<int, 8> MappedOps(Mask);4278 for (int &Idx : MappedOps) {4279 if (Idx >= (int)SrcNumElts)4280 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;4281 else if (Idx >= 0)4282 Idx -= StartIdx[0];4283 }4284 4285 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));4286 return;4287 }4288 4289 // We can't use either concat vectors or extract subvectors so fall back to4290 // replacing the shuffle with extract and build vector.4291 // to insert and build vector.4292 EVT EltVT = VT.getVectorElementType();4293 SmallVector<SDValue,8> Ops;4294 for (int Idx : Mask) {4295 SDValue Res;4296 4297 if (Idx < 0) {4298 Res = DAG.getUNDEF(EltVT);4299 } else {4300 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;4301 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;4302 4303 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,4304 DAG.getVectorIdxConstant(Idx, DL));4305 }4306 4307 Ops.push_back(Res);4308 }4309 4310 setValue(&I, DAG.getBuildVector(VT, DL, Ops));4311}4312 4313void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {4314 ArrayRef<unsigned> Indices = I.getIndices();4315 const Value *Op0 = I.getOperand(0);4316 const Value *Op1 = I.getOperand(1);4317 Type *AggTy = I.getType();4318 Type *ValTy = Op1->getType();4319 bool IntoUndef = isa<UndefValue>(Op0);4320 bool FromUndef = isa<UndefValue>(Op1);4321 4322 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);4323 4324 const TargetLowering &TLI = DAG.getTargetLoweringInfo();4325 SmallVector<EVT, 4> AggValueVTs;4326 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);4327 SmallVector<EVT, 4> ValValueVTs;4328 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);4329 4330 unsigned NumAggValues = AggValueVTs.size();4331 unsigned NumValValues = ValValueVTs.size();4332 SmallVector<SDValue, 4> Values(NumAggValues);4333 4334 // Ignore an insertvalue that produces an empty object4335 if (!NumAggValues) {4336 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));4337 return;4338 }4339 4340 SDValue Agg = getValue(Op0);4341 unsigned i = 0;4342 // Copy the beginning value(s) from the original aggregate.4343 for (; i != LinearIndex; ++i)4344 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :4345 SDValue(Agg.getNode(), Agg.getResNo() + i);4346 // Copy values from the inserted value(s).4347 if (NumValValues) {4348 SDValue Val = getValue(Op1);4349 for (; i != LinearIndex + NumValValues; ++i)4350 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :4351 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);4352 }4353 // Copy remaining value(s) from the original aggregate.4354 for (; i != NumAggValues; ++i)4355 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :4356 SDValue(Agg.getNode(), Agg.getResNo() + i);4357 4358 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),4359 DAG.getVTList(AggValueVTs), Values));4360}4361 4362void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {4363 ArrayRef<unsigned> Indices = I.getIndices();4364 const Value *Op0 = I.getOperand(0);4365 Type *AggTy = Op0->getType();4366 Type *ValTy = I.getType();4367 bool OutOfUndef = isa<UndefValue>(Op0);4368 4369 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);4370 4371 const TargetLowering &TLI = DAG.getTargetLoweringInfo();4372 SmallVector<EVT, 4> ValValueVTs;4373 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);4374 4375 unsigned NumValValues = ValValueVTs.size();4376 4377 // Ignore a extractvalue that produces an empty object4378 if (!NumValValues) {4379 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));4380 return;4381 }4382 4383 SmallVector<SDValue, 4> Values(NumValValues);4384 4385 SDValue Agg = getValue(Op0);4386 // Copy out the selected value(s).4387 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)4388 Values[i - LinearIndex] =4389 OutOfUndef ?4390 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :4391 SDValue(Agg.getNode(), Agg.getResNo() + i);4392 4393 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),4394 DAG.getVTList(ValValueVTs), Values));4395}4396 4397void SelectionDAGBuilder::visitGetElementPtr(const User &I) {4398 Value *Op0 = I.getOperand(0);4399 // Note that the pointer operand may be a vector of pointers. Take the scalar4400 // element which holds a pointer.4401 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();4402 SDValue N = getValue(Op0);4403 SDLoc dl = getCurSDLoc();4404 auto &TLI = DAG.getTargetLoweringInfo();4405 GEPNoWrapFlags NW = cast<GEPOperator>(I).getNoWrapFlags();4406 4407 // For a vector GEP, keep the prefix scalar as long as possible, then4408 // convert any scalars encountered after the first vector operand to vectors.4409 bool IsVectorGEP = I.getType()->isVectorTy();4410 ElementCount VectorElementCount =4411 IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()4412 : ElementCount::getFixed(0);4413 4414 for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);4415 GTI != E; ++GTI) {4416 const Value *Idx = GTI.getOperand();4417 if (StructType *StTy = GTI.getStructTypeOrNull()) {4418 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();4419 if (Field) {4420 // N = N + Offset4421 uint64_t Offset =4422 DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(Field);4423 4424 // In an inbounds GEP with an offset that is nonnegative even when4425 // interpreted as signed, assume there is no unsigned overflow.4426 SDNodeFlags Flags;4427 if (NW.hasNoUnsignedWrap() ||4428 (int64_t(Offset) >= 0 && NW.hasNoUnsignedSignedWrap()))4429 Flags |= SDNodeFlags::NoUnsignedWrap;4430 Flags.setInBounds(NW.isInBounds());4431 4432 N = DAG.getMemBasePlusOffset(4433 N, DAG.getConstant(Offset, dl, N.getValueType()), dl, Flags);4434 }4435 } else {4436 // IdxSize is the width of the arithmetic according to IR semantics.4437 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth4438 // (and fix up the result later).4439 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);4440 MVT IdxTy = MVT::getIntegerVT(IdxSize);4441 TypeSize ElementSize =4442 GTI.getSequentialElementStride(DAG.getDataLayout());4443 // We intentionally mask away the high bits here; ElementSize may not4444 // fit in IdxTy.4445 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue(),4446 /*isSigned=*/false, /*implicitTrunc=*/true);4447 bool ElementScalable = ElementSize.isScalable();4448 4449 // If this is a scalar constant or a splat vector of constants,4450 // handle it quickly.4451 const auto *C = dyn_cast<Constant>(Idx);4452 if (C && isa<VectorType>(C->getType()))4453 C = C->getSplatValue();4454 4455 const auto *CI = dyn_cast_or_null<ConstantInt>(C);4456 if (CI && CI->isZero())4457 continue;4458 if (CI && !ElementScalable) {4459 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);4460 LLVMContext &Context = *DAG.getContext();4461 SDValue OffsVal;4462 if (N.getValueType().isVector())4463 OffsVal = DAG.getConstant(4464 Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));4465 else4466 OffsVal = DAG.getConstant(Offs, dl, IdxTy);4467 4468 // In an inbounds GEP with an offset that is nonnegative even when4469 // interpreted as signed, assume there is no unsigned overflow.4470 SDNodeFlags Flags;4471 if (NW.hasNoUnsignedWrap() ||4472 (Offs.isNonNegative() && NW.hasNoUnsignedSignedWrap()))4473 Flags.setNoUnsignedWrap(true);4474 Flags.setInBounds(NW.isInBounds());4475 4476 OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());4477 4478 N = DAG.getMemBasePlusOffset(N, OffsVal, dl, Flags);4479 continue;4480 }4481 4482 // N = N + Idx * ElementMul;4483 SDValue IdxN = getValue(Idx);4484 4485 if (IdxN.getValueType().isVector() != N.getValueType().isVector()) {4486 if (N.getValueType().isVector()) {4487 EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),4488 VectorElementCount);4489 IdxN = DAG.getSplat(VT, dl, IdxN);4490 } else {4491 EVT VT =4492 EVT::getVectorVT(*Context, N.getValueType(), VectorElementCount);4493 N = DAG.getSplat(VT, dl, N);4494 }4495 }4496 4497 // If the index is smaller or larger than intptr_t, truncate or extend4498 // it.4499 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());4500 4501 SDNodeFlags ScaleFlags;4502 // The multiplication of an index by the type size does not wrap the4503 // pointer index type in a signed sense (mul nsw).4504 ScaleFlags.setNoSignedWrap(NW.hasNoUnsignedSignedWrap());4505 4506 // The multiplication of an index by the type size does not wrap the4507 // pointer index type in an unsigned sense (mul nuw).4508 ScaleFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());4509 4510 if (ElementScalable) {4511 EVT VScaleTy = N.getValueType().getScalarType();4512 SDValue VScale = DAG.getNode(4513 ISD::VSCALE, dl, VScaleTy,4514 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));4515 if (N.getValueType().isVector())4516 VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);4517 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale,4518 ScaleFlags);4519 } else {4520 // If this is a multiply by a power of two, turn it into a shl4521 // immediately. This is a very common case.4522 if (ElementMul != 1) {4523 if (ElementMul.isPowerOf2()) {4524 unsigned Amt = ElementMul.logBase2();4525 IdxN = DAG.getNode(4526 ISD::SHL, dl, N.getValueType(), IdxN,4527 DAG.getShiftAmountConstant(Amt, N.getValueType(), dl),4528 ScaleFlags);4529 } else {4530 SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,4531 IdxN.getValueType());4532 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, Scale,4533 ScaleFlags);4534 }4535 }4536 }4537 4538 // The successive addition of the current address, truncated to the4539 // pointer index type and interpreted as an unsigned number, and each4540 // offset, also interpreted as an unsigned number, does not wrap the4541 // pointer index type (add nuw).4542 SDNodeFlags AddFlags;4543 AddFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());4544 AddFlags.setInBounds(NW.isInBounds());4545 4546 N = DAG.getMemBasePlusOffset(N, IdxN, dl, AddFlags);4547 }4548 }4549 4550 if (IsVectorGEP && !N.getValueType().isVector()) {4551 EVT VT = EVT::getVectorVT(*Context, N.getValueType(), VectorElementCount);4552 N = DAG.getSplat(VT, dl, N);4553 }4554 4555 MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);4556 MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);4557 if (IsVectorGEP) {4558 PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);4559 PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);4560 }4561 4562 if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())4563 N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);4564 4565 setValue(&I, N);4566}4567 4568void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {4569 // If this is a fixed sized alloca in the entry block of the function,4570 // allocate it statically on the stack.4571 if (FuncInfo.StaticAllocaMap.count(&I))4572 return; // getValue will auto-populate this.4573 4574 SDLoc dl = getCurSDLoc();4575 Type *Ty = I.getAllocatedType();4576 const TargetLowering &TLI = DAG.getTargetLoweringInfo();4577 auto &DL = DAG.getDataLayout();4578 TypeSize TySize = DL.getTypeAllocSize(Ty);4579 MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());4580 4581 SDValue AllocSize = getValue(I.getArraySize());4582 4583 EVT IntPtr = TLI.getPointerTy(DL, I.getAddressSpace());4584 if (AllocSize.getValueType() != IntPtr)4585 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);4586 4587 AllocSize = DAG.getNode(4588 ISD::MUL, dl, IntPtr, AllocSize,4589 DAG.getZExtOrTrunc(DAG.getTypeSize(dl, MVT::i64, TySize), dl, IntPtr));4590 4591 // Handle alignment. If the requested alignment is less than or equal to4592 // the stack alignment, ignore it. If the size is greater than or equal to4593 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.4594 Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();4595 if (*Alignment <= StackAlign)4596 Alignment = std::nullopt;4597 4598 const uint64_t StackAlignMask = StackAlign.value() - 1U;4599 // Round the size of the allocation up to the stack alignment size4600 // by add SA-1 to the size. This doesn't overflow because we're computing4601 // an address inside an alloca.4602 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,4603 DAG.getConstant(StackAlignMask, dl, IntPtr),4604 SDNodeFlags::NoUnsignedWrap);4605 4606 // Mask out the low bits for alignment purposes.4607 AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,4608 DAG.getSignedConstant(~StackAlignMask, dl, IntPtr));4609 4610 SDValue Ops[] = {4611 getRoot(), AllocSize,4612 DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};4613 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);4614 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);4615 setValue(&I, DSA);4616 DAG.setRoot(DSA.getValue(1));4617 4618 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());4619}4620 4621static const MDNode *getRangeMetadata(const Instruction &I) {4622 return I.getMetadata(LLVMContext::MD_range);4623}4624 4625static std::optional<ConstantRange> getRange(const Instruction &I) {4626 if (const auto *CB = dyn_cast<CallBase>(&I))4627 if (std::optional<ConstantRange> CR = CB->getRange())4628 return CR;4629 if (const MDNode *Range = getRangeMetadata(I))4630 return getConstantRangeFromMetadata(*Range);4631 return std::nullopt;4632}4633 4634static FPClassTest getNoFPClass(const Instruction &I) {4635 if (const auto *CB = dyn_cast<CallBase>(&I))4636 return CB->getRetNoFPClass();4637 return fcNone;4638}4639 4640void SelectionDAGBuilder::visitLoad(const LoadInst &I) {4641 if (I.isAtomic())4642 return visitAtomicLoad(I);4643 4644 const TargetLowering &TLI = DAG.getTargetLoweringInfo();4645 const Value *SV = I.getOperand(0);4646 if (TLI.supportSwiftError()) {4647 // Swifterror values can come from either a function parameter with4648 // swifterror attribute or an alloca with swifterror attribute.4649 if (const Argument *Arg = dyn_cast<Argument>(SV)) {4650 if (Arg->hasSwiftErrorAttr())4651 return visitLoadFromSwiftError(I);4652 }4653 4654 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {4655 if (Alloca->isSwiftError())4656 return visitLoadFromSwiftError(I);4657 }4658 }4659 4660 SDValue Ptr = getValue(SV);4661 4662 Type *Ty = I.getType();4663 SmallVector<EVT, 4> ValueVTs, MemVTs;4664 SmallVector<TypeSize, 4> Offsets;4665 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);4666 unsigned NumValues = ValueVTs.size();4667 if (NumValues == 0)4668 return;4669 4670 Align Alignment = I.getAlign();4671 AAMDNodes AAInfo = I.getAAMetadata();4672 const MDNode *Ranges = getRangeMetadata(I);4673 bool isVolatile = I.isVolatile();4674 MachineMemOperand::Flags MMOFlags =4675 TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);4676 4677 SDValue Root;4678 bool ConstantMemory = false;4679 if (isVolatile)4680 // Serialize volatile loads with other side effects.4681 Root = getRoot();4682 else if (NumValues > MaxParallelChains)4683 Root = getMemoryRoot();4684 else if (BatchAA &&4685 BatchAA->pointsToConstantMemory(MemoryLocation(4686 SV,4687 LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),4688 AAInfo))) {4689 // Do not serialize (non-volatile) loads of constant memory with anything.4690 Root = DAG.getEntryNode();4691 ConstantMemory = true;4692 MMOFlags |= MachineMemOperand::MOInvariant;4693 } else {4694 // Do not serialize non-volatile loads against each other.4695 Root = DAG.getRoot();4696 }4697 4698 SDLoc dl = getCurSDLoc();4699 4700 if (isVolatile)4701 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);4702 4703 SmallVector<SDValue, 4> Values(NumValues);4704 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));4705 4706 unsigned ChainI = 0;4707 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {4708 // Serializing loads here may result in excessive register pressure, and4709 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling4710 // could recover a bit by hoisting nodes upward in the chain by recognizing4711 // they are side-effect free or do not alias. The optimizer should really4712 // avoid this case by converting large object/array copies to llvm.memcpy4713 // (MaxParallelChains should always remain as failsafe).4714 if (ChainI == MaxParallelChains) {4715 assert(PendingLoads.empty() && "PendingLoads must be serialized first");4716 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,4717 ArrayRef(Chains.data(), ChainI));4718 Root = Chain;4719 ChainI = 0;4720 }4721 4722 // TODO: MachinePointerInfo only supports a fixed length offset.4723 MachinePointerInfo PtrInfo =4724 !Offsets[i].isScalable() || Offsets[i].isZero()4725 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())4726 : MachinePointerInfo();4727 4728 SDValue A = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);4729 SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment,4730 MMOFlags, AAInfo, Ranges);4731 Chains[ChainI] = L.getValue(1);4732 4733 if (MemVTs[i] != ValueVTs[i])4734 L = DAG.getPtrExtOrTrunc(L, dl, ValueVTs[i]);4735 4736 Values[i] = L;4737 }4738 4739 if (!ConstantMemory) {4740 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,4741 ArrayRef(Chains.data(), ChainI));4742 if (isVolatile)4743 DAG.setRoot(Chain);4744 else4745 PendingLoads.push_back(Chain);4746 }4747 4748 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,4749 DAG.getVTList(ValueVTs), Values));4750}4751 4752void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {4753 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&4754 "call visitStoreToSwiftError when backend supports swifterror");4755 4756 SmallVector<EVT, 4> ValueVTs;4757 SmallVector<uint64_t, 4> Offsets;4758 const Value *SrcV = I.getOperand(0);4759 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),4760 SrcV->getType(), ValueVTs, /*MemVTs=*/nullptr, &Offsets, 0);4761 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&4762 "expect a single EVT for swifterror");4763 4764 SDValue Src = getValue(SrcV);4765 // Create a virtual register, then update the virtual register.4766 Register VReg =4767 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());4768 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue4769 // Chain can be getRoot or getControlRoot.4770 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,4771 SDValue(Src.getNode(), Src.getResNo()));4772 DAG.setRoot(CopyNode);4773}4774 4775void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {4776 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&4777 "call visitLoadFromSwiftError when backend supports swifterror");4778 4779 assert(!I.isVolatile() &&4780 !I.hasMetadata(LLVMContext::MD_nontemporal) &&4781 !I.hasMetadata(LLVMContext::MD_invariant_load) &&4782 "Support volatile, non temporal, invariant for load_from_swift_error");4783 4784 const Value *SV = I.getOperand(0);4785 Type *Ty = I.getType();4786 assert(4787 (!BatchAA ||4788 !BatchAA->pointsToConstantMemory(MemoryLocation(4789 SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),4790 I.getAAMetadata()))) &&4791 "load_from_swift_error should not be constant memory");4792 4793 SmallVector<EVT, 4> ValueVTs;4794 SmallVector<uint64_t, 4> Offsets;4795 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,4796 ValueVTs, /*MemVTs=*/nullptr, &Offsets, 0);4797 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&4798 "expect a single EVT for swifterror");4799 4800 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT4801 SDValue L = DAG.getCopyFromReg(4802 getRoot(), getCurSDLoc(),4803 SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);4804 4805 setValue(&I, L);4806}4807 4808void SelectionDAGBuilder::visitStore(const StoreInst &I) {4809 if (I.isAtomic())4810 return visitAtomicStore(I);4811 4812 const Value *SrcV = I.getOperand(0);4813 const Value *PtrV = I.getOperand(1);4814 4815 const TargetLowering &TLI = DAG.getTargetLoweringInfo();4816 if (TLI.supportSwiftError()) {4817 // Swifterror values can come from either a function parameter with4818 // swifterror attribute or an alloca with swifterror attribute.4819 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {4820 if (Arg->hasSwiftErrorAttr())4821 return visitStoreToSwiftError(I);4822 }4823 4824 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {4825 if (Alloca->isSwiftError())4826 return visitStoreToSwiftError(I);4827 }4828 }4829 4830 SmallVector<EVT, 4> ValueVTs, MemVTs;4831 SmallVector<TypeSize, 4> Offsets;4832 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),4833 SrcV->getType(), ValueVTs, &MemVTs, &Offsets);4834 unsigned NumValues = ValueVTs.size();4835 if (NumValues == 0)4836 return;4837 4838 // Get the lowered operands. Note that we do this after4839 // checking if NumResults is zero, because with zero results4840 // the operands won't have values in the map.4841 SDValue Src = getValue(SrcV);4842 SDValue Ptr = getValue(PtrV);4843 4844 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();4845 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));4846 SDLoc dl = getCurSDLoc();4847 Align Alignment = I.getAlign();4848 AAMDNodes AAInfo = I.getAAMetadata();4849 4850 auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());4851 4852 unsigned ChainI = 0;4853 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {4854 // See visitLoad comments.4855 if (ChainI == MaxParallelChains) {4856 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,4857 ArrayRef(Chains.data(), ChainI));4858 Root = Chain;4859 ChainI = 0;4860 }4861 4862 // TODO: MachinePointerInfo only supports a fixed length offset.4863 MachinePointerInfo PtrInfo =4864 !Offsets[i].isScalable() || Offsets[i].isZero()4865 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())4866 : MachinePointerInfo();4867 4868 SDValue Add = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);4869 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);4870 if (MemVTs[i] != ValueVTs[i])4871 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);4872 SDValue St =4873 DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags, AAInfo);4874 Chains[ChainI] = St;4875 }4876 4877 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,4878 ArrayRef(Chains.data(), ChainI));4879 setValue(&I, StoreNode);4880 DAG.setRoot(StoreNode);4881}4882 4883void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,4884 bool IsCompressing) {4885 SDLoc sdl = getCurSDLoc();4886 4887 Value *Src0Operand = I.getArgOperand(0);4888 Value *PtrOperand = I.getArgOperand(1);4889 Value *MaskOperand = I.getArgOperand(2);4890 Align Alignment = I.getParamAlign(1).valueOrOne();4891 4892 SDValue Ptr = getValue(PtrOperand);4893 SDValue Src0 = getValue(Src0Operand);4894 SDValue Mask = getValue(MaskOperand);4895 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());4896 4897 EVT VT = Src0.getValueType();4898 4899 auto MMOFlags = MachineMemOperand::MOStore;4900 if (I.hasMetadata(LLVMContext::MD_nontemporal))4901 MMOFlags |= MachineMemOperand::MONonTemporal;4902 4903 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(4904 MachinePointerInfo(PtrOperand), MMOFlags,4905 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());4906 4907 const auto &TLI = DAG.getTargetLoweringInfo();4908 4909 SDValue StoreNode =4910 !IsCompressing && TTI->hasConditionalLoadStoreForType(4911 I.getArgOperand(0)->getType(), /*IsStore=*/true)4912 ? TLI.visitMaskedStore(DAG, sdl, getMemoryRoot(), MMO, Ptr, Src0,4913 Mask)4914 : DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask,4915 VT, MMO, ISD::UNINDEXED, /*Truncating=*/false,4916 IsCompressing);4917 DAG.setRoot(StoreNode);4918 setValue(&I, StoreNode);4919}4920 4921// Get a uniform base for the Gather/Scatter intrinsic.4922// The first argument of the Gather/Scatter intrinsic is a vector of pointers.4923// We try to represent it as a base pointer + vector of indices.4924// Usually, the vector of pointers comes from a 'getelementptr' instruction.4925// The first operand of the GEP may be a single pointer or a vector of pointers4926// Example:4927// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind4928// or4929// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind4930// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..4931//4932// When the first GEP operand is a single pointer - it is the uniform base we4933// are looking for. If first operand of the GEP is a splat vector - we4934// extract the splat value and use it as a uniform base.4935// In all other cases the function returns 'false'.4936static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,4937 SDValue &Scale, SelectionDAGBuilder *SDB,4938 const BasicBlock *CurBB, uint64_t ElemSize) {4939 SelectionDAG& DAG = SDB->DAG;4940 const TargetLowering &TLI = DAG.getTargetLoweringInfo();4941 const DataLayout &DL = DAG.getDataLayout();4942 4943 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");4944 4945 // Handle splat constant pointer.4946 if (auto *C = dyn_cast<Constant>(Ptr)) {4947 C = C->getSplatValue();4948 if (!C)4949 return false;4950 4951 Base = SDB->getValue(C);4952 4953 ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();4954 EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);4955 Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);4956 Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));4957 return true;4958 }4959 4960 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);4961 if (!GEP || GEP->getParent() != CurBB)4962 return false;4963 4964 if (GEP->getNumOperands() != 2)4965 return false;4966 4967 const Value *BasePtr = GEP->getPointerOperand();4968 const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);4969 4970 // Make sure the base is scalar and the index is a vector.4971 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())4972 return false;4973 4974 TypeSize ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());4975 if (ScaleVal.isScalable())4976 return false;4977 4978 // Target may not support the required addressing mode.4979 if (ScaleVal != 1 &&4980 !TLI.isLegalScaleForGatherScatter(ScaleVal.getFixedValue(), ElemSize))4981 return false;4982 4983 Base = SDB->getValue(BasePtr);4984 Index = SDB->getValue(IndexVal);4985 4986 Scale =4987 DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));4988 return true;4989}4990 4991void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {4992 SDLoc sdl = getCurSDLoc();4993 4994 // llvm.masked.scatter.*(Src0, Ptrs, Mask)4995 const Value *Ptr = I.getArgOperand(1);4996 SDValue Src0 = getValue(I.getArgOperand(0));4997 SDValue Mask = getValue(I.getArgOperand(2));4998 EVT VT = Src0.getValueType();4999 Align Alignment = I.getParamAlign(1).valueOrOne();5000 const TargetLowering &TLI = DAG.getTargetLoweringInfo();5001 5002 SDValue Base;5003 SDValue Index;5004 SDValue Scale;5005 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,5006 I.getParent(), VT.getScalarStoreSize());5007 5008 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();5009 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(5010 MachinePointerInfo(AS), MachineMemOperand::MOStore,5011 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());5012 if (!UniformBase) {5013 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));5014 Index = getValue(Ptr);5015 Scale =5016 DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));5017 }5018 5019 EVT IdxVT = Index.getValueType();5020 EVT EltTy = IdxVT.getVectorElementType();5021 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {5022 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);5023 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);5024 }5025 5026 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };5027 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,5028 Ops, MMO, ISD::SIGNED_SCALED, false);5029 DAG.setRoot(Scatter);5030 setValue(&I, Scatter);5031}5032 5033void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {5034 SDLoc sdl = getCurSDLoc();5035 5036 Value *PtrOperand = I.getArgOperand(0);5037 Value *MaskOperand = I.getArgOperand(1);5038 Value *Src0Operand = I.getArgOperand(2);5039 Align Alignment = I.getParamAlign(0).valueOrOne();5040 5041 SDValue Ptr = getValue(PtrOperand);5042 SDValue Src0 = getValue(Src0Operand);5043 SDValue Mask = getValue(MaskOperand);5044 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());5045 5046 EVT VT = Src0.getValueType();5047 AAMDNodes AAInfo = I.getAAMetadata();5048 const MDNode *Ranges = getRangeMetadata(I);5049 5050 // Do not serialize masked loads of constant memory with anything.5051 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);5052 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);5053 5054 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();5055 5056 auto MMOFlags = MachineMemOperand::MOLoad;5057 if (I.hasMetadata(LLVMContext::MD_nontemporal))5058 MMOFlags |= MachineMemOperand::MONonTemporal;5059 if (I.hasMetadata(LLVMContext::MD_invariant_load))5060 MMOFlags |= MachineMemOperand::MOInvariant;5061 5062 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(5063 MachinePointerInfo(PtrOperand), MMOFlags,5064 LocationSize::beforeOrAfterPointer(), Alignment, AAInfo, Ranges);5065 5066 const auto &TLI = DAG.getTargetLoweringInfo();5067 5068 // The Load/Res may point to different values and both of them are output5069 // variables.5070 SDValue Load;5071 SDValue Res;5072 if (!IsExpanding &&5073 TTI->hasConditionalLoadStoreForType(Src0Operand->getType(),5074 /*IsStore=*/false))5075 Res = TLI.visitMaskedLoad(DAG, sdl, InChain, MMO, Load, Ptr, Src0, Mask);5076 else5077 Res = Load =5078 DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,5079 ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);5080 if (AddToChain)5081 PendingLoads.push_back(Load.getValue(1));5082 setValue(&I, Res);5083}5084 5085void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {5086 SDLoc sdl = getCurSDLoc();5087 5088 // @llvm.masked.gather.*(Ptrs, Mask, Src0)5089 const Value *Ptr = I.getArgOperand(0);5090 SDValue Src0 = getValue(I.getArgOperand(2));5091 SDValue Mask = getValue(I.getArgOperand(1));5092 5093 const TargetLowering &TLI = DAG.getTargetLoweringInfo();5094 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());5095 Align Alignment = I.getParamAlign(0).valueOrOne();5096 5097 const MDNode *Ranges = getRangeMetadata(I);5098 5099 SDValue Root = DAG.getRoot();5100 SDValue Base;5101 SDValue Index;5102 SDValue Scale;5103 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,5104 I.getParent(), VT.getScalarStoreSize());5105 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();5106 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(5107 MachinePointerInfo(AS), MachineMemOperand::MOLoad,5108 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(),5109 Ranges);5110 5111 if (!UniformBase) {5112 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));5113 Index = getValue(Ptr);5114 Scale =5115 DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));5116 }5117 5118 EVT IdxVT = Index.getValueType();5119 EVT EltTy = IdxVT.getVectorElementType();5120 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {5121 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);5122 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);5123 }5124 5125 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };5126 SDValue Gather =5127 DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl, Ops, MMO,5128 ISD::SIGNED_SCALED, ISD::NON_EXTLOAD);5129 5130 PendingLoads.push_back(Gather.getValue(1));5131 setValue(&I, Gather);5132}5133 5134void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {5135 SDLoc dl = getCurSDLoc();5136 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();5137 AtomicOrdering FailureOrdering = I.getFailureOrdering();5138 SyncScope::ID SSID = I.getSyncScopeID();5139 5140 SDValue InChain = getRoot();5141 5142 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();5143 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);5144 5145 const TargetLowering &TLI = DAG.getTargetLoweringInfo();5146 auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());5147 5148 MachineFunction &MF = DAG.getMachineFunction();5149 MachineMemOperand *MMO = MF.getMachineMemOperand(5150 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),5151 DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering,5152 FailureOrdering);5153 5154 SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,5155 dl, MemVT, VTs, InChain,5156 getValue(I.getPointerOperand()),5157 getValue(I.getCompareOperand()),5158 getValue(I.getNewValOperand()), MMO);5159 5160 SDValue OutChain = L.getValue(2);5161 5162 setValue(&I, L);5163 DAG.setRoot(OutChain);5164}5165 5166void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {5167 SDLoc dl = getCurSDLoc();5168 ISD::NodeType NT;5169 switch (I.getOperation()) {5170 default: llvm_unreachable("Unknown atomicrmw operation");5171 case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;5172 case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;5173 case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;5174 case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;5175 case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;5176 case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;5177 case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;5178 case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;5179 case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;5180 case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;5181 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;5182 case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;5183 case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;5184 case AtomicRMWInst::FMax: NT = ISD::ATOMIC_LOAD_FMAX; break;5185 case AtomicRMWInst::FMin: NT = ISD::ATOMIC_LOAD_FMIN; break;5186 case AtomicRMWInst::FMaximum:5187 NT = ISD::ATOMIC_LOAD_FMAXIMUM;5188 break;5189 case AtomicRMWInst::FMinimum:5190 NT = ISD::ATOMIC_LOAD_FMINIMUM;5191 break;5192 case AtomicRMWInst::UIncWrap:5193 NT = ISD::ATOMIC_LOAD_UINC_WRAP;5194 break;5195 case AtomicRMWInst::UDecWrap:5196 NT = ISD::ATOMIC_LOAD_UDEC_WRAP;5197 break;5198 case AtomicRMWInst::USubCond:5199 NT = ISD::ATOMIC_LOAD_USUB_COND;5200 break;5201 case AtomicRMWInst::USubSat:5202 NT = ISD::ATOMIC_LOAD_USUB_SAT;5203 break;5204 }5205 AtomicOrdering Ordering = I.getOrdering();5206 SyncScope::ID SSID = I.getSyncScopeID();5207 5208 SDValue InChain = getRoot();5209 5210 auto MemVT = getValue(I.getValOperand()).getSimpleValueType();5211 const TargetLowering &TLI = DAG.getTargetLoweringInfo();5212 auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());5213 5214 MachineFunction &MF = DAG.getMachineFunction();5215 MachineMemOperand *MMO = MF.getMachineMemOperand(5216 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),5217 DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering);5218 5219 SDValue L =5220 DAG.getAtomic(NT, dl, MemVT, InChain,5221 getValue(I.getPointerOperand()), getValue(I.getValOperand()),5222 MMO);5223 5224 SDValue OutChain = L.getValue(1);5225 5226 setValue(&I, L);5227 DAG.setRoot(OutChain);5228}5229 5230void SelectionDAGBuilder::visitFence(const FenceInst &I) {5231 SDLoc dl = getCurSDLoc();5232 const TargetLowering &TLI = DAG.getTargetLoweringInfo();5233 SDValue Ops[3];5234 Ops[0] = getRoot();5235 Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,5236 TLI.getFenceOperandTy(DAG.getDataLayout()));5237 Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,5238 TLI.getFenceOperandTy(DAG.getDataLayout()));5239 SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);5240 setValue(&I, N);5241 DAG.setRoot(N);5242}5243 5244void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {5245 SDLoc dl = getCurSDLoc();5246 AtomicOrdering Order = I.getOrdering();5247 SyncScope::ID SSID = I.getSyncScopeID();5248 5249 SDValue InChain = getRoot();5250 5251 const TargetLowering &TLI = DAG.getTargetLoweringInfo();5252 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());5253 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());5254 5255 if (!TLI.supportsUnalignedAtomics() &&5256 I.getAlign().value() < MemVT.getSizeInBits() / 8)5257 report_fatal_error("Cannot generate unaligned atomic load");5258 5259 auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);5260 5261 const MDNode *Ranges = getRangeMetadata(I);5262 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(5263 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),5264 I.getAlign(), AAMDNodes(), Ranges, SSID, Order);5265 5266 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);5267 5268 SDValue Ptr = getValue(I.getPointerOperand());5269 SDValue L =5270 DAG.getAtomicLoad(ISD::NON_EXTLOAD, dl, MemVT, MemVT, InChain, Ptr, MMO);5271 5272 SDValue OutChain = L.getValue(1);5273 if (MemVT != VT)5274 L = DAG.getPtrExtOrTrunc(L, dl, VT);5275 5276 setValue(&I, L);5277 DAG.setRoot(OutChain);5278}5279 5280void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {5281 SDLoc dl = getCurSDLoc();5282 5283 AtomicOrdering Ordering = I.getOrdering();5284 SyncScope::ID SSID = I.getSyncScopeID();5285 5286 SDValue InChain = getRoot();5287 5288 const TargetLowering &TLI = DAG.getTargetLoweringInfo();5289 EVT MemVT =5290 TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());5291 5292 if (!TLI.supportsUnalignedAtomics() &&5293 I.getAlign().value() < MemVT.getSizeInBits() / 8)5294 report_fatal_error("Cannot generate unaligned atomic store");5295 5296 auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());5297 5298 MachineFunction &MF = DAG.getMachineFunction();5299 MachineMemOperand *MMO = MF.getMachineMemOperand(5300 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),5301 I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);5302 5303 SDValue Val = getValue(I.getValueOperand());5304 if (Val.getValueType() != MemVT)5305 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);5306 SDValue Ptr = getValue(I.getPointerOperand());5307 5308 SDValue OutChain =5309 DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, Val, Ptr, MMO);5310 5311 setValue(&I, OutChain);5312 DAG.setRoot(OutChain);5313}5314 5315/// Check if this intrinsic call depends on the chain (1st return value)5316/// and if it only *loads* memory.5317/// Ignore the callsite's attributes. A specific call site may be marked with5318/// readnone, but the lowering code will expect the chain based on the5319/// definition.5320std::pair<bool, bool>5321SelectionDAGBuilder::getTargetIntrinsicCallProperties(const CallBase &I) {5322 const Function *F = I.getCalledFunction();5323 bool HasChain = !F->doesNotAccessMemory();5324 bool OnlyLoad =5325 HasChain && F->onlyReadsMemory() && F->willReturn() && F->doesNotThrow();5326 5327 return {HasChain, OnlyLoad};5328}5329 5330SmallVector<SDValue, 8> SelectionDAGBuilder::getTargetIntrinsicOperands(5331 const CallBase &I, bool HasChain, bool OnlyLoad,5332 TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo) {5333 const TargetLowering &TLI = DAG.getTargetLoweringInfo();5334 5335 // Build the operand list.5336 SmallVector<SDValue, 8> Ops;5337 if (HasChain) { // If this intrinsic has side-effects, chainify it.5338 if (OnlyLoad) {5339 // We don't need to serialize loads against other loads.5340 Ops.push_back(DAG.getRoot());5341 } else {5342 Ops.push_back(getRoot());5343 }5344 }5345 5346 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.5347 if (!TgtMemIntrinsicInfo || TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_VOID ||5348 TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_W_CHAIN)5349 Ops.push_back(DAG.getTargetConstant(I.getIntrinsicID(), getCurSDLoc(),5350 TLI.getPointerTy(DAG.getDataLayout())));5351 5352 // Add all operands of the call to the operand list.5353 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {5354 const Value *Arg = I.getArgOperand(i);5355 if (!I.paramHasAttr(i, Attribute::ImmArg)) {5356 Ops.push_back(getValue(Arg));5357 continue;5358 }5359 5360 // Use TargetConstant instead of a regular constant for immarg.5361 EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);5362 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {5363 assert(CI->getBitWidth() <= 64 &&5364 "large intrinsic immediates not handled");5365 Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));5366 } else {5367 Ops.push_back(5368 DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));5369 }5370 }5371 5372 if (std::optional<OperandBundleUse> Bundle =5373 I.getOperandBundle(LLVMContext::OB_deactivation_symbol)) {5374 auto *Sym = Bundle->Inputs[0].get();5375 SDValue SDSym = getValue(Sym);5376 SDSym = DAG.getDeactivationSymbol(cast<GlobalValue>(Sym));5377 Ops.push_back(SDSym);5378 }5379 5380 if (std::optional<OperandBundleUse> Bundle =5381 I.getOperandBundle(LLVMContext::OB_convergencectrl)) {5382 Value *Token = Bundle->Inputs[0].get();5383 SDValue ConvControlToken = getValue(Token);5384 assert(Ops.back().getValueType() != MVT::Glue &&5385 "Did not expect another glue node here.");5386 ConvControlToken =5387 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken);5388 Ops.push_back(ConvControlToken);5389 }5390 5391 return Ops;5392}5393 5394SDVTList SelectionDAGBuilder::getTargetIntrinsicVTList(const CallBase &I,5395 bool HasChain) {5396 const TargetLowering &TLI = DAG.getTargetLoweringInfo();5397 5398 SmallVector<EVT, 4> ValueVTs;5399 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);5400 5401 if (HasChain)5402 ValueVTs.push_back(MVT::Other);5403 5404 return DAG.getVTList(ValueVTs);5405}5406 5407/// Get an INTRINSIC node for a target intrinsic which does not touch memory.5408SDValue SelectionDAGBuilder::getTargetNonMemIntrinsicNode(5409 const Type &IntrinsicVT, bool HasChain, ArrayRef<SDValue> Ops,5410 const SDVTList &VTs) {5411 if (!HasChain)5412 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);5413 if (!IntrinsicVT.isVoidTy())5414 return DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);5415 return DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);5416}5417 5418/// Set root, convert return type if necessary and check alignment.5419SDValue SelectionDAGBuilder::handleTargetIntrinsicRet(const CallBase &I,5420 bool HasChain,5421 bool OnlyLoad,5422 SDValue Result) {5423 if (HasChain) {5424 SDValue Chain = Result.getValue(Result.getNode()->getNumValues() - 1);5425 if (OnlyLoad)5426 PendingLoads.push_back(Chain);5427 else5428 DAG.setRoot(Chain);5429 }5430 5431 if (I.getType()->isVoidTy())5432 return Result;5433 5434 if (MaybeAlign Alignment = I.getRetAlign(); InsertAssertAlign && Alignment) {5435 // Insert `assertalign` node if there's an alignment.5436 Result = DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());5437 } else if (!isa<VectorType>(I.getType())) {5438 Result = lowerRangeToAssertZExt(DAG, I, Result);5439 }5440 5441 return Result;5442}5443 5444/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC5445/// node.5446void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,5447 unsigned Intrinsic) {5448 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);5449 5450 // Info is set by getTgtMemIntrinsic5451 TargetLowering::IntrinsicInfo Info;5452 const TargetLowering &TLI = DAG.getTargetLoweringInfo();5453 bool IsTgtMemIntrinsic =5454 TLI.getTgtMemIntrinsic(Info, I, DAG.getMachineFunction(), Intrinsic);5455 5456 SmallVector<SDValue, 8> Ops = getTargetIntrinsicOperands(5457 I, HasChain, OnlyLoad, IsTgtMemIntrinsic ? &Info : nullptr);5458 SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);5459 5460 // Propagate fast-math-flags from IR to node(s).5461 SDNodeFlags Flags;5462 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))5463 Flags.copyFMF(*FPMO);5464 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);5465 5466 // Create the node.5467 SDValue Result;5468 5469 // In some cases, custom collection of operands from CallInst I may be needed.5470 TLI.CollectTargetIntrinsicOperands(I, Ops, DAG);5471 if (IsTgtMemIntrinsic) {5472 // This is target intrinsic that touches memory5473 //5474 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic5475 // didn't yield anything useful.5476 MachinePointerInfo MPI;5477 if (Info.ptrVal)5478 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);5479 else if (Info.fallbackAddressSpace)5480 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);5481 EVT MemVT = Info.memVT;5482 LocationSize Size = LocationSize::precise(Info.size);5483 if (Size.hasValue() && !Size.getValue())5484 Size = LocationSize::precise(MemVT.getStoreSize());5485 Align Alignment = Info.align.value_or(DAG.getEVTAlign(MemVT));5486 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(5487 MPI, Info.flags, Size, Alignment, I.getAAMetadata(), /*Ranges=*/nullptr,5488 Info.ssid, Info.order, Info.failureOrder);5489 Result =5490 DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, MemVT, MMO);5491 } else {5492 Result = getTargetNonMemIntrinsicNode(*I.getType(), HasChain, Ops, VTs);5493 }5494 5495 Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);5496 5497 setValue(&I, Result);5498}5499 5500/// GetSignificand - Get the significand and build it into a floating-point5501/// number with exponent of 1:5502///5503/// Op = (Op & 0x007fffff) | 0x3f800000;5504///5505/// where Op is the hexadecimal representation of floating point value.5506static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {5507 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,5508 DAG.getConstant(0x007fffff, dl, MVT::i32));5509 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,5510 DAG.getConstant(0x3f800000, dl, MVT::i32));5511 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);5512}5513 5514/// GetExponent - Get the exponent:5515///5516/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);5517///5518/// where Op is the hexadecimal representation of floating point value.5519static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,5520 const TargetLowering &TLI, const SDLoc &dl) {5521 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,5522 DAG.getConstant(0x7f800000, dl, MVT::i32));5523 SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,5524 DAG.getShiftAmountConstant(23, MVT::i32, dl));5525 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,5526 DAG.getConstant(127, dl, MVT::i32));5527 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);5528}5529 5530/// getF32Constant - Get 32-bit floating point constant.5531static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,5532 const SDLoc &dl) {5533 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,5534 MVT::f32);5535}5536 5537static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,5538 SelectionDAG &DAG) {5539 // TODO: What fast-math-flags should be set on the floating-point nodes?5540 5541 // IntegerPartOfX = ((int32_t)(t0);5542 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);5543 5544 // FractionalPartOfX = t0 - (float)IntegerPartOfX;5545 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);5546 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);5547 5548 // IntegerPartOfX <<= 23;5549 IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,5550 DAG.getShiftAmountConstant(23, MVT::i32, dl));5551 5552 SDValue TwoToFractionalPartOfX;5553 if (LimitFloatPrecision <= 6) {5554 // For floating-point precision of 6:5555 //5556 // TwoToFractionalPartOfX =5557 // 0.997535578f +5558 // (0.735607626f + 0.252464424f * x) * x;5559 //5560 // error 0.0144103317, which is 6 bits5561 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,5562 getF32Constant(DAG, 0x3e814304, dl));5563 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,5564 getF32Constant(DAG, 0x3f3c50c8, dl));5565 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);5566 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,5567 getF32Constant(DAG, 0x3f7f5e7e, dl));5568 } else if (LimitFloatPrecision <= 12) {5569 // For floating-point precision of 12:5570 //5571 // TwoToFractionalPartOfX =5572 // 0.999892986f +5573 // (0.696457318f +5574 // (0.224338339f + 0.792043434e-1f * x) * x) * x;5575 //5576 // error 0.000107046256, which is 13 to 14 bits5577 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,5578 getF32Constant(DAG, 0x3da235e3, dl));5579 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,5580 getF32Constant(DAG, 0x3e65b8f3, dl));5581 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);5582 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,5583 getF32Constant(DAG, 0x3f324b07, dl));5584 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);5585 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,5586 getF32Constant(DAG, 0x3f7ff8fd, dl));5587 } else { // LimitFloatPrecision <= 185588 // For floating-point precision of 18:5589 //5590 // TwoToFractionalPartOfX =5591 // 0.999999982f +5592 // (0.693148872f +5593 // (0.240227044f +5594 // (0.554906021e-1f +5595 // (0.961591928e-2f +5596 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;5597 // error 2.47208000*10^(-7), which is better than 18 bits5598 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,5599 getF32Constant(DAG, 0x3924b03e, dl));5600 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,5601 getF32Constant(DAG, 0x3ab24b87, dl));5602 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);5603 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,5604 getF32Constant(DAG, 0x3c1d8c17, dl));5605 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);5606 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,5607 getF32Constant(DAG, 0x3d634a1d, dl));5608 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);5609 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,5610 getF32Constant(DAG, 0x3e75fe14, dl));5611 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);5612 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,5613 getF32Constant(DAG, 0x3f317234, dl));5614 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);5615 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,5616 getF32Constant(DAG, 0x3f800000, dl));5617 }5618 5619 // Add the exponent into the result in integer domain.5620 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);5621 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,5622 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));5623}5624 5625/// expandExp - Lower an exp intrinsic. Handles the special sequences for5626/// limited-precision mode.5627static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,5628 const TargetLowering &TLI, SDNodeFlags Flags) {5629 if (Op.getValueType() == MVT::f32 &&5630 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {5631 5632 // Put the exponent in the right bit position for later addition to the5633 // final result:5634 //5635 // t0 = Op * log2(e)5636 5637 // TODO: What fast-math-flags should be set here?5638 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,5639 DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));5640 return getLimitedPrecisionExp2(t0, dl, DAG);5641 }5642 5643 // No special expansion.5644 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);5645}5646 5647/// expandLog - Lower a log intrinsic. Handles the special sequences for5648/// limited-precision mode.5649static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,5650 const TargetLowering &TLI, SDNodeFlags Flags) {5651 // TODO: What fast-math-flags should be set on the floating-point nodes?5652 5653 if (Op.getValueType() == MVT::f32 &&5654 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {5655 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);5656 5657 // Scale the exponent by log(2).5658 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);5659 SDValue LogOfExponent =5660 DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,5661 DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));5662 5663 // Get the significand and build it into a floating-point number with5664 // exponent of 1.5665 SDValue X = GetSignificand(DAG, Op1, dl);5666 5667 SDValue LogOfMantissa;5668 if (LimitFloatPrecision <= 6) {5669 // For floating-point precision of 6:5670 //5671 // LogofMantissa =5672 // -1.1609546f +5673 // (1.4034025f - 0.23903021f * x) * x;5674 //5675 // error 0.0034276066, which is better than 8 bits5676 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,5677 getF32Constant(DAG, 0xbe74c456, dl));5678 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,5679 getF32Constant(DAG, 0x3fb3a2b1, dl));5680 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);5681 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,5682 getF32Constant(DAG, 0x3f949a29, dl));5683 } else if (LimitFloatPrecision <= 12) {5684 // For floating-point precision of 12:5685 //5686 // LogOfMantissa =5687 // -1.7417939f +5688 // (2.8212026f +5689 // (-1.4699568f +5690 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;5691 //5692 // error 0.000061011436, which is 14 bits5693 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,5694 getF32Constant(DAG, 0xbd67b6d6, dl));5695 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,5696 getF32Constant(DAG, 0x3ee4f4b8, dl));5697 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);5698 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,5699 getF32Constant(DAG, 0x3fbc278b, dl));5700 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);5701 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,5702 getF32Constant(DAG, 0x40348e95, dl));5703 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);5704 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,5705 getF32Constant(DAG, 0x3fdef31a, dl));5706 } else { // LimitFloatPrecision <= 185707 // For floating-point precision of 18:5708 //5709 // LogOfMantissa =5710 // -2.1072184f +5711 // (4.2372794f +5712 // (-3.7029485f +5713 // (2.2781945f +5714 // (-0.87823314f +5715 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;5716 //5717 // error 0.0000023660568, which is better than 18 bits5718 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,5719 getF32Constant(DAG, 0xbc91e5ac, dl));5720 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,5721 getF32Constant(DAG, 0x3e4350aa, dl));5722 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);5723 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,5724 getF32Constant(DAG, 0x3f60d3e3, dl));5725 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);5726 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,5727 getF32Constant(DAG, 0x4011cdf0, dl));5728 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);5729 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,5730 getF32Constant(DAG, 0x406cfd1c, dl));5731 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);5732 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,5733 getF32Constant(DAG, 0x408797cb, dl));5734 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);5735 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,5736 getF32Constant(DAG, 0x4006dcab, dl));5737 }5738 5739 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);5740 }5741 5742 // No special expansion.5743 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);5744}5745 5746/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for5747/// limited-precision mode.5748static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,5749 const TargetLowering &TLI, SDNodeFlags Flags) {5750 // TODO: What fast-math-flags should be set on the floating-point nodes?5751 5752 if (Op.getValueType() == MVT::f32 &&5753 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {5754 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);5755 5756 // Get the exponent.5757 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);5758 5759 // Get the significand and build it into a floating-point number with5760 // exponent of 1.5761 SDValue X = GetSignificand(DAG, Op1, dl);5762 5763 // Different possible minimax approximations of significand in5764 // floating-point for various degrees of accuracy over [1,2].5765 SDValue Log2ofMantissa;5766 if (LimitFloatPrecision <= 6) {5767 // For floating-point precision of 6:5768 //5769 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;5770 //5771 // error 0.0049451742, which is more than 7 bits5772 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,5773 getF32Constant(DAG, 0xbeb08fe0, dl));5774 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,5775 getF32Constant(DAG, 0x40019463, dl));5776 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);5777 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,5778 getF32Constant(DAG, 0x3fd6633d, dl));5779 } else if (LimitFloatPrecision <= 12) {5780 // For floating-point precision of 12:5781 //5782 // Log2ofMantissa =5783 // -2.51285454f +5784 // (4.07009056f +5785 // (-2.12067489f +5786 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;5787 //5788 // error 0.0000876136000, which is better than 13 bits5789 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,5790 getF32Constant(DAG, 0xbda7262e, dl));5791 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,5792 getF32Constant(DAG, 0x3f25280b, dl));5793 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);5794 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,5795 getF32Constant(DAG, 0x4007b923, dl));5796 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);5797 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,5798 getF32Constant(DAG, 0x40823e2f, dl));5799 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);5800 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,5801 getF32Constant(DAG, 0x4020d29c, dl));5802 } else { // LimitFloatPrecision <= 185803 // For floating-point precision of 18:5804 //5805 // Log2ofMantissa =5806 // -3.0400495f +5807 // (6.1129976f +5808 // (-5.3420409f +5809 // (3.2865683f +5810 // (-1.2669343f +5811 // (0.27515199f -5812 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;5813 //5814 // error 0.0000018516, which is better than 18 bits5815 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,5816 getF32Constant(DAG, 0xbcd2769e, dl));5817 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,5818 getF32Constant(DAG, 0x3e8ce0b9, dl));5819 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);5820 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,5821 getF32Constant(DAG, 0x3fa22ae7, dl));5822 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);5823 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,5824 getF32Constant(DAG, 0x40525723, dl));5825 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);5826 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,5827 getF32Constant(DAG, 0x40aaf200, dl));5828 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);5829 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,5830 getF32Constant(DAG, 0x40c39dad, dl));5831 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);5832 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,5833 getF32Constant(DAG, 0x4042902c, dl));5834 }5835 5836 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);5837 }5838 5839 // No special expansion.5840 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);5841}5842 5843/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for5844/// limited-precision mode.5845static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,5846 const TargetLowering &TLI, SDNodeFlags Flags) {5847 // TODO: What fast-math-flags should be set on the floating-point nodes?5848 5849 if (Op.getValueType() == MVT::f32 &&5850 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {5851 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);5852 5853 // Scale the exponent by log10(2) [0.30102999f].5854 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);5855 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,5856 getF32Constant(DAG, 0x3e9a209a, dl));5857 5858 // Get the significand and build it into a floating-point number with5859 // exponent of 1.5860 SDValue X = GetSignificand(DAG, Op1, dl);5861 5862 SDValue Log10ofMantissa;5863 if (LimitFloatPrecision <= 6) {5864 // For floating-point precision of 6:5865 //5866 // Log10ofMantissa =5867 // -0.50419619f +5868 // (0.60948995f - 0.10380950f * x) * x;5869 //5870 // error 0.0014886165, which is 6 bits5871 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,5872 getF32Constant(DAG, 0xbdd49a13, dl));5873 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,5874 getF32Constant(DAG, 0x3f1c0789, dl));5875 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);5876 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,5877 getF32Constant(DAG, 0x3f011300, dl));5878 } else if (LimitFloatPrecision <= 12) {5879 // For floating-point precision of 12:5880 //5881 // Log10ofMantissa =5882 // -0.64831180f +5883 // (0.91751397f +5884 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;5885 //5886 // error 0.00019228036, which is better than 12 bits5887 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,5888 getF32Constant(DAG, 0x3d431f31, dl));5889 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,5890 getF32Constant(DAG, 0x3ea21fb2, dl));5891 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);5892 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,5893 getF32Constant(DAG, 0x3f6ae232, dl));5894 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);5895 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,5896 getF32Constant(DAG, 0x3f25f7c3, dl));5897 } else { // LimitFloatPrecision <= 185898 // For floating-point precision of 18:5899 //5900 // Log10ofMantissa =5901 // -0.84299375f +5902 // (1.5327582f +5903 // (-1.0688956f +5904 // (0.49102474f +5905 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;5906 //5907 // error 0.0000037995730, which is better than 18 bits5908 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,5909 getF32Constant(DAG, 0x3c5d51ce, dl));5910 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,5911 getF32Constant(DAG, 0x3e00685a, dl));5912 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);5913 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,5914 getF32Constant(DAG, 0x3efb6798, dl));5915 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);5916 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,5917 getF32Constant(DAG, 0x3f88d192, dl));5918 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);5919 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,5920 getF32Constant(DAG, 0x3fc4316c, dl));5921 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);5922 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,5923 getF32Constant(DAG, 0x3f57ce70, dl));5924 }5925 5926 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);5927 }5928 5929 // No special expansion.5930 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);5931}5932 5933/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for5934/// limited-precision mode.5935static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,5936 const TargetLowering &TLI, SDNodeFlags Flags) {5937 if (Op.getValueType() == MVT::f32 &&5938 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)5939 return getLimitedPrecisionExp2(Op, dl, DAG);5940 5941 // No special expansion.5942 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);5943}5944 5945/// visitPow - Lower a pow intrinsic. Handles the special sequences for5946/// limited-precision mode with x == 10.0f.5947static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,5948 SelectionDAG &DAG, const TargetLowering &TLI,5949 SDNodeFlags Flags) {5950 bool IsExp10 = false;5951 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&5952 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {5953 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {5954 APFloat Ten(10.0f);5955 IsExp10 = LHSC->isExactlyValue(Ten);5956 }5957 }5958 5959 // TODO: What fast-math-flags should be set on the FMUL node?5960 if (IsExp10) {5961 // Put the exponent in the right bit position for later addition to the5962 // final result:5963 //5964 // #define LOG2OF10 3.3219281f5965 // t0 = Op * LOG2OF10;5966 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,5967 getF32Constant(DAG, 0x40549a78, dl));5968 return getLimitedPrecisionExp2(t0, dl, DAG);5969 }5970 5971 // No special expansion.5972 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);5973}5974 5975/// ExpandPowI - Expand a llvm.powi intrinsic.5976static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,5977 SelectionDAG &DAG) {5978 // If RHS is a constant, we can expand this out to a multiplication tree if5979 // it's beneficial on the target, otherwise we end up lowering to a call to5980 // __powidf2 (for example).5981 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {5982 unsigned Val = RHSC->getSExtValue();5983 5984 // powi(x, 0) -> 1.05985 if (Val == 0)5986 return DAG.getConstantFP(1.0, DL, LHS.getValueType());5987 5988 if (DAG.getTargetLoweringInfo().isBeneficialToExpandPowI(5989 Val, DAG.shouldOptForSize())) {5990 // Get the exponent as a positive value.5991 if ((int)Val < 0)5992 Val = -Val;5993 // We use the simple binary decomposition method to generate the multiply5994 // sequence. There are more optimal ways to do this (for example,5995 // powi(x,15) generates one more multiply than it should), but this has5996 // the benefit of being both really simple and much better than a libcall.5997 SDValue Res; // Logically starts equal to 1.05998 SDValue CurSquare = LHS;5999 // TODO: Intrinsics should have fast-math-flags that propagate to these6000 // nodes.6001 while (Val) {6002 if (Val & 1) {6003 if (Res.getNode())6004 Res =6005 DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare);6006 else6007 Res = CurSquare; // 1.0*CurSquare.6008 }6009 6010 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),6011 CurSquare, CurSquare);6012 Val >>= 1;6013 }6014 6015 // If the original was negative, invert the result, producing 1/(x*x*x).6016 if (RHSC->getSExtValue() < 0)6017 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),6018 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);6019 return Res;6020 }6021 }6022 6023 // Otherwise, expand to a libcall.6024 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);6025}6026 6027static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,6028 SDValue LHS, SDValue RHS, SDValue Scale,6029 SelectionDAG &DAG, const TargetLowering &TLI) {6030 EVT VT = LHS.getValueType();6031 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;6032 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;6033 LLVMContext &Ctx = *DAG.getContext();6034 6035 // If the type is legal but the operation isn't, this node might survive all6036 // the way to operation legalization. If we end up there and we do not have6037 // the ability to widen the type (if VT*2 is not legal), we cannot expand the6038 // node.6039 6040 // Coax the legalizer into expanding the node during type legalization instead6041 // by bumping the size by one bit. This will force it to Promote, enabling the6042 // early expansion and avoiding the need to expand later.6043 6044 // We don't have to do this if Scale is 0; that can always be expanded, unless6045 // it's a saturating signed operation. Those can experience true integer6046 // division overflow, a case which we must avoid.6047 6048 // FIXME: We wouldn't have to do this (or any of the early6049 // expansion/promotion) if it was possible to expand a libcall of an6050 // illegal type during operation legalization. But it's not, so things6051 // get a bit hacky.6052 unsigned ScaleInt = Scale->getAsZExtVal();6053 if ((ScaleInt > 0 || (Saturating && Signed)) &&6054 (TLI.isTypeLegal(VT) ||6055 (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {6056 TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(6057 Opcode, VT, ScaleInt);6058 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {6059 EVT PromVT;6060 if (VT.isScalarInteger())6061 PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);6062 else if (VT.isVector()) {6063 PromVT = VT.getVectorElementType();6064 PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);6065 PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());6066 } else6067 llvm_unreachable("Wrong VT for DIVFIX?");6068 LHS = DAG.getExtOrTrunc(Signed, LHS, DL, PromVT);6069 RHS = DAG.getExtOrTrunc(Signed, RHS, DL, PromVT);6070 EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());6071 // For saturating operations, we need to shift up the LHS to get the6072 // proper saturation width, and then shift down again afterwards.6073 if (Saturating)6074 LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,6075 DAG.getConstant(1, DL, ShiftTy));6076 SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);6077 if (Saturating)6078 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,6079 DAG.getConstant(1, DL, ShiftTy));6080 return DAG.getZExtOrTrunc(Res, DL, VT);6081 }6082 }6083 6084 return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);6085}6086 6087// getUnderlyingArgRegs - Find underlying registers used for a truncated,6088// bitcasted, or split argument. Returns a list of <Register, size in bits>6089static void6090getUnderlyingArgRegs(SmallVectorImpl<std::pair<Register, TypeSize>> &Regs,6091 const SDValue &N) {6092 switch (N.getOpcode()) {6093 case ISD::CopyFromReg: {6094 SDValue Op = N.getOperand(1);6095 Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),6096 Op.getValueType().getSizeInBits());6097 return;6098 }6099 case ISD::BITCAST:6100 case ISD::AssertZext:6101 case ISD::AssertSext:6102 case ISD::TRUNCATE:6103 getUnderlyingArgRegs(Regs, N.getOperand(0));6104 return;6105 case ISD::BUILD_PAIR:6106 case ISD::BUILD_VECTOR:6107 case ISD::CONCAT_VECTORS:6108 for (SDValue Op : N->op_values())6109 getUnderlyingArgRegs(Regs, Op);6110 return;6111 default:6112 return;6113 }6114}6115 6116/// If the DbgValueInst is a dbg_value of a function argument, create the6117/// corresponding DBG_VALUE machine instruction for it now. At the end of6118/// instruction selection, they will be inserted to the entry BB.6119/// We don't currently support this for variadic dbg_values, as they shouldn't6120/// appear for function arguments or in the prologue.6121bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(6122 const Value *V, DILocalVariable *Variable, DIExpression *Expr,6123 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {6124 const Argument *Arg = dyn_cast<Argument>(V);6125 if (!Arg)6126 return false;6127 6128 MachineFunction &MF = DAG.getMachineFunction();6129 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();6130 6131 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind6132 // we've been asked to pursue.6133 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,6134 bool Indirect) {6135 if (Reg.isVirtual() && MF.useDebugInstrRef()) {6136 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF6137 // pointing at the VReg, which will be patched up later.6138 auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);6139 SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg(6140 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,6141 /* isKill */ false, /* isDead */ false,6142 /* isUndef */ false, /* isEarlyClobber */ false,6143 /* SubReg */ 0, /* isDebug */ true)});6144 6145 auto *NewDIExpr = FragExpr;6146 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into6147 // the DIExpression.6148 if (Indirect)6149 NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);6150 SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0});6151 NewDIExpr = DIExpression::prependOpcodes(NewDIExpr, Ops);6152 return BuildMI(MF, DL, Inst, false, MOs, Variable, NewDIExpr);6153 } else {6154 // Create a completely standard DBG_VALUE.6155 auto &Inst = TII->get(TargetOpcode::DBG_VALUE);6156 return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);6157 }6158 };6159 6160 if (Kind == FuncArgumentDbgValueKind::Value) {6161 // ArgDbgValues are hoisted to the beginning of the entry block. So we6162 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in6163 // the entry block.6164 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();6165 if (!IsInEntryBlock)6166 return false;6167 6168 // ArgDbgValues are hoisted to the beginning of the entry block. So we6169 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a6170 // variable that also is a param.6171 //6172 // Although, if we are at the top of the entry block already, we can still6173 // emit using ArgDbgValue. This might catch some situations when the6174 // dbg.value refers to an argument that isn't used in the entry block, so6175 // any CopyToReg node would be optimized out and the only way to express6176 // this DBG_VALUE is by using the physical reg (or FI) as done in this6177 // method. ArgDbgValues are hoisted to the beginning of the entry block. So6178 // we should only emit as ArgDbgValue if the Variable is an argument to the6179 // current function, and the dbg.value intrinsic is found in the entry6180 // block.6181 bool VariableIsFunctionInputArg = Variable->isParameter() &&6182 !DL->getInlinedAt();6183 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;6184 if (!IsInPrologue && !VariableIsFunctionInputArg)6185 return false;6186 6187 // Here we assume that a function argument on IR level only can be used to6188 // describe one input parameter on source level. If we for example have6189 // source code like this6190 //6191 // struct A { long x, y; };6192 // void foo(struct A a, long b) {6193 // ...6194 // b = a.x;6195 // ...6196 // }6197 //6198 // and IR like this6199 //6200 // define void @foo(i32 %a1, i32 %a2, i32 %b) {6201 // entry:6202 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment6203 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment6204 // call void @llvm.dbg.value(metadata i32 %b, "b",6205 // ...6206 // call void @llvm.dbg.value(metadata i32 %a1, "b"6207 // ...6208 //6209 // then the last dbg.value is describing a parameter "b" using a value that6210 // is an argument. But since we already has used %a1 to describe a parameter6211 // we should not handle that last dbg.value here (that would result in an6212 // incorrect hoisting of the DBG_VALUE to the function entry).6213 // Notice that we allow one dbg.value per IR level argument, to accommodate6214 // for the situation with fragments above.6215 // If there is no node for the value being handled, we return true to skip6216 // the normal generation of debug info, as it would kill existing debug6217 // info for the parameter in case of duplicates.6218 if (VariableIsFunctionInputArg) {6219 unsigned ArgNo = Arg->getArgNo();6220 if (ArgNo >= FuncInfo.DescribedArgs.size())6221 FuncInfo.DescribedArgs.resize(ArgNo + 1, false);6222 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))6223 return !NodeMap[V].getNode();6224 FuncInfo.DescribedArgs.set(ArgNo);6225 }6226 }6227 6228 bool IsIndirect = false;6229 std::optional<MachineOperand> Op;6230 // Some arguments' frame index is recorded during argument lowering.6231 int FI = FuncInfo.getArgumentFrameIndex(Arg);6232 if (FI != std::numeric_limits<int>::max())6233 Op = MachineOperand::CreateFI(FI);6234 6235 SmallVector<std::pair<Register, TypeSize>, 8> ArgRegsAndSizes;6236 if (!Op && N.getNode()) {6237 getUnderlyingArgRegs(ArgRegsAndSizes, N);6238 Register Reg;6239 if (ArgRegsAndSizes.size() == 1)6240 Reg = ArgRegsAndSizes.front().first;6241 6242 if (Reg && Reg.isVirtual()) {6243 MachineRegisterInfo &RegInfo = MF.getRegInfo();6244 Register PR = RegInfo.getLiveInPhysReg(Reg);6245 if (PR)6246 Reg = PR;6247 }6248 if (Reg) {6249 Op = MachineOperand::CreateReg(Reg, false);6250 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;6251 }6252 }6253 6254 if (!Op && N.getNode()) {6255 // Check if frame index is available.6256 SDValue LCandidate = peekThroughBitcasts(N);6257 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))6258 if (FrameIndexSDNode *FINode =6259 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))6260 Op = MachineOperand::CreateFI(FINode->getIndex());6261 }6262 6263 if (!Op) {6264 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg6265 auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<Register, TypeSize>>6266 SplitRegs) {6267 unsigned Offset = 0;6268 for (const auto &RegAndSize : SplitRegs) {6269 // If the expression is already a fragment, the current register6270 // offset+size might extend beyond the fragment. In this case, only6271 // the register bits that are inside the fragment are relevant.6272 int RegFragmentSizeInBits = RegAndSize.second;6273 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {6274 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;6275 // The register is entirely outside the expression fragment,6276 // so is irrelevant for debug info.6277 if (Offset >= ExprFragmentSizeInBits)6278 break;6279 // The register is partially outside the expression fragment, only6280 // the low bits within the fragment are relevant for debug info.6281 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {6282 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;6283 }6284 }6285 6286 auto FragmentExpr = DIExpression::createFragmentExpression(6287 Expr, Offset, RegFragmentSizeInBits);6288 Offset += RegAndSize.second;6289 // If a valid fragment expression cannot be created, the variable's6290 // correct value cannot be determined and so it is set as poison.6291 if (!FragmentExpr) {6292 SDDbgValue *SDV = DAG.getConstantDbgValue(6293 Variable, Expr, PoisonValue::get(V->getType()), DL, SDNodeOrder);6294 DAG.AddDbgValue(SDV, false);6295 continue;6296 }6297 MachineInstr *NewMI =6298 MakeVRegDbgValue(RegAndSize.first, *FragmentExpr,6299 Kind != FuncArgumentDbgValueKind::Value);6300 FuncInfo.ArgDbgValues.push_back(NewMI);6301 }6302 };6303 6304 // Check if ValueMap has reg number.6305 DenseMap<const Value *, Register>::const_iterator6306 VMI = FuncInfo.ValueMap.find(V);6307 if (VMI != FuncInfo.ValueMap.end()) {6308 const auto &TLI = DAG.getTargetLoweringInfo();6309 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,6310 V->getType(), std::nullopt);6311 if (RFV.occupiesMultipleRegs()) {6312 splitMultiRegDbgValue(RFV.getRegsAndSizes());6313 return true;6314 }6315 6316 Op = MachineOperand::CreateReg(VMI->second, false);6317 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;6318 } else if (ArgRegsAndSizes.size() > 1) {6319 // This was split due to the calling convention, and no virtual register6320 // mapping exists for the value.6321 splitMultiRegDbgValue(ArgRegsAndSizes);6322 return true;6323 }6324 }6325 6326 if (!Op)6327 return false;6328 6329 assert(Variable->isValidLocationForIntrinsic(DL) &&6330 "Expected inlined-at fields to agree");6331 MachineInstr *NewMI = nullptr;6332 6333 if (Op->isReg())6334 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);6335 else6336 NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,6337 Variable, Expr);6338 6339 // Otherwise, use ArgDbgValues.6340 FuncInfo.ArgDbgValues.push_back(NewMI);6341 return true;6342}6343 6344/// Return the appropriate SDDbgValue based on N.6345SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,6346 DILocalVariable *Variable,6347 DIExpression *Expr,6348 const DebugLoc &dl,6349 unsigned DbgSDNodeOrder) {6350 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {6351 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe6352 // stack slot locations.6353 //6354 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting6355 // debug values here after optimization:6356 //6357 // dbg.value(i32* %px, !"int *px", !DIExpression()), and6358 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))6359 //6360 // Both describe the direct values of their associated variables.6361 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),6362 /*IsIndirect*/ false, dl, DbgSDNodeOrder);6363 }6364 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),6365 /*IsIndirect*/ false, dl, DbgSDNodeOrder);6366}6367 6368static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {6369 switch (Intrinsic) {6370 case Intrinsic::smul_fix:6371 return ISD::SMULFIX;6372 case Intrinsic::umul_fix:6373 return ISD::UMULFIX;6374 case Intrinsic::smul_fix_sat:6375 return ISD::SMULFIXSAT;6376 case Intrinsic::umul_fix_sat:6377 return ISD::UMULFIXSAT;6378 case Intrinsic::sdiv_fix:6379 return ISD::SDIVFIX;6380 case Intrinsic::udiv_fix:6381 return ISD::UDIVFIX;6382 case Intrinsic::sdiv_fix_sat:6383 return ISD::SDIVFIXSAT;6384 case Intrinsic::udiv_fix_sat:6385 return ISD::UDIVFIXSAT;6386 default:6387 llvm_unreachable("Unhandled fixed point intrinsic");6388 }6389}6390 6391/// Given a @llvm.call.preallocated.setup, return the corresponding6392/// preallocated call.6393static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {6394 assert(cast<CallBase>(PreallocatedSetup)6395 ->getCalledFunction()6396 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&6397 "expected call_preallocated_setup Value");6398 for (const auto *U : PreallocatedSetup->users()) {6399 auto *UseCall = cast<CallBase>(U);6400 const Function *Fn = UseCall->getCalledFunction();6401 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {6402 return UseCall;6403 }6404 }6405 llvm_unreachable("expected corresponding call to preallocated setup/arg");6406}6407 6408/// If DI is a debug value with an EntryValue expression, lower it using the6409/// corresponding physical register of the associated Argument value6410/// (guaranteed to exist by the verifier).6411bool SelectionDAGBuilder::visitEntryValueDbgValue(6412 ArrayRef<const Value *> Values, DILocalVariable *Variable,6413 DIExpression *Expr, DebugLoc DbgLoc) {6414 if (!Expr->isEntryValue() || !hasSingleElement(Values))6415 return false;6416 6417 // These properties are guaranteed by the verifier.6418 const Argument *Arg = cast<Argument>(Values[0]);6419 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));6420 6421 auto ArgIt = FuncInfo.ValueMap.find(Arg);6422 if (ArgIt == FuncInfo.ValueMap.end()) {6423 LLVM_DEBUG(6424 dbgs() << "Dropping dbg.value: expression is entry_value but "6425 "couldn't find an associated register for the Argument\n");6426 return true;6427 }6428 Register ArgVReg = ArgIt->getSecond();6429 6430 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())6431 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {6432 SDDbgValue *SDV = DAG.getVRegDbgValue(6433 Variable, Expr, PhysReg, false /*IsIndidrect*/, DbgLoc, SDNodeOrder);6434 DAG.AddDbgValue(SDV, false /*treat as dbg.declare byval parameter*/);6435 return true;6436 }6437 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "6438 "couldn't find a physical register\n");6439 return true;6440}6441 6442/// Lower the call to the specified intrinsic function.6443void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,6444 unsigned Intrinsic) {6445 SDLoc sdl = getCurSDLoc();6446 switch (Intrinsic) {6447 case Intrinsic::experimental_convergence_anchor:6448 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ANCHOR, sdl, MVT::Untyped));6449 break;6450 case Intrinsic::experimental_convergence_entry:6451 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ENTRY, sdl, MVT::Untyped));6452 break;6453 case Intrinsic::experimental_convergence_loop: {6454 auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl);6455 auto *Token = Bundle->Inputs[0].get();6456 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_LOOP, sdl, MVT::Untyped,6457 getValue(Token)));6458 break;6459 }6460 }6461}6462 6463void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,6464 unsigned IntrinsicID) {6465 // For now, we're only lowering an 'add' histogram.6466 // We can add others later, e.g. saturating adds, min/max.6467 assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&6468 "Tried to lower unsupported histogram type");6469 SDLoc sdl = getCurSDLoc();6470 Value *Ptr = I.getOperand(0);6471 SDValue Inc = getValue(I.getOperand(1));6472 SDValue Mask = getValue(I.getOperand(2));6473 6474 const TargetLowering &TLI = DAG.getTargetLoweringInfo();6475 DataLayout TargetDL = DAG.getDataLayout();6476 EVT VT = Inc.getValueType();6477 Align Alignment = DAG.getEVTAlign(VT);6478 6479 const MDNode *Ranges = getRangeMetadata(I);6480 6481 SDValue Root = DAG.getRoot();6482 SDValue Base;6483 SDValue Index;6484 SDValue Scale;6485 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,6486 I.getParent(), VT.getScalarStoreSize());6487 6488 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();6489 6490 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(6491 MachinePointerInfo(AS),6492 MachineMemOperand::MOLoad | MachineMemOperand::MOStore,6493 MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);6494 6495 if (!UniformBase) {6496 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));6497 Index = getValue(Ptr);6498 Scale =6499 DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));6500 }6501 6502 EVT IdxVT = Index.getValueType();6503 EVT EltTy = IdxVT.getVectorElementType();6504 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {6505 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);6506 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);6507 }6508 6509 SDValue ID = DAG.getTargetConstant(IntrinsicID, sdl, MVT::i32);6510 6511 SDValue Ops[] = {Root, Inc, Mask, Base, Index, Scale, ID};6512 SDValue Histogram = DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), VT, sdl,6513 Ops, MMO, ISD::SIGNED_SCALED);6514 6515 setValue(&I, Histogram);6516 DAG.setRoot(Histogram);6517}6518 6519void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,6520 unsigned Intrinsic) {6521 assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&6522 "Tried lowering invalid vector extract last");6523 SDLoc sdl = getCurSDLoc();6524 const DataLayout &Layout = DAG.getDataLayout();6525 SDValue Data = getValue(I.getOperand(0));6526 SDValue Mask = getValue(I.getOperand(1));6527 6528 const TargetLowering &TLI = DAG.getTargetLoweringInfo();6529 EVT ResVT = TLI.getValueType(Layout, I.getType());6530 6531 EVT ExtVT = TLI.getVectorIdxTy(Layout);6532 SDValue Idx = DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, sdl, ExtVT, Mask);6533 SDValue Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, sdl, ResVT, Data, Idx);6534 6535 Value *Default = I.getOperand(2);6536 if (!isa<PoisonValue>(Default) && !isa<UndefValue>(Default)) {6537 SDValue PassThru = getValue(Default);6538 EVT BoolVT = Mask.getValueType().getScalarType();6539 SDValue AnyActive = DAG.getNode(ISD::VECREDUCE_OR, sdl, BoolVT, Mask);6540 Result = DAG.getSelect(sdl, ResVT, AnyActive, Result, PassThru);6541 }6542 6543 setValue(&I, Result);6544}6545 6546/// Lower the call to the specified intrinsic function.6547void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,6548 unsigned Intrinsic) {6549 const TargetLowering &TLI = DAG.getTargetLoweringInfo();6550 SDLoc sdl = getCurSDLoc();6551 DebugLoc dl = getCurDebugLoc();6552 SDValue Res;6553 6554 SDNodeFlags Flags;6555 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))6556 Flags.copyFMF(*FPOp);6557 6558 switch (Intrinsic) {6559 default:6560 // By default, turn this into a target intrinsic node.6561 visitTargetIntrinsic(I, Intrinsic);6562 return;6563 case Intrinsic::vscale: {6564 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());6565 setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));6566 return;6567 }6568 case Intrinsic::vastart: visitVAStart(I); return;6569 case Intrinsic::vaend: visitVAEnd(I); return;6570 case Intrinsic::vacopy: visitVACopy(I); return;6571 case Intrinsic::returnaddress:6572 setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,6573 TLI.getValueType(DAG.getDataLayout(), I.getType()),6574 getValue(I.getArgOperand(0))));6575 return;6576 case Intrinsic::addressofreturnaddress:6577 setValue(&I,6578 DAG.getNode(ISD::ADDROFRETURNADDR, sdl,6579 TLI.getValueType(DAG.getDataLayout(), I.getType())));6580 return;6581 case Intrinsic::sponentry:6582 setValue(&I,6583 DAG.getNode(ISD::SPONENTRY, sdl,6584 TLI.getValueType(DAG.getDataLayout(), I.getType())));6585 return;6586 case Intrinsic::frameaddress:6587 setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,6588 TLI.getFrameIndexTy(DAG.getDataLayout()),6589 getValue(I.getArgOperand(0))));6590 return;6591 case Intrinsic::read_volatile_register:6592 case Intrinsic::read_register: {6593 Value *Reg = I.getArgOperand(0);6594 SDValue Chain = getRoot();6595 SDValue RegName =6596 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));6597 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());6598 Res = DAG.getNode(ISD::READ_REGISTER, sdl,6599 DAG.getVTList(VT, MVT::Other), Chain, RegName);6600 setValue(&I, Res);6601 DAG.setRoot(Res.getValue(1));6602 return;6603 }6604 case Intrinsic::write_register: {6605 Value *Reg = I.getArgOperand(0);6606 Value *RegValue = I.getArgOperand(1);6607 SDValue Chain = getRoot();6608 SDValue RegName =6609 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));6610 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,6611 RegName, getValue(RegValue)));6612 return;6613 }6614 case Intrinsic::memcpy:6615 case Intrinsic::memcpy_inline: {6616 const auto &MCI = cast<MemCpyInst>(I);6617 SDValue Dst = getValue(I.getArgOperand(0));6618 SDValue Src = getValue(I.getArgOperand(1));6619 SDValue Size = getValue(I.getArgOperand(2));6620 assert((!MCI.isForceInlined() || isa<ConstantSDNode>(Size)) &&6621 "memcpy_inline needs constant size");6622 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.6623 Align DstAlign = MCI.getDestAlign().valueOrOne();6624 Align SrcAlign = MCI.getSourceAlign().valueOrOne();6625 Align Alignment = std::min(DstAlign, SrcAlign);6626 bool isVol = MCI.isVolatile();6627 // FIXME: Support passing different dest/src alignments to the memcpy DAG6628 // node.6629 SDValue Root = isVol ? getRoot() : getMemoryRoot();6630 SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, isVol,6631 MCI.isForceInlined(), &I, std::nullopt,6632 MachinePointerInfo(I.getArgOperand(0)),6633 MachinePointerInfo(I.getArgOperand(1)),6634 I.getAAMetadata(), BatchAA);6635 updateDAGForMaybeTailCall(MC);6636 return;6637 }6638 case Intrinsic::memset:6639 case Intrinsic::memset_inline: {6640 const auto &MSII = cast<MemSetInst>(I);6641 SDValue Dst = getValue(I.getArgOperand(0));6642 SDValue Value = getValue(I.getArgOperand(1));6643 SDValue Size = getValue(I.getArgOperand(2));6644 assert((!MSII.isForceInlined() || isa<ConstantSDNode>(Size)) &&6645 "memset_inline needs constant size");6646 // @llvm.memset defines 0 and 1 to both mean no alignment.6647 Align DstAlign = MSII.getDestAlign().valueOrOne();6648 bool isVol = MSII.isVolatile();6649 SDValue Root = isVol ? getRoot() : getMemoryRoot();6650 SDValue MC = DAG.getMemset(6651 Root, sdl, Dst, Value, Size, DstAlign, isVol, MSII.isForceInlined(),6652 &I, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());6653 updateDAGForMaybeTailCall(MC);6654 return;6655 }6656 case Intrinsic::memmove: {6657 const auto &MMI = cast<MemMoveInst>(I);6658 SDValue Op1 = getValue(I.getArgOperand(0));6659 SDValue Op2 = getValue(I.getArgOperand(1));6660 SDValue Op3 = getValue(I.getArgOperand(2));6661 // @llvm.memmove defines 0 and 1 to both mean no alignment.6662 Align DstAlign = MMI.getDestAlign().valueOrOne();6663 Align SrcAlign = MMI.getSourceAlign().valueOrOne();6664 Align Alignment = std::min(DstAlign, SrcAlign);6665 bool isVol = MMI.isVolatile();6666 // FIXME: Support passing different dest/src alignments to the memmove DAG6667 // node.6668 SDValue Root = isVol ? getRoot() : getMemoryRoot();6669 SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol, &I,6670 /* OverrideTailCall */ std::nullopt,6671 MachinePointerInfo(I.getArgOperand(0)),6672 MachinePointerInfo(I.getArgOperand(1)),6673 I.getAAMetadata(), BatchAA);6674 updateDAGForMaybeTailCall(MM);6675 return;6676 }6677 case Intrinsic::memcpy_element_unordered_atomic: {6678 auto &MI = cast<AnyMemCpyInst>(I);6679 SDValue Dst = getValue(MI.getRawDest());6680 SDValue Src = getValue(MI.getRawSource());6681 SDValue Length = getValue(MI.getLength());6682 6683 Type *LengthTy = MI.getLength()->getType();6684 unsigned ElemSz = MI.getElementSizeInBytes();6685 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());6686 SDValue MC =6687 DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,6688 isTC, MachinePointerInfo(MI.getRawDest()),6689 MachinePointerInfo(MI.getRawSource()));6690 updateDAGForMaybeTailCall(MC);6691 return;6692 }6693 case Intrinsic::memmove_element_unordered_atomic: {6694 auto &MI = cast<AnyMemMoveInst>(I);6695 SDValue Dst = getValue(MI.getRawDest());6696 SDValue Src = getValue(MI.getRawSource());6697 SDValue Length = getValue(MI.getLength());6698 6699 Type *LengthTy = MI.getLength()->getType();6700 unsigned ElemSz = MI.getElementSizeInBytes();6701 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());6702 SDValue MC =6703 DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,6704 isTC, MachinePointerInfo(MI.getRawDest()),6705 MachinePointerInfo(MI.getRawSource()));6706 updateDAGForMaybeTailCall(MC);6707 return;6708 }6709 case Intrinsic::memset_element_unordered_atomic: {6710 auto &MI = cast<AnyMemSetInst>(I);6711 SDValue Dst = getValue(MI.getRawDest());6712 SDValue Val = getValue(MI.getValue());6713 SDValue Length = getValue(MI.getLength());6714 6715 Type *LengthTy = MI.getLength()->getType();6716 unsigned ElemSz = MI.getElementSizeInBytes();6717 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());6718 SDValue MC =6719 DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz,6720 isTC, MachinePointerInfo(MI.getRawDest()));6721 updateDAGForMaybeTailCall(MC);6722 return;6723 }6724 case Intrinsic::call_preallocated_setup: {6725 const CallBase *PreallocatedCall = FindPreallocatedCall(&I);6726 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);6727 SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,6728 getRoot(), SrcValue);6729 setValue(&I, Res);6730 DAG.setRoot(Res);6731 return;6732 }6733 case Intrinsic::call_preallocated_arg: {6734 const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));6735 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);6736 SDValue Ops[3];6737 Ops[0] = getRoot();6738 Ops[1] = SrcValue;6739 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,6740 MVT::i32); // arg index6741 SDValue Res = DAG.getNode(6742 ISD::PREALLOCATED_ARG, sdl,6743 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);6744 setValue(&I, Res);6745 DAG.setRoot(Res.getValue(1));6746 return;6747 }6748 6749 case Intrinsic::eh_typeid_for: {6750 // Find the type id for the given typeinfo.6751 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));6752 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);6753 Res = DAG.getConstant(TypeID, sdl, MVT::i32);6754 setValue(&I, Res);6755 return;6756 }6757 6758 case Intrinsic::eh_return_i32:6759 case Intrinsic::eh_return_i64:6760 DAG.getMachineFunction().setCallsEHReturn(true);6761 DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,6762 MVT::Other,6763 getControlRoot(),6764 getValue(I.getArgOperand(0)),6765 getValue(I.getArgOperand(1))));6766 return;6767 case Intrinsic::eh_unwind_init:6768 DAG.getMachineFunction().setCallsUnwindInit(true);6769 return;6770 case Intrinsic::eh_dwarf_cfa:6771 setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,6772 TLI.getPointerTy(DAG.getDataLayout()),6773 getValue(I.getArgOperand(0))));6774 return;6775 case Intrinsic::eh_sjlj_callsite: {6776 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));6777 assert(FuncInfo.getCurrentCallSite() == 0 && "Overlapping call sites!");6778 6779 FuncInfo.setCurrentCallSite(CI->getZExtValue());6780 return;6781 }6782 case Intrinsic::eh_sjlj_functioncontext: {6783 // Get and store the index of the function context.6784 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();6785 AllocaInst *FnCtx =6786 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());6787 int FI = FuncInfo.StaticAllocaMap[FnCtx];6788 MFI.setFunctionContextIndex(FI);6789 return;6790 }6791 case Intrinsic::eh_sjlj_setjmp: {6792 SDValue Ops[2];6793 Ops[0] = getRoot();6794 Ops[1] = getValue(I.getArgOperand(0));6795 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,6796 DAG.getVTList(MVT::i32, MVT::Other), Ops);6797 setValue(&I, Op.getValue(0));6798 DAG.setRoot(Op.getValue(1));6799 return;6800 }6801 case Intrinsic::eh_sjlj_longjmp:6802 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,6803 getRoot(), getValue(I.getArgOperand(0))));6804 return;6805 case Intrinsic::eh_sjlj_setup_dispatch:6806 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,6807 getRoot()));6808 return;6809 case Intrinsic::masked_gather:6810 visitMaskedGather(I);6811 return;6812 case Intrinsic::masked_load:6813 visitMaskedLoad(I);6814 return;6815 case Intrinsic::masked_scatter:6816 visitMaskedScatter(I);6817 return;6818 case Intrinsic::masked_store:6819 visitMaskedStore(I);6820 return;6821 case Intrinsic::masked_expandload:6822 visitMaskedLoad(I, true /* IsExpanding */);6823 return;6824 case Intrinsic::masked_compressstore:6825 visitMaskedStore(I, true /* IsCompressing */);6826 return;6827 case Intrinsic::powi:6828 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),6829 getValue(I.getArgOperand(1)), DAG));6830 return;6831 case Intrinsic::log:6832 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));6833 return;6834 case Intrinsic::log2:6835 setValue(&I,6836 expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));6837 return;6838 case Intrinsic::log10:6839 setValue(&I,6840 expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));6841 return;6842 case Intrinsic::exp:6843 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));6844 return;6845 case Intrinsic::exp2:6846 setValue(&I,6847 expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));6848 return;6849 case Intrinsic::pow:6850 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),6851 getValue(I.getArgOperand(1)), DAG, TLI, Flags));6852 return;6853 case Intrinsic::sqrt:6854 case Intrinsic::fabs:6855 case Intrinsic::sin:6856 case Intrinsic::cos:6857 case Intrinsic::tan:6858 case Intrinsic::asin:6859 case Intrinsic::acos:6860 case Intrinsic::atan:6861 case Intrinsic::sinh:6862 case Intrinsic::cosh:6863 case Intrinsic::tanh:6864 case Intrinsic::exp10:6865 case Intrinsic::floor:6866 case Intrinsic::ceil:6867 case Intrinsic::trunc:6868 case Intrinsic::rint:6869 case Intrinsic::nearbyint:6870 case Intrinsic::round:6871 case Intrinsic::roundeven:6872 case Intrinsic::canonicalize: {6873 unsigned Opcode;6874 // clang-format off6875 switch (Intrinsic) {6876 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.6877 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;6878 case Intrinsic::fabs: Opcode = ISD::FABS; break;6879 case Intrinsic::sin: Opcode = ISD::FSIN; break;6880 case Intrinsic::cos: Opcode = ISD::FCOS; break;6881 case Intrinsic::tan: Opcode = ISD::FTAN; break;6882 case Intrinsic::asin: Opcode = ISD::FASIN; break;6883 case Intrinsic::acos: Opcode = ISD::FACOS; break;6884 case Intrinsic::atan: Opcode = ISD::FATAN; break;6885 case Intrinsic::sinh: Opcode = ISD::FSINH; break;6886 case Intrinsic::cosh: Opcode = ISD::FCOSH; break;6887 case Intrinsic::tanh: Opcode = ISD::FTANH; break;6888 case Intrinsic::exp10: Opcode = ISD::FEXP10; break;6889 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;6890 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;6891 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;6892 case Intrinsic::rint: Opcode = ISD::FRINT; break;6893 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;6894 case Intrinsic::round: Opcode = ISD::FROUND; break;6895 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;6896 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;6897 }6898 // clang-format on6899 6900 setValue(&I, DAG.getNode(Opcode, sdl,6901 getValue(I.getArgOperand(0)).getValueType(),6902 getValue(I.getArgOperand(0)), Flags));6903 return;6904 }6905 case Intrinsic::atan2:6906 setValue(&I, DAG.getNode(ISD::FATAN2, sdl,6907 getValue(I.getArgOperand(0)).getValueType(),6908 getValue(I.getArgOperand(0)),6909 getValue(I.getArgOperand(1)), Flags));6910 return;6911 case Intrinsic::lround:6912 case Intrinsic::llround:6913 case Intrinsic::lrint:6914 case Intrinsic::llrint: {6915 unsigned Opcode;6916 // clang-format off6917 switch (Intrinsic) {6918 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.6919 case Intrinsic::lround: Opcode = ISD::LROUND; break;6920 case Intrinsic::llround: Opcode = ISD::LLROUND; break;6921 case Intrinsic::lrint: Opcode = ISD::LRINT; break;6922 case Intrinsic::llrint: Opcode = ISD::LLRINT; break;6923 }6924 // clang-format on6925 6926 EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());6927 setValue(&I, DAG.getNode(Opcode, sdl, RetVT,6928 getValue(I.getArgOperand(0))));6929 return;6930 }6931 case Intrinsic::minnum:6932 setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,6933 getValue(I.getArgOperand(0)).getValueType(),6934 getValue(I.getArgOperand(0)),6935 getValue(I.getArgOperand(1)), Flags));6936 return;6937 case Intrinsic::maxnum:6938 setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,6939 getValue(I.getArgOperand(0)).getValueType(),6940 getValue(I.getArgOperand(0)),6941 getValue(I.getArgOperand(1)), Flags));6942 return;6943 case Intrinsic::minimum:6944 setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl,6945 getValue(I.getArgOperand(0)).getValueType(),6946 getValue(I.getArgOperand(0)),6947 getValue(I.getArgOperand(1)), Flags));6948 return;6949 case Intrinsic::maximum:6950 setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl,6951 getValue(I.getArgOperand(0)).getValueType(),6952 getValue(I.getArgOperand(0)),6953 getValue(I.getArgOperand(1)), Flags));6954 return;6955 case Intrinsic::minimumnum:6956 setValue(&I, DAG.getNode(ISD::FMINIMUMNUM, sdl,6957 getValue(I.getArgOperand(0)).getValueType(),6958 getValue(I.getArgOperand(0)),6959 getValue(I.getArgOperand(1)), Flags));6960 return;6961 case Intrinsic::maximumnum:6962 setValue(&I, DAG.getNode(ISD::FMAXIMUMNUM, sdl,6963 getValue(I.getArgOperand(0)).getValueType(),6964 getValue(I.getArgOperand(0)),6965 getValue(I.getArgOperand(1)), Flags));6966 return;6967 case Intrinsic::copysign:6968 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,6969 getValue(I.getArgOperand(0)).getValueType(),6970 getValue(I.getArgOperand(0)),6971 getValue(I.getArgOperand(1)), Flags));6972 return;6973 case Intrinsic::ldexp:6974 setValue(&I, DAG.getNode(ISD::FLDEXP, sdl,6975 getValue(I.getArgOperand(0)).getValueType(),6976 getValue(I.getArgOperand(0)),6977 getValue(I.getArgOperand(1)), Flags));6978 return;6979 case Intrinsic::modf:6980 case Intrinsic::sincos:6981 case Intrinsic::sincospi:6982 case Intrinsic::frexp: {6983 unsigned Opcode;6984 switch (Intrinsic) {6985 default:6986 llvm_unreachable("unexpected intrinsic");6987 case Intrinsic::sincos:6988 Opcode = ISD::FSINCOS;6989 break;6990 case Intrinsic::sincospi:6991 Opcode = ISD::FSINCOSPI;6992 break;6993 case Intrinsic::modf:6994 Opcode = ISD::FMODF;6995 break;6996 case Intrinsic::frexp:6997 Opcode = ISD::FFREXP;6998 break;6999 }7000 SmallVector<EVT, 2> ValueVTs;7001 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);7002 SDVTList VTs = DAG.getVTList(ValueVTs);7003 setValue(7004 &I, DAG.getNode(Opcode, sdl, VTs, getValue(I.getArgOperand(0)), Flags));7005 return;7006 }7007 case Intrinsic::arithmetic_fence: {7008 setValue(&I, DAG.getNode(ISD::ARITH_FENCE, sdl,7009 getValue(I.getArgOperand(0)).getValueType(),7010 getValue(I.getArgOperand(0)), Flags));7011 return;7012 }7013 case Intrinsic::fma:7014 setValue(&I, DAG.getNode(7015 ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),7016 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),7017 getValue(I.getArgOperand(2)), Flags));7018 return;7019#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \7020 case Intrinsic::INTRINSIC:7021#include "llvm/IR/ConstrainedOps.def"7022 visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));7023 return;7024#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:7025#include "llvm/IR/VPIntrinsics.def"7026 visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));7027 return;7028 case Intrinsic::fptrunc_round: {7029 // Get the last argument, the metadata and convert it to an integer in the7030 // call7031 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();7032 std::optional<RoundingMode> RoundMode =7033 convertStrToRoundingMode(cast<MDString>(MD)->getString());7034 7035 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());7036 7037 // Propagate fast-math-flags from IR to node(s).7038 SDNodeFlags Flags;7039 Flags.copyFMF(*cast<FPMathOperator>(&I));7040 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);7041 7042 SDValue Result;7043 Result = DAG.getNode(7044 ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),7045 DAG.getTargetConstant((int)*RoundMode, sdl, MVT::i32));7046 setValue(&I, Result);7047 7048 return;7049 }7050 case Intrinsic::fmuladd: {7051 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());7052 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&7053 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {7054 setValue(&I, DAG.getNode(ISD::FMA, sdl,7055 getValue(I.getArgOperand(0)).getValueType(),7056 getValue(I.getArgOperand(0)),7057 getValue(I.getArgOperand(1)),7058 getValue(I.getArgOperand(2)), Flags));7059 } else if (TLI.isOperationLegalOrCustom(ISD::FMULADD, VT)) {7060 // TODO: Support splitting the vector.7061 setValue(&I, DAG.getNode(ISD::FMULADD, sdl,7062 getValue(I.getArgOperand(0)).getValueType(),7063 getValue(I.getArgOperand(0)),7064 getValue(I.getArgOperand(1)),7065 getValue(I.getArgOperand(2)), Flags));7066 } else {7067 // TODO: Intrinsic calls should have fast-math-flags.7068 SDValue Mul = DAG.getNode(7069 ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),7070 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);7071 SDValue Add = DAG.getNode(ISD::FADD, sdl,7072 getValue(I.getArgOperand(0)).getValueType(),7073 Mul, getValue(I.getArgOperand(2)), Flags);7074 setValue(&I, Add);7075 }7076 return;7077 }7078 case Intrinsic::convert_to_fp16:7079 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,7080 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,7081 getValue(I.getArgOperand(0)),7082 DAG.getTargetConstant(0, sdl,7083 MVT::i32))));7084 return;7085 case Intrinsic::convert_from_fp16:7086 setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,7087 TLI.getValueType(DAG.getDataLayout(), I.getType()),7088 DAG.getNode(ISD::BITCAST, sdl, MVT::f16,7089 getValue(I.getArgOperand(0)))));7090 return;7091 case Intrinsic::fptosi_sat: {7092 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());7093 setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, VT,7094 getValue(I.getArgOperand(0)),7095 DAG.getValueType(VT.getScalarType())));7096 return;7097 }7098 case Intrinsic::fptoui_sat: {7099 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());7100 setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, VT,7101 getValue(I.getArgOperand(0)),7102 DAG.getValueType(VT.getScalarType())));7103 return;7104 }7105 case Intrinsic::set_rounding:7106 Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,7107 {getRoot(), getValue(I.getArgOperand(0))});7108 setValue(&I, Res);7109 DAG.setRoot(Res.getValue(0));7110 return;7111 case Intrinsic::is_fpclass: {7112 const DataLayout DLayout = DAG.getDataLayout();7113 EVT DestVT = TLI.getValueType(DLayout, I.getType());7114 EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType());7115 FPClassTest Test = static_cast<FPClassTest>(7116 cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());7117 MachineFunction &MF = DAG.getMachineFunction();7118 const Function &F = MF.getFunction();7119 SDValue Op = getValue(I.getArgOperand(0));7120 SDNodeFlags Flags;7121 Flags.setNoFPExcept(7122 !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));7123 // If ISD::IS_FPCLASS should be expanded, do it right now, because the7124 // expansion can use illegal types. Making expansion early allows7125 // legalizing these types prior to selection.7126 if (!TLI.isOperationLegal(ISD::IS_FPCLASS, ArgVT) &&7127 !TLI.isOperationCustom(ISD::IS_FPCLASS, ArgVT)) {7128 SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG);7129 setValue(&I, Result);7130 return;7131 }7132 7133 SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32);7134 SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags);7135 setValue(&I, V);7136 return;7137 }7138 case Intrinsic::get_fpenv: {7139 const DataLayout DLayout = DAG.getDataLayout();7140 EVT EnvVT = TLI.getValueType(DLayout, I.getType());7141 Align TempAlign = DAG.getEVTAlign(EnvVT);7142 SDValue Chain = getRoot();7143 // Use GET_FPENV if it is legal or custom. Otherwise use memory-based node7144 // and temporary storage in stack.7145 if (TLI.isOperationLegalOrCustom(ISD::GET_FPENV, EnvVT)) {7146 Res = DAG.getNode(7147 ISD::GET_FPENV, sdl,7148 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),7149 MVT::Other),7150 Chain);7151 } else {7152 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());7153 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();7154 auto MPI =7155 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);7156 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(7157 MPI, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),7158 TempAlign);7159 Chain = DAG.getGetFPEnv(Chain, sdl, Temp, EnvVT, MMO);7160 Res = DAG.getLoad(EnvVT, sdl, Chain, Temp, MPI);7161 }7162 setValue(&I, Res);7163 DAG.setRoot(Res.getValue(1));7164 return;7165 }7166 case Intrinsic::set_fpenv: {7167 const DataLayout DLayout = DAG.getDataLayout();7168 SDValue Env = getValue(I.getArgOperand(0));7169 EVT EnvVT = Env.getValueType();7170 Align TempAlign = DAG.getEVTAlign(EnvVT);7171 SDValue Chain = getRoot();7172 // If SET_FPENV is custom or legal, use it. Otherwise use loading7173 // environment from memory.7174 if (TLI.isOperationLegalOrCustom(ISD::SET_FPENV, EnvVT)) {7175 Chain = DAG.getNode(ISD::SET_FPENV, sdl, MVT::Other, Chain, Env);7176 } else {7177 // Allocate space in stack, copy environment bits into it and use this7178 // memory in SET_FPENV_MEM.7179 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());7180 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();7181 auto MPI =7182 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);7183 Chain = DAG.getStore(Chain, sdl, Env, Temp, MPI, TempAlign,7184 MachineMemOperand::MOStore);7185 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(7186 MPI, MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),7187 TempAlign);7188 Chain = DAG.getSetFPEnv(Chain, sdl, Temp, EnvVT, MMO);7189 }7190 DAG.setRoot(Chain);7191 return;7192 }7193 case Intrinsic::reset_fpenv:7194 DAG.setRoot(DAG.getNode(ISD::RESET_FPENV, sdl, MVT::Other, getRoot()));7195 return;7196 case Intrinsic::get_fpmode:7197 Res = DAG.getNode(7198 ISD::GET_FPMODE, sdl,7199 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),7200 MVT::Other),7201 DAG.getRoot());7202 setValue(&I, Res);7203 DAG.setRoot(Res.getValue(1));7204 return;7205 case Intrinsic::set_fpmode:7206 Res = DAG.getNode(ISD::SET_FPMODE, sdl, MVT::Other, {DAG.getRoot()},7207 getValue(I.getArgOperand(0)));7208 DAG.setRoot(Res);7209 return;7210 case Intrinsic::reset_fpmode: {7211 Res = DAG.getNode(ISD::RESET_FPMODE, sdl, MVT::Other, getRoot());7212 DAG.setRoot(Res);7213 return;7214 }7215 case Intrinsic::pcmarker: {7216 SDValue Tmp = getValue(I.getArgOperand(0));7217 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));7218 return;7219 }7220 case Intrinsic::readcyclecounter: {7221 SDValue Op = getRoot();7222 Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,7223 DAG.getVTList(MVT::i64, MVT::Other), Op);7224 setValue(&I, Res);7225 DAG.setRoot(Res.getValue(1));7226 return;7227 }7228 case Intrinsic::readsteadycounter: {7229 SDValue Op = getRoot();7230 Res = DAG.getNode(ISD::READSTEADYCOUNTER, sdl,7231 DAG.getVTList(MVT::i64, MVT::Other), Op);7232 setValue(&I, Res);7233 DAG.setRoot(Res.getValue(1));7234 return;7235 }7236 case Intrinsic::bitreverse:7237 setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,7238 getValue(I.getArgOperand(0)).getValueType(),7239 getValue(I.getArgOperand(0))));7240 return;7241 case Intrinsic::bswap:7242 setValue(&I, DAG.getNode(ISD::BSWAP, sdl,7243 getValue(I.getArgOperand(0)).getValueType(),7244 getValue(I.getArgOperand(0))));7245 return;7246 case Intrinsic::cttz: {7247 SDValue Arg = getValue(I.getArgOperand(0));7248 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));7249 EVT Ty = Arg.getValueType();7250 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,7251 sdl, Ty, Arg));7252 return;7253 }7254 case Intrinsic::ctlz: {7255 SDValue Arg = getValue(I.getArgOperand(0));7256 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));7257 EVT Ty = Arg.getValueType();7258 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,7259 sdl, Ty, Arg));7260 return;7261 }7262 case Intrinsic::ctpop: {7263 SDValue Arg = getValue(I.getArgOperand(0));7264 EVT Ty = Arg.getValueType();7265 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));7266 return;7267 }7268 case Intrinsic::fshl:7269 case Intrinsic::fshr: {7270 bool IsFSHL = Intrinsic == Intrinsic::fshl;7271 SDValue X = getValue(I.getArgOperand(0));7272 SDValue Y = getValue(I.getArgOperand(1));7273 SDValue Z = getValue(I.getArgOperand(2));7274 EVT VT = X.getValueType();7275 7276 if (X == Y) {7277 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;7278 setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));7279 } else {7280 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;7281 setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));7282 }7283 return;7284 }7285 case Intrinsic::sadd_sat: {7286 SDValue Op1 = getValue(I.getArgOperand(0));7287 SDValue Op2 = getValue(I.getArgOperand(1));7288 setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));7289 return;7290 }7291 case Intrinsic::uadd_sat: {7292 SDValue Op1 = getValue(I.getArgOperand(0));7293 SDValue Op2 = getValue(I.getArgOperand(1));7294 setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));7295 return;7296 }7297 case Intrinsic::ssub_sat: {7298 SDValue Op1 = getValue(I.getArgOperand(0));7299 SDValue Op2 = getValue(I.getArgOperand(1));7300 setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));7301 return;7302 }7303 case Intrinsic::usub_sat: {7304 SDValue Op1 = getValue(I.getArgOperand(0));7305 SDValue Op2 = getValue(I.getArgOperand(1));7306 setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));7307 return;7308 }7309 case Intrinsic::sshl_sat: {7310 SDValue Op1 = getValue(I.getArgOperand(0));7311 SDValue Op2 = getValue(I.getArgOperand(1));7312 setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2));7313 return;7314 }7315 case Intrinsic::ushl_sat: {7316 SDValue Op1 = getValue(I.getArgOperand(0));7317 SDValue Op2 = getValue(I.getArgOperand(1));7318 setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2));7319 return;7320 }7321 case Intrinsic::smul_fix:7322 case Intrinsic::umul_fix:7323 case Intrinsic::smul_fix_sat:7324 case Intrinsic::umul_fix_sat: {7325 SDValue Op1 = getValue(I.getArgOperand(0));7326 SDValue Op2 = getValue(I.getArgOperand(1));7327 SDValue Op3 = getValue(I.getArgOperand(2));7328 setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl,7329 Op1.getValueType(), Op1, Op2, Op3));7330 return;7331 }7332 case Intrinsic::sdiv_fix:7333 case Intrinsic::udiv_fix:7334 case Intrinsic::sdiv_fix_sat:7335 case Intrinsic::udiv_fix_sat: {7336 SDValue Op1 = getValue(I.getArgOperand(0));7337 SDValue Op2 = getValue(I.getArgOperand(1));7338 SDValue Op3 = getValue(I.getArgOperand(2));7339 setValue(&I, expandDivFix(FixedPointIntrinsicToOpcode(Intrinsic), sdl,7340 Op1, Op2, Op3, DAG, TLI));7341 return;7342 }7343 case Intrinsic::smax: {7344 SDValue Op1 = getValue(I.getArgOperand(0));7345 SDValue Op2 = getValue(I.getArgOperand(1));7346 setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));7347 return;7348 }7349 case Intrinsic::smin: {7350 SDValue Op1 = getValue(I.getArgOperand(0));7351 SDValue Op2 = getValue(I.getArgOperand(1));7352 setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));7353 return;7354 }7355 case Intrinsic::umax: {7356 SDValue Op1 = getValue(I.getArgOperand(0));7357 SDValue Op2 = getValue(I.getArgOperand(1));7358 setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));7359 return;7360 }7361 case Intrinsic::umin: {7362 SDValue Op1 = getValue(I.getArgOperand(0));7363 SDValue Op2 = getValue(I.getArgOperand(1));7364 setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));7365 return;7366 }7367 case Intrinsic::abs: {7368 // TODO: Preserve "int min is poison" arg in SDAG?7369 SDValue Op1 = getValue(I.getArgOperand(0));7370 setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1));7371 return;7372 }7373 case Intrinsic::scmp: {7374 SDValue Op1 = getValue(I.getArgOperand(0));7375 SDValue Op2 = getValue(I.getArgOperand(1));7376 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());7377 setValue(&I, DAG.getNode(ISD::SCMP, sdl, DestVT, Op1, Op2));7378 break;7379 }7380 case Intrinsic::ucmp: {7381 SDValue Op1 = getValue(I.getArgOperand(0));7382 SDValue Op2 = getValue(I.getArgOperand(1));7383 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());7384 setValue(&I, DAG.getNode(ISD::UCMP, sdl, DestVT, Op1, Op2));7385 break;7386 }7387 case Intrinsic::stacksave: {7388 SDValue Op = getRoot();7389 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());7390 Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op);7391 setValue(&I, Res);7392 DAG.setRoot(Res.getValue(1));7393 return;7394 }7395 case Intrinsic::stackrestore:7396 Res = getValue(I.getArgOperand(0));7397 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));7398 return;7399 case Intrinsic::get_dynamic_area_offset: {7400 SDValue Op = getRoot();7401 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());7402 Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),7403 Op);7404 DAG.setRoot(Op);7405 setValue(&I, Res);7406 return;7407 }7408 case Intrinsic::stackguard: {7409 MachineFunction &MF = DAG.getMachineFunction();7410 const Module &M = *MF.getFunction().getParent();7411 EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());7412 SDValue Chain = getRoot();7413 if (TLI.useLoadStackGuardNode(M)) {7414 Res = getLoadStackGuard(DAG, sdl, Chain);7415 Res = DAG.getPtrExtOrTrunc(Res, sdl, PtrTy);7416 } else {7417 const Value *Global = TLI.getSDagStackGuard(M);7418 if (!Global) {7419 LLVMContext &Ctx = *DAG.getContext();7420 Ctx.diagnose(DiagnosticInfoGeneric("unable to lower stackguard"));7421 setValue(&I, DAG.getPOISON(PtrTy));7422 return;7423 }7424 7425 Align Align = DAG.getDataLayout().getPrefTypeAlign(Global->getType());7426 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),7427 MachinePointerInfo(Global, 0), Align,7428 MachineMemOperand::MOVolatile);7429 }7430 if (TLI.useStackGuardXorFP())7431 Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);7432 DAG.setRoot(Chain);7433 setValue(&I, Res);7434 return;7435 }7436 case Intrinsic::stackprotector: {7437 // Emit code into the DAG to store the stack guard onto the stack.7438 MachineFunction &MF = DAG.getMachineFunction();7439 MachineFrameInfo &MFI = MF.getFrameInfo();7440 const Module &M = *MF.getFunction().getParent();7441 SDValue Src, Chain = getRoot();7442 7443 if (TLI.useLoadStackGuardNode(M))7444 Src = getLoadStackGuard(DAG, sdl, Chain);7445 else7446 Src = getValue(I.getArgOperand(0)); // The guard's value.7447 7448 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));7449 7450 int FI = FuncInfo.StaticAllocaMap[Slot];7451 MFI.setStackProtectorIndex(FI);7452 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());7453 7454 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);7455 7456 // Store the stack protector onto the stack.7457 Res = DAG.getStore(7458 Chain, sdl, Src, FIN,7459 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),7460 MaybeAlign(), MachineMemOperand::MOVolatile);7461 setValue(&I, Res);7462 DAG.setRoot(Res);7463 return;7464 }7465 case Intrinsic::objectsize:7466 llvm_unreachable("llvm.objectsize.* should have been lowered already");7467 7468 case Intrinsic::is_constant:7469 llvm_unreachable("llvm.is.constant.* should have been lowered already");7470 7471 case Intrinsic::annotation:7472 case Intrinsic::ptr_annotation:7473 case Intrinsic::launder_invariant_group:7474 case Intrinsic::strip_invariant_group:7475 // Drop the intrinsic, but forward the value7476 setValue(&I, getValue(I.getOperand(0)));7477 return;7478 7479 case Intrinsic::type_test:7480 case Intrinsic::public_type_test:7481 setValue(&I, getValue(ConstantInt::getTrue(I.getType())));7482 return;7483 7484 case Intrinsic::assume:7485 case Intrinsic::experimental_noalias_scope_decl:7486 case Intrinsic::var_annotation:7487 case Intrinsic::sideeffect:7488 // Discard annotate attributes, noalias scope declarations, assumptions, and7489 // artificial side-effects.7490 return;7491 7492 case Intrinsic::codeview_annotation: {7493 // Emit a label associated with this metadata.7494 MachineFunction &MF = DAG.getMachineFunction();7495 MCSymbol *Label = MF.getContext().createTempSymbol("annotation", true);7496 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();7497 MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));7498 Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);7499 DAG.setRoot(Res);7500 return;7501 }7502 7503 case Intrinsic::init_trampoline: {7504 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());7505 7506 SDValue Ops[6];7507 Ops[0] = getRoot();7508 Ops[1] = getValue(I.getArgOperand(0));7509 Ops[2] = getValue(I.getArgOperand(1));7510 Ops[3] = getValue(I.getArgOperand(2));7511 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));7512 Ops[5] = DAG.getSrcValue(F);7513 7514 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);7515 7516 DAG.setRoot(Res);7517 return;7518 }7519 case Intrinsic::adjust_trampoline:7520 setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,7521 TLI.getPointerTy(DAG.getDataLayout()),7522 getValue(I.getArgOperand(0))));7523 return;7524 case Intrinsic::gcroot: {7525 assert(DAG.getMachineFunction().getFunction().hasGC() &&7526 "only valid in functions with gc specified, enforced by Verifier");7527 assert(GFI && "implied by previous");7528 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();7529 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));7530 7531 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());7532 GFI->addStackRoot(FI->getIndex(), TypeMap);7533 return;7534 }7535 case Intrinsic::gcread:7536 case Intrinsic::gcwrite:7537 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");7538 case Intrinsic::get_rounding:7539 Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot());7540 setValue(&I, Res);7541 DAG.setRoot(Res.getValue(1));7542 return;7543 7544 case Intrinsic::expect:7545 case Intrinsic::expect_with_probability:7546 // Just replace __builtin_expect(exp, c) and7547 // __builtin_expect_with_probability(exp, c, p) with EXP.7548 setValue(&I, getValue(I.getArgOperand(0)));7549 return;7550 7551 case Intrinsic::ubsantrap:7552 case Intrinsic::debugtrap:7553 case Intrinsic::trap: {7554 StringRef TrapFuncName =7555 I.getAttributes().getFnAttr("trap-func-name").getValueAsString();7556 if (TrapFuncName.empty()) {7557 switch (Intrinsic) {7558 case Intrinsic::trap:7559 DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));7560 break;7561 case Intrinsic::debugtrap:7562 DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));7563 break;7564 case Intrinsic::ubsantrap:7565 DAG.setRoot(DAG.getNode(7566 ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),7567 DAG.getTargetConstant(7568 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,7569 MVT::i32)));7570 break;7571 default: llvm_unreachable("unknown trap intrinsic");7572 }7573 DAG.addNoMergeSiteInfo(DAG.getRoot().getNode(),7574 I.hasFnAttr(Attribute::NoMerge));7575 return;7576 }7577 TargetLowering::ArgListTy Args;7578 if (Intrinsic == Intrinsic::ubsantrap) {7579 Value *Arg = I.getArgOperand(0);7580 Args.emplace_back(Arg, getValue(Arg));7581 }7582 7583 TargetLowering::CallLoweringInfo CLI(DAG);7584 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(7585 CallingConv::C, I.getType(),7586 DAG.getExternalSymbol(TrapFuncName.data(),7587 TLI.getPointerTy(DAG.getDataLayout())),7588 std::move(Args));7589 CLI.NoMerge = I.hasFnAttr(Attribute::NoMerge);7590 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);7591 DAG.setRoot(Result.second);7592 return;7593 }7594 7595 case Intrinsic::allow_runtime_check:7596 case Intrinsic::allow_ubsan_check:7597 setValue(&I, getValue(ConstantInt::getTrue(I.getType())));7598 return;7599 7600 case Intrinsic::uadd_with_overflow:7601 case Intrinsic::sadd_with_overflow:7602 case Intrinsic::usub_with_overflow:7603 case Intrinsic::ssub_with_overflow:7604 case Intrinsic::umul_with_overflow:7605 case Intrinsic::smul_with_overflow: {7606 ISD::NodeType Op;7607 switch (Intrinsic) {7608 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.7609 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;7610 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;7611 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;7612 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;7613 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;7614 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;7615 }7616 SDValue Op1 = getValue(I.getArgOperand(0));7617 SDValue Op2 = getValue(I.getArgOperand(1));7618 7619 EVT ResultVT = Op1.getValueType();7620 EVT OverflowVT = MVT::i1;7621 if (ResultVT.isVector())7622 OverflowVT = EVT::getVectorVT(7623 *Context, OverflowVT, ResultVT.getVectorElementCount());7624 7625 SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);7626 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));7627 return;7628 }7629 case Intrinsic::prefetch: {7630 SDValue Ops[5];7631 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();7632 auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore;7633 Ops[0] = DAG.getRoot();7634 Ops[1] = getValue(I.getArgOperand(0));7635 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,7636 MVT::i32);7637 Ops[3] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(2)), sdl,7638 MVT::i32);7639 Ops[4] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(3)), sdl,7640 MVT::i32);7641 SDValue Result = DAG.getMemIntrinsicNode(7642 ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,7643 EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),7644 /* align */ std::nullopt, Flags);7645 7646 // Chain the prefetch in parallel with any pending loads, to stay out of7647 // the way of later optimizations.7648 PendingLoads.push_back(Result);7649 Result = getRoot();7650 DAG.setRoot(Result);7651 return;7652 }7653 case Intrinsic::lifetime_start:7654 case Intrinsic::lifetime_end: {7655 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);7656 // Stack coloring is not enabled in O0, discard region information.7657 if (TM.getOptLevel() == CodeGenOptLevel::None)7658 return;7659 7660 const AllocaInst *LifetimeObject = dyn_cast<AllocaInst>(I.getArgOperand(0));7661 if (!LifetimeObject)7662 return;7663 7664 // First check that the Alloca is static, otherwise it won't have a7665 // valid frame index.7666 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);7667 if (SI == FuncInfo.StaticAllocaMap.end())7668 return;7669 7670 const int FrameIndex = SI->second;7671 Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex);7672 DAG.setRoot(Res);7673 return;7674 }7675 case Intrinsic::pseudoprobe: {7676 auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();7677 auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();7678 auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();7679 Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr);7680 DAG.setRoot(Res);7681 return;7682 }7683 case Intrinsic::invariant_start:7684 // Discard region information.7685 setValue(&I,7686 DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType())));7687 return;7688 case Intrinsic::invariant_end:7689 // Discard region information.7690 return;7691 case Intrinsic::clear_cache: {7692 SDValue InputChain = DAG.getRoot();7693 SDValue StartVal = getValue(I.getArgOperand(0));7694 SDValue EndVal = getValue(I.getArgOperand(1));7695 Res = DAG.getNode(ISD::CLEAR_CACHE, sdl, DAG.getVTList(MVT::Other),7696 {InputChain, StartVal, EndVal});7697 setValue(&I, Res);7698 DAG.setRoot(Res);7699 return;7700 }7701 case Intrinsic::donothing:7702 case Intrinsic::seh_try_begin:7703 case Intrinsic::seh_scope_begin:7704 case Intrinsic::seh_try_end:7705 case Intrinsic::seh_scope_end:7706 // ignore7707 return;7708 case Intrinsic::experimental_stackmap:7709 visitStackmap(I);7710 return;7711 case Intrinsic::experimental_patchpoint_void:7712 case Intrinsic::experimental_patchpoint:7713 visitPatchpoint(I);7714 return;7715 case Intrinsic::experimental_gc_statepoint:7716 LowerStatepoint(cast<GCStatepointInst>(I));7717 return;7718 case Intrinsic::experimental_gc_result:7719 visitGCResult(cast<GCResultInst>(I));7720 return;7721 case Intrinsic::experimental_gc_relocate:7722 visitGCRelocate(cast<GCRelocateInst>(I));7723 return;7724 case Intrinsic::instrprof_cover:7725 llvm_unreachable("instrprof failed to lower a cover");7726 case Intrinsic::instrprof_increment:7727 llvm_unreachable("instrprof failed to lower an increment");7728 case Intrinsic::instrprof_timestamp:7729 llvm_unreachable("instrprof failed to lower a timestamp");7730 case Intrinsic::instrprof_value_profile:7731 llvm_unreachable("instrprof failed to lower a value profiling call");7732 case Intrinsic::instrprof_mcdc_parameters:7733 llvm_unreachable("instrprof failed to lower mcdc parameters");7734 case Intrinsic::instrprof_mcdc_tvbitmap_update:7735 llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update");7736 case Intrinsic::localescape: {7737 MachineFunction &MF = DAG.getMachineFunction();7738 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();7739 7740 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission7741 // is the same on all targets.7742 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {7743 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();7744 if (isa<ConstantPointerNull>(Arg))7745 continue; // Skip null pointers. They represent a hole in index space.7746 AllocaInst *Slot = cast<AllocaInst>(Arg);7747 assert(FuncInfo.StaticAllocaMap.count(Slot) &&7748 "can only escape static allocas");7749 int FI = FuncInfo.StaticAllocaMap[Slot];7750 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(7751 GlobalValue::dropLLVMManglingEscape(MF.getName()), Idx);7752 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,7753 TII->get(TargetOpcode::LOCAL_ESCAPE))7754 .addSym(FrameAllocSym)7755 .addFrameIndex(FI);7756 }7757 7758 return;7759 }7760 7761 case Intrinsic::localrecover: {7762 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)7763 MachineFunction &MF = DAG.getMachineFunction();7764 7765 // Get the symbol that defines the frame offset.7766 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());7767 auto *Idx = cast<ConstantInt>(I.getArgOperand(2));7768 unsigned IdxVal =7769 unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));7770 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(7771 GlobalValue::dropLLVMManglingEscape(Fn->getName()), IdxVal);7772 7773 Value *FP = I.getArgOperand(1);7774 SDValue FPVal = getValue(FP);7775 EVT PtrVT = FPVal.getValueType();7776 7777 // Create a MCSymbol for the label to avoid any target lowering7778 // that would make this PC relative.7779 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);7780 SDValue OffsetVal =7781 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);7782 7783 // Add the offset to the FP.7784 SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);7785 setValue(&I, Add);7786 7787 return;7788 }7789 7790 case Intrinsic::fake_use: {7791 Value *V = I.getArgOperand(0);7792 SDValue Ops[2];7793 // For Values not declared or previously used in this basic block, the7794 // NodeMap will not have an entry, and `getValue` will assert if V has no7795 // valid register value.7796 auto FakeUseValue = [&]() -> SDValue {7797 SDValue &N = NodeMap[V];7798 if (N.getNode())7799 return N;7800 7801 // If there's a virtual register allocated and initialized for this7802 // value, use it.7803 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))7804 return copyFromReg;7805 // FIXME: Do we want to preserve constants? It seems pointless.7806 if (isa<Constant>(V))7807 return getValue(V);7808 return SDValue();7809 }();7810 if (!FakeUseValue || FakeUseValue.isUndef())7811 return;7812 Ops[0] = getRoot();7813 Ops[1] = FakeUseValue;7814 // Also, do not translate a fake use with an undef operand, or any other7815 // empty SDValues.7816 if (!Ops[1] || Ops[1].isUndef())7817 return;7818 DAG.setRoot(DAG.getNode(ISD::FAKE_USE, sdl, MVT::Other, Ops));7819 return;7820 }7821 7822 case Intrinsic::reloc_none: {7823 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();7824 StringRef SymbolName = cast<MDString>(MD)->getString();7825 SDValue Ops[2] = {7826 getRoot(),7827 DAG.getTargetExternalSymbol(7828 SymbolName.data(), TLI.getProgramPointerTy(DAG.getDataLayout()))};7829 DAG.setRoot(DAG.getNode(ISD::RELOC_NONE, sdl, MVT::Other, Ops));7830 return;7831 }7832 7833 case Intrinsic::eh_exceptionpointer:7834 case Intrinsic::eh_exceptioncode: {7835 // Get the exception pointer vreg, copy from it, and resize it to fit.7836 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));7837 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());7838 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);7839 Register VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);7840 SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT);7841 if (Intrinsic == Intrinsic::eh_exceptioncode)7842 N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);7843 setValue(&I, N);7844 return;7845 }7846 case Intrinsic::xray_customevent: {7847 // Here we want to make sure that the intrinsic behaves as if it has a7848 // specific calling convention.7849 const auto &Triple = DAG.getTarget().getTargetTriple();7850 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64)7851 return;7852 7853 SmallVector<SDValue, 8> Ops;7854 7855 // We want to say that we always want the arguments in registers.7856 SDValue LogEntryVal = getValue(I.getArgOperand(0));7857 SDValue StrSizeVal = getValue(I.getArgOperand(1));7858 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);7859 SDValue Chain = getRoot();7860 Ops.push_back(LogEntryVal);7861 Ops.push_back(StrSizeVal);7862 Ops.push_back(Chain);7863 7864 // We need to enforce the calling convention for the callsite, so that7865 // argument ordering is enforced correctly, and that register allocation can7866 // see that some registers may be assumed clobbered and have to preserve7867 // them across calls to the intrinsic.7868 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,7869 sdl, NodeTys, Ops);7870 SDValue patchableNode = SDValue(MN, 0);7871 DAG.setRoot(patchableNode);7872 setValue(&I, patchableNode);7873 return;7874 }7875 case Intrinsic::xray_typedevent: {7876 // Here we want to make sure that the intrinsic behaves as if it has a7877 // specific calling convention.7878 const auto &Triple = DAG.getTarget().getTargetTriple();7879 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64)7880 return;7881 7882 SmallVector<SDValue, 8> Ops;7883 7884 // We want to say that we always want the arguments in registers.7885 // It's unclear to me how manipulating the selection DAG here forces callers7886 // to provide arguments in registers instead of on the stack.7887 SDValue LogTypeId = getValue(I.getArgOperand(0));7888 SDValue LogEntryVal = getValue(I.getArgOperand(1));7889 SDValue StrSizeVal = getValue(I.getArgOperand(2));7890 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);7891 SDValue Chain = getRoot();7892 Ops.push_back(LogTypeId);7893 Ops.push_back(LogEntryVal);7894 Ops.push_back(StrSizeVal);7895 Ops.push_back(Chain);7896 7897 // We need to enforce the calling convention for the callsite, so that7898 // argument ordering is enforced correctly, and that register allocation can7899 // see that some registers may be assumed clobbered and have to preserve7900 // them across calls to the intrinsic.7901 MachineSDNode *MN = DAG.getMachineNode(7902 TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);7903 SDValue patchableNode = SDValue(MN, 0);7904 DAG.setRoot(patchableNode);7905 setValue(&I, patchableNode);7906 return;7907 }7908 case Intrinsic::experimental_deoptimize:7909 LowerDeoptimizeCall(&I);7910 return;7911 case Intrinsic::stepvector:7912 visitStepVector(I);7913 return;7914 case Intrinsic::vector_reduce_fadd:7915 case Intrinsic::vector_reduce_fmul:7916 case Intrinsic::vector_reduce_add:7917 case Intrinsic::vector_reduce_mul:7918 case Intrinsic::vector_reduce_and:7919 case Intrinsic::vector_reduce_or:7920 case Intrinsic::vector_reduce_xor:7921 case Intrinsic::vector_reduce_smax:7922 case Intrinsic::vector_reduce_smin:7923 case Intrinsic::vector_reduce_umax:7924 case Intrinsic::vector_reduce_umin:7925 case Intrinsic::vector_reduce_fmax:7926 case Intrinsic::vector_reduce_fmin:7927 case Intrinsic::vector_reduce_fmaximum:7928 case Intrinsic::vector_reduce_fminimum:7929 visitVectorReduce(I, Intrinsic);7930 return;7931 7932 case Intrinsic::icall_branch_funnel: {7933 SmallVector<SDValue, 16> Ops;7934 Ops.push_back(getValue(I.getArgOperand(0)));7935 7936 int64_t Offset;7937 auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(7938 I.getArgOperand(1), Offset, DAG.getDataLayout()));7939 if (!Base)7940 report_fatal_error(7941 "llvm.icall.branch.funnel operand must be a GlobalValue");7942 Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));7943 7944 struct BranchFunnelTarget {7945 int64_t Offset;7946 SDValue Target;7947 };7948 SmallVector<BranchFunnelTarget, 8> Targets;7949 7950 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {7951 auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(7952 I.getArgOperand(Op), Offset, DAG.getDataLayout()));7953 if (ElemBase != Base)7954 report_fatal_error("all llvm.icall.branch.funnel operands must refer "7955 "to the same GlobalValue");7956 7957 SDValue Val = getValue(I.getArgOperand(Op + 1));7958 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);7959 if (!GA)7960 report_fatal_error(7961 "llvm.icall.branch.funnel operand must be a GlobalValue");7962 Targets.push_back({Offset, DAG.getTargetGlobalAddress(7963 GA->getGlobal(), sdl, Val.getValueType(),7964 GA->getOffset())});7965 }7966 llvm::sort(Targets,7967 [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {7968 return T1.Offset < T2.Offset;7969 });7970 7971 for (auto &T : Targets) {7972 Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));7973 Ops.push_back(T.Target);7974 }7975 7976 Ops.push_back(DAG.getRoot()); // Chain7977 SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,7978 MVT::Other, Ops),7979 0);7980 DAG.setRoot(N);7981 setValue(&I, N);7982 HasTailCall = true;7983 return;7984 }7985 7986 case Intrinsic::wasm_landingpad_index:7987 // Information this intrinsic contained has been transferred to7988 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely7989 // delete it now.7990 return;7991 7992 case Intrinsic::aarch64_settag:7993 case Intrinsic::aarch64_settag_zero: {7994 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();7995 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;7996 SDValue Val = TSI.EmitTargetCodeForSetTag(7997 DAG, sdl, getRoot(), getValue(I.getArgOperand(0)),7998 getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)),7999 ZeroMemory);8000 DAG.setRoot(Val);8001 setValue(&I, Val);8002 return;8003 }8004 case Intrinsic::amdgcn_cs_chain: {8005 // At this point we don't care if it's amdgpu_cs_chain or8006 // amdgpu_cs_chain_preserve.8007 CallingConv::ID CC = CallingConv::AMDGPU_CS_Chain;8008 8009 Type *RetTy = I.getType();8010 assert(RetTy->isVoidTy() && "Should not return");8011 8012 SDValue Callee = getValue(I.getOperand(0));8013 8014 // We only have 2 actual args: one for the SGPRs and one for the VGPRs.8015 // We'll also tack the value of the EXEC mask at the end.8016 TargetLowering::ArgListTy Args;8017 Args.reserve(3);8018 8019 for (unsigned Idx : {2, 3, 1}) {8020 TargetLowering::ArgListEntry Arg(getValue(I.getOperand(Idx)),8021 I.getOperand(Idx)->getType());8022 Arg.setAttributes(&I, Idx);8023 Args.push_back(Arg);8024 }8025 8026 assert(Args[0].IsInReg && "SGPR args should be marked inreg");8027 assert(!Args[1].IsInReg && "VGPR args should not be marked inreg");8028 Args[2].IsInReg = true; // EXEC should be inreg8029 8030 // Forward the flags and any additional arguments.8031 for (unsigned Idx = 4; Idx < I.arg_size(); ++Idx) {8032 TargetLowering::ArgListEntry Arg(getValue(I.getOperand(Idx)),8033 I.getOperand(Idx)->getType());8034 Arg.setAttributes(&I, Idx);8035 Args.push_back(Arg);8036 }8037 8038 TargetLowering::CallLoweringInfo CLI(DAG);8039 CLI.setDebugLoc(getCurSDLoc())8040 .setChain(getRoot())8041 .setCallee(CC, RetTy, Callee, std::move(Args))8042 .setNoReturn(true)8043 .setTailCall(true)8044 .setConvergent(I.isConvergent());8045 CLI.CB = &I;8046 std::pair<SDValue, SDValue> Result =8047 lowerInvokable(CLI, /*EHPadBB*/ nullptr);8048 (void)Result;8049 assert(!Result.first.getNode() && !Result.second.getNode() &&8050 "Should've lowered as tail call");8051 8052 HasTailCall = true;8053 return;8054 }8055 case Intrinsic::amdgcn_call_whole_wave: {8056 TargetLowering::ArgListTy Args;8057 bool isTailCall = I.isTailCall();8058 8059 // The first argument is the callee. Skip it when assembling the call args.8060 for (unsigned Idx = 1; Idx < I.arg_size(); ++Idx) {8061 TargetLowering::ArgListEntry Arg(getValue(I.getArgOperand(Idx)),8062 I.getArgOperand(Idx)->getType());8063 Arg.setAttributes(&I, Idx);8064 8065 // If we have an explicit sret argument that is an Instruction, (i.e., it8066 // might point to function-local memory), we can't meaningfully tail-call.8067 if (Arg.IsSRet && isa<Instruction>(I.getArgOperand(Idx)))8068 isTailCall = false;8069 8070 Args.push_back(Arg);8071 }8072 8073 SDValue ConvControlToken;8074 if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) {8075 auto *Token = Bundle->Inputs[0].get();8076 ConvControlToken = getValue(Token);8077 }8078 8079 TargetLowering::CallLoweringInfo CLI(DAG);8080 CLI.setDebugLoc(getCurSDLoc())8081 .setChain(getRoot())8082 .setCallee(CallingConv::AMDGPU_Gfx_WholeWave, I.getType(),8083 getValue(I.getArgOperand(0)), std::move(Args))8084 .setTailCall(isTailCall && canTailCall(I))8085 .setIsPreallocated(8086 I.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0)8087 .setConvergent(I.isConvergent())8088 .setConvergenceControlToken(ConvControlToken);8089 CLI.CB = &I;8090 8091 std::pair<SDValue, SDValue> Result =8092 lowerInvokable(CLI, /*EHPadBB=*/nullptr);8093 8094 if (Result.first.getNode())8095 setValue(&I, Result.first);8096 return;8097 }8098 case Intrinsic::ptrmask: {8099 SDValue Ptr = getValue(I.getOperand(0));8100 SDValue Mask = getValue(I.getOperand(1));8101 8102 // On arm64_32, pointers are 32 bits when stored in memory, but8103 // zero-extended to 64 bits when in registers. Thus the mask is 32 bits to8104 // match the index type, but the pointer is 64 bits, so the mask must be8105 // zero-extended up to 64 bits to match the pointer.8106 EVT PtrVT =8107 TLI.getValueType(DAG.getDataLayout(), I.getOperand(0)->getType());8108 EVT MemVT =8109 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());8110 assert(PtrVT == Ptr.getValueType());8111 if (Mask.getValueType().getFixedSizeInBits() < MemVT.getFixedSizeInBits()) {8112 // For AMDGPU buffer descriptors the mask is 48 bits, but the pointer is8113 // 128-bit, so we have to pad the mask with ones for unused bits.8114 auto HighOnes = DAG.getNode(8115 ISD::SHL, sdl, PtrVT, DAG.getAllOnesConstant(sdl, PtrVT),8116 DAG.getShiftAmountConstant(Mask.getValueType().getFixedSizeInBits(),8117 PtrVT, sdl));8118 Mask = DAG.getNode(ISD::OR, sdl, PtrVT,8119 DAG.getZExtOrTrunc(Mask, sdl, PtrVT), HighOnes);8120 } else if (Mask.getValueType() != PtrVT)8121 Mask = DAG.getPtrExtOrTrunc(Mask, sdl, PtrVT);8122 8123 assert(Mask.getValueType() == PtrVT);8124 setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr, Mask));8125 return;8126 }8127 case Intrinsic::threadlocal_address: {8128 setValue(&I, getValue(I.getOperand(0)));8129 return;8130 }8131 case Intrinsic::get_active_lane_mask: {8132 EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());8133 SDValue Index = getValue(I.getOperand(0));8134 SDValue TripCount = getValue(I.getOperand(1));8135 EVT ElementVT = Index.getValueType();8136 8137 if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {8138 setValue(&I, DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, sdl, CCVT, Index,8139 TripCount));8140 return;8141 }8142 8143 EVT VecTy = EVT::getVectorVT(*DAG.getContext(), ElementVT,8144 CCVT.getVectorElementCount());8145 8146 SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index);8147 SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount);8148 SDValue VectorStep = DAG.getStepVector(sdl, VecTy);8149 SDValue VectorInduction = DAG.getNode(8150 ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);8151 SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,8152 VectorTripCount, ISD::CondCode::SETULT);8153 setValue(&I, SetCC);8154 return;8155 }8156 case Intrinsic::experimental_get_vector_length: {8157 assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&8158 "Expected positive VF");8159 unsigned VF = cast<ConstantInt>(I.getOperand(1))->getZExtValue();8160 bool IsScalable = cast<ConstantInt>(I.getOperand(2))->isOne();8161 8162 SDValue Count = getValue(I.getOperand(0));8163 EVT CountVT = Count.getValueType();8164 8165 if (!TLI.shouldExpandGetVectorLength(CountVT, VF, IsScalable)) {8166 visitTargetIntrinsic(I, Intrinsic);8167 return;8168 }8169 8170 // Expand to a umin between the trip count and the maximum elements the type8171 // can hold.8172 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());8173 8174 // Extend the trip count to at least the result VT.8175 if (CountVT.bitsLT(VT)) {8176 Count = DAG.getNode(ISD::ZERO_EXTEND, sdl, VT, Count);8177 CountVT = VT;8178 }8179 8180 SDValue MaxEVL = DAG.getElementCount(sdl, CountVT,8181 ElementCount::get(VF, IsScalable));8182 8183 SDValue UMin = DAG.getNode(ISD::UMIN, sdl, CountVT, Count, MaxEVL);8184 // Clip to the result type if needed.8185 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, sdl, VT, UMin);8186 8187 setValue(&I, Trunc);8188 return;8189 }8190 case Intrinsic::vector_partial_reduce_add: {8191 SDValue Acc = getValue(I.getOperand(0));8192 SDValue Input = getValue(I.getOperand(1));8193 setValue(&I,8194 DAG.getNode(ISD::PARTIAL_REDUCE_UMLA, sdl, Acc.getValueType(), Acc,8195 Input, DAG.getConstant(1, sdl, Input.getValueType())));8196 return;8197 }8198 case Intrinsic::vector_partial_reduce_fadd: {8199 SDValue Acc = getValue(I.getOperand(0));8200 SDValue Input = getValue(I.getOperand(1));8201 setValue(&I, DAG.getNode(8202 ISD::PARTIAL_REDUCE_FMLA, sdl, Acc.getValueType(), Acc,8203 Input, DAG.getConstantFP(1.0, sdl, Input.getValueType())));8204 return;8205 }8206 case Intrinsic::experimental_cttz_elts: {8207 auto DL = getCurSDLoc();8208 SDValue Op = getValue(I.getOperand(0));8209 EVT OpVT = Op.getValueType();8210 8211 if (!TLI.shouldExpandCttzElements(OpVT)) {8212 visitTargetIntrinsic(I, Intrinsic);8213 return;8214 }8215 8216 if (OpVT.getScalarType() != MVT::i1) {8217 // Compare the input vector elements to zero & use to count trailing zeros8218 SDValue AllZero = DAG.getConstant(0, DL, OpVT);8219 OpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,8220 OpVT.getVectorElementCount());8221 Op = DAG.getSetCC(DL, OpVT, Op, AllZero, ISD::SETNE);8222 }8223 8224 // If the zero-is-poison flag is set, we can assume the upper limit8225 // of the result is VF-1.8226 bool ZeroIsPoison =8227 !cast<ConstantSDNode>(getValue(I.getOperand(1)))->isZero();8228 ConstantRange VScaleRange(1, true); // Dummy value.8229 if (isa<ScalableVectorType>(I.getOperand(0)->getType()))8230 VScaleRange = getVScaleRange(I.getCaller(), 64);8231 unsigned EltWidth = TLI.getBitWidthForCttzElements(8232 I.getType(), OpVT.getVectorElementCount(), ZeroIsPoison, &VScaleRange);8233 8234 MVT NewEltTy = MVT::getIntegerVT(EltWidth);8235 8236 // Create the new vector type & get the vector length8237 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), NewEltTy,8238 OpVT.getVectorElementCount());8239 8240 SDValue VL =8241 DAG.getElementCount(DL, NewEltTy, OpVT.getVectorElementCount());8242 8243 SDValue StepVec = DAG.getStepVector(DL, NewVT);8244 SDValue SplatVL = DAG.getSplat(NewVT, DL, VL);8245 SDValue StepVL = DAG.getNode(ISD::SUB, DL, NewVT, SplatVL, StepVec);8246 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, Op);8247 SDValue And = DAG.getNode(ISD::AND, DL, NewVT, StepVL, Ext);8248 SDValue Max = DAG.getNode(ISD::VECREDUCE_UMAX, DL, NewEltTy, And);8249 SDValue Sub = DAG.getNode(ISD::SUB, DL, NewEltTy, VL, Max);8250 8251 EVT RetTy = TLI.getValueType(DAG.getDataLayout(), I.getType());8252 SDValue Ret = DAG.getZExtOrTrunc(Sub, DL, RetTy);8253 8254 setValue(&I, Ret);8255 return;8256 }8257 case Intrinsic::vector_insert: {8258 SDValue Vec = getValue(I.getOperand(0));8259 SDValue SubVec = getValue(I.getOperand(1));8260 SDValue Index = getValue(I.getOperand(2));8261 8262 // The intrinsic's index type is i64, but the SDNode requires an index type8263 // suitable for the target. Convert the index as required.8264 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());8265 if (Index.getValueType() != VectorIdxTy)8266 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);8267 8268 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());8269 setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec,8270 Index));8271 return;8272 }8273 case Intrinsic::vector_extract: {8274 SDValue Vec = getValue(I.getOperand(0));8275 SDValue Index = getValue(I.getOperand(1));8276 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());8277 8278 // The intrinsic's index type is i64, but the SDNode requires an index type8279 // suitable for the target. Convert the index as required.8280 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());8281 if (Index.getValueType() != VectorIdxTy)8282 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);8283 8284 setValue(&I,8285 DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index));8286 return;8287 }8288 case Intrinsic::experimental_vector_match: {8289 SDValue Op1 = getValue(I.getOperand(0));8290 SDValue Op2 = getValue(I.getOperand(1));8291 SDValue Mask = getValue(I.getOperand(2));8292 EVT Op1VT = Op1.getValueType();8293 EVT Op2VT = Op2.getValueType();8294 EVT ResVT = Mask.getValueType();8295 unsigned SearchSize = Op2VT.getVectorNumElements();8296 8297 // If the target has native support for this vector match operation, lower8298 // the intrinsic untouched; otherwise, expand it below.8299 if (!TLI.shouldExpandVectorMatch(Op1VT, SearchSize)) {8300 visitTargetIntrinsic(I, Intrinsic);8301 return;8302 }8303 8304 SDValue Ret = DAG.getConstant(0, sdl, ResVT);8305 8306 for (unsigned i = 0; i < SearchSize; ++i) {8307 SDValue Op2Elem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, sdl,8308 Op2VT.getVectorElementType(), Op2,8309 DAG.getVectorIdxConstant(i, sdl));8310 SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, sdl, Op1VT, Op2Elem);8311 SDValue Cmp = DAG.getSetCC(sdl, ResVT, Op1, Splat, ISD::SETEQ);8312 Ret = DAG.getNode(ISD::OR, sdl, ResVT, Ret, Cmp);8313 }8314 8315 setValue(&I, DAG.getNode(ISD::AND, sdl, ResVT, Ret, Mask));8316 return;8317 }8318 case Intrinsic::vector_reverse:8319 visitVectorReverse(I);8320 return;8321 case Intrinsic::vector_splice:8322 visitVectorSplice(I);8323 return;8324 case Intrinsic::callbr_landingpad:8325 visitCallBrLandingPad(I);8326 return;8327 case Intrinsic::vector_interleave2:8328 visitVectorInterleave(I, 2);8329 return;8330 case Intrinsic::vector_interleave3:8331 visitVectorInterleave(I, 3);8332 return;8333 case Intrinsic::vector_interleave4:8334 visitVectorInterleave(I, 4);8335 return;8336 case Intrinsic::vector_interleave5:8337 visitVectorInterleave(I, 5);8338 return;8339 case Intrinsic::vector_interleave6:8340 visitVectorInterleave(I, 6);8341 return;8342 case Intrinsic::vector_interleave7:8343 visitVectorInterleave(I, 7);8344 return;8345 case Intrinsic::vector_interleave8:8346 visitVectorInterleave(I, 8);8347 return;8348 case Intrinsic::vector_deinterleave2:8349 visitVectorDeinterleave(I, 2);8350 return;8351 case Intrinsic::vector_deinterleave3:8352 visitVectorDeinterleave(I, 3);8353 return;8354 case Intrinsic::vector_deinterleave4:8355 visitVectorDeinterleave(I, 4);8356 return;8357 case Intrinsic::vector_deinterleave5:8358 visitVectorDeinterleave(I, 5);8359 return;8360 case Intrinsic::vector_deinterleave6:8361 visitVectorDeinterleave(I, 6);8362 return;8363 case Intrinsic::vector_deinterleave7:8364 visitVectorDeinterleave(I, 7);8365 return;8366 case Intrinsic::vector_deinterleave8:8367 visitVectorDeinterleave(I, 8);8368 return;8369 case Intrinsic::experimental_vector_compress:8370 setValue(&I, DAG.getNode(ISD::VECTOR_COMPRESS, sdl,8371 getValue(I.getArgOperand(0)).getValueType(),8372 getValue(I.getArgOperand(0)),8373 getValue(I.getArgOperand(1)),8374 getValue(I.getArgOperand(2)), Flags));8375 return;8376 case Intrinsic::experimental_convergence_anchor:8377 case Intrinsic::experimental_convergence_entry:8378 case Intrinsic::experimental_convergence_loop:8379 visitConvergenceControl(I, Intrinsic);8380 return;8381 case Intrinsic::experimental_vector_histogram_add: {8382 visitVectorHistogram(I, Intrinsic);8383 return;8384 }8385 case Intrinsic::experimental_vector_extract_last_active: {8386 visitVectorExtractLastActive(I, Intrinsic);8387 return;8388 }8389 case Intrinsic::loop_dependence_war_mask:8390 setValue(&I,8391 DAG.getNode(ISD::LOOP_DEPENDENCE_WAR_MASK, sdl,8392 EVT::getEVT(I.getType()), getValue(I.getOperand(0)),8393 getValue(I.getOperand(1)), getValue(I.getOperand(2))));8394 return;8395 case Intrinsic::loop_dependence_raw_mask:8396 setValue(&I,8397 DAG.getNode(ISD::LOOP_DEPENDENCE_RAW_MASK, sdl,8398 EVT::getEVT(I.getType()), getValue(I.getOperand(0)),8399 getValue(I.getOperand(1)), getValue(I.getOperand(2))));8400 return;8401 }8402}8403 8404void SelectionDAGBuilder::pushFPOpOutChain(SDValue Result,8405 fp::ExceptionBehavior EB) {8406 assert(Result.getNode()->getNumValues() == 2);8407 SDValue OutChain = Result.getValue(1);8408 assert(OutChain.getValueType() == MVT::Other);8409 8410 // Instead of updating the root immediately, push the produced chain to the8411 // appropriate list, deferring the update until the root is requested. In this8412 // case, the nodes from the lists are chained using TokenFactor, indicating8413 // that the operations are independent.8414 //8415 // In particular, the root is updated before any call that might access the8416 // floating-point environment, except for constrained intrinsics.8417 switch (EB) {8418 case fp::ExceptionBehavior::ebMayTrap:8419 case fp::ExceptionBehavior::ebIgnore:8420 PendingConstrainedFP.push_back(OutChain);8421 break;8422 case fp::ExceptionBehavior::ebStrict:8423 PendingConstrainedFPStrict.push_back(OutChain);8424 break;8425 }8426}8427 8428void SelectionDAGBuilder::visitConstrainedFPIntrinsic(8429 const ConstrainedFPIntrinsic &FPI) {8430 SDLoc sdl = getCurSDLoc();8431 8432 // We do not need to serialize constrained FP intrinsics against8433 // each other or against (nonvolatile) loads, so they can be8434 // chained like loads.8435 fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();8436 SDValue Chain = getFPOperationRoot(EB);8437 SmallVector<SDValue, 4> Opers;8438 Opers.push_back(Chain);8439 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)8440 Opers.push_back(getValue(FPI.getArgOperand(I)));8441 8442 const TargetLowering &TLI = DAG.getTargetLoweringInfo();8443 EVT VT = TLI.getValueType(DAG.getDataLayout(), FPI.getType());8444 SDVTList VTs = DAG.getVTList(VT, MVT::Other);8445 8446 SDNodeFlags Flags;8447 if (EB == fp::ExceptionBehavior::ebIgnore)8448 Flags.setNoFPExcept(true);8449 8450 if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI))8451 Flags.copyFMF(*FPOp);8452 8453 unsigned Opcode;8454 switch (FPI.getIntrinsicID()) {8455 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.8456#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \8457 case Intrinsic::INTRINSIC: \8458 Opcode = ISD::STRICT_##DAGN; \8459 break;8460#include "llvm/IR/ConstrainedOps.def"8461 case Intrinsic::experimental_constrained_fmuladd: {8462 Opcode = ISD::STRICT_FMA;8463 // Break fmuladd into fmul and fadd.8464 if (TM.Options.AllowFPOpFusion == FPOpFusion::Strict ||8465 !TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {8466 Opers.pop_back();8467 SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags);8468 pushFPOpOutChain(Mul, EB);8469 Opcode = ISD::STRICT_FADD;8470 Opers.clear();8471 Opers.push_back(Mul.getValue(1));8472 Opers.push_back(Mul.getValue(0));8473 Opers.push_back(getValue(FPI.getArgOperand(2)));8474 }8475 break;8476 }8477 }8478 8479 // A few strict DAG nodes carry additional operands that are not8480 // set up by the default code above.8481 switch (Opcode) {8482 default: break;8483 case ISD::STRICT_FP_ROUND:8484 Opers.push_back(8485 DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())));8486 break;8487 case ISD::STRICT_FSETCC:8488 case ISD::STRICT_FSETCCS: {8489 auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);8490 ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate());8491 if (TM.Options.NoNaNsFPMath)8492 Condition = getFCmpCodeWithoutNaN(Condition);8493 Opers.push_back(DAG.getCondCode(Condition));8494 break;8495 }8496 }8497 8498 SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags);8499 pushFPOpOutChain(Result, EB);8500 8501 SDValue FPResult = Result.getValue(0);8502 setValue(&FPI, FPResult);8503}8504 8505static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {8506 std::optional<unsigned> ResOPC;8507 switch (VPIntrin.getIntrinsicID()) {8508 case Intrinsic::vp_ctlz: {8509 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();8510 ResOPC = IsZeroUndef ? ISD::VP_CTLZ_ZERO_UNDEF : ISD::VP_CTLZ;8511 break;8512 }8513 case Intrinsic::vp_cttz: {8514 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();8515 ResOPC = IsZeroUndef ? ISD::VP_CTTZ_ZERO_UNDEF : ISD::VP_CTTZ;8516 break;8517 }8518 case Intrinsic::vp_cttz_elts: {8519 bool IsZeroPoison = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();8520 ResOPC = IsZeroPoison ? ISD::VP_CTTZ_ELTS_ZERO_UNDEF : ISD::VP_CTTZ_ELTS;8521 break;8522 }8523#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \8524 case Intrinsic::VPID: \8525 ResOPC = ISD::VPSD; \8526 break;8527#include "llvm/IR/VPIntrinsics.def"8528 }8529 8530 if (!ResOPC)8531 llvm_unreachable(8532 "Inconsistency: no SDNode available for this VPIntrinsic!");8533 8534 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||8535 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {8536 if (VPIntrin.getFastMathFlags().allowReassoc())8537 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD8538 : ISD::VP_REDUCE_FMUL;8539 }8540 8541 return *ResOPC;8542}8543 8544void SelectionDAGBuilder::visitVPLoad(8545 const VPIntrinsic &VPIntrin, EVT VT,8546 const SmallVectorImpl<SDValue> &OpValues) {8547 SDLoc DL = getCurSDLoc();8548 Value *PtrOperand = VPIntrin.getArgOperand(0);8549 MaybeAlign Alignment = VPIntrin.getPointerAlignment();8550 AAMDNodes AAInfo = VPIntrin.getAAMetadata();8551 const MDNode *Ranges = getRangeMetadata(VPIntrin);8552 SDValue LD;8553 // Do not serialize variable-length loads of constant memory with8554 // anything.8555 if (!Alignment)8556 Alignment = DAG.getEVTAlign(VT);8557 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);8558 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);8559 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();8560 const TargetLowering &TLI = DAG.getTargetLoweringInfo();8561 MachineMemOperand::Flags MMOFlags =8562 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);8563 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(8564 MachinePointerInfo(PtrOperand), MMOFlags,8565 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);8566 LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],8567 MMO, false /*IsExpanding */);8568 if (AddToChain)8569 PendingLoads.push_back(LD.getValue(1));8570 setValue(&VPIntrin, LD);8571}8572 8573void SelectionDAGBuilder::visitVPLoadFF(8574 const VPIntrinsic &VPIntrin, EVT VT, EVT EVLVT,8575 const SmallVectorImpl<SDValue> &OpValues) {8576 assert(OpValues.size() == 3 && "Unexpected number of operands");8577 SDLoc DL = getCurSDLoc();8578 Value *PtrOperand = VPIntrin.getArgOperand(0);8579 MaybeAlign Alignment = VPIntrin.getPointerAlignment();8580 AAMDNodes AAInfo = VPIntrin.getAAMetadata();8581 const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);8582 SDValue LD;8583 // Do not serialize variable-length loads of constant memory with8584 // anything.8585 if (!Alignment)8586 Alignment = DAG.getEVTAlign(VT);8587 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);8588 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);8589 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();8590 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(8591 MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,8592 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);8593 LD = DAG.getLoadFFVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],8594 MMO);8595 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, EVLVT, LD.getValue(1));8596 if (AddToChain)8597 PendingLoads.push_back(LD.getValue(2));8598 setValue(&VPIntrin, DAG.getMergeValues({LD.getValue(0), Trunc}, DL));8599}8600 8601void SelectionDAGBuilder::visitVPGather(8602 const VPIntrinsic &VPIntrin, EVT VT,8603 const SmallVectorImpl<SDValue> &OpValues) {8604 SDLoc DL = getCurSDLoc();8605 const TargetLowering &TLI = DAG.getTargetLoweringInfo();8606 Value *PtrOperand = VPIntrin.getArgOperand(0);8607 MaybeAlign Alignment = VPIntrin.getPointerAlignment();8608 AAMDNodes AAInfo = VPIntrin.getAAMetadata();8609 const MDNode *Ranges = getRangeMetadata(VPIntrin);8610 SDValue LD;8611 if (!Alignment)8612 Alignment = DAG.getEVTAlign(VT.getScalarType());8613 unsigned AS =8614 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();8615 MachineMemOperand::Flags MMOFlags =8616 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);8617 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(8618 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),8619 *Alignment, AAInfo, Ranges);8620 SDValue Base, Index, Scale;8621 bool UniformBase =8622 getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),8623 VT.getScalarStoreSize());8624 if (!UniformBase) {8625 Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));8626 Index = getValue(PtrOperand);8627 Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));8628 }8629 EVT IdxVT = Index.getValueType();8630 EVT EltTy = IdxVT.getVectorElementType();8631 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {8632 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);8633 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);8634 }8635 LD = DAG.getGatherVP(8636 DAG.getVTList(VT, MVT::Other), VT, DL,8637 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,8638 ISD::SIGNED_SCALED);8639 PendingLoads.push_back(LD.getValue(1));8640 setValue(&VPIntrin, LD);8641}8642 8643void SelectionDAGBuilder::visitVPStore(8644 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {8645 SDLoc DL = getCurSDLoc();8646 Value *PtrOperand = VPIntrin.getArgOperand(1);8647 EVT VT = OpValues[0].getValueType();8648 MaybeAlign Alignment = VPIntrin.getPointerAlignment();8649 AAMDNodes AAInfo = VPIntrin.getAAMetadata();8650 SDValue ST;8651 if (!Alignment)8652 Alignment = DAG.getEVTAlign(VT);8653 SDValue Ptr = OpValues[1];8654 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());8655 const TargetLowering &TLI = DAG.getTargetLoweringInfo();8656 MachineMemOperand::Flags MMOFlags =8657 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);8658 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(8659 MachinePointerInfo(PtrOperand), MMOFlags,8660 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);8661 ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,8662 OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,8663 /* IsTruncating */ false, /*IsCompressing*/ false);8664 DAG.setRoot(ST);8665 setValue(&VPIntrin, ST);8666}8667 8668void SelectionDAGBuilder::visitVPScatter(8669 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {8670 SDLoc DL = getCurSDLoc();8671 const TargetLowering &TLI = DAG.getTargetLoweringInfo();8672 Value *PtrOperand = VPIntrin.getArgOperand(1);8673 EVT VT = OpValues[0].getValueType();8674 MaybeAlign Alignment = VPIntrin.getPointerAlignment();8675 AAMDNodes AAInfo = VPIntrin.getAAMetadata();8676 SDValue ST;8677 if (!Alignment)8678 Alignment = DAG.getEVTAlign(VT.getScalarType());8679 unsigned AS =8680 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();8681 MachineMemOperand::Flags MMOFlags =8682 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);8683 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(8684 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),8685 *Alignment, AAInfo);8686 SDValue Base, Index, Scale;8687 bool UniformBase =8688 getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),8689 VT.getScalarStoreSize());8690 if (!UniformBase) {8691 Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));8692 Index = getValue(PtrOperand);8693 Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));8694 }8695 EVT IdxVT = Index.getValueType();8696 EVT EltTy = IdxVT.getVectorElementType();8697 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {8698 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);8699 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);8700 }8701 ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,8702 {getMemoryRoot(), OpValues[0], Base, Index, Scale,8703 OpValues[2], OpValues[3]},8704 MMO, ISD::SIGNED_SCALED);8705 DAG.setRoot(ST);8706 setValue(&VPIntrin, ST);8707}8708 8709void SelectionDAGBuilder::visitVPStridedLoad(8710 const VPIntrinsic &VPIntrin, EVT VT,8711 const SmallVectorImpl<SDValue> &OpValues) {8712 SDLoc DL = getCurSDLoc();8713 Value *PtrOperand = VPIntrin.getArgOperand(0);8714 MaybeAlign Alignment = VPIntrin.getPointerAlignment();8715 if (!Alignment)8716 Alignment = DAG.getEVTAlign(VT.getScalarType());8717 AAMDNodes AAInfo = VPIntrin.getAAMetadata();8718 const MDNode *Ranges = getRangeMetadata(VPIntrin);8719 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);8720 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);8721 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();8722 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();8723 const TargetLowering &TLI = DAG.getTargetLoweringInfo();8724 MachineMemOperand::Flags MMOFlags =8725 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);8726 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(8727 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),8728 *Alignment, AAInfo, Ranges);8729 8730 SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],8731 OpValues[2], OpValues[3], MMO,8732 false /*IsExpanding*/);8733 8734 if (AddToChain)8735 PendingLoads.push_back(LD.getValue(1));8736 setValue(&VPIntrin, LD);8737}8738 8739void SelectionDAGBuilder::visitVPStridedStore(8740 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {8741 SDLoc DL = getCurSDLoc();8742 Value *PtrOperand = VPIntrin.getArgOperand(1);8743 EVT VT = OpValues[0].getValueType();8744 MaybeAlign Alignment = VPIntrin.getPointerAlignment();8745 if (!Alignment)8746 Alignment = DAG.getEVTAlign(VT.getScalarType());8747 AAMDNodes AAInfo = VPIntrin.getAAMetadata();8748 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();8749 const TargetLowering &TLI = DAG.getTargetLoweringInfo();8750 MachineMemOperand::Flags MMOFlags =8751 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);8752 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(8753 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),8754 *Alignment, AAInfo);8755 8756 SDValue ST = DAG.getStridedStoreVP(8757 getMemoryRoot(), DL, OpValues[0], OpValues[1],8758 DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3],8759 OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,8760 /*IsCompressing*/ false);8761 8762 DAG.setRoot(ST);8763 setValue(&VPIntrin, ST);8764}8765 8766void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) {8767 const TargetLowering &TLI = DAG.getTargetLoweringInfo();8768 SDLoc DL = getCurSDLoc();8769 8770 ISD::CondCode Condition;8771 CmpInst::Predicate CondCode = VPIntrin.getPredicate();8772 bool IsFP = VPIntrin.getOperand(0)->getType()->isFPOrFPVectorTy();8773 if (IsFP) {8774 // FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan)8775 // flags, but calls that don't return floating-point types can't be8776 // FPMathOperators, like vp.fcmp. This affects constrained fcmp too.8777 Condition = getFCmpCondCode(CondCode);8778 if (TM.Options.NoNaNsFPMath)8779 Condition = getFCmpCodeWithoutNaN(Condition);8780 } else {8781 Condition = getICmpCondCode(CondCode);8782 }8783 8784 SDValue Op1 = getValue(VPIntrin.getOperand(0));8785 SDValue Op2 = getValue(VPIntrin.getOperand(1));8786 // #2 is the condition code8787 SDValue MaskOp = getValue(VPIntrin.getOperand(3));8788 SDValue EVL = getValue(VPIntrin.getOperand(4));8789 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();8790 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&8791 "Unexpected target EVL type");8792 EVL = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, EVL);8793 8794 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),8795 VPIntrin.getType());8796 setValue(&VPIntrin,8797 DAG.getSetCCVP(DL, DestVT, Op1, Op2, Condition, MaskOp, EVL));8798}8799 8800void SelectionDAGBuilder::visitVectorPredicationIntrinsic(8801 const VPIntrinsic &VPIntrin) {8802 SDLoc DL = getCurSDLoc();8803 unsigned Opcode = getISDForVPIntrinsic(VPIntrin);8804 8805 auto IID = VPIntrin.getIntrinsicID();8806 8807 if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(&VPIntrin))8808 return visitVPCmp(*CmpI);8809 8810 SmallVector<EVT, 4> ValueVTs;8811 const TargetLowering &TLI = DAG.getTargetLoweringInfo();8812 ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);8813 SDVTList VTs = DAG.getVTList(ValueVTs);8814 8815 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID);8816 8817 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();8818 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&8819 "Unexpected target EVL type");8820 8821 // Request operands.8822 SmallVector<SDValue, 7> OpValues;8823 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {8824 auto Op = getValue(VPIntrin.getArgOperand(I));8825 if (I == EVLParamPos)8826 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);8827 OpValues.push_back(Op);8828 }8829 8830 switch (Opcode) {8831 default: {8832 SDNodeFlags SDFlags;8833 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))8834 SDFlags.copyFMF(*FPMO);8835 SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags);8836 setValue(&VPIntrin, Result);8837 break;8838 }8839 case ISD::VP_LOAD:8840 visitVPLoad(VPIntrin, ValueVTs[0], OpValues);8841 break;8842 case ISD::VP_LOAD_FF:8843 visitVPLoadFF(VPIntrin, ValueVTs[0], ValueVTs[1], OpValues);8844 break;8845 case ISD::VP_GATHER:8846 visitVPGather(VPIntrin, ValueVTs[0], OpValues);8847 break;8848 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:8849 visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);8850 break;8851 case ISD::VP_STORE:8852 visitVPStore(VPIntrin, OpValues);8853 break;8854 case ISD::VP_SCATTER:8855 visitVPScatter(VPIntrin, OpValues);8856 break;8857 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:8858 visitVPStridedStore(VPIntrin, OpValues);8859 break;8860 case ISD::VP_FMULADD: {8861 assert(OpValues.size() == 5 && "Unexpected number of operands");8862 SDNodeFlags SDFlags;8863 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))8864 SDFlags.copyFMF(*FPMO);8865 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&8866 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), ValueVTs[0])) {8867 setValue(&VPIntrin, DAG.getNode(ISD::VP_FMA, DL, VTs, OpValues, SDFlags));8868 } else {8869 SDValue Mul = DAG.getNode(8870 ISD::VP_FMUL, DL, VTs,8871 {OpValues[0], OpValues[1], OpValues[3], OpValues[4]}, SDFlags);8872 SDValue Add =8873 DAG.getNode(ISD::VP_FADD, DL, VTs,8874 {Mul, OpValues[2], OpValues[3], OpValues[4]}, SDFlags);8875 setValue(&VPIntrin, Add);8876 }8877 break;8878 }8879 case ISD::VP_IS_FPCLASS: {8880 const DataLayout DLayout = DAG.getDataLayout();8881 EVT DestVT = TLI.getValueType(DLayout, VPIntrin.getType());8882 auto Constant = OpValues[1]->getAsZExtVal();8883 SDValue Check = DAG.getTargetConstant(Constant, DL, MVT::i32);8884 SDValue V = DAG.getNode(ISD::VP_IS_FPCLASS, DL, DestVT,8885 {OpValues[0], Check, OpValues[2], OpValues[3]});8886 setValue(&VPIntrin, V);8887 return;8888 }8889 case ISD::VP_INTTOPTR: {8890 SDValue N = OpValues[0];8891 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), VPIntrin.getType());8892 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), VPIntrin.getType());8893 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],8894 OpValues[2]);8895 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],8896 OpValues[2]);8897 setValue(&VPIntrin, N);8898 break;8899 }8900 case ISD::VP_PTRTOINT: {8901 SDValue N = OpValues[0];8902 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),8903 VPIntrin.getType());8904 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(),8905 VPIntrin.getOperand(0)->getType());8906 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],8907 OpValues[2]);8908 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],8909 OpValues[2]);8910 setValue(&VPIntrin, N);8911 break;8912 }8913 case ISD::VP_ABS:8914 case ISD::VP_CTLZ:8915 case ISD::VP_CTLZ_ZERO_UNDEF:8916 case ISD::VP_CTTZ:8917 case ISD::VP_CTTZ_ZERO_UNDEF:8918 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:8919 case ISD::VP_CTTZ_ELTS: {8920 SDValue Result =8921 DAG.getNode(Opcode, DL, VTs, {OpValues[0], OpValues[2], OpValues[3]});8922 setValue(&VPIntrin, Result);8923 break;8924 }8925 }8926}8927 8928SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,8929 const BasicBlock *EHPadBB,8930 MCSymbol *&BeginLabel) {8931 MachineFunction &MF = DAG.getMachineFunction();8932 8933 // Insert a label before the invoke call to mark the try range. This can be8934 // used to detect deletion of the invoke via the MachineModuleInfo.8935 BeginLabel = MF.getContext().createTempSymbol();8936 8937 // For SjLj, keep track of which landing pads go with which invokes8938 // so as to maintain the ordering of pads in the LSDA.8939 unsigned CallSiteIndex = FuncInfo.getCurrentCallSite();8940 if (CallSiteIndex) {8941 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);8942 LPadToCallSiteMap[FuncInfo.getMBB(EHPadBB)].push_back(CallSiteIndex);8943 8944 // Now that the call site is handled, stop tracking it.8945 FuncInfo.setCurrentCallSite(0);8946 }8947 8948 return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel);8949}8950 8951SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,8952 const BasicBlock *EHPadBB,8953 MCSymbol *BeginLabel) {8954 assert(BeginLabel && "BeginLabel should've been set");8955 8956 MachineFunction &MF = DAG.getMachineFunction();8957 8958 // Insert a label at the end of the invoke call to mark the try range. This8959 // can be used to detect deletion of the invoke via the MachineModuleInfo.8960 MCSymbol *EndLabel = MF.getContext().createTempSymbol();8961 Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);8962 8963 // Inform MachineModuleInfo of range.8964 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());8965 // There is a platform (e.g. wasm) that uses funclet style IR but does not8966 // actually use outlined funclets and their LSDA info style.8967 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {8968 assert(II && "II should've been set");8969 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();8970 EHInfo->addIPToStateRange(II, BeginLabel, EndLabel);8971 } else if (!isScopedEHPersonality(Pers)) {8972 assert(EHPadBB);8973 MF.addInvoke(FuncInfo.getMBB(EHPadBB), BeginLabel, EndLabel);8974 }8975 8976 return Chain;8977}8978 8979std::pair<SDValue, SDValue>8980SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,8981 const BasicBlock *EHPadBB) {8982 MCSymbol *BeginLabel = nullptr;8983 8984 if (EHPadBB) {8985 // Both PendingLoads and PendingExports must be flushed here;8986 // this call might not return.8987 (void)getRoot();8988 DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel));8989 CLI.setChain(getRoot());8990 }8991 8992 const TargetLowering &TLI = DAG.getTargetLoweringInfo();8993 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);8994 8995 assert((CLI.IsTailCall || Result.second.getNode()) &&8996 "Non-null chain expected with non-tail call!");8997 assert((Result.second.getNode() || !Result.first.getNode()) &&8998 "Null value expected with tail call!");8999 9000 if (!Result.second.getNode()) {9001 // As a special case, a null chain means that a tail call has been emitted9002 // and the DAG root is already updated.9003 HasTailCall = true;9004 9005 // Since there's no actual continuation from this block, nothing can be9006 // relying on us setting vregs for them.9007 PendingExports.clear();9008 } else {9009 DAG.setRoot(Result.second);9010 }9011 9012 if (EHPadBB) {9013 DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB,9014 BeginLabel));9015 Result.second = getRoot();9016 }9017 9018 return Result;9019}9020 9021bool SelectionDAGBuilder::canTailCall(const CallBase &CB) const {9022 bool isMustTailCall = CB.isMustTailCall();9023 9024 // Avoid emitting tail calls in functions with the disable-tail-calls9025 // attribute.9026 const Function *Caller = CB.getParent()->getParent();9027 if (!isMustTailCall &&9028 Caller->getFnAttribute("disable-tail-calls").getValueAsBool())9029 return false;9030 9031 // We can't tail call inside a function with a swifterror argument. Lowering9032 // does not support this yet. It would have to move into the swifterror9033 // register before the call.9034 if (DAG.getTargetLoweringInfo().supportSwiftError() &&9035 Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))9036 return false;9037 9038 // Check if target-independent constraints permit a tail call here.9039 // Target-dependent constraints are checked within TLI->LowerCallTo.9040 return isInTailCallPosition(CB, DAG.getTarget());9041}9042 9043void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,9044 bool isTailCall, bool isMustTailCall,9045 const BasicBlock *EHPadBB,9046 const TargetLowering::PtrAuthInfo *PAI) {9047 auto &DL = DAG.getDataLayout();9048 FunctionType *FTy = CB.getFunctionType();9049 Type *RetTy = CB.getType();9050 9051 TargetLowering::ArgListTy Args;9052 Args.reserve(CB.arg_size());9053 9054 const Value *SwiftErrorVal = nullptr;9055 const TargetLowering &TLI = DAG.getTargetLoweringInfo();9056 9057 if (isTailCall)9058 isTailCall = canTailCall(CB);9059 9060 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {9061 const Value *V = *I;9062 9063 // Skip empty types9064 if (V->getType()->isEmptyTy())9065 continue;9066 9067 SDValue ArgNode = getValue(V);9068 TargetLowering::ArgListEntry Entry(ArgNode, V->getType());9069 Entry.setAttributes(&CB, I - CB.arg_begin());9070 9071 // Use swifterror virtual register as input to the call.9072 if (Entry.IsSwiftError && TLI.supportSwiftError()) {9073 SwiftErrorVal = V;9074 // We find the virtual register for the actual swifterror argument.9075 // Instead of using the Value, we use the virtual register instead.9076 Entry.Node =9077 DAG.getRegister(SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V),9078 EVT(TLI.getPointerTy(DL)));9079 }9080 9081 Args.push_back(Entry);9082 9083 // If we have an explicit sret argument that is an Instruction, (i.e., it9084 // might point to function-local memory), we can't meaningfully tail-call.9085 if (Entry.IsSRet && isa<Instruction>(V))9086 isTailCall = false;9087 }9088 9089 // If call site has a cfguardtarget operand bundle, create and add an9090 // additional ArgListEntry.9091 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {9092 Value *V = Bundle->Inputs[0];9093 TargetLowering::ArgListEntry Entry(V, getValue(V));9094 Entry.IsCFGuardTarget = true;9095 Args.push_back(Entry);9096 }9097 9098 // Disable tail calls if there is an swifterror argument. Targets have not9099 // been updated to support tail calls.9100 if (TLI.supportSwiftError() && SwiftErrorVal)9101 isTailCall = false;9102 9103 ConstantInt *CFIType = nullptr;9104 if (CB.isIndirectCall()) {9105 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi)) {9106 if (!TLI.supportKCFIBundles())9107 report_fatal_error(9108 "Target doesn't support calls with kcfi operand bundles.");9109 CFIType = cast<ConstantInt>(Bundle->Inputs[0]);9110 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");9111 }9112 }9113 9114 SDValue ConvControlToken;9115 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {9116 auto *Token = Bundle->Inputs[0].get();9117 ConvControlToken = getValue(Token);9118 }9119 9120 GlobalValue *DeactivationSymbol = nullptr;9121 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_deactivation_symbol)) {9122 DeactivationSymbol = cast<GlobalValue>(Bundle->Inputs[0].get());9123 }9124 9125 TargetLowering::CallLoweringInfo CLI(DAG);9126 CLI.setDebugLoc(getCurSDLoc())9127 .setChain(getRoot())9128 .setCallee(RetTy, FTy, Callee, std::move(Args), CB)9129 .setTailCall(isTailCall)9130 .setConvergent(CB.isConvergent())9131 .setIsPreallocated(9132 CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0)9133 .setCFIType(CFIType)9134 .setConvergenceControlToken(ConvControlToken)9135 .setDeactivationSymbol(DeactivationSymbol);9136 9137 // Set the pointer authentication info if we have it.9138 if (PAI) {9139 if (!TLI.supportPtrAuthBundles())9140 report_fatal_error(9141 "This target doesn't support calls with ptrauth operand bundles.");9142 CLI.setPtrAuth(*PAI);9143 }9144 9145 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);9146 9147 if (Result.first.getNode()) {9148 Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);9149 Result.first = lowerNoFPClassToAssertNoFPClass(DAG, CB, Result.first);9150 setValue(&CB, Result.first);9151 }9152 9153 // The last element of CLI.InVals has the SDValue for swifterror return.9154 // Here we copy it to a virtual register and update SwiftErrorMap for9155 // book-keeping.9156 if (SwiftErrorVal && TLI.supportSwiftError()) {9157 // Get the last element of InVals.9158 SDValue Src = CLI.InVals.back();9159 Register VReg =9160 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);9161 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);9162 DAG.setRoot(CopyNode);9163 }9164}9165 9166static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,9167 SelectionDAGBuilder &Builder) {9168 // Check to see if this load can be trivially constant folded, e.g. if the9169 // input is from a string literal.9170 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {9171 // Cast pointer to the type we really want to load.9172 Type *LoadTy =9173 Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());9174 if (LoadVT.isVector())9175 LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements());9176 if (const Constant *LoadCst =9177 ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),9178 LoadTy, Builder.DAG.getDataLayout()))9179 return Builder.getValue(LoadCst);9180 }9181 9182 // Otherwise, we have to emit the load. If the pointer is to unfoldable but9183 // still constant memory, the input chain can be the entry node.9184 SDValue Root;9185 bool ConstantMemory = false;9186 9187 // Do not serialize (non-volatile) loads of constant memory with anything.9188 if (Builder.BatchAA && Builder.BatchAA->pointsToConstantMemory(PtrVal)) {9189 Root = Builder.DAG.getEntryNode();9190 ConstantMemory = true;9191 } else {9192 // Do not serialize non-volatile loads against each other.9193 Root = Builder.DAG.getRoot();9194 }9195 9196 SDValue Ptr = Builder.getValue(PtrVal);9197 SDValue LoadVal =9198 Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,9199 MachinePointerInfo(PtrVal), Align(1));9200 9201 if (!ConstantMemory)9202 Builder.PendingLoads.push_back(LoadVal.getValue(1));9203 return LoadVal;9204}9205 9206/// Record the value for an instruction that produces an integer result,9207/// converting the type where necessary.9208void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,9209 SDValue Value,9210 bool IsSigned) {9211 EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),9212 I.getType(), true);9213 Value = DAG.getExtOrTrunc(IsSigned, Value, getCurSDLoc(), VT);9214 setValue(&I, Value);9215}9216 9217/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return9218/// true and lower it. Otherwise return false, and it will be lowered like a9219/// normal call.9220/// The caller already checked that \p I calls the appropriate LibFunc with a9221/// correct prototype.9222bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {9223 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);9224 const Value *Size = I.getArgOperand(2);9225 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));9226 if (CSize && CSize->getZExtValue() == 0) {9227 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),9228 I.getType(), true);9229 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));9230 return true;9231 }9232 9233 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();9234 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(9235 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),9236 getValue(Size), &I);9237 if (Res.first.getNode()) {9238 processIntegerCallValue(I, Res.first, true);9239 PendingLoads.push_back(Res.second);9240 return true;9241 }9242 9243 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 09244 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 09245 if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))9246 return false;9247 9248 // If the target has a fast compare for the given size, it will return a9249 // preferred load type for that size. Require that the load VT is legal and9250 // that the target supports unaligned loads of that type. Otherwise, return9251 // INVALID.9252 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {9253 const TargetLowering &TLI = DAG.getTargetLoweringInfo();9254 MVT LVT = TLI.hasFastEqualityCompare(NumBits);9255 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {9256 // TODO: Handle 5 byte compare as 4-byte + 1 byte.9257 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.9258 // TODO: Check alignment of src and dest ptrs.9259 unsigned DstAS = LHS->getType()->getPointerAddressSpace();9260 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();9261 if (!TLI.isTypeLegal(LVT) ||9262 !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||9263 !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))9264 LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;9265 }9266 9267 return LVT;9268 };9269 9270 // This turns into unaligned loads. We only do this if the target natively9271 // supports the MVT we'll be loading or if it is small enough (<= 4) that9272 // we'll only produce a small number of byte loads.9273 MVT LoadVT;9274 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;9275 switch (NumBitsToCompare) {9276 default:9277 return false;9278 case 16:9279 LoadVT = MVT::i16;9280 break;9281 case 32:9282 LoadVT = MVT::i32;9283 break;9284 case 64:9285 case 128:9286 case 256:9287 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);9288 break;9289 }9290 9291 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)9292 return false;9293 9294 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);9295 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);9296 9297 // Bitcast to a wide integer type if the loads are vectors.9298 if (LoadVT.isVector()) {9299 EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());9300 LoadL = DAG.getBitcast(CmpVT, LoadL);9301 LoadR = DAG.getBitcast(CmpVT, LoadR);9302 }9303 9304 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);9305 processIntegerCallValue(I, Cmp, false);9306 return true;9307}9308 9309/// See if we can lower a memchr call into an optimized form. If so, return9310/// true and lower it. Otherwise return false, and it will be lowered like a9311/// normal call.9312/// The caller already checked that \p I calls the appropriate LibFunc with a9313/// correct prototype.9314bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {9315 const Value *Src = I.getArgOperand(0);9316 const Value *Char = I.getArgOperand(1);9317 const Value *Length = I.getArgOperand(2);9318 9319 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();9320 std::pair<SDValue, SDValue> Res =9321 TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),9322 getValue(Src), getValue(Char), getValue(Length),9323 MachinePointerInfo(Src));9324 if (Res.first.getNode()) {9325 setValue(&I, Res.first);9326 PendingLoads.push_back(Res.second);9327 return true;9328 }9329 9330 return false;9331}9332 9333/// See if we can lower a mempcpy call into an optimized form. If so, return9334/// true and lower it. Otherwise return false, and it will be lowered like a9335/// normal call.9336/// The caller already checked that \p I calls the appropriate LibFunc with a9337/// correct prototype.9338bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {9339 SDValue Dst = getValue(I.getArgOperand(0));9340 SDValue Src = getValue(I.getArgOperand(1));9341 SDValue Size = getValue(I.getArgOperand(2));9342 9343 Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();9344 Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();9345 // DAG::getMemcpy needs Alignment to be defined.9346 Align Alignment = std::min(DstAlign, SrcAlign);9347 9348 SDLoc sdl = getCurSDLoc();9349 9350 // In the mempcpy context we need to pass in a false value for isTailCall9351 // because the return pointer needs to be adjusted by the size of9352 // the copied memory.9353 SDValue Root = getMemoryRoot();9354 SDValue MC = DAG.getMemcpy(9355 Root, sdl, Dst, Src, Size, Alignment, false, false, /*CI=*/nullptr,9356 std::nullopt, MachinePointerInfo(I.getArgOperand(0)),9357 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata());9358 assert(MC.getNode() != nullptr &&9359 "** memcpy should not be lowered as TailCall in mempcpy context **");9360 DAG.setRoot(MC);9361 9362 // Check if Size needs to be truncated or extended.9363 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());9364 9365 // Adjust return pointer to point just past the last dst byte.9366 SDValue DstPlusSize = DAG.getMemBasePlusOffset(Dst, Size, sdl);9367 setValue(&I, DstPlusSize);9368 return true;9369}9370 9371/// See if we can lower a strcpy call into an optimized form. If so, return9372/// true and lower it, otherwise return false and it will be lowered like a9373/// normal call.9374/// The caller already checked that \p I calls the appropriate LibFunc with a9375/// correct prototype.9376bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {9377 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);9378 9379 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();9380 std::pair<SDValue, SDValue> Res =9381 TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),9382 getValue(Arg0), getValue(Arg1),9383 MachinePointerInfo(Arg0),9384 MachinePointerInfo(Arg1), isStpcpy);9385 if (Res.first.getNode()) {9386 setValue(&I, Res.first);9387 DAG.setRoot(Res.second);9388 return true;9389 }9390 9391 return false;9392}9393 9394/// See if we can lower a strcmp call into an optimized form. If so, return9395/// true and lower it, otherwise return false and it will be lowered like a9396/// normal call.9397/// The caller already checked that \p I calls the appropriate LibFunc with a9398/// correct prototype.9399bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {9400 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);9401 9402 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();9403 std::pair<SDValue, SDValue> Res =9404 TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),9405 getValue(Arg0), getValue(Arg1),9406 MachinePointerInfo(Arg0),9407 MachinePointerInfo(Arg1));9408 if (Res.first.getNode()) {9409 processIntegerCallValue(I, Res.first, true);9410 PendingLoads.push_back(Res.second);9411 return true;9412 }9413 9414 return false;9415}9416 9417/// See if we can lower a strlen call into an optimized form. If so, return9418/// true and lower it, otherwise return false and it will be lowered like a9419/// normal call.9420/// The caller already checked that \p I calls the appropriate LibFunc with a9421/// correct prototype.9422bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {9423 const Value *Arg0 = I.getArgOperand(0);9424 9425 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();9426 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrlen(9427 DAG, getCurSDLoc(), DAG.getRoot(), getValue(Arg0), &I);9428 if (Res.first.getNode()) {9429 processIntegerCallValue(I, Res.first, false);9430 PendingLoads.push_back(Res.second);9431 return true;9432 }9433 9434 return false;9435}9436 9437/// See if we can lower a strnlen call into an optimized form. If so, return9438/// true and lower it, otherwise return false and it will be lowered like a9439/// normal call.9440/// The caller already checked that \p I calls the appropriate LibFunc with a9441/// correct prototype.9442bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {9443 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);9444 9445 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();9446 std::pair<SDValue, SDValue> Res =9447 TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),9448 getValue(Arg0), getValue(Arg1),9449 MachinePointerInfo(Arg0));9450 if (Res.first.getNode()) {9451 processIntegerCallValue(I, Res.first, false);9452 PendingLoads.push_back(Res.second);9453 return true;9454 }9455 9456 return false;9457}9458 9459/// See if we can lower a unary floating-point operation into an SDNode with9460/// the specified Opcode. If so, return true and lower it, otherwise return9461/// false and it will be lowered like a normal call.9462/// The caller already checked that \p I calls the appropriate LibFunc with a9463/// correct prototype.9464bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,9465 unsigned Opcode) {9466 // We already checked this call's prototype; verify it doesn't modify errno.9467 // Do not perform optimizations for call sites that require strict9468 // floating-point semantics.9469 if (!I.onlyReadsMemory() || I.isStrictFP())9470 return false;9471 9472 SDNodeFlags Flags;9473 Flags.copyFMF(cast<FPMathOperator>(I));9474 9475 SDValue Tmp = getValue(I.getArgOperand(0));9476 setValue(&I,9477 DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));9478 return true;9479}9480 9481/// See if we can lower a binary floating-point operation into an SDNode with9482/// the specified Opcode. If so, return true and lower it. Otherwise return9483/// false, and it will be lowered like a normal call.9484/// The caller already checked that \p I calls the appropriate LibFunc with a9485/// correct prototype.9486bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,9487 unsigned Opcode) {9488 // We already checked this call's prototype; verify it doesn't modify errno.9489 // Do not perform optimizations for call sites that require strict9490 // floating-point semantics.9491 if (!I.onlyReadsMemory() || I.isStrictFP())9492 return false;9493 9494 SDNodeFlags Flags;9495 Flags.copyFMF(cast<FPMathOperator>(I));9496 9497 SDValue Tmp0 = getValue(I.getArgOperand(0));9498 SDValue Tmp1 = getValue(I.getArgOperand(1));9499 EVT VT = Tmp0.getValueType();9500 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));9501 return true;9502}9503 9504void SelectionDAGBuilder::visitCall(const CallInst &I) {9505 // Handle inline assembly differently.9506 if (I.isInlineAsm()) {9507 visitInlineAsm(I);9508 return;9509 }9510 9511 diagnoseDontCall(I);9512 9513 if (Function *F = I.getCalledFunction()) {9514 if (F->isDeclaration()) {9515 // Is this an LLVM intrinsic?9516 if (unsigned IID = F->getIntrinsicID()) {9517 visitIntrinsicCall(I, IID);9518 return;9519 }9520 }9521 9522 // Check for well-known libc/libm calls. If the function is internal, it9523 // can't be a library call. Don't do the check if marked as nobuiltin for9524 // some reason.9525 LibFunc Func;9526 if (!I.isNoBuiltin() && !F->hasLocalLinkage() && F->hasName() &&9527 LibInfo->getLibFunc(*F, Func) && LibInfo->hasOptimizedCodeGen(Func)) {9528 switch (Func) {9529 default: break;9530 case LibFunc_bcmp:9531 if (visitMemCmpBCmpCall(I))9532 return;9533 break;9534 case LibFunc_copysign:9535 case LibFunc_copysignf:9536 case LibFunc_copysignl:9537 // We already checked this call's prototype; verify it doesn't modify9538 // errno.9539 if (I.onlyReadsMemory()) {9540 SDValue LHS = getValue(I.getArgOperand(0));9541 SDValue RHS = getValue(I.getArgOperand(1));9542 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),9543 LHS.getValueType(), LHS, RHS));9544 return;9545 }9546 break;9547 case LibFunc_fabs:9548 case LibFunc_fabsf:9549 case LibFunc_fabsl:9550 if (visitUnaryFloatCall(I, ISD::FABS))9551 return;9552 break;9553 case LibFunc_fmin:9554 case LibFunc_fminf:9555 case LibFunc_fminl:9556 if (visitBinaryFloatCall(I, ISD::FMINNUM))9557 return;9558 break;9559 case LibFunc_fmax:9560 case LibFunc_fmaxf:9561 case LibFunc_fmaxl:9562 if (visitBinaryFloatCall(I, ISD::FMAXNUM))9563 return;9564 break;9565 case LibFunc_fminimum_num:9566 case LibFunc_fminimum_numf:9567 case LibFunc_fminimum_numl:9568 if (visitBinaryFloatCall(I, ISD::FMINIMUMNUM))9569 return;9570 break;9571 case LibFunc_fmaximum_num:9572 case LibFunc_fmaximum_numf:9573 case LibFunc_fmaximum_numl:9574 if (visitBinaryFloatCall(I, ISD::FMAXIMUMNUM))9575 return;9576 break;9577 case LibFunc_sin:9578 case LibFunc_sinf:9579 case LibFunc_sinl:9580 if (visitUnaryFloatCall(I, ISD::FSIN))9581 return;9582 break;9583 case LibFunc_cos:9584 case LibFunc_cosf:9585 case LibFunc_cosl:9586 if (visitUnaryFloatCall(I, ISD::FCOS))9587 return;9588 break;9589 case LibFunc_tan:9590 case LibFunc_tanf:9591 case LibFunc_tanl:9592 if (visitUnaryFloatCall(I, ISD::FTAN))9593 return;9594 break;9595 case LibFunc_asin:9596 case LibFunc_asinf:9597 case LibFunc_asinl:9598 if (visitUnaryFloatCall(I, ISD::FASIN))9599 return;9600 break;9601 case LibFunc_acos:9602 case LibFunc_acosf:9603 case LibFunc_acosl:9604 if (visitUnaryFloatCall(I, ISD::FACOS))9605 return;9606 break;9607 case LibFunc_atan:9608 case LibFunc_atanf:9609 case LibFunc_atanl:9610 if (visitUnaryFloatCall(I, ISD::FATAN))9611 return;9612 break;9613 case LibFunc_atan2:9614 case LibFunc_atan2f:9615 case LibFunc_atan2l:9616 if (visitBinaryFloatCall(I, ISD::FATAN2))9617 return;9618 break;9619 case LibFunc_sinh:9620 case LibFunc_sinhf:9621 case LibFunc_sinhl:9622 if (visitUnaryFloatCall(I, ISD::FSINH))9623 return;9624 break;9625 case LibFunc_cosh:9626 case LibFunc_coshf:9627 case LibFunc_coshl:9628 if (visitUnaryFloatCall(I, ISD::FCOSH))9629 return;9630 break;9631 case LibFunc_tanh:9632 case LibFunc_tanhf:9633 case LibFunc_tanhl:9634 if (visitUnaryFloatCall(I, ISD::FTANH))9635 return;9636 break;9637 case LibFunc_sqrt:9638 case LibFunc_sqrtf:9639 case LibFunc_sqrtl:9640 case LibFunc_sqrt_finite:9641 case LibFunc_sqrtf_finite:9642 case LibFunc_sqrtl_finite:9643 if (visitUnaryFloatCall(I, ISD::FSQRT))9644 return;9645 break;9646 case LibFunc_floor:9647 case LibFunc_floorf:9648 case LibFunc_floorl:9649 if (visitUnaryFloatCall(I, ISD::FFLOOR))9650 return;9651 break;9652 case LibFunc_nearbyint:9653 case LibFunc_nearbyintf:9654 case LibFunc_nearbyintl:9655 if (visitUnaryFloatCall(I, ISD::FNEARBYINT))9656 return;9657 break;9658 case LibFunc_ceil:9659 case LibFunc_ceilf:9660 case LibFunc_ceill:9661 if (visitUnaryFloatCall(I, ISD::FCEIL))9662 return;9663 break;9664 case LibFunc_rint:9665 case LibFunc_rintf:9666 case LibFunc_rintl:9667 if (visitUnaryFloatCall(I, ISD::FRINT))9668 return;9669 break;9670 case LibFunc_round:9671 case LibFunc_roundf:9672 case LibFunc_roundl:9673 if (visitUnaryFloatCall(I, ISD::FROUND))9674 return;9675 break;9676 case LibFunc_trunc:9677 case LibFunc_truncf:9678 case LibFunc_truncl:9679 if (visitUnaryFloatCall(I, ISD::FTRUNC))9680 return;9681 break;9682 case LibFunc_log2:9683 case LibFunc_log2f:9684 case LibFunc_log2l:9685 if (visitUnaryFloatCall(I, ISD::FLOG2))9686 return;9687 break;9688 case LibFunc_exp2:9689 case LibFunc_exp2f:9690 case LibFunc_exp2l:9691 if (visitUnaryFloatCall(I, ISD::FEXP2))9692 return;9693 break;9694 case LibFunc_exp10:9695 case LibFunc_exp10f:9696 case LibFunc_exp10l:9697 if (visitUnaryFloatCall(I, ISD::FEXP10))9698 return;9699 break;9700 case LibFunc_ldexp:9701 case LibFunc_ldexpf:9702 case LibFunc_ldexpl:9703 if (visitBinaryFloatCall(I, ISD::FLDEXP))9704 return;9705 break;9706 case LibFunc_memcmp:9707 if (visitMemCmpBCmpCall(I))9708 return;9709 break;9710 case LibFunc_mempcpy:9711 if (visitMemPCpyCall(I))9712 return;9713 break;9714 case LibFunc_memchr:9715 if (visitMemChrCall(I))9716 return;9717 break;9718 case LibFunc_strcpy:9719 if (visitStrCpyCall(I, false))9720 return;9721 break;9722 case LibFunc_stpcpy:9723 if (visitStrCpyCall(I, true))9724 return;9725 break;9726 case LibFunc_strcmp:9727 if (visitStrCmpCall(I))9728 return;9729 break;9730 case LibFunc_strlen:9731 if (visitStrLenCall(I))9732 return;9733 break;9734 case LibFunc_strnlen:9735 if (visitStrNLenCall(I))9736 return;9737 break;9738 }9739 }9740 }9741 9742 if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {9743 LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), /*EHPadBB=*/nullptr);9744 return;9745 }9746 9747 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't9748 // have to do anything here to lower funclet bundles.9749 // CFGuardTarget bundles are lowered in LowerCallTo.9750 failForInvalidBundles(9751 I, "calls",9752 {LLVMContext::OB_deopt, LLVMContext::OB_funclet,9753 LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,9754 LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,9755 LLVMContext::OB_convergencectrl, LLVMContext::OB_deactivation_symbol});9756 9757 SDValue Callee = getValue(I.getCalledOperand());9758 9759 if (I.hasDeoptState())9760 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);9761 else9762 // Check if we can potentially perform a tail call. More detailed checking9763 // is be done within LowerCallTo, after more information about the call is9764 // known.9765 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());9766}9767 9768void SelectionDAGBuilder::LowerCallSiteWithPtrAuthBundle(9769 const CallBase &CB, const BasicBlock *EHPadBB) {9770 auto PAB = CB.getOperandBundle("ptrauth");9771 const Value *CalleeV = CB.getCalledOperand();9772 9773 // Gather the call ptrauth data from the operand bundle:9774 // [ i32 <key>, i64 <discriminator> ]9775 const auto *Key = cast<ConstantInt>(PAB->Inputs[0]);9776 const Value *Discriminator = PAB->Inputs[1];9777 9778 assert(Key->getType()->isIntegerTy(32) && "Invalid ptrauth key");9779 assert(Discriminator->getType()->isIntegerTy(64) &&9780 "Invalid ptrauth discriminator");9781 9782 // Look through ptrauth constants to find the raw callee.9783 // Do a direct unauthenticated call if we found it and everything matches.9784 if (const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(CalleeV))9785 if (CalleeCPA->isKnownCompatibleWith(Key, Discriminator,9786 DAG.getDataLayout()))9787 return LowerCallTo(CB, getValue(CalleeCPA->getPointer()), CB.isTailCall(),9788 CB.isMustTailCall(), EHPadBB);9789 9790 // Functions should never be ptrauth-called directly.9791 assert(!isa<Function>(CalleeV) && "invalid direct ptrauth call");9792 9793 // Otherwise, do an authenticated indirect call.9794 TargetLowering::PtrAuthInfo PAI = {Key->getZExtValue(),9795 getValue(Discriminator)};9796 9797 LowerCallTo(CB, getValue(CalleeV), CB.isTailCall(), CB.isMustTailCall(),9798 EHPadBB, &PAI);9799}9800 9801namespace {9802 9803/// AsmOperandInfo - This contains information for each constraint that we are9804/// lowering.9805class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {9806public:9807 /// CallOperand - If this is the result output operand or a clobber9808 /// this is null, otherwise it is the incoming operand to the CallInst.9809 /// This gets modified as the asm is processed.9810 SDValue CallOperand;9811 9812 /// AssignedRegs - If this is a register or register class operand, this9813 /// contains the set of register corresponding to the operand.9814 RegsForValue AssignedRegs;9815 9816 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)9817 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {9818 }9819 9820 /// Whether or not this operand accesses memory9821 bool hasMemory(const TargetLowering &TLI) const {9822 // Indirect operand accesses access memory.9823 if (isIndirect)9824 return true;9825 9826 for (const auto &Code : Codes)9827 if (TLI.getConstraintType(Code) == TargetLowering::C_Memory)9828 return true;9829 9830 return false;9831 }9832};9833 9834 9835} // end anonymous namespace9836 9837/// Make sure that the output operand \p OpInfo and its corresponding input9838/// operand \p MatchingOpInfo have compatible constraint types (otherwise error9839/// out).9840static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,9841 SDISelAsmOperandInfo &MatchingOpInfo,9842 SelectionDAG &DAG) {9843 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)9844 return;9845 9846 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();9847 const auto &TLI = DAG.getTargetLoweringInfo();9848 9849 std::pair<unsigned, const TargetRegisterClass *> MatchRC =9850 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,9851 OpInfo.ConstraintVT);9852 std::pair<unsigned, const TargetRegisterClass *> InputRC =9853 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,9854 MatchingOpInfo.ConstraintVT);9855 const bool OutOpIsIntOrFP =9856 OpInfo.ConstraintVT.isInteger() || OpInfo.ConstraintVT.isFloatingPoint();9857 const bool InOpIsIntOrFP = MatchingOpInfo.ConstraintVT.isInteger() ||9858 MatchingOpInfo.ConstraintVT.isFloatingPoint();9859 if ((OutOpIsIntOrFP != InOpIsIntOrFP) || (MatchRC.second != InputRC.second)) {9860 // FIXME: error out in a more elegant fashion9861 report_fatal_error("Unsupported asm: input constraint"9862 " with a matching output constraint of"9863 " incompatible type!");9864 }9865 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;9866}9867 9868/// Get a direct memory input to behave well as an indirect operand.9869/// This may introduce stores, hence the need for a \p Chain.9870/// \return The (possibly updated) chain.9871static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,9872 SDISelAsmOperandInfo &OpInfo,9873 SelectionDAG &DAG) {9874 const TargetLowering &TLI = DAG.getTargetLoweringInfo();9875 9876 // If we don't have an indirect input, put it in the constpool if we can,9877 // otherwise spill it to a stack slot.9878 // TODO: This isn't quite right. We need to handle these according to9879 // the addressing mode that the constraint wants. Also, this may take9880 // an additional register for the computation and we don't want that9881 // either.9882 9883 // If the operand is a float, integer, or vector constant, spill to a9884 // constant pool entry to get its address.9885 const Value *OpVal = OpInfo.CallOperandVal;9886 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||9887 isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {9888 OpInfo.CallOperand = DAG.getConstantPool(9889 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));9890 return Chain;9891 }9892 9893 // Otherwise, create a stack slot and emit a store to it before the asm.9894 Type *Ty = OpVal->getType();9895 auto &DL = DAG.getDataLayout();9896 TypeSize TySize = DL.getTypeAllocSize(Ty);9897 MachineFunction &MF = DAG.getMachineFunction();9898 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();9899 int StackID = 0;9900 if (TySize.isScalable())9901 StackID = TFI->getStackIDForScalableVectors();9902 int SSFI = MF.getFrameInfo().CreateStackObject(TySize.getKnownMinValue(),9903 DL.getPrefTypeAlign(Ty), false,9904 nullptr, StackID);9905 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));9906 Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,9907 MachinePointerInfo::getFixedStack(MF, SSFI),9908 TLI.getMemValueType(DL, Ty));9909 OpInfo.CallOperand = StackSlot;9910 9911 return Chain;9912}9913 9914/// GetRegistersForValue - Assign registers (virtual or physical) for the9915/// specified operand. We prefer to assign virtual registers, to allow the9916/// register allocator to handle the assignment process. However, if the asm9917/// uses features that we can't model on machineinstrs, we have SDISel do the9918/// allocation. This produces generally horrible, but correct, code.9919///9920/// OpInfo describes the operand9921/// RefOpInfo describes the matching operand if any, the operand otherwise9922static std::optional<unsigned>9923getRegistersForValue(SelectionDAG &DAG, const SDLoc &DL,9924 SDISelAsmOperandInfo &OpInfo,9925 SDISelAsmOperandInfo &RefOpInfo) {9926 LLVMContext &Context = *DAG.getContext();9927 const TargetLowering &TLI = DAG.getTargetLoweringInfo();9928 9929 MachineFunction &MF = DAG.getMachineFunction();9930 SmallVector<Register, 4> Regs;9931 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();9932 9933 // No work to do for memory/address operands.9934 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||9935 OpInfo.ConstraintType == TargetLowering::C_Address)9936 return std::nullopt;9937 9938 // If this is a constraint for a single physreg, or a constraint for a9939 // register class, find it.9940 unsigned AssignedReg;9941 const TargetRegisterClass *RC;9942 std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(9943 &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);9944 // RC is unset only on failure. Return immediately.9945 if (!RC)9946 return std::nullopt;9947 9948 // Get the actual register value type. This is important, because the user9949 // may have asked for (e.g.) the AX register in i32 type. We need to9950 // remember that AX is actually i16 to get the right extension.9951 const MVT RegVT = *TRI.legalclasstypes_begin(*RC);9952 9953 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {9954 // If this is an FP operand in an integer register (or visa versa), or more9955 // generally if the operand value disagrees with the register class we plan9956 // to stick it in, fix the operand type.9957 //9958 // If this is an input value, the bitcast to the new type is done now.9959 // Bitcast for output value is done at the end of visitInlineAsm().9960 if ((OpInfo.Type == InlineAsm::isOutput ||9961 OpInfo.Type == InlineAsm::isInput) &&9962 !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {9963 // Try to convert to the first EVT that the reg class contains. If the9964 // types are identical size, use a bitcast to convert (e.g. two differing9965 // vector types). Note: output bitcast is done at the end of9966 // visitInlineAsm().9967 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {9968 // Exclude indirect inputs while they are unsupported because the code9969 // to perform the load is missing and thus OpInfo.CallOperand still9970 // refers to the input address rather than the pointed-to value.9971 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)9972 OpInfo.CallOperand =9973 DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);9974 OpInfo.ConstraintVT = RegVT;9975 // If the operand is an FP value and we want it in integer registers,9976 // use the corresponding integer type. This turns an f64 value into9977 // i64, which can be passed with two i32 values on a 32-bit machine.9978 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {9979 MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());9980 if (OpInfo.Type == InlineAsm::isInput)9981 OpInfo.CallOperand =9982 DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);9983 OpInfo.ConstraintVT = VT;9984 }9985 }9986 }9987 9988 // No need to allocate a matching input constraint since the constraint it's9989 // matching to has already been allocated.9990 if (OpInfo.isMatchingInputConstraint())9991 return std::nullopt;9992 9993 EVT ValueVT = OpInfo.ConstraintVT;9994 if (OpInfo.ConstraintVT == MVT::Other)9995 ValueVT = RegVT;9996 9997 // Initialize NumRegs.9998 unsigned NumRegs = 1;9999 if (OpInfo.ConstraintVT != MVT::Other)10000 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);10001 10002 // If this is a constraint for a specific physical register, like {r17},10003 // assign it now.10004 10005 // If this associated to a specific register, initialize iterator to correct10006 // place. If virtual, make sure we have enough registers10007 10008 // Initialize iterator if necessary10009 TargetRegisterClass::iterator I = RC->begin();10010 MachineRegisterInfo &RegInfo = MF.getRegInfo();10011 10012 // Do not check for single registers.10013 if (AssignedReg) {10014 I = std::find(I, RC->end(), AssignedReg);10015 if (I == RC->end()) {10016 // RC does not contain the selected register, which indicates a10017 // mismatch between the register and the required type/bitwidth.10018 return {AssignedReg};10019 }10020 }10021 10022 for (; NumRegs; --NumRegs, ++I) {10023 assert(I != RC->end() && "Ran out of registers to allocate!");10024 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC);10025 Regs.push_back(R);10026 }10027 10028 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);10029 return std::nullopt;10030}10031 10032static unsigned10033findMatchingInlineAsmOperand(unsigned OperandNo,10034 const std::vector<SDValue> &AsmNodeOperands) {10035 // Scan until we find the definition we already emitted of this operand.10036 unsigned CurOp = InlineAsm::Op_FirstOperand;10037 for (; OperandNo; --OperandNo) {10038 // Advance to the next operand.10039 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();10040 const InlineAsm::Flag F(OpFlag);10041 assert(10042 (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) &&10043 "Skipped past definitions?");10044 CurOp += F.getNumOperandRegisters() + 1;10045 }10046 return CurOp;10047}10048 10049namespace {10050 10051class ExtraFlags {10052 unsigned Flags = 0;10053 10054public:10055 explicit ExtraFlags(const CallBase &Call) {10056 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());10057 if (IA->hasSideEffects())10058 Flags |= InlineAsm::Extra_HasSideEffects;10059 if (IA->isAlignStack())10060 Flags |= InlineAsm::Extra_IsAlignStack;10061 if (Call.isConvergent())10062 Flags |= InlineAsm::Extra_IsConvergent;10063 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;10064 }10065 10066 void update(const TargetLowering::AsmOperandInfo &OpInfo) {10067 // Ideally, we would only check against memory constraints. However, the10068 // meaning of an Other constraint can be target-specific and we can't easily10069 // reason about it. Therefore, be conservative and set MayLoad/MayStore10070 // for Other constraints as well.10071 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||10072 OpInfo.ConstraintType == TargetLowering::C_Other) {10073 if (OpInfo.Type == InlineAsm::isInput)10074 Flags |= InlineAsm::Extra_MayLoad;10075 else if (OpInfo.Type == InlineAsm::isOutput)10076 Flags |= InlineAsm::Extra_MayStore;10077 else if (OpInfo.Type == InlineAsm::isClobber)10078 Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);10079 }10080 }10081 10082 unsigned get() const { return Flags; }10083};10084 10085} // end anonymous namespace10086 10087static bool isFunction(SDValue Op) {10088 if (Op && Op.getOpcode() == ISD::GlobalAddress) {10089 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {10090 auto Fn = dyn_cast_or_null<Function>(GA->getGlobal());10091 10092 // In normal "call dllimport func" instruction (non-inlineasm) it force10093 // indirect access by specifing call opcode. And usually specially print10094 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can10095 // not do in this way now. (In fact, this is similar with "Data Access"10096 // action). So here we ignore dllimport function.10097 if (Fn && !Fn->hasDLLImportStorageClass())10098 return true;10099 }10100 }10101 return false;10102}10103 10104/// visitInlineAsm - Handle a call to an InlineAsm object.10105void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,10106 const BasicBlock *EHPadBB) {10107 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());10108 10109 /// ConstraintOperands - Information about all of the constraints.10110 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;10111 10112 const TargetLowering &TLI = DAG.getTargetLoweringInfo();10113 TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(10114 DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), Call);10115 10116 // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack,10117 // AsmDialect, MayLoad, MayStore).10118 bool HasSideEffect = IA->hasSideEffects();10119 ExtraFlags ExtraInfo(Call);10120 10121 for (auto &T : TargetConstraints) {10122 ConstraintOperands.push_back(SDISelAsmOperandInfo(T));10123 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();10124 10125 if (OpInfo.CallOperandVal)10126 OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);10127 10128 if (!HasSideEffect)10129 HasSideEffect = OpInfo.hasMemory(TLI);10130 10131 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.10132 // FIXME: Could we compute this on OpInfo rather than T?10133 10134 // Compute the constraint code and ConstraintType to use.10135 TLI.ComputeConstraintToUse(T, SDValue());10136 10137 if (T.ConstraintType == TargetLowering::C_Immediate &&10138 OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand))10139 // We've delayed emitting a diagnostic like the "n" constraint because10140 // inlining could cause an integer showing up.10141 return emitInlineAsmError(Call, "constraint '" + Twine(T.ConstraintCode) +10142 "' expects an integer constant "10143 "expression");10144 10145 ExtraInfo.update(T);10146 }10147 10148 // We won't need to flush pending loads if this asm doesn't touch10149 // memory and is nonvolatile.10150 SDValue Glue, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot();10151 10152 bool EmitEHLabels = isa<InvokeInst>(Call);10153 if (EmitEHLabels) {10154 assert(EHPadBB && "InvokeInst must have an EHPadBB");10155 }10156 bool IsCallBr = isa<CallBrInst>(Call);10157 10158 if (IsCallBr || EmitEHLabels) {10159 // If this is a callbr or invoke we need to flush pending exports since10160 // inlineasm_br and invoke are terminators.10161 // We need to do this before nodes are glued to the inlineasm_br node.10162 Chain = getControlRoot();10163 }10164 10165 MCSymbol *BeginLabel = nullptr;10166 if (EmitEHLabels) {10167 Chain = lowerStartEH(Chain, EHPadBB, BeginLabel);10168 }10169 10170 int OpNo = -1;10171 SmallVector<StringRef> AsmStrs;10172 IA->collectAsmStrs(AsmStrs);10173 10174 // Second pass over the constraints: compute which constraint option to use.10175 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {10176 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)10177 OpNo++;10178 10179 // If this is an output operand with a matching input operand, look up the10180 // matching input. If their types mismatch, e.g. one is an integer, the10181 // other is floating point, or their sizes are different, flag it as an10182 // error.10183 if (OpInfo.hasMatchingInput()) {10184 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];10185 patchMatchingInput(OpInfo, Input, DAG);10186 }10187 10188 // Compute the constraint code and ConstraintType to use.10189 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);10190 10191 if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&10192 OpInfo.Type == InlineAsm::isClobber) ||10193 OpInfo.ConstraintType == TargetLowering::C_Address)10194 continue;10195 10196 // In Linux PIC model, there are 4 cases about value/label addressing:10197 //10198 // 1: Function call or Label jmp inside the module.10199 // 2: Data access (such as global variable, static variable) inside module.10200 // 3: Function call or Label jmp outside the module.10201 // 4: Data access (such as global variable) outside the module.10202 //10203 // Due to current llvm inline asm architecture designed to not "recognize"10204 // the asm code, there are quite troubles for us to treat mem addressing10205 // differently for same value/adress used in different instuctions.10206 // For example, in pic model, call a func may in plt way or direclty10207 // pc-related, but lea/mov a function adress may use got.10208 //10209 // Here we try to "recognize" function call for the case 1 and case 3 in10210 // inline asm. And try to adjust the constraint for them.10211 //10212 // TODO: Due to current inline asm didn't encourage to jmp to the outsider10213 // label, so here we don't handle jmp function label now, but we need to10214 // enhance it (especilly in PIC model) if we meet meaningful requirements.10215 if (OpInfo.isIndirect && isFunction(OpInfo.CallOperand) &&10216 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&10217 TM.getCodeModel() != CodeModel::Large) {10218 OpInfo.isIndirect = false;10219 OpInfo.ConstraintType = TargetLowering::C_Address;10220 }10221 10222 // If this is a memory input, and if the operand is not indirect, do what we10223 // need to provide an address for the memory input.10224 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&10225 !OpInfo.isIndirect) {10226 assert((OpInfo.isMultipleAlternative ||10227 (OpInfo.Type == InlineAsm::isInput)) &&10228 "Can only indirectify direct input operands!");10229 10230 // Memory operands really want the address of the value.10231 Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);10232 10233 // There is no longer a Value* corresponding to this operand.10234 OpInfo.CallOperandVal = nullptr;10235 10236 // It is now an indirect operand.10237 OpInfo.isIndirect = true;10238 }10239 10240 }10241 10242 // AsmNodeOperands - The operands for the ISD::INLINEASM node.10243 std::vector<SDValue> AsmNodeOperands;10244 AsmNodeOperands.push_back(SDValue()); // reserve space for input chain10245 AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(10246 IA->getAsmString().data(), TLI.getProgramPointerTy(DAG.getDataLayout())));10247 10248 // If we have a !srcloc metadata node associated with it, we want to attach10249 // this to the ultimately generated inline asm machineinstr. To do this, we10250 // pass in the third operand as this (potentially null) inline asm MDNode.10251 const MDNode *SrcLoc = Call.getMetadata("srcloc");10252 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));10253 10254 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore10255 // bits as operand 3.10256 AsmNodeOperands.push_back(DAG.getTargetConstant(10257 ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));10258 10259 // Third pass: Loop over operands to prepare DAG-level operands.. As part of10260 // this, assign virtual and physical registers for inputs and otput.10261 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {10262 // Assign Registers.10263 SDISelAsmOperandInfo &RefOpInfo =10264 OpInfo.isMatchingInputConstraint()10265 ? ConstraintOperands[OpInfo.getMatchedOperand()]10266 : OpInfo;10267 const auto RegError =10268 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo);10269 if (RegError) {10270 const MachineFunction &MF = DAG.getMachineFunction();10271 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();10272 const char *RegName = TRI.getName(*RegError);10273 emitInlineAsmError(Call, "register '" + Twine(RegName) +10274 "' allocated for constraint '" +10275 Twine(OpInfo.ConstraintCode) +10276 "' does not match required type");10277 return;10278 }10279 10280 auto DetectWriteToReservedRegister = [&]() {10281 const MachineFunction &MF = DAG.getMachineFunction();10282 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();10283 for (Register Reg : OpInfo.AssignedRegs.Regs) {10284 if (Reg.isPhysical() && TRI.isInlineAsmReadOnlyReg(MF, Reg)) {10285 const char *RegName = TRI.getName(Reg);10286 emitInlineAsmError(Call, "write to reserved register '" +10287 Twine(RegName) + "'");10288 return true;10289 }10290 }10291 return false;10292 };10293 assert((OpInfo.ConstraintType != TargetLowering::C_Address ||10294 (OpInfo.Type == InlineAsm::isInput &&10295 !OpInfo.isMatchingInputConstraint())) &&10296 "Only address as input operand is allowed.");10297 10298 switch (OpInfo.Type) {10299 case InlineAsm::isOutput:10300 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {10301 const InlineAsm::ConstraintCode ConstraintID =10302 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);10303 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&10304 "Failed to convert memory constraint code to constraint id.");10305 10306 // Add information to the INLINEASM node to know about this output.10307 InlineAsm::Flag OpFlags(InlineAsm::Kind::Mem, 1);10308 OpFlags.setMemConstraint(ConstraintID);10309 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),10310 MVT::i32));10311 AsmNodeOperands.push_back(OpInfo.CallOperand);10312 } else {10313 // Otherwise, this outputs to a register (directly for C_Register /10314 // C_RegisterClass, and a target-defined fashion for10315 // C_Immediate/C_Other). Find a register that we can use.10316 if (OpInfo.AssignedRegs.Regs.empty()) {10317 emitInlineAsmError(10318 Call, "couldn't allocate output register for constraint '" +10319 Twine(OpInfo.ConstraintCode) + "'");10320 return;10321 }10322 10323 if (DetectWriteToReservedRegister())10324 return;10325 10326 // Add information to the INLINEASM node to know that this register is10327 // set.10328 OpInfo.AssignedRegs.AddInlineAsmOperands(10329 OpInfo.isEarlyClobber ? InlineAsm::Kind::RegDefEarlyClobber10330 : InlineAsm::Kind::RegDef,10331 false, 0, getCurSDLoc(), DAG, AsmNodeOperands);10332 }10333 break;10334 10335 case InlineAsm::isInput:10336 case InlineAsm::isLabel: {10337 SDValue InOperandVal = OpInfo.CallOperand;10338 10339 if (OpInfo.isMatchingInputConstraint()) {10340 // If this is required to match an output register we have already set,10341 // just use its register.10342 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),10343 AsmNodeOperands);10344 InlineAsm::Flag Flag(AsmNodeOperands[CurOp]->getAsZExtVal());10345 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {10346 if (OpInfo.isIndirect) {10347 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c10348 emitInlineAsmError(Call, "inline asm not supported yet: "10349 "don't know how to handle tied "10350 "indirect register inputs");10351 return;10352 }10353 10354 SmallVector<Register, 4> Regs;10355 MachineFunction &MF = DAG.getMachineFunction();10356 MachineRegisterInfo &MRI = MF.getRegInfo();10357 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();10358 auto *R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]);10359 Register TiedReg = R->getReg();10360 MVT RegVT = R->getSimpleValueType(0);10361 const TargetRegisterClass *RC =10362 TiedReg.isVirtual() ? MRI.getRegClass(TiedReg)10363 : RegVT != MVT::Untyped ? TLI.getRegClassFor(RegVT)10364 : TRI.getMinimalPhysRegClass(TiedReg);10365 for (unsigned i = 0, e = Flag.getNumOperandRegisters(); i != e; ++i)10366 Regs.push_back(MRI.createVirtualRegister(RC));10367 10368 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());10369 10370 SDLoc dl = getCurSDLoc();10371 // Use the produced MatchedRegs object to10372 MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue, &Call);10373 MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind::RegUse, true,10374 OpInfo.getMatchedOperand(), dl, DAG,10375 AsmNodeOperands);10376 break;10377 }10378 10379 assert(Flag.isMemKind() && "Unknown matching constraint!");10380 assert(Flag.getNumOperandRegisters() == 1 &&10381 "Unexpected number of operands");10382 // Add information to the INLINEASM node to know about this input.10383 // See InlineAsm.h isUseOperandTiedToDef.10384 Flag.clearMemConstraint();10385 Flag.setMatchingOp(OpInfo.getMatchedOperand());10386 AsmNodeOperands.push_back(DAG.getTargetConstant(10387 Flag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));10388 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);10389 break;10390 }10391 10392 // Treat indirect 'X' constraint as memory.10393 if (OpInfo.ConstraintType == TargetLowering::C_Other &&10394 OpInfo.isIndirect)10395 OpInfo.ConstraintType = TargetLowering::C_Memory;10396 10397 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||10398 OpInfo.ConstraintType == TargetLowering::C_Other) {10399 std::vector<SDValue> Ops;10400 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,10401 Ops, DAG);10402 if (Ops.empty()) {10403 if (OpInfo.ConstraintType == TargetLowering::C_Immediate)10404 if (isa<ConstantSDNode>(InOperandVal)) {10405 emitInlineAsmError(Call, "value out of range for constraint '" +10406 Twine(OpInfo.ConstraintCode) + "'");10407 return;10408 }10409 10410 emitInlineAsmError(Call,10411 "invalid operand for inline asm constraint '" +10412 Twine(OpInfo.ConstraintCode) + "'");10413 return;10414 }10415 10416 // Add information to the INLINEASM node to know about this input.10417 InlineAsm::Flag ResOpType(InlineAsm::Kind::Imm, Ops.size());10418 AsmNodeOperands.push_back(DAG.getTargetConstant(10419 ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));10420 llvm::append_range(AsmNodeOperands, Ops);10421 break;10422 }10423 10424 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {10425 assert((OpInfo.isIndirect ||10426 OpInfo.ConstraintType != TargetLowering::C_Memory) &&10427 "Operand must be indirect to be a mem!");10428 assert(InOperandVal.getValueType() ==10429 TLI.getPointerTy(DAG.getDataLayout()) &&10430 "Memory operands expect pointer values");10431 10432 const InlineAsm::ConstraintCode ConstraintID =10433 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);10434 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&10435 "Failed to convert memory constraint code to constraint id.");10436 10437 // Add information to the INLINEASM node to know about this input.10438 InlineAsm::Flag ResOpType(InlineAsm::Kind::Mem, 1);10439 ResOpType.setMemConstraint(ConstraintID);10440 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,10441 getCurSDLoc(),10442 MVT::i32));10443 AsmNodeOperands.push_back(InOperandVal);10444 break;10445 }10446 10447 if (OpInfo.ConstraintType == TargetLowering::C_Address) {10448 const InlineAsm::ConstraintCode ConstraintID =10449 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);10450 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&10451 "Failed to convert memory constraint code to constraint id.");10452 10453 InlineAsm::Flag ResOpType(InlineAsm::Kind::Mem, 1);10454 10455 SDValue AsmOp = InOperandVal;10456 if (isFunction(InOperandVal)) {10457 auto *GA = cast<GlobalAddressSDNode>(InOperandVal);10458 ResOpType = InlineAsm::Flag(InlineAsm::Kind::Func, 1);10459 AsmOp = DAG.getTargetGlobalAddress(GA->getGlobal(), getCurSDLoc(),10460 InOperandVal.getValueType(),10461 GA->getOffset());10462 }10463 10464 // Add information to the INLINEASM node to know about this input.10465 ResOpType.setMemConstraint(ConstraintID);10466 10467 AsmNodeOperands.push_back(10468 DAG.getTargetConstant(ResOpType, getCurSDLoc(), MVT::i32));10469 10470 AsmNodeOperands.push_back(AsmOp);10471 break;10472 }10473 10474 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&10475 OpInfo.ConstraintType != TargetLowering::C_Register) {10476 emitInlineAsmError(Call, "unknown asm constraint '" +10477 Twine(OpInfo.ConstraintCode) + "'");10478 return;10479 }10480 10481 // TODO: Support this.10482 if (OpInfo.isIndirect) {10483 emitInlineAsmError(10484 Call, "Don't know how to handle indirect register inputs yet "10485 "for constraint '" +10486 Twine(OpInfo.ConstraintCode) + "'");10487 return;10488 }10489 10490 // Copy the input into the appropriate registers.10491 if (OpInfo.AssignedRegs.Regs.empty()) {10492 emitInlineAsmError(Call,10493 "couldn't allocate input reg for constraint '" +10494 Twine(OpInfo.ConstraintCode) + "'");10495 return;10496 }10497 10498 if (DetectWriteToReservedRegister())10499 return;10500 10501 SDLoc dl = getCurSDLoc();10502 10503 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue,10504 &Call);10505 10506 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind::RegUse, false,10507 0, dl, DAG, AsmNodeOperands);10508 break;10509 }10510 case InlineAsm::isClobber:10511 // Add the clobbered value to the operand list, so that the register10512 // allocator is aware that the physreg got clobbered.10513 if (!OpInfo.AssignedRegs.Regs.empty())10514 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind::Clobber,10515 false, 0, getCurSDLoc(), DAG,10516 AsmNodeOperands);10517 break;10518 }10519 }10520 10521 // Finish up input operands. Set the input chain and add the flag last.10522 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;10523 if (Glue.getNode()) AsmNodeOperands.push_back(Glue);10524 10525 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;10526 Chain = DAG.getNode(ISDOpc, getCurSDLoc(),10527 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);10528 Glue = Chain.getValue(1);10529 10530 // Do additional work to generate outputs.10531 10532 SmallVector<EVT, 1> ResultVTs;10533 SmallVector<SDValue, 1> ResultValues;10534 SmallVector<SDValue, 8> OutChains;10535 10536 llvm::Type *CallResultType = Call.getType();10537 ArrayRef<Type *> ResultTypes;10538 if (StructType *StructResult = dyn_cast<StructType>(CallResultType))10539 ResultTypes = StructResult->elements();10540 else if (!CallResultType->isVoidTy())10541 ResultTypes = ArrayRef(CallResultType);10542 10543 auto CurResultType = ResultTypes.begin();10544 auto handleRegAssign = [&](SDValue V) {10545 assert(CurResultType != ResultTypes.end() && "Unexpected value");10546 assert((*CurResultType)->isSized() && "Unexpected unsized type");10547 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);10548 ++CurResultType;10549 // If the type of the inline asm call site return value is different but has10550 // same size as the type of the asm output bitcast it. One example of this10551 // is for vectors with different width / number of elements. This can10552 // happen for register classes that can contain multiple different value10553 // types. The preg or vreg allocated may not have the same VT as was10554 // expected.10555 //10556 // This can also happen for a return value that disagrees with the register10557 // class it is put in, eg. a double in a general-purpose register on a10558 // 32-bit machine.10559 if (ResultVT != V.getValueType() &&10560 ResultVT.getSizeInBits() == V.getValueSizeInBits())10561 V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);10562 else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&10563 V.getValueType().isInteger()) {10564 // If a result value was tied to an input value, the computed result10565 // may have a wider width than the expected result. Extract the10566 // relevant portion.10567 V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);10568 }10569 assert(ResultVT == V.getValueType() && "Asm result value mismatch!");10570 ResultVTs.push_back(ResultVT);10571 ResultValues.push_back(V);10572 };10573 10574 // Deal with output operands.10575 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {10576 if (OpInfo.Type == InlineAsm::isOutput) {10577 SDValue Val;10578 // Skip trivial output operands.10579 if (OpInfo.AssignedRegs.Regs.empty())10580 continue;10581 10582 switch (OpInfo.ConstraintType) {10583 case TargetLowering::C_Register:10584 case TargetLowering::C_RegisterClass:10585 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),10586 Chain, &Glue, &Call);10587 break;10588 case TargetLowering::C_Immediate:10589 case TargetLowering::C_Other:10590 Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, getCurSDLoc(),10591 OpInfo, DAG);10592 break;10593 case TargetLowering::C_Memory:10594 break; // Already handled.10595 case TargetLowering::C_Address:10596 break; // Silence warning.10597 case TargetLowering::C_Unknown:10598 assert(false && "Unexpected unknown constraint");10599 }10600 10601 // Indirect output manifest as stores. Record output chains.10602 if (OpInfo.isIndirect) {10603 const Value *Ptr = OpInfo.CallOperandVal;10604 assert(Ptr && "Expected value CallOperandVal for indirect asm operand");10605 SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),10606 MachinePointerInfo(Ptr));10607 OutChains.push_back(Store);10608 } else {10609 // generate CopyFromRegs to associated registers.10610 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");10611 if (Val.getOpcode() == ISD::MERGE_VALUES) {10612 for (const SDValue &V : Val->op_values())10613 handleRegAssign(V);10614 } else10615 handleRegAssign(Val);10616 }10617 }10618 }10619 10620 // Set results.10621 if (!ResultValues.empty()) {10622 assert(CurResultType == ResultTypes.end() &&10623 "Mismatch in number of ResultTypes");10624 assert(ResultValues.size() == ResultTypes.size() &&10625 "Mismatch in number of output operands in asm result");10626 10627 SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),10628 DAG.getVTList(ResultVTs), ResultValues);10629 setValue(&Call, V);10630 }10631 10632 // Collect store chains.10633 if (!OutChains.empty())10634 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);10635 10636 if (EmitEHLabels) {10637 Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel);10638 }10639 10640 // Only Update Root if inline assembly has a memory effect.10641 if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr ||10642 EmitEHLabels)10643 DAG.setRoot(Chain);10644}10645 10646void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,10647 const Twine &Message) {10648 LLVMContext &Ctx = *DAG.getContext();10649 Ctx.diagnose(DiagnosticInfoInlineAsm(Call, Message));10650 10651 // Make sure we leave the DAG in a valid state10652 const TargetLowering &TLI = DAG.getTargetLoweringInfo();10653 SmallVector<EVT, 1> ValueVTs;10654 ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs);10655 10656 if (ValueVTs.empty())10657 return;10658 10659 SmallVector<SDValue, 1> Ops;10660 for (const EVT &VT : ValueVTs)10661 Ops.push_back(DAG.getUNDEF(VT));10662 10663 setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));10664}10665 10666void SelectionDAGBuilder::visitVAStart(const CallInst &I) {10667 DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),10668 MVT::Other, getRoot(),10669 getValue(I.getArgOperand(0)),10670 DAG.getSrcValue(I.getArgOperand(0))));10671}10672 10673void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {10674 const TargetLowering &TLI = DAG.getTargetLoweringInfo();10675 const DataLayout &DL = DAG.getDataLayout();10676 SDValue V = DAG.getVAArg(10677 TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),10678 getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),10679 DL.getABITypeAlign(I.getType()).value());10680 DAG.setRoot(V.getValue(1));10681 10682 if (I.getType()->isPointerTy())10683 V = DAG.getPtrExtOrTrunc(10684 V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType()));10685 setValue(&I, V);10686}10687 10688void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {10689 DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),10690 MVT::Other, getRoot(),10691 getValue(I.getArgOperand(0)),10692 DAG.getSrcValue(I.getArgOperand(0))));10693}10694 10695void SelectionDAGBuilder::visitVACopy(const CallInst &I) {10696 DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),10697 MVT::Other, getRoot(),10698 getValue(I.getArgOperand(0)),10699 getValue(I.getArgOperand(1)),10700 DAG.getSrcValue(I.getArgOperand(0)),10701 DAG.getSrcValue(I.getArgOperand(1))));10702}10703 10704SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,10705 const Instruction &I,10706 SDValue Op) {10707 std::optional<ConstantRange> CR = getRange(I);10708 10709 if (!CR || CR->isFullSet() || CR->isEmptySet() || CR->isUpperWrapped())10710 return Op;10711 10712 APInt Lo = CR->getUnsignedMin();10713 if (!Lo.isMinValue())10714 return Op;10715 10716 APInt Hi = CR->getUnsignedMax();10717 unsigned Bits = std::max(Hi.getActiveBits(),10718 static_cast<unsigned>(IntegerType::MIN_INT_BITS));10719 10720 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);10721 10722 SDLoc SL = getCurSDLoc();10723 10724 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,10725 DAG.getValueType(SmallVT));10726 unsigned NumVals = Op.getNode()->getNumValues();10727 if (NumVals == 1)10728 return ZExt;10729 10730 SmallVector<SDValue, 4> Ops;10731 10732 Ops.push_back(ZExt);10733 for (unsigned I = 1; I != NumVals; ++I)10734 Ops.push_back(Op.getValue(I));10735 10736 return DAG.getMergeValues(Ops, SL);10737}10738 10739SDValue SelectionDAGBuilder::lowerNoFPClassToAssertNoFPClass(10740 SelectionDAG &DAG, const Instruction &I, SDValue Op) {10741 FPClassTest Classes = getNoFPClass(I);10742 if (Classes == fcNone)10743 return Op;10744 10745 SDLoc SL = getCurSDLoc();10746 SDValue TestConst = DAG.getTargetConstant(Classes, SDLoc(), MVT::i32);10747 10748 if (Op.getOpcode() != ISD::MERGE_VALUES) {10749 return DAG.getNode(ISD::AssertNoFPClass, SL, Op.getValueType(), Op,10750 TestConst);10751 }10752 10753 SmallVector<SDValue, 8> Ops(Op.getNumOperands());10754 for (unsigned I = 0, E = Ops.size(); I != E; ++I) {10755 SDValue MergeOp = Op.getOperand(I);10756 Ops[I] = DAG.getNode(ISD::AssertNoFPClass, SL, MergeOp.getValueType(),10757 MergeOp, TestConst);10758 }10759 10760 return DAG.getMergeValues(Ops, SL);10761}10762 10763/// Populate a CallLowerinInfo (into \p CLI) based on the properties of10764/// the call being lowered.10765///10766/// This is a helper for lowering intrinsics that follow a target calling10767/// convention or require stack pointer adjustment. Only a subset of the10768/// intrinsic's operands need to participate in the calling convention.10769void SelectionDAGBuilder::populateCallLoweringInfo(10770 TargetLowering::CallLoweringInfo &CLI, const CallBase *Call,10771 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,10772 AttributeSet RetAttrs, bool IsPatchPoint) {10773 TargetLowering::ArgListTy Args;10774 Args.reserve(NumArgs);10775 10776 // Populate the argument list.10777 // Attributes for args start at offset 1, after the return attribute.10778 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;10779 ArgI != ArgE; ++ArgI) {10780 const Value *V = Call->getOperand(ArgI);10781 10782 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");10783 10784 TargetLowering::ArgListEntry Entry(getValue(V), V->getType());10785 Entry.setAttributes(Call, ArgI);10786 Args.push_back(Entry);10787 }10788 10789 CLI.setDebugLoc(getCurSDLoc())10790 .setChain(getRoot())10791 .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args),10792 RetAttrs)10793 .setDiscardResult(Call->use_empty())10794 .setIsPatchPoint(IsPatchPoint)10795 .setIsPreallocated(10796 Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);10797}10798 10799/// Add a stack map intrinsic call's live variable operands to a stackmap10800/// or patchpoint target node's operand list.10801///10802/// Constants are converted to TargetConstants purely as an optimization to10803/// avoid constant materialization and register allocation.10804///10805/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not10806/// generate addess computation nodes, and so FinalizeISel can convert the10807/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids10808/// address materialization and register allocation, but may also be required10809/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an10810/// alloca in the entry block, then the runtime may assume that the alloca's10811/// StackMap location can be read immediately after compilation and that the10812/// location is valid at any point during execution (this is similar to the10813/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were10814/// only available in a register, then the runtime would need to trap when10815/// execution reaches the StackMap in order to read the alloca's location.10816static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,10817 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,10818 SelectionDAGBuilder &Builder) {10819 SelectionDAG &DAG = Builder.DAG;10820 for (unsigned I = StartIdx; I < Call.arg_size(); I++) {10821 SDValue Op = Builder.getValue(Call.getArgOperand(I));10822 10823 // Things on the stack are pointer-typed, meaning that they are already10824 // legal and can be emitted directly to target nodes.10825 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {10826 Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType()));10827 } else {10828 // Otherwise emit a target independent node to be legalised.10829 Ops.push_back(Builder.getValue(Call.getArgOperand(I)));10830 }10831 }10832}10833 10834/// Lower llvm.experimental.stackmap.10835void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {10836 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,10837 // [live variables...])10838 10839 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");10840 10841 SDValue Chain, InGlue, Callee;10842 SmallVector<SDValue, 32> Ops;10843 10844 SDLoc DL = getCurSDLoc();10845 Callee = getValue(CI.getCalledOperand());10846 10847 // The stackmap intrinsic only records the live variables (the arguments10848 // passed to it) and emits NOPS (if requested). Unlike the patchpoint10849 // intrinsic, this won't be lowered to a function call. This means we don't10850 // have to worry about calling conventions and target specific lowering code.10851 // Instead we perform the call lowering right here.10852 //10853 // chain, flag = CALLSEQ_START(chain, 0, 0)10854 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)10855 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)10856 //10857 Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);10858 InGlue = Chain.getValue(1);10859 10860 // Add the STACKMAP operands, starting with DAG house-keeping.10861 Ops.push_back(Chain);10862 Ops.push_back(InGlue);10863 10864 // Add the <id>, <numShadowBytes> operands.10865 //10866 // These do not require legalisation, and can be emitted directly to target10867 // constant nodes.10868 SDValue ID = getValue(CI.getArgOperand(0));10869 assert(ID.getValueType() == MVT::i64);10870 SDValue IDConst =10871 DAG.getTargetConstant(ID->getAsZExtVal(), DL, ID.getValueType());10872 Ops.push_back(IDConst);10873 10874 SDValue Shad = getValue(CI.getArgOperand(1));10875 assert(Shad.getValueType() == MVT::i32);10876 SDValue ShadConst =10877 DAG.getTargetConstant(Shad->getAsZExtVal(), DL, Shad.getValueType());10878 Ops.push_back(ShadConst);10879 10880 // Add the live variables.10881 addStackMapLiveVars(CI, 2, DL, Ops, *this);10882 10883 // Create the STACKMAP node.10884 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);10885 Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);10886 InGlue = Chain.getValue(1);10887 10888 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, DL);10889 10890 // Stackmaps don't generate values, so nothing goes into the NodeMap.10891 10892 // Set the root to the target-lowered call chain.10893 DAG.setRoot(Chain);10894 10895 // Inform the Frame Information that we have a stackmap in this function.10896 FuncInfo.MF->getFrameInfo().setHasStackMap();10897}10898 10899/// Lower llvm.experimental.patchpoint directly to its target opcode.10900void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,10901 const BasicBlock *EHPadBB) {10902 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,10903 // i32 <numBytes>,10904 // i8* <target>,10905 // i32 <numArgs>,10906 // [Args...],10907 // [live variables...])10908 10909 CallingConv::ID CC = CB.getCallingConv();10910 bool IsAnyRegCC = CC == CallingConv::AnyReg;10911 bool HasDef = !CB.getType()->isVoidTy();10912 SDLoc dl = getCurSDLoc();10913 SDValue Callee = getValue(CB.getArgOperand(PatchPointOpers::TargetPos));10914 10915 // Handle immediate and symbolic callees.10916 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))10917 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,10918 /*isTarget=*/true);10919 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))10920 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),10921 SDLoc(SymbolicCallee),10922 SymbolicCallee->getValueType(0));10923 10924 // Get the real number of arguments participating in the call <numArgs>10925 SDValue NArgVal = getValue(CB.getArgOperand(PatchPointOpers::NArgPos));10926 unsigned NumArgs = NArgVal->getAsZExtVal();10927 10928 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>10929 // Intrinsics include all meta-operands up to but not including CC.10930 unsigned NumMetaOpers = PatchPointOpers::CCPos;10931 assert(CB.arg_size() >= NumMetaOpers + NumArgs &&10932 "Not enough arguments provided to the patchpoint intrinsic");10933 10934 // For AnyRegCC the arguments are lowered later on manually.10935 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;10936 Type *ReturnTy =10937 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType();10938 10939 TargetLowering::CallLoweringInfo CLI(DAG);10940 populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,10941 ReturnTy, CB.getAttributes().getRetAttrs(), true);10942 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);10943 10944 SDNode *CallEnd = Result.second.getNode();10945 if (CallEnd->getOpcode() == ISD::EH_LABEL)10946 CallEnd = CallEnd->getOperand(0).getNode();10947 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))10948 CallEnd = CallEnd->getOperand(0).getNode();10949 10950 /// Get a call instruction from the call sequence chain.10951 /// Tail calls are not allowed.10952 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&10953 "Expected a callseq node.");10954 SDNode *Call = CallEnd->getOperand(0).getNode();10955 bool HasGlue = Call->getGluedNode();10956 10957 // Replace the target specific call node with the patchable intrinsic.10958 SmallVector<SDValue, 8> Ops;10959 10960 // Push the chain.10961 Ops.push_back(*(Call->op_begin()));10962 10963 // Optionally, push the glue (if any).10964 if (HasGlue)10965 Ops.push_back(*(Call->op_end() - 1));10966 10967 // Push the register mask info.10968 if (HasGlue)10969 Ops.push_back(*(Call->op_end() - 2));10970 else10971 Ops.push_back(*(Call->op_end() - 1));10972 10973 // Add the <id> and <numBytes> constants.10974 SDValue IDVal = getValue(CB.getArgOperand(PatchPointOpers::IDPos));10975 Ops.push_back(DAG.getTargetConstant(IDVal->getAsZExtVal(), dl, MVT::i64));10976 SDValue NBytesVal = getValue(CB.getArgOperand(PatchPointOpers::NBytesPos));10977 Ops.push_back(DAG.getTargetConstant(NBytesVal->getAsZExtVal(), dl, MVT::i32));10978 10979 // Add the callee.10980 Ops.push_back(Callee);10981 10982 // Adjust <numArgs> to account for any arguments that have been passed on the10983 // stack instead.10984 // Call Node: Chain, Target, {Args}, RegMask, [Glue]10985 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);10986 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;10987 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));10988 10989 // Add the calling convention10990 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));10991 10992 // Add the arguments we omitted previously. The register allocator should10993 // place these in any free register.10994 if (IsAnyRegCC)10995 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)10996 Ops.push_back(getValue(CB.getArgOperand(i)));10997 10998 // Push the arguments from the call instruction.10999 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;11000 Ops.append(Call->op_begin() + 2, e);11001 11002 // Push live variables for the stack map.11003 addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);11004 11005 SDVTList NodeTys;11006 if (IsAnyRegCC && HasDef) {11007 // Create the return types based on the intrinsic definition11008 const TargetLowering &TLI = DAG.getTargetLoweringInfo();11009 SmallVector<EVT, 3> ValueVTs;11010 ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);11011 assert(ValueVTs.size() == 1 && "Expected only one return value type.");11012 11013 // There is always a chain and a glue type at the end11014 ValueVTs.push_back(MVT::Other);11015 ValueVTs.push_back(MVT::Glue);11016 NodeTys = DAG.getVTList(ValueVTs);11017 } else11018 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);11019 11020 // Replace the target specific call node with a PATCHPOINT node.11021 SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops);11022 11023 // Update the NodeMap.11024 if (HasDef) {11025 if (IsAnyRegCC)11026 setValue(&CB, SDValue(PPV.getNode(), 0));11027 else11028 setValue(&CB, Result.first);11029 }11030 11031 // Fixup the consumers of the intrinsic. The chain and glue may be used in the11032 // call sequence. Furthermore the location of the chain and glue can change11033 // when the AnyReg calling convention is used and the intrinsic returns a11034 // value.11035 if (IsAnyRegCC && HasDef) {11036 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};11037 SDValue To[] = {PPV.getValue(1), PPV.getValue(2)};11038 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);11039 } else11040 DAG.ReplaceAllUsesWith(Call, PPV.getNode());11041 DAG.DeleteNode(Call);11042 11043 // Inform the Frame Information that we have a patchpoint in this function.11044 FuncInfo.MF->getFrameInfo().setHasPatchPoint();11045}11046 11047void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,11048 unsigned Intrinsic) {11049 const TargetLowering &TLI = DAG.getTargetLoweringInfo();11050 SDValue Op1 = getValue(I.getArgOperand(0));11051 SDValue Op2;11052 if (I.arg_size() > 1)11053 Op2 = getValue(I.getArgOperand(1));11054 SDLoc dl = getCurSDLoc();11055 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());11056 SDValue Res;11057 SDNodeFlags SDFlags;11058 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))11059 SDFlags.copyFMF(*FPMO);11060 11061 switch (Intrinsic) {11062 case Intrinsic::vector_reduce_fadd:11063 if (SDFlags.hasAllowReassociation())11064 Res = DAG.getNode(ISD::FADD, dl, VT, Op1,11065 DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),11066 SDFlags);11067 else11068 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags);11069 break;11070 case Intrinsic::vector_reduce_fmul:11071 if (SDFlags.hasAllowReassociation())11072 Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,11073 DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),11074 SDFlags);11075 else11076 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags);11077 break;11078 case Intrinsic::vector_reduce_add:11079 Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);11080 break;11081 case Intrinsic::vector_reduce_mul:11082 Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);11083 break;11084 case Intrinsic::vector_reduce_and:11085 Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);11086 break;11087 case Intrinsic::vector_reduce_or:11088 Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);11089 break;11090 case Intrinsic::vector_reduce_xor:11091 Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);11092 break;11093 case Intrinsic::vector_reduce_smax:11094 Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);11095 break;11096 case Intrinsic::vector_reduce_smin:11097 Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);11098 break;11099 case Intrinsic::vector_reduce_umax:11100 Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);11101 break;11102 case Intrinsic::vector_reduce_umin:11103 Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);11104 break;11105 case Intrinsic::vector_reduce_fmax:11106 Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);11107 break;11108 case Intrinsic::vector_reduce_fmin:11109 Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);11110 break;11111 case Intrinsic::vector_reduce_fmaximum:11112 Res = DAG.getNode(ISD::VECREDUCE_FMAXIMUM, dl, VT, Op1, SDFlags);11113 break;11114 case Intrinsic::vector_reduce_fminimum:11115 Res = DAG.getNode(ISD::VECREDUCE_FMINIMUM, dl, VT, Op1, SDFlags);11116 break;11117 default:11118 llvm_unreachable("Unhandled vector reduce intrinsic");11119 }11120 setValue(&I, Res);11121}11122 11123/// Returns an AttributeList representing the attributes applied to the return11124/// value of the given call.11125static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {11126 SmallVector<Attribute::AttrKind, 2> Attrs;11127 if (CLI.RetSExt)11128 Attrs.push_back(Attribute::SExt);11129 if (CLI.RetZExt)11130 Attrs.push_back(Attribute::ZExt);11131 if (CLI.IsInReg)11132 Attrs.push_back(Attribute::InReg);11133 11134 return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,11135 Attrs);11136}11137 11138/// TargetLowering::LowerCallTo - This is the default LowerCallTo11139/// implementation, which just calls LowerCall.11140/// FIXME: When all targets are11141/// migrated to using LowerCall, this hook should be integrated into SDISel.11142std::pair<SDValue, SDValue>11143TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {11144 LLVMContext &Context = CLI.RetTy->getContext();11145 11146 // Handle the incoming return values from the call.11147 CLI.Ins.clear();11148 SmallVector<Type *, 4> RetOrigTys;11149 SmallVector<TypeSize, 4> Offsets;11150 auto &DL = CLI.DAG.getDataLayout();11151 ComputeValueTypes(DL, CLI.OrigRetTy, RetOrigTys, &Offsets);11152 11153 SmallVector<EVT, 4> RetVTs;11154 if (CLI.RetTy != CLI.OrigRetTy) {11155 assert(RetOrigTys.size() == 1 &&11156 "Only supported for non-aggregate returns");11157 RetVTs.push_back(getValueType(DL, CLI.RetTy));11158 } else {11159 for (Type *Ty : RetOrigTys)11160 RetVTs.push_back(getValueType(DL, Ty));11161 }11162 11163 if (CLI.IsPostTypeLegalization) {11164 // If we are lowering a libcall after legalization, split the return type.11165 SmallVector<Type *, 4> OldRetOrigTys;11166 SmallVector<EVT, 4> OldRetVTs;11167 SmallVector<TypeSize, 4> OldOffsets;11168 RetOrigTys.swap(OldRetOrigTys);11169 RetVTs.swap(OldRetVTs);11170 Offsets.swap(OldOffsets);11171 11172 for (size_t i = 0, e = OldRetVTs.size(); i != e; ++i) {11173 EVT RetVT = OldRetVTs[i];11174 uint64_t Offset = OldOffsets[i];11175 MVT RegisterVT = getRegisterType(Context, RetVT);11176 unsigned NumRegs = getNumRegisters(Context, RetVT);11177 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;11178 RetOrigTys.append(NumRegs, OldRetOrigTys[i]);11179 RetVTs.append(NumRegs, RegisterVT);11180 for (unsigned j = 0; j != NumRegs; ++j)11181 Offsets.push_back(TypeSize::getFixed(Offset + j * RegisterVTByteSZ));11182 }11183 }11184 11185 SmallVector<ISD::OutputArg, 4> Outs;11186 GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);11187 11188 bool CanLowerReturn =11189 this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),11190 CLI.IsVarArg, Outs, Context, CLI.RetTy);11191 11192 SDValue DemoteStackSlot;11193 int DemoteStackIdx = -100;11194 if (!CanLowerReturn) {11195 // FIXME: equivalent assert?11196 // assert(!CS.hasInAllocaArgument() &&11197 // "sret demotion is incompatible with inalloca");11198 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);11199 Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);11200 MachineFunction &MF = CLI.DAG.getMachineFunction();11201 DemoteStackIdx =11202 MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);11203 Type *StackSlotPtrType = PointerType::get(Context, DL.getAllocaAddrSpace());11204 11205 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));11206 ArgListEntry Entry(DemoteStackSlot, StackSlotPtrType);11207 Entry.IsSRet = true;11208 Entry.Alignment = Alignment;11209 CLI.getArgs().insert(CLI.getArgs().begin(), Entry);11210 CLI.NumFixedArgs += 1;11211 CLI.getArgs()[0].IndirectType = CLI.RetTy;11212 CLI.RetTy = CLI.OrigRetTy = Type::getVoidTy(Context);11213 11214 // sret demotion isn't compatible with tail-calls, since the sret argument11215 // points into the callers stack frame.11216 CLI.IsTailCall = false;11217 } else {11218 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(11219 CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);11220 for (unsigned I = 0, E = RetVTs.size(); I != E; ++I) {11221 ISD::ArgFlagsTy Flags;11222 if (NeedsRegBlock) {11223 Flags.setInConsecutiveRegs();11224 if (I == RetVTs.size() - 1)11225 Flags.setInConsecutiveRegsLast();11226 }11227 EVT VT = RetVTs[I];11228 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);11229 unsigned NumRegs =11230 getNumRegistersForCallingConv(Context, CLI.CallConv, VT);11231 for (unsigned i = 0; i != NumRegs; ++i) {11232 ISD::InputArg Ret(Flags, RegisterVT, VT, RetOrigTys[I],11233 CLI.IsReturnValueUsed, ISD::InputArg::NoArgIndex, 0);11234 if (CLI.RetTy->isPointerTy()) {11235 Ret.Flags.setPointer();11236 Ret.Flags.setPointerAddrSpace(11237 cast<PointerType>(CLI.RetTy)->getAddressSpace());11238 }11239 if (CLI.RetSExt)11240 Ret.Flags.setSExt();11241 if (CLI.RetZExt)11242 Ret.Flags.setZExt();11243 if (CLI.IsInReg)11244 Ret.Flags.setInReg();11245 CLI.Ins.push_back(Ret);11246 }11247 }11248 }11249 11250 // We push in swifterror return as the last element of CLI.Ins.11251 ArgListTy &Args = CLI.getArgs();11252 if (supportSwiftError()) {11253 for (const ArgListEntry &Arg : Args) {11254 if (Arg.IsSwiftError) {11255 ISD::ArgFlagsTy Flags;11256 Flags.setSwiftError();11257 ISD::InputArg Ret(Flags, getPointerTy(DL), EVT(getPointerTy(DL)),11258 PointerType::getUnqual(Context),11259 /*Used=*/true, ISD::InputArg::NoArgIndex, 0);11260 CLI.Ins.push_back(Ret);11261 }11262 }11263 }11264 11265 // Handle all of the outgoing arguments.11266 CLI.Outs.clear();11267 CLI.OutVals.clear();11268 for (unsigned i = 0, e = Args.size(); i != e; ++i) {11269 SmallVector<Type *, 4> OrigArgTys;11270 ComputeValueTypes(DL, Args[i].OrigTy, OrigArgTys);11271 // FIXME: Split arguments if CLI.IsPostTypeLegalization11272 Type *FinalType = Args[i].Ty;11273 if (Args[i].IsByVal)11274 FinalType = Args[i].IndirectType;11275 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(11276 FinalType, CLI.CallConv, CLI.IsVarArg, DL);11277 for (unsigned Value = 0, NumValues = OrigArgTys.size(); Value != NumValues;11278 ++Value) {11279 Type *OrigArgTy = OrigArgTys[Value];11280 Type *ArgTy = OrigArgTy;11281 if (Args[i].Ty != Args[i].OrigTy) {11282 assert(Value == 0 && "Only supported for non-aggregate arguments");11283 ArgTy = Args[i].Ty;11284 }11285 11286 EVT VT = getValueType(DL, ArgTy);11287 SDValue Op = SDValue(Args[i].Node.getNode(),11288 Args[i].Node.getResNo() + Value);11289 ISD::ArgFlagsTy Flags;11290 11291 // Certain targets (such as MIPS), may have a different ABI alignment11292 // for a type depending on the context. Give the target a chance to11293 // specify the alignment it wants.11294 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));11295 Flags.setOrigAlign(OriginalAlignment);11296 11297 if (i >= CLI.NumFixedArgs)11298 Flags.setVarArg();11299 if (ArgTy->isPointerTy()) {11300 Flags.setPointer();11301 Flags.setPointerAddrSpace(cast<PointerType>(ArgTy)->getAddressSpace());11302 }11303 if (Args[i].IsZExt)11304 Flags.setZExt();11305 if (Args[i].IsSExt)11306 Flags.setSExt();11307 if (Args[i].IsNoExt)11308 Flags.setNoExt();11309 if (Args[i].IsInReg) {11310 // If we are using vectorcall calling convention, a structure that is11311 // passed InReg - is surely an HVA11312 if (CLI.CallConv == CallingConv::X86_VectorCall &&11313 isa<StructType>(FinalType)) {11314 // The first value of a structure is marked11315 if (0 == Value)11316 Flags.setHvaStart();11317 Flags.setHva();11318 }11319 // Set InReg Flag11320 Flags.setInReg();11321 }11322 if (Args[i].IsSRet)11323 Flags.setSRet();11324 if (Args[i].IsSwiftSelf)11325 Flags.setSwiftSelf();11326 if (Args[i].IsSwiftAsync)11327 Flags.setSwiftAsync();11328 if (Args[i].IsSwiftError)11329 Flags.setSwiftError();11330 if (Args[i].IsCFGuardTarget)11331 Flags.setCFGuardTarget();11332 if (Args[i].IsByVal)11333 Flags.setByVal();11334 if (Args[i].IsByRef)11335 Flags.setByRef();11336 if (Args[i].IsPreallocated) {11337 Flags.setPreallocated();11338 // Set the byval flag for CCAssignFn callbacks that don't know about11339 // preallocated. This way we can know how many bytes we should've11340 // allocated and how many bytes a callee cleanup function will pop. If11341 // we port preallocated to more targets, we'll have to add custom11342 // preallocated handling in the various CC lowering callbacks.11343 Flags.setByVal();11344 }11345 if (Args[i].IsInAlloca) {11346 Flags.setInAlloca();11347 // Set the byval flag for CCAssignFn callbacks that don't know about11348 // inalloca. This way we can know how many bytes we should've allocated11349 // and how many bytes a callee cleanup function will pop. If we port11350 // inalloca to more targets, we'll have to add custom inalloca handling11351 // in the various CC lowering callbacks.11352 Flags.setByVal();11353 }11354 Align MemAlign;11355 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {11356 unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType);11357 Flags.setByValSize(FrameSize);11358 11359 // info is not there but there are cases it cannot get right.11360 if (auto MA = Args[i].Alignment)11361 MemAlign = *MA;11362 else11363 MemAlign = getByValTypeAlignment(Args[i].IndirectType, DL);11364 } else if (auto MA = Args[i].Alignment) {11365 MemAlign = *MA;11366 } else {11367 MemAlign = OriginalAlignment;11368 }11369 Flags.setMemAlign(MemAlign);11370 if (Args[i].IsNest)11371 Flags.setNest();11372 if (NeedsRegBlock)11373 Flags.setInConsecutiveRegs();11374 11375 MVT PartVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);11376 unsigned NumParts =11377 getNumRegistersForCallingConv(Context, CLI.CallConv, VT);11378 SmallVector<SDValue, 4> Parts(NumParts);11379 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;11380 11381 if (Args[i].IsSExt)11382 ExtendKind = ISD::SIGN_EXTEND;11383 else if (Args[i].IsZExt)11384 ExtendKind = ISD::ZERO_EXTEND;11385 11386 // Conservatively only handle 'returned' on non-vectors that can be lowered,11387 // for now.11388 if (Args[i].IsReturned && !Op.getValueType().isVector() &&11389 CanLowerReturn) {11390 assert((CLI.RetTy == Args[i].Ty ||11391 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&11392 CLI.RetTy->getPointerAddressSpace() ==11393 Args[i].Ty->getPointerAddressSpace())) &&11394 RetVTs.size() == NumValues && "unexpected use of 'returned'");11395 // Before passing 'returned' to the target lowering code, ensure that11396 // either the register MVT and the actual EVT are the same size or that11397 // the return value and argument are extended in the same way; in these11398 // cases it's safe to pass the argument register value unchanged as the11399 // return register value (although it's at the target's option whether11400 // to do so)11401 // TODO: allow code generation to take advantage of partially preserved11402 // registers rather than clobbering the entire register when the11403 // parameter extension method is not compatible with the return11404 // extension method11405 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||11406 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&11407 CLI.RetZExt == Args[i].IsZExt))11408 Flags.setReturned();11409 }11410 11411 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB,11412 CLI.CallConv, ExtendKind);11413 11414 for (unsigned j = 0; j != NumParts; ++j) {11415 // if it isn't first piece, alignment must be 111416 // For scalable vectors the scalable part is currently handled11417 // by individual targets, so we just use the known minimum size here.11418 ISD::OutputArg MyFlags(11419 Flags, Parts[j].getValueType().getSimpleVT(), VT, OrigArgTy, i,11420 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());11421 if (NumParts > 1 && j == 0)11422 MyFlags.Flags.setSplit();11423 else if (j != 0) {11424 MyFlags.Flags.setOrigAlign(Align(1));11425 if (j == NumParts - 1)11426 MyFlags.Flags.setSplitEnd();11427 }11428 11429 CLI.Outs.push_back(MyFlags);11430 CLI.OutVals.push_back(Parts[j]);11431 }11432 11433 if (NeedsRegBlock && Value == NumValues - 1)11434 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();11435 }11436 }11437 11438 SmallVector<SDValue, 4> InVals;11439 CLI.Chain = LowerCall(CLI, InVals);11440 11441 // Update CLI.InVals to use outside of this function.11442 CLI.InVals = InVals;11443 11444 // Verify that the target's LowerCall behaved as expected.11445 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&11446 "LowerCall didn't return a valid chain!");11447 assert((!CLI.IsTailCall || InVals.empty()) &&11448 "LowerCall emitted a return value for a tail call!");11449 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&11450 "LowerCall didn't emit the correct number of values!");11451 11452 // For a tail call, the return value is merely live-out and there aren't11453 // any nodes in the DAG representing it. Return a special value to11454 // indicate that a tail call has been emitted and no more Instructions11455 // should be processed in the current block.11456 if (CLI.IsTailCall) {11457 CLI.DAG.setRoot(CLI.Chain);11458 return std::make_pair(SDValue(), SDValue());11459 }11460 11461#ifndef NDEBUG11462 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {11463 assert(InVals[i].getNode() && "LowerCall emitted a null value!");11464 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&11465 "LowerCall emitted a value with the wrong type!");11466 }11467#endif11468 11469 SmallVector<SDValue, 4> ReturnValues;11470 if (!CanLowerReturn) {11471 // The instruction result is the result of loading from the11472 // hidden sret parameter.11473 MVT PtrVT = getPointerTy(DL, DL.getAllocaAddrSpace());11474 11475 unsigned NumValues = RetVTs.size();11476 ReturnValues.resize(NumValues);11477 SmallVector<SDValue, 4> Chains(NumValues);11478 11479 // An aggregate return value cannot wrap around the address space, so11480 // offsets to its parts don't wrap either.11481 MachineFunction &MF = CLI.DAG.getMachineFunction();11482 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);11483 for (unsigned i = 0; i < NumValues; ++i) {11484 SDValue Add = CLI.DAG.getMemBasePlusOffset(11485 DemoteStackSlot, CLI.DAG.getConstant(Offsets[i], CLI.DL, PtrVT),11486 CLI.DL, SDNodeFlags::NoUnsignedWrap);11487 SDValue L = CLI.DAG.getLoad(11488 RetVTs[i], CLI.DL, CLI.Chain, Add,11489 MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(),11490 DemoteStackIdx, Offsets[i]),11491 HiddenSRetAlign);11492 ReturnValues[i] = L;11493 Chains[i] = L.getValue(1);11494 }11495 11496 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);11497 } else {11498 // Collect the legal value parts into potentially illegal values11499 // that correspond to the original function's return values.11500 std::optional<ISD::NodeType> AssertOp;11501 if (CLI.RetSExt)11502 AssertOp = ISD::AssertSext;11503 else if (CLI.RetZExt)11504 AssertOp = ISD::AssertZext;11505 unsigned CurReg = 0;11506 for (EVT VT : RetVTs) {11507 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);11508 unsigned NumRegs =11509 getNumRegistersForCallingConv(Context, CLI.CallConv, VT);11510 11511 ReturnValues.push_back(getCopyFromParts(11512 CLI.DAG, CLI.DL, &InVals[CurReg], NumRegs, RegisterVT, VT, nullptr,11513 CLI.Chain, CLI.CallConv, AssertOp));11514 CurReg += NumRegs;11515 }11516 11517 // For a function returning void, there is no return value. We can't create11518 // such a node, so we just return a null return value in that case. In11519 // that case, nothing will actually look at the value.11520 if (ReturnValues.empty())11521 return std::make_pair(SDValue(), CLI.Chain);11522 }11523 11524 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,11525 CLI.DAG.getVTList(RetVTs), ReturnValues);11526 return std::make_pair(Res, CLI.Chain);11527}11528 11529/// Places new result values for the node in Results (their number11530/// and types must exactly match those of the original return values of11531/// the node), or leaves Results empty, which indicates that the node is not11532/// to be custom lowered after all.11533void TargetLowering::LowerOperationWrapper(SDNode *N,11534 SmallVectorImpl<SDValue> &Results,11535 SelectionDAG &DAG) const {11536 SDValue Res = LowerOperation(SDValue(N, 0), DAG);11537 11538 if (!Res.getNode())11539 return;11540 11541 // If the original node has one result, take the return value from11542 // LowerOperation as is. It might not be result number 0.11543 if (N->getNumValues() == 1) {11544 Results.push_back(Res);11545 return;11546 }11547 11548 // If the original node has multiple results, then the return node should11549 // have the same number of results.11550 assert((N->getNumValues() == Res->getNumValues()) &&11551 "Lowering returned the wrong number of results!");11552 11553 // Places new result values base on N result number.11554 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)11555 Results.push_back(Res.getValue(I));11556}11557 11558SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {11559 llvm_unreachable("LowerOperation not implemented for this target!");11560}11561 11562void SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V,11563 Register Reg,11564 ISD::NodeType ExtendType) {11565 SDValue Op = getNonRegisterValue(V);11566 assert((Op.getOpcode() != ISD::CopyFromReg ||11567 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&11568 "Copy from a reg to the same reg!");11569 assert(!Reg.isPhysical() && "Is a physreg");11570 11571 const TargetLowering &TLI = DAG.getTargetLoweringInfo();11572 // If this is an InlineAsm we have to match the registers required, not the11573 // notional registers required by the type.11574 11575 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),11576 std::nullopt); // This is not an ABI copy.11577 SDValue Chain = DAG.getEntryNode();11578 11579 if (ExtendType == ISD::ANY_EXTEND) {11580 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V);11581 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())11582 ExtendType = PreferredExtendIt->second;11583 }11584 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);11585 PendingExports.push_back(Chain);11586}11587 11588#include "llvm/CodeGen/SelectionDAGISel.h"11589 11590/// isOnlyUsedInEntryBlock - If the specified argument is only used in the11591/// entry block, return true. This includes arguments used by switches, since11592/// the switch may expand into multiple basic blocks.11593static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {11594 // With FastISel active, we may be splitting blocks, so force creation11595 // of virtual registers for all non-dead arguments.11596 if (FastISel)11597 return A->use_empty();11598 11599 const BasicBlock &Entry = A->getParent()->front();11600 for (const User *U : A->users())11601 if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))11602 return false; // Use not in entry block.11603 11604 return true;11605}11606 11607using ArgCopyElisionMapTy =11608 DenseMap<const Argument *,11609 std::pair<const AllocaInst *, const StoreInst *>>;11610 11611/// Scan the entry block of the function in FuncInfo for arguments that look11612/// like copies into a local alloca. Record any copied arguments in11613/// ArgCopyElisionCandidates.11614static void11615findArgumentCopyElisionCandidates(const DataLayout &DL,11616 FunctionLoweringInfo *FuncInfo,11617 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {11618 // Record the state of every static alloca used in the entry block. Argument11619 // allocas are all used in the entry block, so we need approximately as many11620 // entries as we have arguments.11621 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };11622 SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas;11623 unsigned NumArgs = FuncInfo->Fn->arg_size();11624 StaticAllocas.reserve(NumArgs * 2);11625 11626 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {11627 if (!V)11628 return nullptr;11629 V = V->stripPointerCasts();11630 const auto *AI = dyn_cast<AllocaInst>(V);11631 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))11632 return nullptr;11633 auto Iter = StaticAllocas.insert({AI, Unknown});11634 return &Iter.first->second;11635 };11636 11637 // Look for stores of arguments to static allocas. Look through bitcasts and11638 // GEPs to handle type coercions, as long as the alloca is fully initialized11639 // by the store. Any non-store use of an alloca escapes it and any subsequent11640 // unanalyzed store might write it.11641 // FIXME: Handle structs initialized with multiple stores.11642 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {11643 // Look for stores, and handle non-store uses conservatively.11644 const auto *SI = dyn_cast<StoreInst>(&I);11645 if (!SI) {11646 // We will look through cast uses, so ignore them completely.11647 if (I.isCast())11648 continue;11649 // Ignore debug info and pseudo op intrinsics, they don't escape or store11650 // to allocas.11651 if (I.isDebugOrPseudoInst())11652 continue;11653 // This is an unknown instruction. Assume it escapes or writes to all11654 // static alloca operands.11655 for (const Use &U : I.operands()) {11656 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))11657 *Info = StaticAllocaInfo::Clobbered;11658 }11659 continue;11660 }11661 11662 // If the stored value is a static alloca, mark it as escaped.11663 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))11664 *Info = StaticAllocaInfo::Clobbered;11665 11666 // Check if the destination is a static alloca.11667 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();11668 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);11669 if (!Info)11670 continue;11671 const AllocaInst *AI = cast<AllocaInst>(Dst);11672 11673 // Skip allocas that have been initialized or clobbered.11674 if (*Info != StaticAllocaInfo::Unknown)11675 continue;11676 11677 // Check if the stored value is an argument, and that this store fully11678 // initializes the alloca.11679 // If the argument type has padding bits we can't directly forward a pointer11680 // as the upper bits may contain garbage.11681 // Don't elide copies from the same argument twice.11682 const Value *Val = SI->getValueOperand()->stripPointerCasts();11683 const auto *Arg = dyn_cast<Argument>(Val);11684 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||11685 Arg->getType()->isEmptyTy() ||11686 DL.getTypeStoreSize(Arg->getType()) !=11687 DL.getTypeAllocSize(AI->getAllocatedType()) ||11688 !DL.typeSizeEqualsStoreSize(Arg->getType()) ||11689 ArgCopyElisionCandidates.count(Arg)) {11690 *Info = StaticAllocaInfo::Clobbered;11691 continue;11692 }11693 11694 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI11695 << '\n');11696 11697 // Mark this alloca and store for argument copy elision.11698 *Info = StaticAllocaInfo::Elidable;11699 ArgCopyElisionCandidates.insert({Arg, {AI, SI}});11700 11701 // Stop scanning if we've seen all arguments. This will happen early in -O011702 // builds, which is useful, because -O0 builds have large entry blocks and11703 // many allocas.11704 if (ArgCopyElisionCandidates.size() == NumArgs)11705 break;11706 }11707}11708 11709/// Try to elide argument copies from memory into a local alloca. Succeeds if11710/// ArgVal is a load from a suitable fixed stack object.11711static void tryToElideArgumentCopy(11712 FunctionLoweringInfo &FuncInfo, SmallVectorImpl<SDValue> &Chains,11713 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,11714 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,11715 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,11716 ArrayRef<SDValue> ArgVals, bool &ArgHasUses) {11717 // Check if this is a load from a fixed stack object.11718 auto *LNode = dyn_cast<LoadSDNode>(ArgVals[0]);11719 if (!LNode)11720 return;11721 auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());11722 if (!FINode)11723 return;11724 11725 // Check that the fixed stack object is the right size and alignment.11726 // Look at the alignment that the user wrote on the alloca instead of looking11727 // at the stack object.11728 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);11729 assert(ArgCopyIter != ArgCopyElisionCandidates.end());11730 const AllocaInst *AI = ArgCopyIter->second.first;11731 int FixedIndex = FINode->getIndex();11732 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];11733 int OldIndex = AllocaIndex;11734 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();11735 if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {11736 LLVM_DEBUG(11737 dbgs() << " argument copy elision failed due to bad fixed stack "11738 "object size\n");11739 return;11740 }11741 Align RequiredAlignment = AI->getAlign();11742 if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {11743 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "11744 "greater than stack argument alignment ("11745 << DebugStr(RequiredAlignment) << " vs "11746 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");11747 return;11748 }11749 11750 // Perform the elision. Delete the old stack object and replace its only use11751 // in the variable info map. Mark the stack object as mutable and aliased.11752 LLVM_DEBUG({11753 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'11754 << " Replacing frame index " << OldIndex << " with " << FixedIndex11755 << '\n';11756 });11757 MFI.RemoveStackObject(OldIndex);11758 MFI.setIsImmutableObjectIndex(FixedIndex, false);11759 MFI.setIsAliasedObjectIndex(FixedIndex, true);11760 AllocaIndex = FixedIndex;11761 ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});11762 for (SDValue ArgVal : ArgVals)11763 Chains.push_back(ArgVal.getValue(1));11764 11765 // Avoid emitting code for the store implementing the copy.11766 const StoreInst *SI = ArgCopyIter->second.second;11767 ElidedArgCopyInstrs.insert(SI);11768 11769 // Check for uses of the argument again so that we can avoid exporting ArgVal11770 // if it is't used by anything other than the store.11771 for (const Value *U : Arg.users()) {11772 if (U != SI) {11773 ArgHasUses = true;11774 break;11775 }11776 }11777}11778 11779void SelectionDAGISel::LowerArguments(const Function &F) {11780 SelectionDAG &DAG = SDB->DAG;11781 SDLoc dl = SDB->getCurSDLoc();11782 const DataLayout &DL = DAG.getDataLayout();11783 SmallVector<ISD::InputArg, 16> Ins;11784 11785 // In Naked functions we aren't going to save any registers.11786 if (F.hasFnAttribute(Attribute::Naked))11787 return;11788 11789 if (!FuncInfo->CanLowerReturn) {11790 // Put in an sret pointer parameter before all the other parameters.11791 MVT ValueVT = TLI->getPointerTy(DL, DL.getAllocaAddrSpace());11792 11793 ISD::ArgFlagsTy Flags;11794 Flags.setSRet();11795 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVT);11796 ISD::InputArg RetArg(Flags, RegisterVT, ValueVT, F.getReturnType(), true,11797 ISD::InputArg::NoArgIndex, 0);11798 Ins.push_back(RetArg);11799 }11800 11801 // Look for stores of arguments to static allocas. Mark such arguments with a11802 // flag to ask the target to give us the memory location of that argument if11803 // available.11804 ArgCopyElisionMapTy ArgCopyElisionCandidates;11805 findArgumentCopyElisionCandidates(DL, FuncInfo.get(),11806 ArgCopyElisionCandidates);11807 11808 // Set up the incoming argument description vector.11809 for (const Argument &Arg : F.args()) {11810 unsigned ArgNo = Arg.getArgNo();11811 SmallVector<Type *, 4> Types;11812 ComputeValueTypes(DAG.getDataLayout(), Arg.getType(), Types);11813 bool isArgValueUsed = !Arg.use_empty();11814 unsigned PartBase = 0;11815 Type *FinalType = Arg.getType();11816 if (Arg.hasAttribute(Attribute::ByVal))11817 FinalType = Arg.getParamByValType();11818 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(11819 FinalType, F.getCallingConv(), F.isVarArg(), DL);11820 for (unsigned Value = 0, NumValues = Types.size(); Value != NumValues;11821 ++Value) {11822 Type *ArgTy = Types[Value];11823 EVT VT = TLI->getValueType(DL, ArgTy);11824 ISD::ArgFlagsTy Flags;11825 11826 if (ArgTy->isPointerTy()) {11827 Flags.setPointer();11828 Flags.setPointerAddrSpace(cast<PointerType>(ArgTy)->getAddressSpace());11829 }11830 if (Arg.hasAttribute(Attribute::ZExt))11831 Flags.setZExt();11832 if (Arg.hasAttribute(Attribute::SExt))11833 Flags.setSExt();11834 if (Arg.hasAttribute(Attribute::InReg)) {11835 // If we are using vectorcall calling convention, a structure that is11836 // passed InReg - is surely an HVA11837 if (F.getCallingConv() == CallingConv::X86_VectorCall &&11838 isa<StructType>(Arg.getType())) {11839 // The first value of a structure is marked11840 if (0 == Value)11841 Flags.setHvaStart();11842 Flags.setHva();11843 }11844 // Set InReg Flag11845 Flags.setInReg();11846 }11847 if (Arg.hasAttribute(Attribute::StructRet))11848 Flags.setSRet();11849 if (Arg.hasAttribute(Attribute::SwiftSelf))11850 Flags.setSwiftSelf();11851 if (Arg.hasAttribute(Attribute::SwiftAsync))11852 Flags.setSwiftAsync();11853 if (Arg.hasAttribute(Attribute::SwiftError))11854 Flags.setSwiftError();11855 if (Arg.hasAttribute(Attribute::ByVal))11856 Flags.setByVal();11857 if (Arg.hasAttribute(Attribute::ByRef))11858 Flags.setByRef();11859 if (Arg.hasAttribute(Attribute::InAlloca)) {11860 Flags.setInAlloca();11861 // Set the byval flag for CCAssignFn callbacks that don't know about11862 // inalloca. This way we can know how many bytes we should've allocated11863 // and how many bytes a callee cleanup function will pop. If we port11864 // inalloca to more targets, we'll have to add custom inalloca handling11865 // in the various CC lowering callbacks.11866 Flags.setByVal();11867 }11868 if (Arg.hasAttribute(Attribute::Preallocated)) {11869 Flags.setPreallocated();11870 // Set the byval flag for CCAssignFn callbacks that don't know about11871 // preallocated. This way we can know how many bytes we should've11872 // allocated and how many bytes a callee cleanup function will pop. If11873 // we port preallocated to more targets, we'll have to add custom11874 // preallocated handling in the various CC lowering callbacks.11875 Flags.setByVal();11876 }11877 11878 // Certain targets (such as MIPS), may have a different ABI alignment11879 // for a type depending on the context. Give the target a chance to11880 // specify the alignment it wants.11881 const Align OriginalAlignment(11882 TLI->getABIAlignmentForCallingConv(ArgTy, DL));11883 Flags.setOrigAlign(OriginalAlignment);11884 11885 Align MemAlign;11886 Type *ArgMemTy = nullptr;11887 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||11888 Flags.isByRef()) {11889 if (!ArgMemTy)11890 ArgMemTy = Arg.getPointeeInMemoryValueType();11891 11892 uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);11893 11894 // For in-memory arguments, size and alignment should be passed from FE.11895 // BE will guess if this info is not there but there are cases it cannot11896 // get right.11897 if (auto ParamAlign = Arg.getParamStackAlign())11898 MemAlign = *ParamAlign;11899 else if ((ParamAlign = Arg.getParamAlign()))11900 MemAlign = *ParamAlign;11901 else11902 MemAlign = TLI->getByValTypeAlignment(ArgMemTy, DL);11903 if (Flags.isByRef())11904 Flags.setByRefSize(MemSize);11905 else11906 Flags.setByValSize(MemSize);11907 } else if (auto ParamAlign = Arg.getParamStackAlign()) {11908 MemAlign = *ParamAlign;11909 } else {11910 MemAlign = OriginalAlignment;11911 }11912 Flags.setMemAlign(MemAlign);11913 11914 if (Arg.hasAttribute(Attribute::Nest))11915 Flags.setNest();11916 if (NeedsRegBlock)11917 Flags.setInConsecutiveRegs();11918 if (ArgCopyElisionCandidates.count(&Arg))11919 Flags.setCopyElisionCandidate();11920 if (Arg.hasAttribute(Attribute::Returned))11921 Flags.setReturned();11922 11923 MVT RegisterVT = TLI->getRegisterTypeForCallingConv(11924 *CurDAG->getContext(), F.getCallingConv(), VT);11925 unsigned NumRegs = TLI->getNumRegistersForCallingConv(11926 *CurDAG->getContext(), F.getCallingConv(), VT);11927 for (unsigned i = 0; i != NumRegs; ++i) {11928 // For scalable vectors, use the minimum size; individual targets11929 // are responsible for handling scalable vector arguments and11930 // return values.11931 ISD::InputArg MyFlags(11932 Flags, RegisterVT, VT, ArgTy, isArgValueUsed, ArgNo,11933 PartBase + i * RegisterVT.getStoreSize().getKnownMinValue());11934 if (NumRegs > 1 && i == 0)11935 MyFlags.Flags.setSplit();11936 // if it isn't first piece, alignment must be 111937 else if (i > 0) {11938 MyFlags.Flags.setOrigAlign(Align(1));11939 if (i == NumRegs - 1)11940 MyFlags.Flags.setSplitEnd();11941 }11942 Ins.push_back(MyFlags);11943 }11944 if (NeedsRegBlock && Value == NumValues - 1)11945 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();11946 PartBase += VT.getStoreSize().getKnownMinValue();11947 }11948 }11949 11950 // Call the target to set up the argument values.11951 SmallVector<SDValue, 8> InVals;11952 SDValue NewRoot = TLI->LowerFormalArguments(11953 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);11954 11955 // Verify that the target's LowerFormalArguments behaved as expected.11956 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&11957 "LowerFormalArguments didn't return a valid chain!");11958 assert(InVals.size() == Ins.size() &&11959 "LowerFormalArguments didn't emit the correct number of values!");11960 LLVM_DEBUG({11961 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {11962 assert(InVals[i].getNode() &&11963 "LowerFormalArguments emitted a null value!");11964 assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&11965 "LowerFormalArguments emitted a value with the wrong type!");11966 }11967 });11968 11969 // Update the DAG with the new chain value resulting from argument lowering.11970 DAG.setRoot(NewRoot);11971 11972 // Set up the argument values.11973 unsigned i = 0;11974 if (!FuncInfo->CanLowerReturn) {11975 // Create a virtual register for the sret pointer, and put in a copy11976 // from the sret argument into it.11977 MVT VT = TLI->getPointerTy(DL, DL.getAllocaAddrSpace());11978 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);11979 std::optional<ISD::NodeType> AssertOp;11980 SDValue ArgValue =11981 getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, nullptr, NewRoot,11982 F.getCallingConv(), AssertOp);11983 11984 MachineFunction& MF = SDB->DAG.getMachineFunction();11985 MachineRegisterInfo& RegInfo = MF.getRegInfo();11986 Register SRetReg =11987 RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));11988 FuncInfo->DemoteRegister = SRetReg;11989 NewRoot =11990 SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);11991 DAG.setRoot(NewRoot);11992 11993 // i indexes lowered arguments. Bump it past the hidden sret argument.11994 ++i;11995 }11996 11997 SmallVector<SDValue, 4> Chains;11998 DenseMap<int, int> ArgCopyElisionFrameIndexMap;11999 for (const Argument &Arg : F.args()) {12000 SmallVector<SDValue, 4> ArgValues;12001 SmallVector<EVT, 4> ValueVTs;12002 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);12003 unsigned NumValues = ValueVTs.size();12004 if (NumValues == 0)12005 continue;12006 12007 bool ArgHasUses = !Arg.use_empty();12008 12009 // Elide the copying store if the target loaded this argument from a12010 // suitable fixed stack object.12011 if (Ins[i].Flags.isCopyElisionCandidate()) {12012 unsigned NumParts = 0;12013 for (EVT VT : ValueVTs)12014 NumParts += TLI->getNumRegistersForCallingConv(*CurDAG->getContext(),12015 F.getCallingConv(), VT);12016 12017 tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap,12018 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,12019 ArrayRef(&InVals[i], NumParts), ArgHasUses);12020 }12021 12022 // If this argument is unused then remember its value. It is used to generate12023 // debugging information.12024 bool isSwiftErrorArg =12025 TLI->supportSwiftError() &&12026 Arg.hasAttribute(Attribute::SwiftError);12027 if (!ArgHasUses && !isSwiftErrorArg) {12028 SDB->setUnusedArgValue(&Arg, InVals[i]);12029 12030 // Also remember any frame index for use in FastISel.12031 if (FrameIndexSDNode *FI =12032 dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))12033 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());12034 }12035 12036 for (unsigned Val = 0; Val != NumValues; ++Val) {12037 EVT VT = ValueVTs[Val];12038 MVT PartVT = TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(),12039 F.getCallingConv(), VT);12040 unsigned NumParts = TLI->getNumRegistersForCallingConv(12041 *CurDAG->getContext(), F.getCallingConv(), VT);12042 12043 // Even an apparent 'unused' swifterror argument needs to be returned. So12044 // we do generate a copy for it that can be used on return from the12045 // function.12046 if (ArgHasUses || isSwiftErrorArg) {12047 std::optional<ISD::NodeType> AssertOp;12048 if (Arg.hasAttribute(Attribute::SExt))12049 AssertOp = ISD::AssertSext;12050 else if (Arg.hasAttribute(Attribute::ZExt))12051 AssertOp = ISD::AssertZext;12052 12053 SDValue OutVal =12054 getCopyFromParts(DAG, dl, &InVals[i], NumParts, PartVT, VT, nullptr,12055 NewRoot, F.getCallingConv(), AssertOp);12056 12057 FPClassTest NoFPClass = Arg.getNoFPClass();12058 if (NoFPClass != fcNone) {12059 SDValue SDNoFPClass = DAG.getTargetConstant(12060 static_cast<uint64_t>(NoFPClass), dl, MVT::i32);12061 OutVal = DAG.getNode(ISD::AssertNoFPClass, dl, OutVal.getValueType(),12062 OutVal, SDNoFPClass);12063 }12064 ArgValues.push_back(OutVal);12065 }12066 12067 i += NumParts;12068 }12069 12070 // We don't need to do anything else for unused arguments.12071 if (ArgValues.empty())12072 continue;12073 12074 // Note down frame index.12075 if (FrameIndexSDNode *FI =12076 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))12077 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());12078 12079 SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues),12080 SDB->getCurSDLoc());12081 12082 SDB->setValue(&Arg, Res);12083 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {12084 // We want to associate the argument with the frame index, among12085 // involved operands, that correspond to the lowest address. The12086 // getCopyFromParts function, called earlier, is swapping the order of12087 // the operands to BUILD_PAIR depending on endianness. The result of12088 // that swapping is that the least significant bits of the argument will12089 // be in the first operand of the BUILD_PAIR node, and the most12090 // significant bits will be in the second operand.12091 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;12092 if (LoadSDNode *LNode =12093 dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))12094 if (FrameIndexSDNode *FI =12095 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))12096 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());12097 }12098 12099 // Analyses past this point are naive and don't expect an assertion.12100 if (Res.getOpcode() == ISD::AssertZext)12101 Res = Res.getOperand(0);12102 12103 // Update the SwiftErrorVRegDefMap.12104 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {12105 Register Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();12106 if (Reg.isVirtual())12107 SwiftError->setCurrentVReg(FuncInfo->MBB, SwiftError->getFunctionArg(),12108 Reg);12109 }12110 12111 // If this argument is live outside of the entry block, insert a copy from12112 // wherever we got it to the vreg that other BB's will reference it as.12113 if (Res.getOpcode() == ISD::CopyFromReg) {12114 // If we can, though, try to skip creating an unnecessary vreg.12115 // FIXME: This isn't very clean... it would be nice to make this more12116 // general.12117 Register Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();12118 if (Reg.isVirtual()) {12119 FuncInfo->ValueMap[&Arg] = Reg;12120 continue;12121 }12122 }12123 if (!isOnlyUsedInEntryBlock(&Arg, TM.Options.EnableFastISel)) {12124 FuncInfo->InitializeRegForValue(&Arg);12125 SDB->CopyToExportRegsIfNeeded(&Arg);12126 }12127 }12128 12129 if (!Chains.empty()) {12130 Chains.push_back(NewRoot);12131 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);12132 }12133 12134 DAG.setRoot(NewRoot);12135 12136 assert(i == InVals.size() && "Argument register count mismatch!");12137 12138 // If any argument copy elisions occurred and we have debug info, update the12139 // stale frame indices used in the dbg.declare variable info table.12140 if (!ArgCopyElisionFrameIndexMap.empty()) {12141 for (MachineFunction::VariableDbgInfo &VI :12142 MF->getInStackSlotVariableDbgInfo()) {12143 auto I = ArgCopyElisionFrameIndexMap.find(VI.getStackSlot());12144 if (I != ArgCopyElisionFrameIndexMap.end())12145 VI.updateStackSlot(I->second);12146 }12147 }12148 12149 // Finally, if the target has anything special to do, allow it to do so.12150 emitFunctionEntryCode();12151}12152 12153/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to12154/// ensure constants are generated when needed. Remember the virtual registers12155/// that need to be added to the Machine PHI nodes as input. We cannot just12156/// directly add them, because expansion might result in multiple MBB's for one12157/// BB. As such, the start of the BB might correspond to a different MBB than12158/// the end.12159void12160SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {12161 const TargetLowering &TLI = DAG.getTargetLoweringInfo();12162 12163 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;12164 12165 // Check PHI nodes in successors that expect a value to be available from this12166 // block.12167 for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) {12168 if (!isa<PHINode>(SuccBB->begin())) continue;12169 MachineBasicBlock *SuccMBB = FuncInfo.getMBB(SuccBB);12170 12171 // If this terminator has multiple identical successors (common for12172 // switches), only handle each succ once.12173 if (!SuccsHandled.insert(SuccMBB).second)12174 continue;12175 12176 MachineBasicBlock::iterator MBBI = SuccMBB->begin();12177 12178 // At this point we know that there is a 1-1 correspondence between LLVM PHI12179 // nodes and Machine PHI nodes, but the incoming operands have not been12180 // emitted yet.12181 for (const PHINode &PN : SuccBB->phis()) {12182 // Ignore dead phi's.12183 if (PN.use_empty())12184 continue;12185 12186 // Skip empty types12187 if (PN.getType()->isEmptyTy())12188 continue;12189 12190 Register Reg;12191 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);12192 12193 if (const auto *C = dyn_cast<Constant>(PHIOp)) {12194 Register &RegOut = ConstantsOut[C];12195 if (!RegOut) {12196 RegOut = FuncInfo.CreateRegs(&PN);12197 // We need to zero/sign extend ConstantInt phi operands to match12198 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.12199 ISD::NodeType ExtendType = ISD::ANY_EXTEND;12200 if (auto *CI = dyn_cast<ConstantInt>(C))12201 ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND12202 : ISD::ZERO_EXTEND;12203 CopyValueToVirtualRegister(C, RegOut, ExtendType);12204 }12205 Reg = RegOut;12206 } else {12207 DenseMap<const Value *, Register>::iterator I =12208 FuncInfo.ValueMap.find(PHIOp);12209 if (I != FuncInfo.ValueMap.end())12210 Reg = I->second;12211 else {12212 assert(isa<AllocaInst>(PHIOp) &&12213 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&12214 "Didn't codegen value into a register!??");12215 Reg = FuncInfo.CreateRegs(&PN);12216 CopyValueToVirtualRegister(PHIOp, Reg);12217 }12218 }12219 12220 // Remember that this register needs to added to the machine PHI node as12221 // the input for this MBB.12222 SmallVector<EVT, 4> ValueVTs;12223 ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);12224 for (EVT VT : ValueVTs) {12225 const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);12226 for (unsigned i = 0; i != NumRegisters; ++i)12227 FuncInfo.PHINodesToUpdate.emplace_back(&*MBBI++, Reg + i);12228 Reg += NumRegisters;12229 }12230 }12231 }12232 12233 ConstantsOut.clear();12234}12235 12236MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {12237 MachineFunction::iterator I(MBB);12238 if (++I == FuncInfo.MF->end())12239 return nullptr;12240 return &*I;12241}12242 12243/// During lowering new call nodes can be created (such as memset, etc.).12244/// Those will become new roots of the current DAG, but complications arise12245/// when they are tail calls. In such cases, the call lowering will update12246/// the root, but the builder still needs to know that a tail call has been12247/// lowered in order to avoid generating an additional return.12248void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {12249 // If the node is null, we do have a tail call.12250 if (MaybeTC.getNode() != nullptr)12251 DAG.setRoot(MaybeTC);12252 else12253 HasTailCall = true;12254}12255 12256void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,12257 MachineBasicBlock *SwitchMBB,12258 MachineBasicBlock *DefaultMBB) {12259 MachineFunction *CurMF = FuncInfo.MF;12260 MachineBasicBlock *NextMBB = nullptr;12261 MachineFunction::iterator BBI(W.MBB);12262 if (++BBI != FuncInfo.MF->end())12263 NextMBB = &*BBI;12264 12265 unsigned Size = W.LastCluster - W.FirstCluster + 1;12266 12267 BranchProbabilityInfo *BPI = FuncInfo.BPI;12268 12269 if (Size == 2 && W.MBB == SwitchMBB) {12270 // If any two of the cases has the same destination, and if one value12271 // is the same as the other, but has one bit unset that the other has set,12272 // use bit manipulation to do two compares at once. For example:12273 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"12274 // TODO: This could be extended to merge any 2 cases in switches with 312275 // cases.12276 // TODO: Handle cases where W.CaseBB != SwitchBB.12277 CaseCluster &Small = *W.FirstCluster;12278 CaseCluster &Big = *W.LastCluster;12279 12280 if (Small.Low == Small.High && Big.Low == Big.High &&12281 Small.MBB == Big.MBB) {12282 const APInt &SmallValue = Small.Low->getValue();12283 const APInt &BigValue = Big.Low->getValue();12284 12285 // Check that there is only one bit different.12286 APInt CommonBit = BigValue ^ SmallValue;12287 if (CommonBit.isPowerOf2()) {12288 SDValue CondLHS = getValue(Cond);12289 EVT VT = CondLHS.getValueType();12290 SDLoc DL = getCurSDLoc();12291 12292 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,12293 DAG.getConstant(CommonBit, DL, VT));12294 SDValue Cond = DAG.getSetCC(12295 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),12296 ISD::SETEQ);12297 12298 // Update successor info.12299 // Both Small and Big will jump to Small.BB, so we sum up the12300 // probabilities.12301 addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);12302 if (BPI)12303 addSuccessorWithProb(12304 SwitchMBB, DefaultMBB,12305 // The default destination is the first successor in IR.12306 BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));12307 else12308 addSuccessorWithProb(SwitchMBB, DefaultMBB);12309 12310 // Insert the true branch.12311 SDValue BrCond =12312 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,12313 DAG.getBasicBlock(Small.MBB));12314 // Insert the false branch.12315 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,12316 DAG.getBasicBlock(DefaultMBB));12317 12318 DAG.setRoot(BrCond);12319 return;12320 }12321 }12322 }12323 12324 if (TM.getOptLevel() != CodeGenOptLevel::None) {12325 // Here, we order cases by probability so the most likely case will be12326 // checked first. However, two clusters can have the same probability in12327 // which case their relative ordering is non-deterministic. So we use Low12328 // as a tie-breaker as clusters are guaranteed to never overlap.12329 llvm::sort(W.FirstCluster, W.LastCluster + 1,12330 [](const CaseCluster &a, const CaseCluster &b) {12331 return a.Prob != b.Prob ?12332 a.Prob > b.Prob :12333 a.Low->getValue().slt(b.Low->getValue());12334 });12335 12336 // Rearrange the case blocks so that the last one falls through if possible12337 // without changing the order of probabilities.12338 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {12339 --I;12340 if (I->Prob > W.LastCluster->Prob)12341 break;12342 if (I->Kind == CC_Range && I->MBB == NextMBB) {12343 std::swap(*I, *W.LastCluster);12344 break;12345 }12346 }12347 }12348 12349 // Compute total probability.12350 BranchProbability DefaultProb = W.DefaultProb;12351 BranchProbability UnhandledProbs = DefaultProb;12352 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)12353 UnhandledProbs += I->Prob;12354 12355 MachineBasicBlock *CurMBB = W.MBB;12356 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {12357 bool FallthroughUnreachable = false;12358 MachineBasicBlock *Fallthrough;12359 if (I == W.LastCluster) {12360 // For the last cluster, fall through to the default destination.12361 Fallthrough = DefaultMBB;12362 FallthroughUnreachable = isa<UnreachableInst>(12363 DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());12364 } else {12365 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());12366 CurMF->insert(BBI, Fallthrough);12367 // Put Cond in a virtual register to make it available from the new blocks.12368 ExportFromCurrentBlock(Cond);12369 }12370 UnhandledProbs -= I->Prob;12371 12372 switch (I->Kind) {12373 case CC_JumpTable: {12374 // FIXME: Optimize away range check based on pivot comparisons.12375 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;12376 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;12377 12378 // The jump block hasn't been inserted yet; insert it here.12379 MachineBasicBlock *JumpMBB = JT->MBB;12380 CurMF->insert(BBI, JumpMBB);12381 12382 auto JumpProb = I->Prob;12383 auto FallthroughProb = UnhandledProbs;12384 12385 // If the default statement is a target of the jump table, we evenly12386 // distribute the default probability to successors of CurMBB. Also12387 // update the probability on the edge from JumpMBB to Fallthrough.12388 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),12389 SE = JumpMBB->succ_end();12390 SI != SE; ++SI) {12391 if (*SI == DefaultMBB) {12392 JumpProb += DefaultProb / 2;12393 FallthroughProb -= DefaultProb / 2;12394 JumpMBB->setSuccProbability(SI, DefaultProb / 2);12395 JumpMBB->normalizeSuccProbs();12396 break;12397 }12398 }12399 12400 // If the default clause is unreachable, propagate that knowledge into12401 // JTH->FallthroughUnreachable which will use it to suppress the range12402 // check.12403 //12404 // However, don't do this if we're doing branch target enforcement,12405 // because a table branch _without_ a range check can be a tempting JOP12406 // gadget - out-of-bounds inputs that are impossible in correct12407 // execution become possible again if an attacker can influence the12408 // control flow. So if an attacker doesn't already have a BTI bypass12409 // available, we don't want them to be able to get one out of this12410 // table branch.12411 if (FallthroughUnreachable) {12412 Function &CurFunc = CurMF->getFunction();12413 if (!CurFunc.hasFnAttribute("branch-target-enforcement"))12414 JTH->FallthroughUnreachable = true;12415 }12416 12417 if (!JTH->FallthroughUnreachable)12418 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);12419 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);12420 CurMBB->normalizeSuccProbs();12421 12422 // The jump table header will be inserted in our current block, do the12423 // range check, and fall through to our fallthrough block.12424 JTH->HeaderBB = CurMBB;12425 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.12426 12427 // If we're in the right place, emit the jump table header right now.12428 if (CurMBB == SwitchMBB) {12429 visitJumpTableHeader(*JT, *JTH, SwitchMBB);12430 JTH->Emitted = true;12431 }12432 break;12433 }12434 case CC_BitTests: {12435 // FIXME: Optimize away range check based on pivot comparisons.12436 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];12437 12438 // The bit test blocks haven't been inserted yet; insert them here.12439 for (BitTestCase &BTC : BTB->Cases)12440 CurMF->insert(BBI, BTC.ThisBB);12441 12442 // Fill in fields of the BitTestBlock.12443 BTB->Parent = CurMBB;12444 BTB->Default = Fallthrough;12445 12446 BTB->DefaultProb = UnhandledProbs;12447 // If the cases in bit test don't form a contiguous range, we evenly12448 // distribute the probability on the edge to Fallthrough to two12449 // successors of CurMBB.12450 if (!BTB->ContiguousRange) {12451 BTB->Prob += DefaultProb / 2;12452 BTB->DefaultProb -= DefaultProb / 2;12453 }12454 12455 if (FallthroughUnreachable)12456 BTB->FallthroughUnreachable = true;12457 12458 // If we're in the right place, emit the bit test header right now.12459 if (CurMBB == SwitchMBB) {12460 visitBitTestHeader(*BTB, SwitchMBB);12461 BTB->Emitted = true;12462 }12463 break;12464 }12465 case CC_Range: {12466 const Value *RHS, *LHS, *MHS;12467 ISD::CondCode CC;12468 if (I->Low == I->High) {12469 // Check Cond == I->Low.12470 CC = ISD::SETEQ;12471 LHS = Cond;12472 RHS=I->Low;12473 MHS = nullptr;12474 } else {12475 // Check I->Low <= Cond <= I->High.12476 CC = ISD::SETLE;12477 LHS = I->Low;12478 MHS = Cond;12479 RHS = I->High;12480 }12481 12482 // If Fallthrough is unreachable, fold away the comparison.12483 if (FallthroughUnreachable)12484 CC = ISD::SETTRUE;12485 12486 // The false probability is the sum of all unhandled cases.12487 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,12488 getCurSDLoc(), I->Prob, UnhandledProbs);12489 12490 if (CurMBB == SwitchMBB)12491 visitSwitchCase(CB, SwitchMBB);12492 else12493 SL->SwitchCases.push_back(CB);12494 12495 break;12496 }12497 }12498 CurMBB = Fallthrough;12499 }12500}12501 12502void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,12503 const SwitchWorkListItem &W,12504 Value *Cond,12505 MachineBasicBlock *SwitchMBB) {12506 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&12507 "Clusters not sorted?");12508 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");12509 12510 auto [LastLeft, FirstRight, LeftProb, RightProb] =12511 SL->computeSplitWorkItemInfo(W);12512 12513 // Use the first element on the right as pivot since we will make less-than12514 // comparisons against it.12515 CaseClusterIt PivotCluster = FirstRight;12516 assert(PivotCluster > W.FirstCluster);12517 assert(PivotCluster <= W.LastCluster);12518 12519 CaseClusterIt FirstLeft = W.FirstCluster;12520 CaseClusterIt LastRight = W.LastCluster;12521 12522 const ConstantInt *Pivot = PivotCluster->Low;12523 12524 // New blocks will be inserted immediately after the current one.12525 MachineFunction::iterator BBI(W.MBB);12526 ++BBI;12527 12528 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,12529 // we can branch to its destination directly if it's squeezed exactly in12530 // between the known lower bound and Pivot - 1.12531 MachineBasicBlock *LeftMBB;12532 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&12533 FirstLeft->Low == W.GE &&12534 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {12535 LeftMBB = FirstLeft->MBB;12536 } else {12537 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());12538 FuncInfo.MF->insert(BBI, LeftMBB);12539 WorkList.push_back(12540 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});12541 // Put Cond in a virtual register to make it available from the new blocks.12542 ExportFromCurrentBlock(Cond);12543 }12544 12545 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a12546 // single cluster, RHS.Low == Pivot, and we can branch to its destination12547 // directly if RHS.High equals the current upper bound.12548 MachineBasicBlock *RightMBB;12549 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&12550 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {12551 RightMBB = FirstRight->MBB;12552 } else {12553 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());12554 FuncInfo.MF->insert(BBI, RightMBB);12555 WorkList.push_back(12556 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});12557 // Put Cond in a virtual register to make it available from the new blocks.12558 ExportFromCurrentBlock(Cond);12559 }12560 12561 // Create the CaseBlock record that will be used to lower the branch.12562 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,12563 getCurSDLoc(), LeftProb, RightProb);12564 12565 if (W.MBB == SwitchMBB)12566 visitSwitchCase(CB, SwitchMBB);12567 else12568 SL->SwitchCases.push_back(CB);12569}12570 12571// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb12572// from the swith statement.12573static BranchProbability scaleCaseProbality(BranchProbability CaseProb,12574 BranchProbability PeeledCaseProb) {12575 if (PeeledCaseProb == BranchProbability::getOne())12576 return BranchProbability::getZero();12577 BranchProbability SwitchProb = PeeledCaseProb.getCompl();12578 12579 uint32_t Numerator = CaseProb.getNumerator();12580 uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());12581 return BranchProbability(Numerator, std::max(Numerator, Denominator));12582}12583 12584// Try to peel the top probability case if it exceeds the threshold.12585// Return current MachineBasicBlock for the switch statement if the peeling12586// does not occur.12587// If the peeling is performed, return the newly created MachineBasicBlock12588// for the peeled switch statement. Also update Clusters to remove the peeled12589// case. PeeledCaseProb is the BranchProbability for the peeled case.12590MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(12591 const SwitchInst &SI, CaseClusterVector &Clusters,12592 BranchProbability &PeeledCaseProb) {12593 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;12594 // Don't perform if there is only one cluster or optimizing for size.12595 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||12596 TM.getOptLevel() == CodeGenOptLevel::None ||12597 SwitchMBB->getParent()->getFunction().hasMinSize())12598 return SwitchMBB;12599 12600 BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);12601 unsigned PeeledCaseIndex = 0;12602 bool SwitchPeeled = false;12603 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {12604 CaseCluster &CC = Clusters[Index];12605 if (CC.Prob < TopCaseProb)12606 continue;12607 TopCaseProb = CC.Prob;12608 PeeledCaseIndex = Index;12609 SwitchPeeled = true;12610 }12611 if (!SwitchPeeled)12612 return SwitchMBB;12613 12614 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "12615 << TopCaseProb << "\n");12616 12617 // Record the MBB for the peeled switch statement.12618 MachineFunction::iterator BBI(SwitchMBB);12619 ++BBI;12620 MachineBasicBlock *PeeledSwitchMBB =12621 FuncInfo.MF->CreateMachineBasicBlock(SwitchMBB->getBasicBlock());12622 FuncInfo.MF->insert(BBI, PeeledSwitchMBB);12623 12624 ExportFromCurrentBlock(SI.getCondition());12625 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;12626 SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,12627 nullptr, nullptr, TopCaseProb.getCompl()};12628 lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);12629 12630 Clusters.erase(PeeledCaseIt);12631 for (CaseCluster &CC : Clusters) {12632 LLVM_DEBUG(12633 dbgs() << "Scale the probablity for one cluster, before scaling: "12634 << CC.Prob << "\n");12635 CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);12636 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");12637 }12638 PeeledCaseProb = TopCaseProb;12639 return PeeledSwitchMBB;12640}12641 12642void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {12643 // Extract cases from the switch.12644 BranchProbabilityInfo *BPI = FuncInfo.BPI;12645 CaseClusterVector Clusters;12646 Clusters.reserve(SI.getNumCases());12647 for (auto I : SI.cases()) {12648 MachineBasicBlock *Succ = FuncInfo.getMBB(I.getCaseSuccessor());12649 const ConstantInt *CaseVal = I.getCaseValue();12650 BranchProbability Prob =12651 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())12652 : BranchProbability(1, SI.getNumCases() + 1);12653 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));12654 }12655 12656 MachineBasicBlock *DefaultMBB = FuncInfo.getMBB(SI.getDefaultDest());12657 12658 // Cluster adjacent cases with the same destination. We do this at all12659 // optimization levels because it's cheap to do and will make codegen faster12660 // if there are many clusters.12661 sortAndRangeify(Clusters);12662 12663 // The branch probablity of the peeled case.12664 BranchProbability PeeledCaseProb = BranchProbability::getZero();12665 MachineBasicBlock *PeeledSwitchMBB =12666 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);12667 12668 // If there is only the default destination, jump there directly.12669 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;12670 if (Clusters.empty()) {12671 assert(PeeledSwitchMBB == SwitchMBB);12672 SwitchMBB->addSuccessor(DefaultMBB);12673 if (DefaultMBB != NextBlock(SwitchMBB)) {12674 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,12675 getControlRoot(), DAG.getBasicBlock(DefaultMBB)));12676 }12677 return;12678 }12679 12680 SL->findJumpTables(Clusters, &SI, getCurSDLoc(), DefaultMBB, DAG.getPSI(),12681 DAG.getBFI());12682 SL->findBitTestClusters(Clusters, &SI);12683 12684 LLVM_DEBUG({12685 dbgs() << "Case clusters: ";12686 for (const CaseCluster &C : Clusters) {12687 if (C.Kind == CC_JumpTable)12688 dbgs() << "JT:";12689 if (C.Kind == CC_BitTests)12690 dbgs() << "BT:";12691 12692 C.Low->getValue().print(dbgs(), true);12693 if (C.Low != C.High) {12694 dbgs() << '-';12695 C.High->getValue().print(dbgs(), true);12696 }12697 dbgs() << ' ';12698 }12699 dbgs() << '\n';12700 });12701 12702 assert(!Clusters.empty());12703 SwitchWorkList WorkList;12704 CaseClusterIt First = Clusters.begin();12705 CaseClusterIt Last = Clusters.end() - 1;12706 auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);12707 // Scale the branchprobability for DefaultMBB if the peel occurs and12708 // DefaultMBB is not replaced.12709 if (PeeledCaseProb != BranchProbability::getZero() &&12710 DefaultMBB == FuncInfo.getMBB(SI.getDefaultDest()))12711 DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);12712 WorkList.push_back(12713 {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});12714 12715 while (!WorkList.empty()) {12716 SwitchWorkListItem W = WorkList.pop_back_val();12717 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;12718 12719 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOptLevel::None &&12720 !DefaultMBB->getParent()->getFunction().hasMinSize()) {12721 // For optimized builds, lower large range as a balanced binary tree.12722 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);12723 continue;12724 }12725 12726 lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);12727 }12728}12729 12730void SelectionDAGBuilder::visitStepVector(const CallInst &I) {12731 const TargetLowering &TLI = DAG.getTargetLoweringInfo();12732 auto DL = getCurSDLoc();12733 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());12734 setValue(&I, DAG.getStepVector(DL, ResultVT));12735}12736 12737void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {12738 const TargetLowering &TLI = DAG.getTargetLoweringInfo();12739 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());12740 12741 SDLoc DL = getCurSDLoc();12742 SDValue V = getValue(I.getOperand(0));12743 assert(VT == V.getValueType() && "Malformed vector.reverse!");12744 12745 if (VT.isScalableVector()) {12746 setValue(&I, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V));12747 return;12748 }12749 12750 // Use VECTOR_SHUFFLE for the fixed-length vector12751 // to maintain existing behavior.12752 SmallVector<int, 8> Mask;12753 unsigned NumElts = VT.getVectorMinNumElements();12754 for (unsigned i = 0; i != NumElts; ++i)12755 Mask.push_back(NumElts - 1 - i);12756 12757 setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));12758}12759 12760void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,12761 unsigned Factor) {12762 auto DL = getCurSDLoc();12763 SDValue InVec = getValue(I.getOperand(0));12764 12765 SmallVector<EVT, 4> ValueVTs;12766 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),12767 ValueVTs);12768 12769 EVT OutVT = ValueVTs[0];12770 unsigned OutNumElts = OutVT.getVectorMinNumElements();12771 12772 SmallVector<SDValue, 4> SubVecs(Factor);12773 for (unsigned i = 0; i != Factor; ++i) {12774 assert(ValueVTs[i] == OutVT && "Expected VTs to be the same");12775 SubVecs[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,12776 DAG.getVectorIdxConstant(OutNumElts * i, DL));12777 }12778 12779 // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit12780 // from existing legalisation and combines.12781 if (OutVT.isFixedLengthVector() && Factor == 2) {12782 SDValue Even = DAG.getVectorShuffle(OutVT, DL, SubVecs[0], SubVecs[1],12783 createStrideMask(0, 2, OutNumElts));12784 SDValue Odd = DAG.getVectorShuffle(OutVT, DL, SubVecs[0], SubVecs[1],12785 createStrideMask(1, 2, OutNumElts));12786 SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc());12787 setValue(&I, Res);12788 return;12789 }12790 12791 SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,12792 DAG.getVTList(ValueVTs), SubVecs);12793 setValue(&I, Res);12794}12795 12796void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,12797 unsigned Factor) {12798 auto DL = getCurSDLoc();12799 const TargetLowering &TLI = DAG.getTargetLoweringInfo();12800 EVT InVT = getValue(I.getOperand(0)).getValueType();12801 EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType());12802 12803 SmallVector<SDValue, 8> InVecs(Factor);12804 for (unsigned i = 0; i < Factor; ++i) {12805 InVecs[i] = getValue(I.getOperand(i));12806 assert(InVecs[i].getValueType() == InVecs[0].getValueType() &&12807 "Expected VTs to be the same");12808 }12809 12810 // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit12811 // from existing legalisation and combines.12812 if (OutVT.isFixedLengthVector() && Factor == 2) {12813 unsigned NumElts = InVT.getVectorMinNumElements();12814 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVecs);12815 setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT),12816 createInterleaveMask(NumElts, 2)));12817 return;12818 }12819 12820 SmallVector<EVT, 8> ValueVTs(Factor, InVT);12821 SDValue Res =12822 DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, DAG.getVTList(ValueVTs), InVecs);12823 12824 SmallVector<SDValue, 8> Results(Factor);12825 for (unsigned i = 0; i < Factor; ++i)12826 Results[i] = Res.getValue(i);12827 12828 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Results);12829 setValue(&I, Res);12830}12831 12832void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {12833 SmallVector<EVT, 4> ValueVTs;12834 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),12835 ValueVTs);12836 unsigned NumValues = ValueVTs.size();12837 if (NumValues == 0) return;12838 12839 SmallVector<SDValue, 4> Values(NumValues);12840 SDValue Op = getValue(I.getOperand(0));12841 12842 for (unsigned i = 0; i != NumValues; ++i)12843 Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i],12844 SDValue(Op.getNode(), Op.getResNo() + i));12845 12846 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),12847 DAG.getVTList(ValueVTs), Values));12848}12849 12850void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {12851 const TargetLowering &TLI = DAG.getTargetLoweringInfo();12852 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());12853 12854 SDLoc DL = getCurSDLoc();12855 SDValue V1 = getValue(I.getOperand(0));12856 SDValue V2 = getValue(I.getOperand(1));12857 int64_t Imm = cast<ConstantInt>(I.getOperand(2))->getSExtValue();12858 12859 // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.12860 if (VT.isScalableVector()) {12861 setValue(12862 &I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,12863 DAG.getSignedConstant(12864 Imm, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));12865 return;12866 }12867 12868 unsigned NumElts = VT.getVectorNumElements();12869 12870 uint64_t Idx = (NumElts + Imm) % NumElts;12871 12872 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.12873 SmallVector<int, 8> Mask;12874 for (unsigned i = 0; i < NumElts; ++i)12875 Mask.push_back(Idx + i);12876 setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));12877}12878 12879// Consider the following MIR after SelectionDAG, which produces output in12880// phyregs in the first case or virtregs in the second case.12881//12882// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx12883// %5:gr32 = COPY $ebx12884// %6:gr32 = COPY $edx12885// %1:gr32 = COPY %6:gr3212886// %0:gr32 = COPY %5:gr3212887//12888// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr3212889// %1:gr32 = COPY %6:gr3212890// %0:gr32 = COPY %5:gr3212891//12892// Given %0, we'd like to return $ebx in the first case and %5 in the second.12893// Given %1, we'd like to return $edx in the first case and %6 in the second.12894//12895// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap12896// to a single virtreg (such as %0). The remaining outputs monotonically12897// increase in virtreg number from there. If a callbr has no outputs, then it12898// should not have a corresponding callbr landingpad; in fact, the callbr12899// landingpad would not even be able to refer to such a callbr.12900static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) {12901 MachineInstr *MI = MRI.def_begin(Reg)->getParent();12902 // There is definitely at least one copy.12903 assert(MI->getOpcode() == TargetOpcode::COPY &&12904 "start of copy chain MUST be COPY");12905 Reg = MI->getOperand(1).getReg();12906 12907 // If the copied register in the first copy must be virtual.12908 assert(Reg.isVirtual() && "expected COPY of virtual register");12909 MI = MRI.def_begin(Reg)->getParent();12910 12911 // There may be an optional second copy.12912 if (MI->getOpcode() == TargetOpcode::COPY) {12913 assert(Reg.isVirtual() && "expected COPY of virtual register");12914 Reg = MI->getOperand(1).getReg();12915 assert(Reg.isPhysical() && "expected COPY of physical register");12916 } else {12917 // The start of the chain must be an INLINEASM_BR.12918 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&12919 "end of copy chain MUST be INLINEASM_BR");12920 }12921 12922 return Reg;12923}12924 12925// We must do this walk rather than the simpler12926// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));12927// otherwise we will end up with copies of virtregs only valid along direct12928// edges.12929void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {12930 SmallVector<EVT, 8> ResultVTs;12931 SmallVector<SDValue, 8> ResultValues;12932 const auto *CBR =12933 cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator());12934 12935 const TargetLowering &TLI = DAG.getTargetLoweringInfo();12936 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();12937 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();12938 12939 Register InitialDef = FuncInfo.ValueMap[CBR];12940 SDValue Chain = DAG.getRoot();12941 12942 // Re-parse the asm constraints string.12943 TargetLowering::AsmOperandInfoVector TargetConstraints =12944 TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR);12945 for (auto &T : TargetConstraints) {12946 SDISelAsmOperandInfo OpInfo(T);12947 if (OpInfo.Type != InlineAsm::isOutput)12948 continue;12949 12950 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the12951 // individual constraint.12952 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);12953 12954 switch (OpInfo.ConstraintType) {12955 case TargetLowering::C_Register:12956 case TargetLowering::C_RegisterClass: {12957 // Fill in OpInfo.AssignedRegs.Regs.12958 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo);12959 12960 // getRegistersForValue may produce 1 to many registers based on whether12961 // the OpInfo.ConstraintVT is legal on the target or not.12962 for (Register &Reg : OpInfo.AssignedRegs.Regs) {12963 Register OriginalDef = FollowCopyChain(MRI, InitialDef++);12964 if (OriginalDef.isPhysical())12965 FuncInfo.MBB->addLiveIn(OriginalDef);12966 // Update the assigned registers to use the original defs.12967 Reg = OriginalDef;12968 }12969 12970 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(12971 DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR);12972 ResultValues.push_back(V);12973 ResultVTs.push_back(OpInfo.ConstraintVT);12974 break;12975 }12976 case TargetLowering::C_Other: {12977 SDValue Flag;12978 SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(),12979 OpInfo, DAG);12980 ++InitialDef;12981 ResultValues.push_back(V);12982 ResultVTs.push_back(OpInfo.ConstraintVT);12983 break;12984 }12985 default:12986 break;12987 }12988 }12989 SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),12990 DAG.getVTList(ResultVTs), ResultValues);12991 setValue(&I, V);12992}12993