4007 lines · cpp
1//===-- HexagonISelLowering.cpp - Hexagon DAG Lowering Implementation -----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the interfaces that Hexagon uses to lower LLVM code10// into a selection DAG.11//12//===----------------------------------------------------------------------===//13 14#include "HexagonISelLowering.h"15#include "Hexagon.h"16#include "HexagonMachineFunctionInfo.h"17#include "HexagonRegisterInfo.h"18#include "HexagonSubtarget.h"19#include "HexagonTargetMachine.h"20#include "HexagonTargetObjectFile.h"21#include "llvm/ADT/APInt.h"22#include "llvm/ADT/ArrayRef.h"23#include "llvm/ADT/SmallVector.h"24#include "llvm/ADT/StringSwitch.h"25#include "llvm/CodeGen/CallingConvLower.h"26#include "llvm/CodeGen/MachineFrameInfo.h"27#include "llvm/CodeGen/MachineFunction.h"28#include "llvm/CodeGen/MachineMemOperand.h"29#include "llvm/CodeGen/MachineRegisterInfo.h"30#include "llvm/CodeGen/SelectionDAG.h"31#include "llvm/CodeGen/TargetCallingConv.h"32#include "llvm/CodeGen/ValueTypes.h"33#include "llvm/IR/BasicBlock.h"34#include "llvm/IR/CallingConv.h"35#include "llvm/IR/DataLayout.h"36#include "llvm/IR/DerivedTypes.h"37#include "llvm/IR/DiagnosticInfo.h"38#include "llvm/IR/DiagnosticPrinter.h"39#include "llvm/IR/Function.h"40#include "llvm/IR/GlobalValue.h"41#include "llvm/IR/IRBuilder.h"42#include "llvm/IR/InlineAsm.h"43#include "llvm/IR/Instructions.h"44#include "llvm/IR/IntrinsicInst.h"45#include "llvm/IR/Intrinsics.h"46#include "llvm/IR/IntrinsicsHexagon.h"47#include "llvm/IR/Module.h"48#include "llvm/IR/Type.h"49#include "llvm/IR/Value.h"50#include "llvm/Support/Casting.h"51#include "llvm/Support/CodeGen.h"52#include "llvm/Support/CommandLine.h"53#include "llvm/Support/Debug.h"54#include "llvm/Support/ErrorHandling.h"55#include "llvm/Support/MathExtras.h"56#include "llvm/Support/raw_ostream.h"57#include "llvm/Target/TargetMachine.h"58#include <algorithm>59#include <cassert>60#include <cstdint>61#include <limits>62#include <utility>63 64using namespace llvm;65 66#define DEBUG_TYPE "hexagon-lowering"67 68static cl::opt<bool> EmitJumpTables("hexagon-emit-jump-tables",69 cl::init(true), cl::Hidden,70 cl::desc("Control jump table emission on Hexagon target"));71 72static cl::opt<bool>73 EnableHexSDNodeSched("enable-hexagon-sdnode-sched", cl::Hidden,74 cl::desc("Enable Hexagon SDNode scheduling"));75 76static cl::opt<int> MinimumJumpTables("minimum-jump-tables", cl::Hidden,77 cl::init(5),78 cl::desc("Set minimum jump tables"));79 80static cl::opt<int>81 MaxStoresPerMemcpyCL("max-store-memcpy", cl::Hidden, cl::init(6),82 cl::desc("Max #stores to inline memcpy"));83 84static cl::opt<int>85 MaxStoresPerMemcpyOptSizeCL("max-store-memcpy-Os", cl::Hidden, cl::init(4),86 cl::desc("Max #stores to inline memcpy"));87 88static cl::opt<int>89 MaxStoresPerMemmoveCL("max-store-memmove", cl::Hidden, cl::init(6),90 cl::desc("Max #stores to inline memmove"));91 92static cl::opt<int>93 MaxStoresPerMemmoveOptSizeCL("max-store-memmove-Os", cl::Hidden,94 cl::init(4),95 cl::desc("Max #stores to inline memmove"));96 97static cl::opt<int>98 MaxStoresPerMemsetCL("max-store-memset", cl::Hidden, cl::init(8),99 cl::desc("Max #stores to inline memset"));100 101static cl::opt<int>102 MaxStoresPerMemsetOptSizeCL("max-store-memset-Os", cl::Hidden, cl::init(4),103 cl::desc("Max #stores to inline memset"));104 105static cl::opt<bool>106 ConstantLoadsToImm("constant-loads-to-imm", cl::Hidden, cl::init(true),107 cl::desc("Convert constant loads to immediate values."));108 109static cl::opt<bool> AlignLoads("hexagon-align-loads",110 cl::Hidden, cl::init(false),111 cl::desc("Rewrite unaligned loads as a pair of aligned loads"));112 113static cl::opt<bool>114 DisableArgsMinAlignment("hexagon-disable-args-min-alignment", cl::Hidden,115 cl::init(false),116 cl::desc("Disable minimum alignment of 1 for "117 "arguments passed by value on stack"));118 119// Implement calling convention for Hexagon.120 121static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT,122 CCValAssign::LocInfo &LocInfo,123 ISD::ArgFlagsTy &ArgFlags, CCState &State) {124 static const MCPhysReg ArgRegs[] = {125 Hexagon::R0, Hexagon::R1, Hexagon::R2,126 Hexagon::R3, Hexagon::R4, Hexagon::R5127 };128 const unsigned NumArgRegs = std::size(ArgRegs);129 unsigned RegNum = State.getFirstUnallocated(ArgRegs);130 131 // RegNum is an index into ArgRegs: skip a register if RegNum is odd.132 if (RegNum != NumArgRegs && RegNum % 2 == 1)133 State.AllocateReg(ArgRegs[RegNum]);134 135 // Always return false here, as this function only makes sure that the first136 // unallocated register has an even register number and does not actually137 // allocate a register for the current argument.138 return false;139}140 141#include "HexagonGenCallingConv.inc"142 143unsigned HexagonTargetLowering::getVectorTypeBreakdownForCallingConv(144 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,145 unsigned &NumIntermediates, MVT &RegisterVT) const {146 147 bool isBoolVector = VT.getVectorElementType() == MVT::i1;148 bool isPowerOf2 = VT.isPow2VectorType();149 unsigned NumElts = VT.getVectorNumElements();150 151 // Split vectors of type vXi1 into (X/8) vectors of type v8i1,152 // where X is divisible by 8.153 if (isBoolVector && !Subtarget.useHVXOps() && isPowerOf2 && NumElts >= 8) {154 RegisterVT = MVT::v8i8;155 IntermediateVT = MVT::v8i1;156 NumIntermediates = NumElts / 8;157 return NumIntermediates;158 }159 160 // In HVX 64-byte mode, vectors of type vXi1 are split into (X / 64) vectors161 // of type v64i1, provided that X is divisible by 64.162 if (isBoolVector && Subtarget.useHVX64BOps() && isPowerOf2 && NumElts >= 64) {163 RegisterVT = MVT::v64i8;164 IntermediateVT = MVT::v64i1;165 NumIntermediates = NumElts / 64;166 return NumIntermediates;167 }168 169 // In HVX 128-byte mode, vectors of type vXi1 are split into (X / 128) vectors170 // of type v128i1, provided that X is divisible by 128.171 if (isBoolVector && Subtarget.useHVX128BOps() && isPowerOf2 &&172 NumElts >= 128) {173 RegisterVT = MVT::v128i8;174 IntermediateVT = MVT::v128i1;175 NumIntermediates = NumElts / 128;176 return NumIntermediates;177 }178 179 return TargetLowering::getVectorTypeBreakdownForCallingConv(180 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);181}182 183std::pair<MVT, unsigned>184HexagonTargetLowering::handleMaskRegisterForCallingConv(185 const HexagonSubtarget &Subtarget, EVT VT) const {186 assert(VT.getVectorElementType() == MVT::i1);187 188 const unsigned NumElems = VT.getVectorNumElements();189 190 if (!VT.isPow2VectorType())191 return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};192 193 if (!Subtarget.useHVXOps() && NumElems >= 8)194 return {MVT::v8i8, NumElems / 8};195 196 if (Subtarget.useHVX64BOps() && NumElems >= 64)197 return {MVT::v64i8, NumElems / 64};198 199 if (Subtarget.useHVX128BOps() && NumElems >= 128)200 return {MVT::v128i8, NumElems / 128};201 202 return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};203}204 205MVT HexagonTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,206 CallingConv::ID CC,207 EVT VT) const {208 209 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {210 auto [RegisterVT, NumRegisters] =211 handleMaskRegisterForCallingConv(Subtarget, VT);212 if (RegisterVT != MVT::INVALID_SIMPLE_VALUE_TYPE)213 return RegisterVT;214 }215 216 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);217}218 219SDValue220HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG)221 const {222 unsigned IntNo = Op.getConstantOperandVal(0);223 SDLoc dl(Op);224 switch (IntNo) {225 default:226 return SDValue(); // Don't custom lower most intrinsics.227 case Intrinsic::thread_pointer: {228 EVT PtrVT = getPointerTy(DAG.getDataLayout());229 return DAG.getNode(HexagonISD::THREAD_POINTER, dl, PtrVT);230 }231 }232}233 234/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified235/// by "Src" to address "Dst" of size "Size". Alignment information is236/// specified by the specific parameter attribute. The copy will be passed as237/// a byval function parameter. Sometimes what we are copying is the end of a238/// larger object, the part that does not fit in registers.239static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,240 SDValue Chain, ISD::ArgFlagsTy Flags,241 SelectionDAG &DAG, const SDLoc &dl) {242 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);243 return DAG.getMemcpy(244 Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(),245 /*isVolatile=*/false, /*AlwaysInline=*/false,246 /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), MachinePointerInfo());247}248 249bool250HexagonTargetLowering::CanLowerReturn(251 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,252 const SmallVectorImpl<ISD::OutputArg> &Outs,253 LLVMContext &Context, const Type *RetTy) const {254 SmallVector<CCValAssign, 16> RVLocs;255 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);256 257 if (MF.getSubtarget<HexagonSubtarget>().useHVXOps())258 return CCInfo.CheckReturn(Outs, RetCC_Hexagon_HVX);259 return CCInfo.CheckReturn(Outs, RetCC_Hexagon);260}261 262// LowerReturn - Lower ISD::RET. If a struct is larger than 8 bytes and is263// passed by value, the function prototype is modified to return void and264// the value is stored in memory pointed by a pointer passed by caller.265SDValue266HexagonTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,267 bool IsVarArg,268 const SmallVectorImpl<ISD::OutputArg> &Outs,269 const SmallVectorImpl<SDValue> &OutVals,270 const SDLoc &dl, SelectionDAG &DAG) const {271 // CCValAssign - represent the assignment of the return value to locations.272 SmallVector<CCValAssign, 16> RVLocs;273 274 // CCState - Info about the registers and stack slot.275 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,276 *DAG.getContext());277 278 // Analyze return values of ISD::RET279 if (Subtarget.useHVXOps())280 CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon_HVX);281 else282 CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon);283 284 SDValue Glue;285 SmallVector<SDValue, 4> RetOps(1, Chain);286 287 // Copy the result values into the output registers.288 for (unsigned i = 0; i != RVLocs.size(); ++i) {289 CCValAssign &VA = RVLocs[i];290 SDValue Val = OutVals[i];291 292 switch (VA.getLocInfo()) {293 default:294 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.295 llvm_unreachable("Unknown loc info!");296 case CCValAssign::Full:297 break;298 case CCValAssign::BCvt:299 Val = DAG.getBitcast(VA.getLocVT(), Val);300 break;301 case CCValAssign::SExt:302 Val = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Val);303 break;304 case CCValAssign::ZExt:305 Val = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Val);306 break;307 case CCValAssign::AExt:308 Val = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Val);309 break;310 }311 312 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Val, Glue);313 314 // Guarantee that all emitted copies are stuck together with flags.315 Glue = Chain.getValue(1);316 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));317 }318 319 RetOps[0] = Chain; // Update chain.320 321 // Add the glue if we have it.322 if (Glue.getNode())323 RetOps.push_back(Glue);324 325 return DAG.getNode(HexagonISD::RET_GLUE, dl, MVT::Other, RetOps);326}327 328bool HexagonTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {329 // If either no tail call or told not to tail call at all, don't.330 return CI->isTailCall();331}332 333Register HexagonTargetLowering::getRegisterByName(334 const char* RegName, LLT VT, const MachineFunction &) const {335 // Just support r19, the linux kernel uses it.336 Register Reg = StringSwitch<Register>(RegName)337 .Case("r0", Hexagon::R0)338 .Case("r1", Hexagon::R1)339 .Case("r2", Hexagon::R2)340 .Case("r3", Hexagon::R3)341 .Case("r4", Hexagon::R4)342 .Case("r5", Hexagon::R5)343 .Case("r6", Hexagon::R6)344 .Case("r7", Hexagon::R7)345 .Case("r8", Hexagon::R8)346 .Case("r9", Hexagon::R9)347 .Case("r10", Hexagon::R10)348 .Case("r11", Hexagon::R11)349 .Case("r12", Hexagon::R12)350 .Case("r13", Hexagon::R13)351 .Case("r14", Hexagon::R14)352 .Case("r15", Hexagon::R15)353 .Case("r16", Hexagon::R16)354 .Case("r17", Hexagon::R17)355 .Case("r18", Hexagon::R18)356 .Case("r19", Hexagon::R19)357 .Case("r20", Hexagon::R20)358 .Case("r21", Hexagon::R21)359 .Case("r22", Hexagon::R22)360 .Case("r23", Hexagon::R23)361 .Case("r24", Hexagon::R24)362 .Case("r25", Hexagon::R25)363 .Case("r26", Hexagon::R26)364 .Case("r27", Hexagon::R27)365 .Case("r28", Hexagon::R28)366 .Case("r29", Hexagon::R29)367 .Case("r30", Hexagon::R30)368 .Case("r31", Hexagon::R31)369 .Case("r1:0", Hexagon::D0)370 .Case("r3:2", Hexagon::D1)371 .Case("r5:4", Hexagon::D2)372 .Case("r7:6", Hexagon::D3)373 .Case("r9:8", Hexagon::D4)374 .Case("r11:10", Hexagon::D5)375 .Case("r13:12", Hexagon::D6)376 .Case("r15:14", Hexagon::D7)377 .Case("r17:16", Hexagon::D8)378 .Case("r19:18", Hexagon::D9)379 .Case("r21:20", Hexagon::D10)380 .Case("r23:22", Hexagon::D11)381 .Case("r25:24", Hexagon::D12)382 .Case("r27:26", Hexagon::D13)383 .Case("r29:28", Hexagon::D14)384 .Case("r31:30", Hexagon::D15)385 .Case("sp", Hexagon::R29)386 .Case("fp", Hexagon::R30)387 .Case("lr", Hexagon::R31)388 .Case("p0", Hexagon::P0)389 .Case("p1", Hexagon::P1)390 .Case("p2", Hexagon::P2)391 .Case("p3", Hexagon::P3)392 .Case("sa0", Hexagon::SA0)393 .Case("lc0", Hexagon::LC0)394 .Case("sa1", Hexagon::SA1)395 .Case("lc1", Hexagon::LC1)396 .Case("m0", Hexagon::M0)397 .Case("m1", Hexagon::M1)398 .Case("usr", Hexagon::USR)399 .Case("ugp", Hexagon::UGP)400 .Case("cs0", Hexagon::CS0)401 .Case("cs1", Hexagon::CS1)402 .Default(Register());403 return Reg;404}405 406/// LowerCallResult - Lower the result values of an ISD::CALL into the407/// appropriate copies out of appropriate physical registers. This assumes that408/// Chain/Glue are the input chain/glue to use, and that TheCall is the call409/// being lowered. Returns a SDNode with the same number of values as the410/// ISD::CALL.411SDValue HexagonTargetLowering::LowerCallResult(412 SDValue Chain, SDValue Glue, CallingConv::ID CallConv, bool IsVarArg,413 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,414 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,415 const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const {416 // Assign locations to each value returned by this call.417 SmallVector<CCValAssign, 16> RVLocs;418 419 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,420 *DAG.getContext());421 422 if (Subtarget.useHVXOps())423 CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon_HVX);424 else425 CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon);426 427 // Copy all of the result registers out of their specified physreg.428 for (unsigned i = 0; i != RVLocs.size(); ++i) {429 SDValue RetVal;430 if (RVLocs[i].getValVT() == MVT::i1) {431 // Return values of type MVT::i1 require special handling. The reason432 // is that MVT::i1 is associated with the PredRegs register class, but433 // values of that type are still returned in R0. Generate an explicit434 // copy into a predicate register from R0, and treat the value of the435 // predicate register as the call result.436 auto &MRI = DAG.getMachineFunction().getRegInfo();437 SDValue FR0 = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(),438 MVT::i32, Glue);439 // FR0 = (Value, Chain, Glue)440 Register PredR = MRI.createVirtualRegister(&Hexagon::PredRegsRegClass);441 SDValue TPR = DAG.getCopyToReg(FR0.getValue(1), dl, PredR,442 FR0.getValue(0), FR0.getValue(2));443 // TPR = (Chain, Glue)444 // Don't glue this CopyFromReg, because it copies from a virtual445 // register. If it is glued to the call, InstrEmitter will add it446 // as an implicit def to the call (EmitMachineNode).447 RetVal = DAG.getCopyFromReg(TPR.getValue(0), dl, PredR, MVT::i1);448 Glue = TPR.getValue(1);449 Chain = TPR.getValue(0);450 } else {451 RetVal = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(),452 RVLocs[i].getValVT(), Glue);453 Glue = RetVal.getValue(2);454 Chain = RetVal.getValue(1);455 }456 InVals.push_back(RetVal.getValue(0));457 }458 459 return Chain;460}461 462/// LowerCall - Functions arguments are copied from virtual regs to463/// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted.464SDValue465HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,466 SmallVectorImpl<SDValue> &InVals) const {467 SelectionDAG &DAG = CLI.DAG;468 SDLoc &dl = CLI.DL;469 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;470 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;471 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;472 SDValue Chain = CLI.Chain;473 SDValue Callee = CLI.Callee;474 CallingConv::ID CallConv = CLI.CallConv;475 bool IsVarArg = CLI.IsVarArg;476 bool DoesNotReturn = CLI.DoesNotReturn;477 478 bool IsStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();479 MachineFunction &MF = DAG.getMachineFunction();480 MachineFrameInfo &MFI = MF.getFrameInfo();481 auto PtrVT = getPointerTy(MF.getDataLayout());482 483 if (GlobalAddressSDNode *GAN = dyn_cast<GlobalAddressSDNode>(Callee))484 Callee = DAG.getTargetGlobalAddress(GAN->getGlobal(), dl, MVT::i32);485 486 // Linux ABI treats var-arg calls the same way as regular ones.487 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;488 489 // Analyze operands of the call, assigning locations to each operand.490 SmallVector<CCValAssign, 16> ArgLocs;491 CCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext());492 493 if (Subtarget.useHVXOps())494 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_HVX);495 else if (DisableArgsMinAlignment)496 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_Legacy);497 else498 CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon);499 500 if (CLI.IsTailCall) {501 bool StructAttrFlag = MF.getFunction().hasStructRetAttr();502 CLI.IsTailCall = IsEligibleForTailCallOptimization(Callee, CallConv,503 IsVarArg, IsStructRet, StructAttrFlag, Outs,504 OutVals, Ins, DAG);505 for (const CCValAssign &VA : ArgLocs) {506 if (VA.isMemLoc()) {507 CLI.IsTailCall = false;508 break;509 }510 }511 LLVM_DEBUG(dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"512 : "Argument must be passed on stack. "513 "Not eligible for Tail Call\n"));514 }515 // Get a count of how many bytes are to be pushed on the stack.516 unsigned NumBytes = CCInfo.getStackSize();517 SmallVector<std::pair<unsigned, SDValue>, 16> RegsToPass;518 SmallVector<SDValue, 8> MemOpChains;519 520 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();521 SDValue StackPtr =522 DAG.getCopyFromReg(Chain, dl, HRI.getStackRegister(), PtrVT);523 524 bool NeedsArgAlign = false;525 Align LargestAlignSeen;526 // Walk the register/memloc assignments, inserting copies/loads.527 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {528 CCValAssign &VA = ArgLocs[i];529 SDValue Arg = OutVals[i];530 ISD::ArgFlagsTy Flags = Outs[i].Flags;531 // Record if we need > 8 byte alignment on an argument.532 bool ArgAlign = Subtarget.isHVXVectorType(VA.getValVT());533 NeedsArgAlign |= ArgAlign;534 535 // Promote the value if needed.536 switch (VA.getLocInfo()) {537 default:538 // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.539 llvm_unreachable("Unknown loc info!");540 case CCValAssign::Full:541 break;542 case CCValAssign::BCvt:543 Arg = DAG.getBitcast(VA.getLocVT(), Arg);544 break;545 case CCValAssign::SExt:546 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);547 break;548 case CCValAssign::ZExt:549 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);550 break;551 case CCValAssign::AExt:552 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);553 break;554 }555 556 if (VA.isMemLoc()) {557 unsigned LocMemOffset = VA.getLocMemOffset();558 SDValue MemAddr = DAG.getConstant(LocMemOffset, dl,559 StackPtr.getValueType());560 MemAddr = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, MemAddr);561 if (ArgAlign)562 LargestAlignSeen = std::max(563 LargestAlignSeen, Align(VA.getLocVT().getStoreSizeInBits() / 8));564 if (Flags.isByVal()) {565 // The argument is a struct passed by value. According to LLVM, "Arg"566 // is a pointer.567 MemOpChains.push_back(CreateCopyOfByValArgument(Arg, MemAddr, Chain,568 Flags, DAG, dl));569 } else {570 MachinePointerInfo LocPI = MachinePointerInfo::getStack(571 DAG.getMachineFunction(), LocMemOffset);572 SDValue S = DAG.getStore(Chain, dl, Arg, MemAddr, LocPI);573 MemOpChains.push_back(S);574 }575 continue;576 }577 578 // Arguments that can be passed on register must be kept at RegsToPass579 // vector.580 if (VA.isRegLoc())581 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));582 }583 584 if (NeedsArgAlign && Subtarget.hasV60Ops()) {585 LLVM_DEBUG(dbgs() << "Function needs byte stack align due to call args\n");586 Align VecAlign = HRI.getSpillAlign(Hexagon::HvxVRRegClass);587 LargestAlignSeen = std::max(LargestAlignSeen, VecAlign);588 MFI.ensureMaxAlignment(LargestAlignSeen);589 }590 // Transform all store nodes into one single node because all store591 // nodes are independent of each other.592 if (!MemOpChains.empty())593 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);594 595 SDValue Glue;596 if (!CLI.IsTailCall) {597 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl);598 Glue = Chain.getValue(1);599 }600 601 // Build a sequence of copy-to-reg nodes chained together with token602 // chain and flag operands which copy the outgoing args into registers.603 // The Glue is necessary since all emitted instructions must be604 // stuck together.605 if (!CLI.IsTailCall) {606 for (const auto &R : RegsToPass) {607 Chain = DAG.getCopyToReg(Chain, dl, R.first, R.second, Glue);608 Glue = Chain.getValue(1);609 }610 } else {611 // For tail calls lower the arguments to the 'real' stack slot.612 //613 // Force all the incoming stack arguments to be loaded from the stack614 // before any new outgoing arguments are stored to the stack, because the615 // outgoing stack slots may alias the incoming argument stack slots, and616 // the alias isn't otherwise explicit. This is slightly more conservative617 // than necessary, because it means that each store effectively depends618 // on every argument instead of just those arguments it would clobber.619 //620 // Do not flag preceding copytoreg stuff together with the following stuff.621 Glue = SDValue();622 for (const auto &R : RegsToPass) {623 Chain = DAG.getCopyToReg(Chain, dl, R.first, R.second, Glue);624 Glue = Chain.getValue(1);625 }626 Glue = SDValue();627 }628 629 bool LongCalls = MF.getSubtarget<HexagonSubtarget>().useLongCalls();630 unsigned Flags = LongCalls ? HexagonII::HMOTF_ConstExtended : 0;631 632 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every633 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol634 // node so that legalize doesn't hack it.635 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {636 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl, PtrVT, 0, Flags);637 } else if (ExternalSymbolSDNode *S =638 dyn_cast<ExternalSymbolSDNode>(Callee)) {639 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, Flags);640 }641 642 // Returns a chain & a flag for retval copy to use.643 SmallVector<SDValue, 8> Ops;644 Ops.push_back(Chain);645 Ops.push_back(Callee);646 647 // Add argument registers to the end of the list so that they are648 // known live into the call.649 for (const auto &R : RegsToPass)650 Ops.push_back(DAG.getRegister(R.first, R.second.getValueType()));651 652 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallConv);653 assert(Mask && "Missing call preserved mask for calling convention");654 Ops.push_back(DAG.getRegisterMask(Mask));655 656 if (Glue.getNode())657 Ops.push_back(Glue);658 659 if (CLI.IsTailCall) {660 MFI.setHasTailCall();661 return DAG.getNode(HexagonISD::TC_RETURN, dl, MVT::Other, Ops);662 }663 664 // Set this here because we need to know this for "hasFP" in frame lowering.665 // The target-independent code calls getFrameRegister before setting it, and666 // getFrameRegister uses hasFP to determine whether the function has FP.667 MFI.setHasCalls(true);668 669 unsigned OpCode = DoesNotReturn ? HexagonISD::CALLnr : HexagonISD::CALL;670 Chain = DAG.getNode(OpCode, dl, {MVT::Other, MVT::Glue}, Ops);671 Glue = Chain.getValue(1);672 673 // Create the CALLSEQ_END node.674 Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, Glue, dl);675 Glue = Chain.getValue(1);676 677 // Handle result values, copying them out of physregs into vregs that we678 // return.679 return LowerCallResult(Chain, Glue, CallConv, IsVarArg, Ins, dl, DAG,680 InVals, OutVals, Callee);681}682 683/// Returns true by value, base pointer and offset pointer and addressing684/// mode by reference if this node can be combined with a load / store to685/// form a post-indexed load / store.686bool HexagonTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,687 SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM,688 SelectionDAG &DAG) const {689 LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(N);690 if (!LSN)691 return false;692 EVT VT = LSN->getMemoryVT();693 if (!VT.isSimple())694 return false;695 bool IsLegalType = VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||696 VT == MVT::i64 || VT == MVT::f32 || VT == MVT::f64 ||697 VT == MVT::v2i16 || VT == MVT::v2i32 || VT == MVT::v4i8 ||698 VT == MVT::v4i16 || VT == MVT::v8i8 ||699 Subtarget.isHVXVectorType(VT.getSimpleVT());700 if (!IsLegalType)701 return false;702 703 if (Op->getOpcode() != ISD::ADD)704 return false;705 Base = Op->getOperand(0);706 Offset = Op->getOperand(1);707 if (!isa<ConstantSDNode>(Offset.getNode()))708 return false;709 AM = ISD::POST_INC;710 711 int32_t V = cast<ConstantSDNode>(Offset.getNode())->getSExtValue();712 return Subtarget.getInstrInfo()->isValidAutoIncImm(VT, V);713}714 715SDValue HexagonTargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const {716 if (DAG.getMachineFunction().getFunction().hasOptSize())717 return SDValue();718 else719 return Op;720}721 722SDValue723HexagonTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {724 MachineFunction &MF = DAG.getMachineFunction();725 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();726 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();727 unsigned LR = HRI.getRARegister();728 729 if ((Op.getOpcode() != ISD::INLINEASM &&730 Op.getOpcode() != ISD::INLINEASM_BR) || HMFI.hasClobberLR())731 return Op;732 733 unsigned NumOps = Op.getNumOperands();734 if (Op.getOperand(NumOps-1).getValueType() == MVT::Glue)735 --NumOps; // Ignore the flag operand.736 737 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {738 const InlineAsm::Flag Flags(Op.getConstantOperandVal(i));739 unsigned NumVals = Flags.getNumOperandRegisters();740 ++i; // Skip the ID value.741 742 switch (Flags.getKind()) {743 default:744 llvm_unreachable("Bad flags!");745 case InlineAsm::Kind::RegUse:746 case InlineAsm::Kind::Imm:747 case InlineAsm::Kind::Mem:748 i += NumVals;749 break;750 case InlineAsm::Kind::Clobber:751 case InlineAsm::Kind::RegDef:752 case InlineAsm::Kind::RegDefEarlyClobber: {753 for (; NumVals; --NumVals, ++i) {754 Register Reg = cast<RegisterSDNode>(Op.getOperand(i))->getReg();755 if (Reg != LR)756 continue;757 HMFI.setHasClobberLR(true);758 return Op;759 }760 break;761 }762 }763 }764 765 return Op;766}767 768// Need to transform ISD::PREFETCH into something that doesn't inherit769// all of the properties of ISD::PREFETCH, specifically SDNPMayLoad and770// SDNPMayStore.771SDValue HexagonTargetLowering::LowerPREFETCH(SDValue Op,772 SelectionDAG &DAG) const {773 SDValue Chain = Op.getOperand(0);774 SDValue Addr = Op.getOperand(1);775 // Lower it to DCFETCH($reg, #0). A "pat" will try to merge the offset in,776 // if the "reg" is fed by an "add".777 SDLoc DL(Op);778 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);779 return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero);780}781 782// Custom-handle ISD::READCYCLECOUNTER because the target-independent SDNode783// is marked as having side-effects, while the register read on Hexagon does784// not have any. TableGen refuses to accept the direct pattern from that node785// to the A4_tfrcpp.786SDValue HexagonTargetLowering::LowerREADCYCLECOUNTER(SDValue Op,787 SelectionDAG &DAG) const {788 SDValue Chain = Op.getOperand(0);789 SDLoc dl(Op);790 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);791 return DAG.getNode(HexagonISD::READCYCLE, dl, VTs, Chain);792}793 794// Custom-handle ISD::READSTEADYCOUNTER because the target-independent SDNode795// is marked as having side-effects, while the register read on Hexagon does796// not have any. TableGen refuses to accept the direct pattern from that node797// to the A4_tfrcpp.798SDValue HexagonTargetLowering::LowerREADSTEADYCOUNTER(SDValue Op,799 SelectionDAG &DAG) const {800 SDValue Chain = Op.getOperand(0);801 SDLoc dl(Op);802 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);803 return DAG.getNode(HexagonISD::READTIMER, dl, VTs, Chain);804}805 806SDValue HexagonTargetLowering::LowerINTRINSIC_VOID(SDValue Op,807 SelectionDAG &DAG) const {808 SDValue Chain = Op.getOperand(0);809 unsigned IntNo = Op.getConstantOperandVal(1);810 // Lower the hexagon_prefetch builtin to DCFETCH, as above.811 if (IntNo == Intrinsic::hexagon_prefetch) {812 SDValue Addr = Op.getOperand(2);813 SDLoc DL(Op);814 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);815 return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero);816 }817 return SDValue();818}819 820SDValue821HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,822 SelectionDAG &DAG) const {823 SDValue Chain = Op.getOperand(0);824 SDValue Size = Op.getOperand(1);825 SDValue Align = Op.getOperand(2);826 SDLoc dl(Op);827 828 ConstantSDNode *AlignConst = dyn_cast<ConstantSDNode>(Align);829 assert(AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC");830 831 unsigned A = AlignConst->getSExtValue();832 auto &HFI = *Subtarget.getFrameLowering();833 // "Zero" means natural stack alignment.834 if (A == 0)835 A = HFI.getStackAlign().value();836 837 LLVM_DEBUG({838 dbgs () << __func__ << " Align: " << A << " Size: ";839 Size.getNode()->dump(&DAG);840 dbgs() << "\n";841 });842 843 SDValue AC = DAG.getConstant(A, dl, MVT::i32);844 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);845 SDValue AA = DAG.getNode(HexagonISD::ALLOCA, dl, VTs, Chain, Size, AC);846 847 DAG.ReplaceAllUsesOfValueWith(Op, AA);848 return AA;849}850 851SDValue HexagonTargetLowering::LowerFormalArguments(852 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,853 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,854 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {855 MachineFunction &MF = DAG.getMachineFunction();856 MachineFrameInfo &MFI = MF.getFrameInfo();857 MachineRegisterInfo &MRI = MF.getRegInfo();858 859 // Linux ABI treats var-arg calls the same way as regular ones.860 bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;861 862 // Assign locations to all of the incoming arguments.863 SmallVector<CCValAssign, 16> ArgLocs;864 CCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext());865 866 if (Subtarget.useHVXOps())867 CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_HVX);868 else if (DisableArgsMinAlignment)869 CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_Legacy);870 else871 CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon);872 873 // For LLVM, in the case when returning a struct by value (>8byte),874 // the first argument is a pointer that points to the location on caller's875 // stack where the return value will be stored. For Hexagon, the location on876 // caller's stack is passed only when the struct size is smaller than (and877 // equal to) 8 bytes. If not, no address will be passed into callee and878 // callee return the result directly through R0/R1.879 auto NextSingleReg = [] (const TargetRegisterClass &RC, unsigned Reg) {880 switch (RC.getID()) {881 case Hexagon::IntRegsRegClassID:882 return Reg - Hexagon::R0 + 1;883 case Hexagon::DoubleRegsRegClassID:884 return (Reg - Hexagon::D0 + 1) * 2;885 case Hexagon::HvxVRRegClassID:886 return Reg - Hexagon::V0 + 1;887 case Hexagon::HvxWRRegClassID:888 return (Reg - Hexagon::W0 + 1) * 2;889 }890 llvm_unreachable("Unexpected register class");891 };892 893 auto &HFL = const_cast<HexagonFrameLowering&>(*Subtarget.getFrameLowering());894 auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();895 HFL.FirstVarArgSavedReg = 0;896 HMFI.setFirstNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));897 898 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {899 CCValAssign &VA = ArgLocs[i];900 ISD::ArgFlagsTy Flags = Ins[i].Flags;901 bool ByVal = Flags.isByVal();902 903 // Arguments passed in registers:904 // 1. 32- and 64-bit values and HVX vectors are passed directly,905 // 2. Large structs are passed via an address, and the address is906 // passed in a register.907 if (VA.isRegLoc() && ByVal && Flags.getByValSize() <= 8)908 llvm_unreachable("ByValSize must be bigger than 8 bytes");909 910 bool InReg = VA.isRegLoc() &&911 (!ByVal || (ByVal && Flags.getByValSize() > 8));912 913 if (InReg) {914 MVT RegVT = VA.getLocVT();915 if (VA.getLocInfo() == CCValAssign::BCvt)916 RegVT = VA.getValVT();917 918 const TargetRegisterClass *RC = getRegClassFor(RegVT);919 Register VReg = MRI.createVirtualRegister(RC);920 SDValue Copy = DAG.getCopyFromReg(Chain, dl, VReg, RegVT);921 922 // Treat values of type MVT::i1 specially: they are passed in923 // registers of type i32, but they need to remain as values of924 // type i1 for consistency of the argument lowering.925 if (VA.getValVT() == MVT::i1) {926 assert(RegVT.getSizeInBits() <= 32);927 SDValue T = DAG.getNode(ISD::AND, dl, RegVT,928 Copy, DAG.getConstant(1, dl, RegVT));929 Copy = DAG.getSetCC(dl, MVT::i1, T, DAG.getConstant(0, dl, RegVT),930 ISD::SETNE);931 } else {932#ifndef NDEBUG933 unsigned RegSize = RegVT.getSizeInBits();934 assert(RegSize == 32 || RegSize == 64 ||935 Subtarget.isHVXVectorType(RegVT));936#endif937 }938 InVals.push_back(Copy);939 MRI.addLiveIn(VA.getLocReg(), VReg);940 HFL.FirstVarArgSavedReg = NextSingleReg(*RC, VA.getLocReg());941 } else {942 assert(VA.isMemLoc() && "Argument should be passed in memory");943 944 // If it's a byval parameter, then we need to compute the945 // "real" size, not the size of the pointer.946 unsigned ObjSize = Flags.isByVal()947 ? Flags.getByValSize()948 : VA.getLocVT().getStoreSizeInBits() / 8;949 950 // Create the frame index object for this incoming parameter.951 int Offset = HEXAGON_LRFP_SIZE + VA.getLocMemOffset();952 int FI = MFI.CreateFixedObject(ObjSize, Offset, true);953 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);954 955 if (Flags.isByVal()) {956 // If it's a pass-by-value aggregate, then do not dereference the stack957 // location. Instead, we should generate a reference to the stack958 // location.959 InVals.push_back(FIN);960 } else {961 SDValue L = DAG.getLoad(VA.getValVT(), dl, Chain, FIN,962 MachinePointerInfo::getFixedStack(MF, FI, 0));963 InVals.push_back(L);964 }965 }966 }967 968 if (IsVarArg && Subtarget.isEnvironmentMusl()) {969 for (int i = HFL.FirstVarArgSavedReg; i < 6; i++)970 MRI.addLiveIn(Hexagon::R0+i);971 }972 973 if (IsVarArg && Subtarget.isEnvironmentMusl()) {974 HMFI.setFirstNamedArgFrameIndex(HMFI.getFirstNamedArgFrameIndex() - 1);975 HMFI.setLastNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));976 977 // Create Frame index for the start of register saved area.978 int NumVarArgRegs = 6 - HFL.FirstVarArgSavedReg;979 bool RequiresPadding = (NumVarArgRegs & 1);980 int RegSaveAreaSizePlusPadding = RequiresPadding981 ? (NumVarArgRegs + 1) * 4982 : NumVarArgRegs * 4;983 984 if (RegSaveAreaSizePlusPadding > 0) {985 // The offset to saved register area should be 8 byte aligned.986 int RegAreaStart = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();987 if (!(RegAreaStart % 8))988 RegAreaStart = (RegAreaStart + 7) & -8;989 990 int RegSaveAreaFrameIndex =991 MFI.CreateFixedObject(RegSaveAreaSizePlusPadding, RegAreaStart, true);992 HMFI.setRegSavedAreaStartFrameIndex(RegSaveAreaFrameIndex);993 994 // This will point to the next argument passed via stack.995 int Offset = RegAreaStart + RegSaveAreaSizePlusPadding;996 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, Offset, true);997 HMFI.setVarArgsFrameIndex(FI);998 } else {999 // This will point to the next argument passed via stack, when1000 // there is no saved register area.1001 int Offset = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();1002 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, Offset, true);1003 HMFI.setRegSavedAreaStartFrameIndex(FI);1004 HMFI.setVarArgsFrameIndex(FI);1005 }1006 }1007 1008 1009 if (IsVarArg && !Subtarget.isEnvironmentMusl()) {1010 // This will point to the next argument passed via stack.1011 int Offset = HEXAGON_LRFP_SIZE + CCInfo.getStackSize();1012 int FI = MFI.CreateFixedObject(Hexagon_PointerSize, Offset, true);1013 HMFI.setVarArgsFrameIndex(FI);1014 }1015 1016 return Chain;1017}1018 1019SDValue1020HexagonTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {1021 // VASTART stores the address of the VarArgsFrameIndex slot into the1022 // memory location argument.1023 MachineFunction &MF = DAG.getMachineFunction();1024 HexagonMachineFunctionInfo *QFI = MF.getInfo<HexagonMachineFunctionInfo>();1025 SDValue Addr = DAG.getFrameIndex(QFI->getVarArgsFrameIndex(), MVT::i32);1026 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();1027 1028 if (!Subtarget.isEnvironmentMusl()) {1029 return DAG.getStore(Op.getOperand(0), SDLoc(Op), Addr, Op.getOperand(1),1030 MachinePointerInfo(SV));1031 }1032 auto &FuncInfo = *MF.getInfo<HexagonMachineFunctionInfo>();1033 auto &HFL = *Subtarget.getFrameLowering();1034 SDLoc DL(Op);1035 SmallVector<SDValue, 8> MemOps;1036 1037 // Get frame index of va_list.1038 SDValue FIN = Op.getOperand(1);1039 1040 // If first Vararg register is odd, add 4 bytes to start of1041 // saved register area to point to the first register location.1042 // This is because the saved register area has to be 8 byte aligned.1043 // In case of an odd start register, there will be 4 bytes of padding in1044 // the beginning of saved register area. If all registers area used up,1045 // the following condition will handle it correctly.1046 SDValue SavedRegAreaStartFrameIndex =1047 DAG.getFrameIndex(FuncInfo.getRegSavedAreaStartFrameIndex(), MVT::i32);1048 1049 auto PtrVT = getPointerTy(DAG.getDataLayout());1050 1051 if (HFL.FirstVarArgSavedReg & 1)1052 SavedRegAreaStartFrameIndex =1053 DAG.getNode(ISD::ADD, DL, PtrVT,1054 DAG.getFrameIndex(FuncInfo.getRegSavedAreaStartFrameIndex(),1055 MVT::i32),1056 DAG.getIntPtrConstant(4, DL));1057 1058 // Store the saved register area start pointer.1059 SDValue Store =1060 DAG.getStore(Op.getOperand(0), DL,1061 SavedRegAreaStartFrameIndex,1062 FIN, MachinePointerInfo(SV));1063 MemOps.push_back(Store);1064 1065 // Store saved register area end pointer.1066 FIN = DAG.getNode(ISD::ADD, DL, PtrVT,1067 FIN, DAG.getIntPtrConstant(4, DL));1068 Store = DAG.getStore(Op.getOperand(0), DL,1069 DAG.getFrameIndex(FuncInfo.getVarArgsFrameIndex(),1070 PtrVT),1071 FIN, MachinePointerInfo(SV, 4));1072 MemOps.push_back(Store);1073 1074 // Store overflow area pointer.1075 FIN = DAG.getNode(ISD::ADD, DL, PtrVT,1076 FIN, DAG.getIntPtrConstant(4, DL));1077 Store = DAG.getStore(Op.getOperand(0), DL,1078 DAG.getFrameIndex(FuncInfo.getVarArgsFrameIndex(),1079 PtrVT),1080 FIN, MachinePointerInfo(SV, 8));1081 MemOps.push_back(Store);1082 1083 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);1084}1085 1086SDValue1087HexagonTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {1088 // Assert that the linux ABI is enabled for the current compilation.1089 assert(Subtarget.isEnvironmentMusl() && "Linux ABI should be enabled");1090 SDValue Chain = Op.getOperand(0);1091 SDValue DestPtr = Op.getOperand(1);1092 SDValue SrcPtr = Op.getOperand(2);1093 const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();1094 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();1095 SDLoc DL(Op);1096 // Size of the va_list is 12 bytes as it has 3 pointers. Therefore,1097 // we need to memcopy 12 bytes from va_list to another similar list.1098 return DAG.getMemcpy(1099 Chain, DL, DestPtr, SrcPtr, DAG.getIntPtrConstant(12, DL), Align(4),1100 /*isVolatile*/ false, false, /*CI=*/nullptr, std::nullopt,1101 MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));1102}1103 1104SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {1105 const SDLoc &dl(Op);1106 SDValue LHS = Op.getOperand(0);1107 SDValue RHS = Op.getOperand(1);1108 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();1109 MVT ResTy = ty(Op);1110 MVT OpTy = ty(LHS);1111 1112 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {1113 MVT ElemTy = OpTy.getVectorElementType();1114 assert(ElemTy.isScalarInteger());1115 MVT WideTy = MVT::getVectorVT(MVT::getIntegerVT(2*ElemTy.getSizeInBits()),1116 OpTy.getVectorNumElements());1117 return DAG.getSetCC(dl, ResTy,1118 DAG.getSExtOrTrunc(LHS, SDLoc(LHS), WideTy),1119 DAG.getSExtOrTrunc(RHS, SDLoc(RHS), WideTy), CC);1120 }1121 1122 // Treat all other vector types as legal.1123 if (ResTy.isVector())1124 return Op;1125 1126 // Comparisons of short integers should use sign-extend, not zero-extend,1127 // since we can represent small negative values in the compare instructions.1128 // The LLVM default is to use zero-extend arbitrarily in these cases.1129 auto isSExtFree = [this](SDValue N) {1130 switch (N.getOpcode()) {1131 case ISD::TRUNCATE: {1132 // A sign-extend of a truncate of a sign-extend is free.1133 SDValue Op = N.getOperand(0);1134 if (Op.getOpcode() != ISD::AssertSext)1135 return false;1136 EVT OrigTy = cast<VTSDNode>(Op.getOperand(1))->getVT();1137 unsigned ThisBW = ty(N).getSizeInBits();1138 unsigned OrigBW = OrigTy.getSizeInBits();1139 // The type that was sign-extended to get the AssertSext must be1140 // narrower than the type of N (so that N has still the same value1141 // as the original).1142 return ThisBW >= OrigBW;1143 }1144 case ISD::LOAD:1145 // We have sign-extended loads.1146 return true;1147 }1148 return false;1149 };1150 1151 if (OpTy == MVT::i8 || OpTy == MVT::i16) {1152 ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS);1153 bool IsNegative = C && C->getAPIntValue().isNegative();1154 if (IsNegative || isSExtFree(LHS) || isSExtFree(RHS))1155 return DAG.getSetCC(dl, ResTy,1156 DAG.getSExtOrTrunc(LHS, SDLoc(LHS), MVT::i32),1157 DAG.getSExtOrTrunc(RHS, SDLoc(RHS), MVT::i32), CC);1158 }1159 1160 return SDValue();1161}1162 1163SDValue1164HexagonTargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {1165 SDValue PredOp = Op.getOperand(0);1166 SDValue Op1 = Op.getOperand(1), Op2 = Op.getOperand(2);1167 MVT OpTy = ty(Op1);1168 const SDLoc &dl(Op);1169 1170 if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {1171 MVT ElemTy = OpTy.getVectorElementType();1172 assert(ElemTy.isScalarInteger());1173 MVT WideTy = MVT::getVectorVT(MVT::getIntegerVT(2*ElemTy.getSizeInBits()),1174 OpTy.getVectorNumElements());1175 // Generate (trunc (select (_, sext, sext))).1176 return DAG.getSExtOrTrunc(1177 DAG.getSelect(dl, WideTy, PredOp,1178 DAG.getSExtOrTrunc(Op1, dl, WideTy),1179 DAG.getSExtOrTrunc(Op2, dl, WideTy)),1180 dl, OpTy);1181 }1182 1183 return SDValue();1184}1185 1186SDValue1187HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {1188 EVT ValTy = Op.getValueType();1189 ConstantPoolSDNode *CPN = cast<ConstantPoolSDNode>(Op);1190 Constant *CVal = nullptr;1191 bool isVTi1Type = false;1192 if (auto *CV = dyn_cast<ConstantVector>(CPN->getConstVal())) {1193 if (cast<VectorType>(CV->getType())->getElementType()->isIntegerTy(1)) {1194 IRBuilder<> IRB(CV->getContext());1195 SmallVector<Constant*, 128> NewConst;1196 unsigned VecLen = CV->getNumOperands();1197 assert(isPowerOf2_32(VecLen) &&1198 "conversion only supported for pow2 VectorSize");1199 for (unsigned i = 0; i < VecLen; ++i)1200 NewConst.push_back(IRB.getInt8(CV->getOperand(i)->isZeroValue()));1201 1202 CVal = ConstantVector::get(NewConst);1203 isVTi1Type = true;1204 }1205 }1206 Align Alignment = CPN->getAlign();1207 bool IsPositionIndependent = isPositionIndependent();1208 unsigned char TF = IsPositionIndependent ? HexagonII::MO_PCREL : 0;1209 1210 unsigned Offset = 0;1211 SDValue T;1212 if (CPN->isMachineConstantPoolEntry())1213 T = DAG.getTargetConstantPool(CPN->getMachineCPVal(), ValTy, Alignment,1214 Offset, TF);1215 else if (isVTi1Type)1216 T = DAG.getTargetConstantPool(CVal, ValTy, Alignment, Offset, TF);1217 else1218 T = DAG.getTargetConstantPool(CPN->getConstVal(), ValTy, Alignment, Offset,1219 TF);1220 1221 assert(cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF &&1222 "Inconsistent target flag encountered");1223 1224 if (IsPositionIndependent)1225 return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), ValTy, T);1226 return DAG.getNode(HexagonISD::CP, SDLoc(Op), ValTy, T);1227}1228 1229SDValue1230HexagonTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {1231 EVT VT = Op.getValueType();1232 int Idx = cast<JumpTableSDNode>(Op)->getIndex();1233 if (isPositionIndependent()) {1234 SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL);1235 return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), VT, T);1236 }1237 1238 SDValue T = DAG.getTargetJumpTable(Idx, VT);1239 return DAG.getNode(HexagonISD::JT, SDLoc(Op), VT, T);1240}1241 1242SDValue1243HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {1244 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();1245 MachineFunction &MF = DAG.getMachineFunction();1246 MachineFrameInfo &MFI = MF.getFrameInfo();1247 MFI.setReturnAddressIsTaken(true);1248 1249 EVT VT = Op.getValueType();1250 SDLoc dl(Op);1251 unsigned Depth = Op.getConstantOperandVal(0);1252 if (Depth) {1253 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);1254 SDValue Offset = DAG.getConstant(4, dl, MVT::i32);1255 return DAG.getLoad(VT, dl, DAG.getEntryNode(),1256 DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset),1257 MachinePointerInfo());1258 }1259 1260 // Return LR, which contains the return address. Mark it an implicit live-in.1261 Register Reg = MF.addLiveIn(HRI.getRARegister(), getRegClassFor(MVT::i32));1262 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);1263}1264 1265SDValue1266HexagonTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {1267 const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();1268 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();1269 MFI.setFrameAddressIsTaken(true);1270 1271 EVT VT = Op.getValueType();1272 SDLoc dl(Op);1273 unsigned Depth = Op.getConstantOperandVal(0);1274 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl,1275 HRI.getFrameRegister(), VT);1276 while (Depth--)1277 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,1278 MachinePointerInfo());1279 return FrameAddr;1280}1281 1282SDValue1283HexagonTargetLowering::LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const {1284 SDLoc dl(Op);1285 return DAG.getNode(HexagonISD::BARRIER, dl, MVT::Other, Op.getOperand(0));1286}1287 1288SDValue1289HexagonTargetLowering::LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const {1290 SDLoc dl(Op);1291 auto *GAN = cast<GlobalAddressSDNode>(Op);1292 auto PtrVT = getPointerTy(DAG.getDataLayout());1293 auto *GV = GAN->getGlobal();1294 int64_t Offset = GAN->getOffset();1295 1296 auto &HLOF = *HTM.getObjFileLowering();1297 Reloc::Model RM = HTM.getRelocationModel();1298 1299 if (RM == Reloc::Static) {1300 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset);1301 const GlobalObject *GO = GV->getAliaseeObject();1302 if (GO && Subtarget.useSmallData() && HLOF.isGlobalInSmallSection(GO, HTM))1303 return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, GA);1304 return DAG.getNode(HexagonISD::CONST32, dl, PtrVT, GA);1305 }1306 1307 bool UsePCRel = getTargetMachine().shouldAssumeDSOLocal(GV);1308 if (UsePCRel) {1309 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset,1310 HexagonII::MO_PCREL);1311 return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, GA);1312 }1313 1314 // Use GOT index.1315 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT);1316 SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, HexagonII::MO_GOT);1317 SDValue Off = DAG.getConstant(Offset, dl, MVT::i32);1318 return DAG.getNode(HexagonISD::AT_GOT, dl, PtrVT, GOT, GA, Off);1319}1320 1321// Specifies that for loads and stores VT can be promoted to PromotedLdStVT.1322SDValue1323HexagonTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {1324 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();1325 SDLoc dl(Op);1326 EVT PtrVT = getPointerTy(DAG.getDataLayout());1327 1328 Reloc::Model RM = HTM.getRelocationModel();1329 if (RM == Reloc::Static) {1330 SDValue A = DAG.getTargetBlockAddress(BA, PtrVT);1331 return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, A);1332 }1333 1334 SDValue A = DAG.getTargetBlockAddress(BA, PtrVT, 0, HexagonII::MO_PCREL);1335 return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, A);1336}1337 1338SDValue1339HexagonTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG)1340 const {1341 EVT PtrVT = getPointerTy(DAG.getDataLayout());1342 SDValue GOTSym = DAG.getTargetExternalSymbol(HEXAGON_GOT_SYM_NAME, PtrVT,1343 HexagonII::MO_PCREL);1344 return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), PtrVT, GOTSym);1345}1346 1347SDValue1348HexagonTargetLowering::GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain,1349 GlobalAddressSDNode *GA, SDValue Glue, EVT PtrVT, unsigned ReturnReg,1350 unsigned char OperandFlags) const {1351 MachineFunction &MF = DAG.getMachineFunction();1352 MachineFrameInfo &MFI = MF.getFrameInfo();1353 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);1354 SDLoc dl(GA);1355 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,1356 GA->getValueType(0),1357 GA->getOffset(),1358 OperandFlags);1359 // Create Operands for the call.The Operands should have the following:1360 // 1. Chain SDValue1361 // 2. Callee which in this case is the Global address value.1362 // 3. Registers live into the call.In this case its R0, as we1363 // have just one argument to be passed.1364 // 4. Glue.1365 // Note: The order is important.1366 1367 const auto &HRI = *Subtarget.getRegisterInfo();1368 const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallingConv::C);1369 assert(Mask && "Missing call preserved mask for calling convention");1370 SDValue Ops[] = { Chain, TGA, DAG.getRegister(Hexagon::R0, PtrVT),1371 DAG.getRegisterMask(Mask), Glue };1372 Chain = DAG.getNode(HexagonISD::CALL, dl, NodeTys, Ops);1373 1374 // Inform MFI that function has calls.1375 MFI.setAdjustsStack(true);1376 1377 Glue = Chain.getValue(1);1378 return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Glue);1379}1380 1381//1382// Lower using the initial executable model for TLS addresses1383//1384SDValue1385HexagonTargetLowering::LowerToTLSInitialExecModel(GlobalAddressSDNode *GA,1386 SelectionDAG &DAG) const {1387 SDLoc dl(GA);1388 int64_t Offset = GA->getOffset();1389 auto PtrVT = getPointerTy(DAG.getDataLayout());1390 1391 // Get the thread pointer.1392 SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT);1393 1394 bool IsPositionIndependent = isPositionIndependent();1395 unsigned char TF =1396 IsPositionIndependent ? HexagonII::MO_IEGOT : HexagonII::MO_IE;1397 1398 // First generate the TLS symbol address1399 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT,1400 Offset, TF);1401 1402 SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);1403 1404 if (IsPositionIndependent) {1405 // Generate the GOT pointer in case of position independent code1406 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Sym, DAG);1407 1408 // Add the TLS Symbol address to GOT pointer.This gives1409 // GOT relative relocation for the symbol.1410 Sym = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym);1411 }1412 1413 // Load the offset value for TLS symbol.This offset is relative to1414 // thread pointer.1415 SDValue LoadOffset =1416 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Sym, MachinePointerInfo());1417 1418 // Address of the thread local variable is the add of thread1419 // pointer and the offset of the variable.1420 return DAG.getNode(ISD::ADD, dl, PtrVT, TP, LoadOffset);1421}1422 1423//1424// Lower using the local executable model for TLS addresses1425//1426SDValue1427HexagonTargetLowering::LowerToTLSLocalExecModel(GlobalAddressSDNode *GA,1428 SelectionDAG &DAG) const {1429 SDLoc dl(GA);1430 int64_t Offset = GA->getOffset();1431 auto PtrVT = getPointerTy(DAG.getDataLayout());1432 1433 // Get the thread pointer.1434 SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT);1435 // Generate the TLS symbol address1436 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset,1437 HexagonII::MO_TPREL);1438 SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);1439 1440 // Address of the thread local variable is the add of thread1441 // pointer and the offset of the variable.1442 return DAG.getNode(ISD::ADD, dl, PtrVT, TP, Sym);1443}1444 1445//1446// Lower using the general dynamic model for TLS addresses1447//1448SDValue1449HexagonTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,1450 SelectionDAG &DAG) const {1451 SDLoc dl(GA);1452 int64_t Offset = GA->getOffset();1453 auto PtrVT = getPointerTy(DAG.getDataLayout());1454 1455 // First generate the TLS symbol address1456 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset,1457 HexagonII::MO_GDGOT);1458 1459 // Then, generate the GOT pointer1460 SDValue GOT = LowerGLOBAL_OFFSET_TABLE(TGA, DAG);1461 1462 // Add the TLS symbol and the GOT pointer1463 SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);1464 SDValue Chain = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym);1465 1466 // Copy over the argument to R01467 SDValue InGlue;1468 Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, Hexagon::R0, Chain, InGlue);1469 InGlue = Chain.getValue(1);1470 1471 unsigned Flags = DAG.getSubtarget<HexagonSubtarget>().useLongCalls()1472 ? HexagonII::MO_GDPLT | HexagonII::HMOTF_ConstExtended1473 : HexagonII::MO_GDPLT;1474 1475 return GetDynamicTLSAddr(DAG, Chain, GA, InGlue, PtrVT,1476 Hexagon::R0, Flags);1477}1478 1479//1480// Lower TLS addresses.1481//1482// For now for dynamic models, we only support the general dynamic model.1483//1484SDValue1485HexagonTargetLowering::LowerGlobalTLSAddress(SDValue Op,1486 SelectionDAG &DAG) const {1487 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);1488 1489 switch (HTM.getTLSModel(GA->getGlobal())) {1490 case TLSModel::GeneralDynamic:1491 case TLSModel::LocalDynamic:1492 return LowerToTLSGeneralDynamicModel(GA, DAG);1493 case TLSModel::InitialExec:1494 return LowerToTLSInitialExecModel(GA, DAG);1495 case TLSModel::LocalExec:1496 return LowerToTLSLocalExecModel(GA, DAG);1497 }1498 llvm_unreachable("Bogus TLS model");1499}1500 1501//===----------------------------------------------------------------------===//1502// TargetLowering Implementation1503//===----------------------------------------------------------------------===//1504 1505HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,1506 const HexagonSubtarget &ST)1507 : TargetLowering(TM, ST),1508 HTM(static_cast<const HexagonTargetMachine &>(TM)), Subtarget(ST) {1509 auto &HRI = *Subtarget.getRegisterInfo();1510 1511 setPrefLoopAlignment(Align(16));1512 setMinFunctionAlignment(Align(4));1513 setPrefFunctionAlignment(Align(16));1514 setStackPointerRegisterToSaveRestore(HRI.getStackRegister());1515 setBooleanContents(TargetLoweringBase::UndefinedBooleanContent);1516 setBooleanVectorContents(TargetLoweringBase::UndefinedBooleanContent);1517 1518 setMaxAtomicSizeInBitsSupported(64);1519 setMinCmpXchgSizeInBits(32);1520 1521 if (EnableHexSDNodeSched)1522 setSchedulingPreference(Sched::VLIW);1523 else1524 setSchedulingPreference(Sched::Source);1525 1526 // Limits for inline expansion of memcpy/memmove1527 MaxStoresPerMemcpy = MaxStoresPerMemcpyCL;1528 MaxStoresPerMemcpyOptSize = MaxStoresPerMemcpyOptSizeCL;1529 MaxStoresPerMemmove = MaxStoresPerMemmoveCL;1530 MaxStoresPerMemmoveOptSize = MaxStoresPerMemmoveOptSizeCL;1531 MaxStoresPerMemset = MaxStoresPerMemsetCL;1532 MaxStoresPerMemsetOptSize = MaxStoresPerMemsetOptSizeCL;1533 1534 //1535 // Set up register classes.1536 //1537 1538 addRegisterClass(MVT::i1, &Hexagon::PredRegsRegClass);1539 addRegisterClass(MVT::v2i1, &Hexagon::PredRegsRegClass); // bbbbaaaa1540 addRegisterClass(MVT::v4i1, &Hexagon::PredRegsRegClass); // ddccbbaa1541 addRegisterClass(MVT::v8i1, &Hexagon::PredRegsRegClass); // hgfedcba1542 addRegisterClass(MVT::i32, &Hexagon::IntRegsRegClass);1543 addRegisterClass(MVT::v2i16, &Hexagon::IntRegsRegClass);1544 addRegisterClass(MVT::v4i8, &Hexagon::IntRegsRegClass);1545 addRegisterClass(MVT::i64, &Hexagon::DoubleRegsRegClass);1546 addRegisterClass(MVT::v8i8, &Hexagon::DoubleRegsRegClass);1547 addRegisterClass(MVT::v4i16, &Hexagon::DoubleRegsRegClass);1548 addRegisterClass(MVT::v2i32, &Hexagon::DoubleRegsRegClass);1549 1550 addRegisterClass(MVT::f32, &Hexagon::IntRegsRegClass);1551 addRegisterClass(MVT::f64, &Hexagon::DoubleRegsRegClass);1552 1553 //1554 // Handling of scalar operations.1555 //1556 // All operations default to "legal", except:1557 // - indexed loads and stores (pre-/post-incremented),1558 // - ANY_EXTEND_VECTOR_INREG, ATOMIC_CMP_SWAP_WITH_SUCCESS, CONCAT_VECTORS,1559 // ConstantFP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN,1560 // FLOG, FLOG2, FLOG10, FMAXIMUMNUM, FMINIMUMNUM, FNEARBYINT, FRINT, FROUND,1561 // TRAP, FTRUNC, PREFETCH, SIGN_EXTEND_VECTOR_INREG,1562 // ZERO_EXTEND_VECTOR_INREG,1563 // which default to "expand" for at least one type.1564 1565 // Misc operations.1566 setOperationAction(ISD::ConstantFP, MVT::f32, Legal);1567 setOperationAction(ISD::ConstantFP, MVT::f64, Legal);1568 setOperationAction(ISD::TRAP, MVT::Other, Legal);1569 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);1570 setOperationAction(ISD::ConstantPool, MVT::i32, Custom);1571 setOperationAction(ISD::JumpTable, MVT::i32, Custom);1572 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);1573 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);1574 setOperationAction(ISD::INLINEASM, MVT::Other, Custom);1575 setOperationAction(ISD::INLINEASM_BR, MVT::Other, Custom);1576 setOperationAction(ISD::PREFETCH, MVT::Other, Custom);1577 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom);1578 setOperationAction(ISD::READSTEADYCOUNTER, MVT::i64, Custom);1579 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);1580 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);1581 setOperationAction(ISD::EH_RETURN, MVT::Other, Custom);1582 setOperationAction(ISD::GLOBAL_OFFSET_TABLE, MVT::i32, Custom);1583 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom);1584 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);1585 1586 // Custom legalize GlobalAddress nodes into CONST32.1587 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);1588 setOperationAction(ISD::GlobalAddress, MVT::i8, Custom);1589 setOperationAction(ISD::BlockAddress, MVT::i32, Custom);1590 1591 // Hexagon needs to optimize cases with negative constants.1592 setOperationAction(ISD::SETCC, MVT::i8, Custom);1593 setOperationAction(ISD::SETCC, MVT::i16, Custom);1594 setOperationAction(ISD::SETCC, MVT::v4i8, Custom);1595 setOperationAction(ISD::SETCC, MVT::v2i16, Custom);1596 1597 // VASTART needs to be custom lowered to use the VarArgsFrameIndex.1598 setOperationAction(ISD::VASTART, MVT::Other, Custom);1599 setOperationAction(ISD::VAEND, MVT::Other, Expand);1600 setOperationAction(ISD::VAARG, MVT::Other, Expand);1601 if (Subtarget.isEnvironmentMusl())1602 setOperationAction(ISD::VACOPY, MVT::Other, Custom);1603 else1604 setOperationAction(ISD::VACOPY, MVT::Other, Expand);1605 1606 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);1607 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);1608 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);1609 1610 if (EmitJumpTables)1611 setMinimumJumpTableEntries(MinimumJumpTables);1612 else1613 setMinimumJumpTableEntries(std::numeric_limits<unsigned>::max());1614 setOperationAction(ISD::BR_JT, MVT::Other, Expand);1615 1616 for (unsigned LegalIntOp :1617 {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) {1618 setOperationAction(LegalIntOp, MVT::i32, Legal);1619 setOperationAction(LegalIntOp, MVT::i64, Legal);1620 }1621 1622 // Hexagon has A4_addp_c and A4_subp_c that take and generate a carry bit,1623 // but they only operate on i64.1624 for (MVT VT : MVT::integer_valuetypes()) {1625 setOperationAction(ISD::UADDO, VT, Custom);1626 setOperationAction(ISD::USUBO, VT, Custom);1627 setOperationAction(ISD::SADDO, VT, Expand);1628 setOperationAction(ISD::SSUBO, VT, Expand);1629 setOperationAction(ISD::UADDO_CARRY, VT, Expand);1630 setOperationAction(ISD::USUBO_CARRY, VT, Expand);1631 }1632 setOperationAction(ISD::UADDO_CARRY, MVT::i64, Custom);1633 setOperationAction(ISD::USUBO_CARRY, MVT::i64, Custom);1634 1635 setOperationAction(ISD::CTLZ, MVT::i8, Promote);1636 setOperationAction(ISD::CTLZ, MVT::i16, Promote);1637 setOperationAction(ISD::CTTZ, MVT::i8, Promote);1638 setOperationAction(ISD::CTTZ, MVT::i16, Promote);1639 1640 // Popcount can count # of 1s in i64 but returns i32.1641 setOperationAction(ISD::CTPOP, MVT::i8, Promote);1642 setOperationAction(ISD::CTPOP, MVT::i16, Promote);1643 setOperationAction(ISD::CTPOP, MVT::i32, Promote);1644 setOperationAction(ISD::CTPOP, MVT::i64, Legal);1645 1646 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);1647 setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);1648 setOperationAction(ISD::BSWAP, MVT::i32, Legal);1649 setOperationAction(ISD::BSWAP, MVT::i64, Legal);1650 1651 setOperationAction(ISD::FSHL, MVT::i32, Legal);1652 setOperationAction(ISD::FSHL, MVT::i64, Legal);1653 setOperationAction(ISD::FSHR, MVT::i32, Legal);1654 setOperationAction(ISD::FSHR, MVT::i64, Legal);1655 1656 for (unsigned IntExpOp :1657 {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM,1658 ISD::SDIVREM, ISD::UDIVREM, ISD::ROTL, ISD::ROTR,1659 ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS,1660 ISD::SMUL_LOHI, ISD::UMUL_LOHI}) {1661 for (MVT VT : MVT::integer_valuetypes())1662 setOperationAction(IntExpOp, VT, Expand);1663 }1664 1665 for (unsigned FPExpOp :1666 {ISD::FDIV, ISD::FREM, ISD::FSQRT, ISD::FSIN, ISD::FCOS, ISD::FSINCOS,1667 ISD::FPOW, ISD::FCOPYSIGN}) {1668 for (MVT VT : MVT::fp_valuetypes())1669 setOperationAction(FPExpOp, VT, Expand);1670 }1671 1672 // No extending loads from i32.1673 for (MVT VT : MVT::integer_valuetypes()) {1674 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i32, Expand);1675 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand);1676 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i32, Expand);1677 }1678 // Turn FP truncstore into trunc + store.1679 setTruncStoreAction(MVT::f64, MVT::f32, Expand);1680 setTruncStoreAction(MVT::f32, MVT::bf16, Expand);1681 setTruncStoreAction(MVT::f64, MVT::bf16, Expand);1682 // Turn FP extload into load/fpextend.1683 for (MVT VT : MVT::fp_valuetypes())1684 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);1685 1686 // Expand BR_CC and SELECT_CC for all integer and fp types.1687 for (MVT VT : MVT::integer_valuetypes()) {1688 setOperationAction(ISD::BR_CC, VT, Expand);1689 setOperationAction(ISD::SELECT_CC, VT, Expand);1690 }1691 for (MVT VT : MVT::fp_valuetypes()) {1692 setOperationAction(ISD::BR_CC, VT, Expand);1693 setOperationAction(ISD::SELECT_CC, VT, Expand);1694 }1695 setOperationAction(ISD::BR_CC, MVT::Other, Expand);1696 1697 //1698 // Handling of vector operations.1699 //1700 1701 // Set the action for vector operations to "expand", then override it with1702 // either "custom" or "legal" for specific cases.1703 // clang-format off1704 static const unsigned VectExpOps[] = {1705 // Integer arithmetic:1706 ISD::ADD, ISD::SUB, ISD::MUL, ISD::SDIV, ISD::UDIV,1707 ISD::SREM, ISD::UREM, ISD::SDIVREM, ISD::UDIVREM, ISD::SADDO,1708 ISD::UADDO, ISD::SSUBO, ISD::USUBO, ISD::SMUL_LOHI, ISD::UMUL_LOHI,1709 // Logical/bit:1710 ISD::AND, ISD::OR, ISD::XOR, ISD::ROTL, ISD::ROTR,1711 ISD::CTPOP, ISD::CTLZ, ISD::CTTZ, ISD::BSWAP, ISD::BITREVERSE,1712 // Floating point arithmetic/math functions:1713 ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FMA, ISD::FDIV,1714 ISD::FREM, ISD::FNEG, ISD::FABS, ISD::FSQRT, ISD::FSIN,1715 ISD::FCOS, ISD::FPOW, ISD::FLOG, ISD::FLOG2,1716 ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FCEIL, ISD::FTRUNC,1717 ISD::FRINT, ISD::FNEARBYINT, ISD::FROUND, ISD::FFLOOR,1718 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM,1719 ISD::FSINCOS, ISD::FLDEXP,1720 // Misc:1721 ISD::BR_CC, ISD::SELECT_CC, ISD::ConstantPool,1722 // Vector:1723 ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR,1724 ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT,1725 ISD::EXTRACT_SUBVECTOR, ISD::INSERT_SUBVECTOR,1726 ISD::CONCAT_VECTORS, ISD::VECTOR_SHUFFLE,1727 ISD::SPLAT_VECTOR,1728 };1729 // clang-format on1730 1731 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {1732 for (unsigned VectExpOp : VectExpOps)1733 setOperationAction(VectExpOp, VT, Expand);1734 1735 // Expand all extending loads and truncating stores:1736 for (MVT TargetVT : MVT::fixedlen_vector_valuetypes()) {1737 if (TargetVT == VT)1738 continue;1739 setLoadExtAction(ISD::EXTLOAD, TargetVT, VT, Expand);1740 setLoadExtAction(ISD::ZEXTLOAD, TargetVT, VT, Expand);1741 setLoadExtAction(ISD::SEXTLOAD, TargetVT, VT, Expand);1742 setTruncStoreAction(VT, TargetVT, Expand);1743 }1744 1745 // Normalize all inputs to SELECT to be vectors of i32.1746 if (VT.getVectorElementType() != MVT::i32) {1747 MVT VT32 = MVT::getVectorVT(MVT::i32, VT.getSizeInBits()/32);1748 setOperationAction(ISD::SELECT, VT, Promote);1749 AddPromotedToType(ISD::SELECT, VT, VT32);1750 }1751 setOperationAction(ISD::SRA, VT, Custom);1752 setOperationAction(ISD::SHL, VT, Custom);1753 setOperationAction(ISD::SRL, VT, Custom);1754 }1755 1756 setOperationAction(ISD::SADDSAT, MVT::i32, Legal);1757 setOperationAction(ISD::SADDSAT, MVT::i64, Legal);1758 1759 // Extending loads from (native) vectors of i8 into (native) vectors of i161760 // are legal.1761 setLoadExtAction(ISD::EXTLOAD, MVT::v2i16, MVT::v2i8, Legal);1762 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i16, MVT::v2i8, Legal);1763 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i16, MVT::v2i8, Legal);1764 setLoadExtAction(ISD::EXTLOAD, MVT::v4i16, MVT::v4i8, Legal);1765 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, MVT::v4i8, Legal);1766 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, MVT::v4i8, Legal);1767 1768 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Legal);1769 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Legal);1770 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Legal);1771 1772 // Types natively supported:1773 for (MVT NativeVT : {MVT::v8i1, MVT::v4i1, MVT::v2i1, MVT::v4i8,1774 MVT::v8i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}) {1775 setOperationAction(ISD::BUILD_VECTOR, NativeVT, Custom);1776 setOperationAction(ISD::EXTRACT_VECTOR_ELT, NativeVT, Custom);1777 setOperationAction(ISD::INSERT_VECTOR_ELT, NativeVT, Custom);1778 setOperationAction(ISD::EXTRACT_SUBVECTOR, NativeVT, Custom);1779 setOperationAction(ISD::INSERT_SUBVECTOR, NativeVT, Custom);1780 setOperationAction(ISD::CONCAT_VECTORS, NativeVT, Custom);1781 1782 setOperationAction(ISD::ADD, NativeVT, Legal);1783 setOperationAction(ISD::SUB, NativeVT, Legal);1784 setOperationAction(ISD::MUL, NativeVT, Legal);1785 setOperationAction(ISD::AND, NativeVT, Legal);1786 setOperationAction(ISD::OR, NativeVT, Legal);1787 setOperationAction(ISD::XOR, NativeVT, Legal);1788 1789 if (NativeVT.getVectorElementType() != MVT::i1) {1790 setOperationAction(ISD::SPLAT_VECTOR, NativeVT, Legal);1791 setOperationAction(ISD::BSWAP, NativeVT, Legal);1792 setOperationAction(ISD::BITREVERSE, NativeVT, Legal);1793 }1794 }1795 1796 for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32}) {1797 setOperationAction(ISD::SMIN, VT, Legal);1798 setOperationAction(ISD::SMAX, VT, Legal);1799 setOperationAction(ISD::UMIN, VT, Legal);1800 setOperationAction(ISD::UMAX, VT, Legal);1801 }1802 1803 // Custom lower unaligned loads.1804 // Also, for both loads and stores, verify the alignment of the address1805 // in case it is a compile-time constant. This is a usability feature to1806 // provide a meaningful error message to users.1807 for (MVT VT : {MVT::i16, MVT::i32, MVT::v4i8, MVT::i64, MVT::v8i8,1808 MVT::v2i16, MVT::v4i16, MVT::v2i32}) {1809 setOperationAction(ISD::LOAD, VT, Custom);1810 setOperationAction(ISD::STORE, VT, Custom);1811 }1812 1813 // Custom-lower load/stores of boolean vectors.1814 for (MVT VT : {MVT::v2i1, MVT::v4i1, MVT::v8i1}) {1815 setOperationAction(ISD::LOAD, VT, Custom);1816 setOperationAction(ISD::STORE, VT, Custom);1817 }1818 1819 // Normalize integer compares to EQ/GT/UGT1820 for (MVT VT : {MVT::v2i16, MVT::v4i8, MVT::v8i8, MVT::v2i32, MVT::v4i16,1821 MVT::v2i32}) {1822 setCondCodeAction(ISD::SETNE, VT, Expand);1823 setCondCodeAction(ISD::SETLE, VT, Expand);1824 setCondCodeAction(ISD::SETGE, VT, Expand);1825 setCondCodeAction(ISD::SETLT, VT, Expand);1826 setCondCodeAction(ISD::SETULE, VT, Expand);1827 setCondCodeAction(ISD::SETUGE, VT, Expand);1828 setCondCodeAction(ISD::SETULT, VT, Expand);1829 }1830 1831 // Normalize boolean compares to [U]LE/[U]LT1832 for (MVT VT : {MVT::i1, MVT::v2i1, MVT::v4i1, MVT::v8i1}) {1833 setCondCodeAction(ISD::SETGE, VT, Expand);1834 setCondCodeAction(ISD::SETGT, VT, Expand);1835 setCondCodeAction(ISD::SETUGE, VT, Expand);1836 setCondCodeAction(ISD::SETUGT, VT, Expand);1837 }1838 1839 // Custom-lower bitcasts from i8 to v8i1.1840 setOperationAction(ISD::BITCAST, MVT::i8, Custom);1841 setOperationAction(ISD::SETCC, MVT::v2i16, Custom);1842 setOperationAction(ISD::VSELECT, MVT::v4i8, Custom);1843 setOperationAction(ISD::VSELECT, MVT::v2i16, Custom);1844 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i8, Custom);1845 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom);1846 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom);1847 1848 // V5+.1849 setOperationAction(ISD::FMA, MVT::f64, Expand);1850 setOperationAction(ISD::FADD, MVT::f64, Expand);1851 setOperationAction(ISD::FSUB, MVT::f64, Expand);1852 setOperationAction(ISD::FMUL, MVT::f64, Expand);1853 setOperationAction(ISD::FDIV, MVT::f32, Custom);1854 1855 setOperationAction(ISD::FMINIMUMNUM, MVT::f32, Legal);1856 setOperationAction(ISD::FMAXIMUMNUM, MVT::f32, Legal);1857 1858 setOperationAction(ISD::FP_TO_UINT, MVT::i1, Promote);1859 setOperationAction(ISD::FP_TO_UINT, MVT::i8, Promote);1860 setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote);1861 setOperationAction(ISD::FP_TO_SINT, MVT::i1, Promote);1862 setOperationAction(ISD::FP_TO_SINT, MVT::i8, Promote);1863 setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote);1864 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote);1865 setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote);1866 setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);1867 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote);1868 setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote);1869 setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote);1870 1871 // Special handling for half-precision floating point conversions.1872 // Lower half float conversions into library calls.1873 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);1874 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);1875 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);1876 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);1877 setOperationAction(ISD::BF16_TO_FP, MVT::f32, Expand);1878 setOperationAction(ISD::BF16_TO_FP, MVT::f64, Expand);1879 setOperationAction(ISD::FP_TO_BF16, MVT::f64, Expand);1880 1881 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);1882 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);1883 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand);1884 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand);1885 1886 setTruncStoreAction(MVT::f32, MVT::f16, Expand);1887 setTruncStoreAction(MVT::f64, MVT::f16, Expand);1888 1889 // Handling of indexed loads/stores: default is "expand".1890 //1891 for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64, MVT::f32, MVT::f64,1892 MVT::v2i16, MVT::v2i32, MVT::v4i8, MVT::v4i16, MVT::v8i8}) {1893 setIndexedLoadAction(ISD::POST_INC, VT, Legal);1894 setIndexedStoreAction(ISD::POST_INC, VT, Legal);1895 }1896 1897 // Subtarget-specific operation actions.1898 //1899 if (Subtarget.hasV60Ops()) {1900 setOperationAction(ISD::ROTL, MVT::i32, Legal);1901 setOperationAction(ISD::ROTL, MVT::i64, Legal);1902 setOperationAction(ISD::ROTR, MVT::i32, Legal);1903 setOperationAction(ISD::ROTR, MVT::i64, Legal);1904 }1905 if (Subtarget.hasV66Ops()) {1906 setOperationAction(ISD::FADD, MVT::f64, Legal);1907 setOperationAction(ISD::FSUB, MVT::f64, Legal);1908 }1909 if (Subtarget.hasV67Ops()) {1910 setOperationAction(ISD::FMINIMUMNUM, MVT::f64, Legal);1911 setOperationAction(ISD::FMAXIMUMNUM, MVT::f64, Legal);1912 setOperationAction(ISD::FMUL, MVT::f64, Legal);1913 }1914 1915 setTargetDAGCombine(ISD::OR);1916 setTargetDAGCombine(ISD::TRUNCATE);1917 setTargetDAGCombine(ISD::VSELECT);1918 1919 if (Subtarget.useHVXOps())1920 initializeHVXLowering();1921 1922 computeRegisterProperties(&HRI);1923}1924 1925const char* HexagonTargetLowering::getTargetNodeName(unsigned Opcode) const {1926 switch ((HexagonISD::NodeType)Opcode) {1927 case HexagonISD::ADDC: return "HexagonISD::ADDC";1928 case HexagonISD::SUBC: return "HexagonISD::SUBC";1929 case HexagonISD::ALLOCA: return "HexagonISD::ALLOCA";1930 case HexagonISD::AT_GOT: return "HexagonISD::AT_GOT";1931 case HexagonISD::AT_PCREL: return "HexagonISD::AT_PCREL";1932 case HexagonISD::BARRIER: return "HexagonISD::BARRIER";1933 case HexagonISD::CALL: return "HexagonISD::CALL";1934 case HexagonISD::CALLnr: return "HexagonISD::CALLnr";1935 case HexagonISD::CALLR: return "HexagonISD::CALLR";1936 case HexagonISD::COMBINE: return "HexagonISD::COMBINE";1937 case HexagonISD::CONST32_GP: return "HexagonISD::CONST32_GP";1938 case HexagonISD::CONST32: return "HexagonISD::CONST32";1939 case HexagonISD::CP: return "HexagonISD::CP";1940 case HexagonISD::DCFETCH: return "HexagonISD::DCFETCH";1941 case HexagonISD::EH_RETURN: return "HexagonISD::EH_RETURN";1942 case HexagonISD::TSTBIT: return "HexagonISD::TSTBIT";1943 case HexagonISD::EXTRACTU: return "HexagonISD::EXTRACTU";1944 case HexagonISD::INSERT: return "HexagonISD::INSERT";1945 case HexagonISD::JT: return "HexagonISD::JT";1946 case HexagonISD::RET_GLUE: return "HexagonISD::RET_GLUE";1947 case HexagonISD::TC_RETURN: return "HexagonISD::TC_RETURN";1948 case HexagonISD::VASL: return "HexagonISD::VASL";1949 case HexagonISD::VASR: return "HexagonISD::VASR";1950 case HexagonISD::VLSR: return "HexagonISD::VLSR";1951 case HexagonISD::MFSHL: return "HexagonISD::MFSHL";1952 case HexagonISD::MFSHR: return "HexagonISD::MFSHR";1953 case HexagonISD::SSAT: return "HexagonISD::SSAT";1954 case HexagonISD::USAT: return "HexagonISD::USAT";1955 case HexagonISD::SMUL_LOHI: return "HexagonISD::SMUL_LOHI";1956 case HexagonISD::UMUL_LOHI: return "HexagonISD::UMUL_LOHI";1957 case HexagonISD::USMUL_LOHI: return "HexagonISD::USMUL_LOHI";1958 case HexagonISD::VEXTRACTW: return "HexagonISD::VEXTRACTW";1959 case HexagonISD::VINSERTW0: return "HexagonISD::VINSERTW0";1960 case HexagonISD::VROR: return "HexagonISD::VROR";1961 case HexagonISD::READCYCLE: return "HexagonISD::READCYCLE";1962 case HexagonISD::READTIMER: return "HexagonISD::READTIMER";1963 case HexagonISD::THREAD_POINTER:1964 return "HexagonISD::THREAD_POINTER";1965 case HexagonISD::PTRUE: return "HexagonISD::PTRUE";1966 case HexagonISD::PFALSE: return "HexagonISD::PFALSE";1967 case HexagonISD::D2P: return "HexagonISD::D2P";1968 case HexagonISD::P2D: return "HexagonISD::P2D";1969 case HexagonISD::V2Q: return "HexagonISD::V2Q";1970 case HexagonISD::Q2V: return "HexagonISD::Q2V";1971 case HexagonISD::QCAT: return "HexagonISD::QCAT";1972 case HexagonISD::QTRUE: return "HexagonISD::QTRUE";1973 case HexagonISD::QFALSE: return "HexagonISD::QFALSE";1974 case HexagonISD::TL_EXTEND: return "HexagonISD::TL_EXTEND";1975 case HexagonISD::TL_TRUNCATE: return "HexagonISD::TL_TRUNCATE";1976 case HexagonISD::TYPECAST: return "HexagonISD::TYPECAST";1977 case HexagonISD::VALIGN: return "HexagonISD::VALIGN";1978 case HexagonISD::VALIGNADDR: return "HexagonISD::VALIGNADDR";1979 case HexagonISD::ISEL: return "HexagonISD::ISEL";1980 case HexagonISD::OP_END: break;1981 }1982 return nullptr;1983}1984 1985bool1986HexagonTargetLowering::validateConstPtrAlignment(SDValue Ptr, Align NeedAlign,1987 const SDLoc &dl, SelectionDAG &DAG) const {1988 auto *CA = dyn_cast<ConstantSDNode>(Ptr);1989 if (!CA)1990 return true;1991 unsigned Addr = CA->getZExtValue();1992 Align HaveAlign =1993 Addr != 0 ? Align(1ull << llvm::countr_zero(Addr)) : NeedAlign;1994 if (HaveAlign >= NeedAlign)1995 return true;1996 1997 static int DK_MisalignedTrap = llvm::getNextAvailablePluginDiagnosticKind();1998 1999 struct DiagnosticInfoMisalignedTrap : public DiagnosticInfo {2000 DiagnosticInfoMisalignedTrap(StringRef M)2001 : DiagnosticInfo(DK_MisalignedTrap, DS_Remark), Msg(M) {}2002 void print(DiagnosticPrinter &DP) const override {2003 DP << Msg;2004 }2005 static bool classof(const DiagnosticInfo *DI) {2006 return DI->getKind() == DK_MisalignedTrap;2007 }2008 StringRef Msg;2009 };2010 2011 std::string ErrMsg;2012 raw_string_ostream O(ErrMsg);2013 O << "Misaligned constant address: " << format_hex(Addr, 10)2014 << " has alignment " << HaveAlign.value()2015 << ", but the memory access requires " << NeedAlign.value();2016 if (DebugLoc DL = dl.getDebugLoc())2017 DL.print(O << ", at ");2018 O << ". The instruction has been replaced with a trap.";2019 2020 DAG.getContext()->diagnose(DiagnosticInfoMisalignedTrap(O.str()));2021 return false;2022}2023 2024SDValue2025HexagonTargetLowering::replaceMemWithUndef(SDValue Op, SelectionDAG &DAG)2026 const {2027 const SDLoc &dl(Op);2028 auto *LS = cast<LSBaseSDNode>(Op.getNode());2029 assert(!LS->isIndexed() && "Not expecting indexed ops on constant address");2030 2031 SDValue Chain = LS->getChain();2032 SDValue Trap = DAG.getNode(ISD::TRAP, dl, MVT::Other, Chain);2033 if (LS->getOpcode() == ISD::LOAD)2034 return DAG.getMergeValues({DAG.getUNDEF(ty(Op)), Trap}, dl);2035 return Trap;2036}2037 2038// Bit-reverse Load Intrinsic: Check if the instruction is a bit reverse load2039// intrinsic.2040static bool isBrevLdIntrinsic(const Value *Inst) {2041 unsigned ID = cast<IntrinsicInst>(Inst)->getIntrinsicID();2042 return (ID == Intrinsic::hexagon_L2_loadrd_pbr ||2043 ID == Intrinsic::hexagon_L2_loadri_pbr ||2044 ID == Intrinsic::hexagon_L2_loadrh_pbr ||2045 ID == Intrinsic::hexagon_L2_loadruh_pbr ||2046 ID == Intrinsic::hexagon_L2_loadrb_pbr ||2047 ID == Intrinsic::hexagon_L2_loadrub_pbr);2048}2049 2050// Bit-reverse Load Intrinsic :Crawl up and figure out the object from previous2051// instruction. So far we only handle bitcast, extract value and bit reverse2052// load intrinsic instructions. Should we handle CGEP ?2053static Value *getBrevLdObject(Value *V) {2054 if (Operator::getOpcode(V) == Instruction::ExtractValue ||2055 Operator::getOpcode(V) == Instruction::BitCast)2056 V = cast<Operator>(V)->getOperand(0);2057 else if (isa<IntrinsicInst>(V) && isBrevLdIntrinsic(V))2058 V = cast<Instruction>(V)->getOperand(0);2059 return V;2060}2061 2062// Bit-reverse Load Intrinsic: For a PHI Node return either an incoming edge or2063// a back edge. If the back edge comes from the intrinsic itself, the incoming2064// edge is returned.2065static Value *returnEdge(const PHINode *PN, Value *IntrBaseVal) {2066 const BasicBlock *Parent = PN->getParent();2067 int Idx = -1;2068 for (unsigned i = 0, e = PN->getNumIncomingValues(); i < e; ++i) {2069 BasicBlock *Blk = PN->getIncomingBlock(i);2070 // Determine if the back edge is originated from intrinsic.2071 if (Blk == Parent) {2072 Value *BackEdgeVal = PN->getIncomingValue(i);2073 Value *BaseVal;2074 // Loop over till we return the same Value or we hit the IntrBaseVal.2075 do {2076 BaseVal = BackEdgeVal;2077 BackEdgeVal = getBrevLdObject(BackEdgeVal);2078 } while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal));2079 // If the getBrevLdObject returns IntrBaseVal, we should return the2080 // incoming edge.2081 if (IntrBaseVal == BackEdgeVal)2082 continue;2083 Idx = i;2084 break;2085 } else // Set the node to incoming edge.2086 Idx = i;2087 }2088 assert(Idx >= 0 && "Unexpected index to incoming argument in PHI");2089 return PN->getIncomingValue(Idx);2090}2091 2092// Bit-reverse Load Intrinsic: Figure out the underlying object the base2093// pointer points to, for the bit-reverse load intrinsic. Setting this to2094// memoperand might help alias analysis to figure out the dependencies.2095static Value *getUnderLyingObjectForBrevLdIntr(Value *V) {2096 Value *IntrBaseVal = V;2097 Value *BaseVal;2098 // Loop over till we return the same Value, implies we either figure out2099 // the object or we hit a PHI2100 do {2101 BaseVal = V;2102 V = getBrevLdObject(V);2103 } while (BaseVal != V);2104 2105 // Identify the object from PHINode.2106 if (const PHINode *PN = dyn_cast<PHINode>(V))2107 return returnEdge(PN, IntrBaseVal);2108 // For non PHI nodes, the object is the last value returned by getBrevLdObject2109 else2110 return V;2111}2112 2113/// Given an intrinsic, checks if on the target the intrinsic will need to map2114/// to a MemIntrinsicNode (touches memory). If this is the case, it returns2115/// true and store the intrinsic information into the IntrinsicInfo that was2116/// passed to the function.2117bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,2118 const CallInst &I,2119 MachineFunction &MF,2120 unsigned Intrinsic) const {2121 switch (Intrinsic) {2122 case Intrinsic::hexagon_L2_loadrd_pbr:2123 case Intrinsic::hexagon_L2_loadri_pbr:2124 case Intrinsic::hexagon_L2_loadrh_pbr:2125 case Intrinsic::hexagon_L2_loadruh_pbr:2126 case Intrinsic::hexagon_L2_loadrb_pbr:2127 case Intrinsic::hexagon_L2_loadrub_pbr: {2128 Info.opc = ISD::INTRINSIC_W_CHAIN;2129 auto &DL = I.getDataLayout();2130 auto &Cont = I.getCalledFunction()->getParent()->getContext();2131 // The intrinsic function call is of the form { ElTy, i8* }2132 // @llvm.hexagon.L2.loadXX.pbr(i8*, i32). The pointer and memory access type2133 // should be derived from ElTy.2134 Type *ElTy = I.getCalledFunction()->getReturnType()->getStructElementType(0);2135 Info.memVT = MVT::getVT(ElTy);2136 llvm::Value *BasePtrVal = I.getOperand(0);2137 Info.ptrVal = getUnderLyingObjectForBrevLdIntr(BasePtrVal);2138 // The offset value comes through Modifier register. For now, assume the2139 // offset is 0.2140 Info.offset = 0;2141 Info.align = DL.getABITypeAlign(Info.memVT.getTypeForEVT(Cont));2142 Info.flags = MachineMemOperand::MOLoad;2143 return true;2144 }2145 case Intrinsic::hexagon_V6_vgathermw:2146 case Intrinsic::hexagon_V6_vgathermw_128B:2147 case Intrinsic::hexagon_V6_vgathermh:2148 case Intrinsic::hexagon_V6_vgathermh_128B:2149 case Intrinsic::hexagon_V6_vgathermhw:2150 case Intrinsic::hexagon_V6_vgathermhw_128B:2151 case Intrinsic::hexagon_V6_vgathermwq:2152 case Intrinsic::hexagon_V6_vgathermwq_128B:2153 case Intrinsic::hexagon_V6_vgathermhq:2154 case Intrinsic::hexagon_V6_vgathermhq_128B:2155 case Intrinsic::hexagon_V6_vgathermhwq:2156 case Intrinsic::hexagon_V6_vgathermhwq_128B:2157 case Intrinsic::hexagon_V6_vgather_vscattermh:2158 case Intrinsic::hexagon_V6_vgather_vscattermh_128B: {2159 const Module &M = *I.getParent()->getParent()->getParent();2160 Info.opc = ISD::INTRINSIC_W_CHAIN;2161 Type *VecTy = I.getArgOperand(1)->getType();2162 Info.memVT = MVT::getVT(VecTy);2163 Info.ptrVal = I.getArgOperand(0);2164 Info.offset = 0;2165 Info.align =2166 MaybeAlign(M.getDataLayout().getTypeAllocSizeInBits(VecTy) / 8);2167 Info.flags = MachineMemOperand::MOLoad |2168 MachineMemOperand::MOStore |2169 MachineMemOperand::MOVolatile;2170 return true;2171 }2172 default:2173 break;2174 }2175 return false;2176}2177 2178bool HexagonTargetLowering::hasBitTest(SDValue X, SDValue Y) const {2179 return X.getValueType().isScalarInteger(); // 'tstbit'2180}2181 2182bool HexagonTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {2183 return isTruncateFree(EVT::getEVT(Ty1), EVT::getEVT(Ty2));2184}2185 2186bool HexagonTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {2187 if (!VT1.isSimple() || !VT2.isSimple())2188 return false;2189 return VT1.getSimpleVT() == MVT::i64 && VT2.getSimpleVT() == MVT::i32;2190}2191 2192bool HexagonTargetLowering::isFMAFasterThanFMulAndFAdd(2193 const MachineFunction &MF, EVT VT) const {2194 return isOperationLegalOrCustom(ISD::FMA, VT);2195}2196 2197// Should we expand the build vector with shuffles?2198bool HexagonTargetLowering::shouldExpandBuildVectorWithShuffles(EVT VT,2199 unsigned DefinedValues) const {2200 return false;2201}2202 2203bool HexagonTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,2204 unsigned Index) const {2205 assert(ResVT.getVectorElementType() == SrcVT.getVectorElementType());2206 if (!ResVT.isSimple() || !SrcVT.isSimple())2207 return false;2208 2209 MVT ResTy = ResVT.getSimpleVT(), SrcTy = SrcVT.getSimpleVT();2210 if (ResTy.getVectorElementType() != MVT::i1)2211 return true;2212 2213 // Non-HVX bool vectors are relatively cheap.2214 return SrcTy.getVectorNumElements() <= 8;2215}2216 2217bool HexagonTargetLowering::isTargetCanonicalConstantNode(SDValue Op) const {2218 return Op.getOpcode() == ISD::CONCAT_VECTORS ||2219 TargetLowering::isTargetCanonicalConstantNode(Op);2220}2221 2222bool HexagonTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask,2223 EVT VT) const {2224 return true;2225}2226 2227TargetLoweringBase::LegalizeTypeAction2228HexagonTargetLowering::getPreferredVectorAction(MVT VT) const {2229 unsigned VecLen = VT.getVectorMinNumElements();2230 MVT ElemTy = VT.getVectorElementType();2231 2232 if (VecLen == 1 || VT.isScalableVector())2233 return TargetLoweringBase::TypeScalarizeVector;2234 2235 if (Subtarget.useHVXOps()) {2236 unsigned Action = getPreferredHvxVectorAction(VT);2237 if (Action != ~0u)2238 return static_cast<TargetLoweringBase::LegalizeTypeAction>(Action);2239 }2240 2241 // Always widen (remaining) vectors of i1.2242 if (ElemTy == MVT::i1)2243 return TargetLoweringBase::TypeWidenVector;2244 // Widen non-power-of-2 vectors. Such types cannot be split right now,2245 // and computeRegisterProperties will override "split" with "widen",2246 // which can cause other issues.2247 if (!isPowerOf2_32(VecLen))2248 return TargetLoweringBase::TypeWidenVector;2249 2250 return TargetLoweringBase::TypeSplitVector;2251}2252 2253TargetLoweringBase::LegalizeAction2254HexagonTargetLowering::getCustomOperationAction(SDNode &Op) const {2255 if (Subtarget.useHVXOps()) {2256 unsigned Action = getCustomHvxOperationAction(Op);2257 if (Action != ~0u)2258 return static_cast<TargetLoweringBase::LegalizeAction>(Action);2259 }2260 return TargetLoweringBase::Legal;2261}2262 2263std::pair<SDValue, int>2264HexagonTargetLowering::getBaseAndOffset(SDValue Addr) const {2265 if (Addr.getOpcode() == ISD::ADD) {2266 SDValue Op1 = Addr.getOperand(1);2267 if (auto *CN = dyn_cast<const ConstantSDNode>(Op1.getNode()))2268 return { Addr.getOperand(0), CN->getSExtValue() };2269 }2270 return { Addr, 0 };2271}2272 2273// Lower a vector shuffle (V1, V2, V3). V1 and V2 are the two vectors2274// to select data from, V3 is the permutation.2275SDValue2276HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)2277 const {2278 const auto *SVN = cast<ShuffleVectorSDNode>(Op);2279 ArrayRef<int> AM = SVN->getMask();2280 assert(AM.size() <= 8 && "Unexpected shuffle mask");2281 unsigned VecLen = AM.size();2282 2283 MVT VecTy = ty(Op);2284 assert(!Subtarget.isHVXVectorType(VecTy, true) &&2285 "HVX shuffles should be legal");2286 assert(VecTy.getSizeInBits() <= 64 && "Unexpected vector length");2287 2288 SDValue Op0 = Op.getOperand(0);2289 SDValue Op1 = Op.getOperand(1);2290 const SDLoc &dl(Op);2291 2292 // If the inputs are not the same as the output, bail. This is not an2293 // error situation, but complicates the handling and the default expansion2294 // (into BUILD_VECTOR) should be adequate.2295 if (ty(Op0) != VecTy || ty(Op1) != VecTy)2296 return SDValue();2297 2298 // Normalize the mask so that the first non-negative index comes from2299 // the first operand.2300 SmallVector<int, 8> Mask(AM);2301 unsigned F = llvm::find_if(AM, [](int M) { return M >= 0; }) - AM.data();2302 if (F == AM.size())2303 return DAG.getUNDEF(VecTy);2304 if (AM[F] >= int(VecLen)) {2305 ShuffleVectorSDNode::commuteMask(Mask);2306 std::swap(Op0, Op1);2307 }2308 2309 // Express the shuffle mask in terms of bytes.2310 SmallVector<int,8> ByteMask;2311 unsigned ElemBytes = VecTy.getVectorElementType().getSizeInBits() / 8;2312 for (int M : Mask) {2313 if (M < 0) {2314 for (unsigned j = 0; j != ElemBytes; ++j)2315 ByteMask.push_back(-1);2316 } else {2317 for (unsigned j = 0; j != ElemBytes; ++j)2318 ByteMask.push_back(M*ElemBytes + j);2319 }2320 }2321 assert(ByteMask.size() <= 8);2322 2323 // All non-undef (non-negative) indexes are well within [0..127], so they2324 // fit in a single byte. Build two 64-bit words:2325 // - MaskIdx where each byte is the corresponding index (for non-negative2326 // indexes), and 0xFF for negative indexes, and2327 // - MaskUnd that has 0xFF for each negative index.2328 uint64_t MaskIdx = 0;2329 uint64_t MaskUnd = 0;2330 for (unsigned i = 0, e = ByteMask.size(); i != e; ++i) {2331 unsigned S = 8*i;2332 uint64_t M = ByteMask[i] & 0xFF;2333 if (M == 0xFF)2334 MaskUnd |= M << S;2335 MaskIdx |= M << S;2336 }2337 2338 if (ByteMask.size() == 4) {2339 // Identity.2340 if (MaskIdx == (0x03020100 | MaskUnd))2341 return Op0;2342 // Byte swap.2343 if (MaskIdx == (0x00010203 | MaskUnd)) {2344 SDValue T0 = DAG.getBitcast(MVT::i32, Op0);2345 SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i32, T0);2346 return DAG.getBitcast(VecTy, T1);2347 }2348 2349 // Byte packs.2350 SDValue Concat10 =2351 getCombine(Op1, Op0, dl, typeJoin({ty(Op1), ty(Op0)}), DAG);2352 if (MaskIdx == (0x06040200 | MaskUnd))2353 return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat10}, DAG);2354 if (MaskIdx == (0x07050301 | MaskUnd))2355 return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat10}, DAG);2356 2357 SDValue Concat01 =2358 getCombine(Op0, Op1, dl, typeJoin({ty(Op0), ty(Op1)}), DAG);2359 if (MaskIdx == (0x02000604 | MaskUnd))2360 return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat01}, DAG);2361 if (MaskIdx == (0x03010705 | MaskUnd))2362 return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat01}, DAG);2363 }2364 2365 if (ByteMask.size() == 8) {2366 // Identity.2367 if (MaskIdx == (0x0706050403020100ull | MaskUnd))2368 return Op0;2369 // Byte swap.2370 if (MaskIdx == (0x0001020304050607ull | MaskUnd)) {2371 SDValue T0 = DAG.getBitcast(MVT::i64, Op0);2372 SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i64, T0);2373 return DAG.getBitcast(VecTy, T1);2374 }2375 2376 // Halfword picks.2377 if (MaskIdx == (0x0d0c050409080100ull | MaskUnd))2378 return getInstr(Hexagon::S2_shuffeh, dl, VecTy, {Op1, Op0}, DAG);2379 if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd))2380 return getInstr(Hexagon::S2_shuffoh, dl, VecTy, {Op1, Op0}, DAG);2381 if (MaskIdx == (0x0d0c090805040100ull | MaskUnd))2382 return getInstr(Hexagon::S2_vtrunewh, dl, VecTy, {Op1, Op0}, DAG);2383 if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd))2384 return getInstr(Hexagon::S2_vtrunowh, dl, VecTy, {Op1, Op0}, DAG);2385 if (MaskIdx == (0x0706030205040100ull | MaskUnd)) {2386 VectorPair P = opSplit(Op0, dl, DAG);2387 return getInstr(Hexagon::S2_packhl, dl, VecTy, {P.second, P.first}, DAG);2388 }2389 2390 // Byte packs.2391 if (MaskIdx == (0x0e060c040a020800ull | MaskUnd))2392 return getInstr(Hexagon::S2_shuffeb, dl, VecTy, {Op1, Op0}, DAG);2393 if (MaskIdx == (0x0f070d050b030901ull | MaskUnd))2394 return getInstr(Hexagon::S2_shuffob, dl, VecTy, {Op1, Op0}, DAG);2395 }2396 2397 return SDValue();2398}2399 2400SDValue2401HexagonTargetLowering::getSplatValue(SDValue Op, SelectionDAG &DAG) const {2402 switch (Op.getOpcode()) {2403 case ISD::BUILD_VECTOR:2404 if (SDValue S = cast<BuildVectorSDNode>(Op)->getSplatValue())2405 return S;2406 break;2407 case ISD::SPLAT_VECTOR:2408 return Op.getOperand(0);2409 }2410 return SDValue();2411}2412 2413// Create a Hexagon-specific node for shifting a vector by an integer.2414SDValue2415HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG)2416 const {2417 unsigned NewOpc;2418 switch (Op.getOpcode()) {2419 case ISD::SHL:2420 NewOpc = HexagonISD::VASL;2421 break;2422 case ISD::SRA:2423 NewOpc = HexagonISD::VASR;2424 break;2425 case ISD::SRL:2426 NewOpc = HexagonISD::VLSR;2427 break;2428 default:2429 llvm_unreachable("Unexpected shift opcode");2430 }2431 2432 if (SDValue Sp = getSplatValue(Op.getOperand(1), DAG))2433 return DAG.getNode(NewOpc, SDLoc(Op), ty(Op), Op.getOperand(0), Sp);2434 return SDValue();2435}2436 2437SDValue2438HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const {2439 const SDLoc &dl(Op);2440 2441 // First try to convert the shift (by vector) to a shift by a scalar.2442 // If we first split the shift, the shift amount will become 'extract2443 // subvector', and will no longer be recognized as scalar.2444 SDValue Res = Op;2445 if (SDValue S = getVectorShiftByInt(Op, DAG))2446 Res = S;2447 2448 unsigned Opc = Res.getOpcode();2449 switch (Opc) {2450 case HexagonISD::VASR:2451 case HexagonISD::VLSR:2452 case HexagonISD::VASL:2453 break;2454 default:2455 // No instructions for shifts by non-scalars.2456 return SDValue();2457 }2458 2459 MVT ResTy = ty(Res);2460 if (ResTy.getVectorElementType() != MVT::i8)2461 return Res;2462 2463 // For shifts of i8, extend the inputs to i16, then truncate back to i8.2464 assert(ResTy.getVectorElementType() == MVT::i8);2465 SDValue Val = Res.getOperand(0), Amt = Res.getOperand(1);2466 2467 auto ShiftPartI8 = [&dl, &DAG, this](unsigned Opc, SDValue V, SDValue A) {2468 MVT Ty = ty(V);2469 MVT ExtTy = MVT::getVectorVT(MVT::i16, Ty.getVectorNumElements());2470 SDValue ExtV = Opc == HexagonISD::VASR ? DAG.getSExtOrTrunc(V, dl, ExtTy)2471 : DAG.getZExtOrTrunc(V, dl, ExtTy);2472 SDValue ExtS = DAG.getNode(Opc, dl, ExtTy, {ExtV, A});2473 return DAG.getZExtOrTrunc(ExtS, dl, Ty);2474 };2475 2476 if (ResTy.getSizeInBits() == 32)2477 return ShiftPartI8(Opc, Val, Amt);2478 2479 auto [LoV, HiV] = opSplit(Val, dl, DAG);2480 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy,2481 {ShiftPartI8(Opc, LoV, Amt), ShiftPartI8(Opc, HiV, Amt)});2482}2483 2484SDValue2485HexagonTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {2486 if (isa<ConstantSDNode>(Op.getOperand(1).getNode()))2487 return Op;2488 return SDValue();2489}2490 2491SDValue2492HexagonTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {2493 MVT ResTy = ty(Op);2494 SDValue InpV = Op.getOperand(0);2495 MVT InpTy = ty(InpV);2496 assert(ResTy.getSizeInBits() == InpTy.getSizeInBits());2497 const SDLoc &dl(Op);2498 2499 // Handle conversion from i8 to v8i1.2500 if (InpTy == MVT::i8) {2501 if (ResTy == MVT::v8i1) {2502 SDValue Sc = DAG.getBitcast(tyScalar(InpTy), InpV);2503 SDValue Ext = DAG.getZExtOrTrunc(Sc, dl, MVT::i32);2504 return getInstr(Hexagon::C2_tfrrp, dl, ResTy, Ext, DAG);2505 }2506 return SDValue();2507 }2508 2509 return Op;2510}2511 2512bool2513HexagonTargetLowering::getBuildVectorConstInts(ArrayRef<SDValue> Values,2514 MVT VecTy, SelectionDAG &DAG,2515 MutableArrayRef<ConstantInt*> Consts) const {2516 MVT ElemTy = VecTy.getVectorElementType();2517 unsigned ElemWidth = ElemTy.getSizeInBits();2518 IntegerType *IntTy = IntegerType::get(*DAG.getContext(), ElemWidth);2519 bool AllConst = true;2520 2521 for (unsigned i = 0, e = Values.size(); i != e; ++i) {2522 SDValue V = Values[i];2523 if (V.isUndef()) {2524 Consts[i] = ConstantInt::get(IntTy, 0);2525 continue;2526 }2527 // Make sure to always cast to IntTy.2528 if (auto *CN = dyn_cast<ConstantSDNode>(V.getNode())) {2529 const ConstantInt *CI = CN->getConstantIntValue();2530 Consts[i] = ConstantInt::get(IntTy, CI->getValue().getSExtValue());2531 } else if (auto *CN = dyn_cast<ConstantFPSDNode>(V.getNode())) {2532 const ConstantFP *CF = CN->getConstantFPValue();2533 APInt A = CF->getValueAPF().bitcastToAPInt();2534 Consts[i] = ConstantInt::get(IntTy, A.getZExtValue());2535 } else {2536 AllConst = false;2537 }2538 }2539 return AllConst;2540}2541 2542SDValue2543HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl,2544 MVT VecTy, SelectionDAG &DAG) const {2545 MVT ElemTy = VecTy.getVectorElementType();2546 assert(VecTy.getVectorNumElements() == Elem.size());2547 2548 SmallVector<ConstantInt*,4> Consts(Elem.size());2549 bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);2550 2551 unsigned First, Num = Elem.size();2552 for (First = 0; First != Num; ++First) {2553 if (!isUndef(Elem[First]))2554 break;2555 }2556 if (First == Num)2557 return DAG.getUNDEF(VecTy);2558 2559 if (AllConst &&2560 llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); }))2561 return getZero(dl, VecTy, DAG);2562 2563 if (ElemTy == MVT::i16 || ElemTy == MVT::f16) {2564 assert(Elem.size() == 2);2565 if (AllConst) {2566 // The 'Consts' array will have all values as integers regardless2567 // of the vector element type.2568 uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) |2569 Consts[1]->getZExtValue() << 16;2570 return DAG.getBitcast(VecTy, DAG.getConstant(V, dl, MVT::i32));2571 }2572 SDValue E0, E1;2573 if (ElemTy == MVT::f16) {2574 E0 = DAG.getZExtOrTrunc(DAG.getBitcast(MVT::i16, Elem[0]), dl, MVT::i32);2575 E1 = DAG.getZExtOrTrunc(DAG.getBitcast(MVT::i16, Elem[1]), dl, MVT::i32);2576 } else {2577 E0 = Elem[0];2578 E1 = Elem[1];2579 }2580 SDValue N = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32, {E1, E0}, DAG);2581 return DAG.getBitcast(VecTy, N);2582 }2583 2584 if (ElemTy == MVT::i8) {2585 // First try generating a constant.2586 if (AllConst) {2587 uint32_t V = (Consts[0]->getZExtValue() & 0xFF) |2588 (Consts[1]->getZExtValue() & 0xFF) << 8 |2589 (Consts[2]->getZExtValue() & 0xFF) << 16 |2590 Consts[3]->getZExtValue() << 24;2591 return DAG.getBitcast(MVT::v4i8, DAG.getConstant(V, dl, MVT::i32));2592 }2593 2594 // Then try splat.2595 bool IsSplat = true;2596 for (unsigned i = First+1; i != Num; ++i) {2597 if (Elem[i] == Elem[First] || isUndef(Elem[i]))2598 continue;2599 IsSplat = false;2600 break;2601 }2602 if (IsSplat) {2603 // Legalize the operand of SPLAT_VECTOR.2604 SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32);2605 return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext);2606 }2607 2608 // Generate2609 // (zxtb(Elem[0]) | (zxtb(Elem[1]) << 8)) |2610 // (zxtb(Elem[2]) | (zxtb(Elem[3]) << 8)) << 162611 assert(Elem.size() == 4);2612 SDValue Vs[4];2613 for (unsigned i = 0; i != 4; ++i) {2614 Vs[i] = DAG.getZExtOrTrunc(Elem[i], dl, MVT::i32);2615 Vs[i] = DAG.getZeroExtendInReg(Vs[i], dl, MVT::i8);2616 }2617 SDValue S8 = DAG.getConstant(8, dl, MVT::i32);2618 SDValue T0 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[1], S8});2619 SDValue T1 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[3], S8});2620 SDValue B0 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[0], T0});2621 SDValue B1 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[2], T1});2622 2623 SDValue R = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32, {B1, B0}, DAG);2624 return DAG.getBitcast(MVT::v4i8, R);2625 }2626 2627#ifndef NDEBUG2628 dbgs() << "VecTy: " << VecTy << '\n';2629#endif2630 llvm_unreachable("Unexpected vector element type");2631}2632 2633SDValue2634HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl,2635 MVT VecTy, SelectionDAG &DAG) const {2636 MVT ElemTy = VecTy.getVectorElementType();2637 assert(VecTy.getVectorNumElements() == Elem.size());2638 2639 SmallVector<ConstantInt*,8> Consts(Elem.size());2640 bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);2641 2642 unsigned First, Num = Elem.size();2643 for (First = 0; First != Num; ++First) {2644 if (!isUndef(Elem[First]))2645 break;2646 }2647 if (First == Num)2648 return DAG.getUNDEF(VecTy);2649 2650 if (AllConst &&2651 llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); }))2652 return getZero(dl, VecTy, DAG);2653 2654 // First try splat if possible.2655 if (ElemTy == MVT::i16 || ElemTy == MVT::f16) {2656 bool IsSplat = true;2657 for (unsigned i = First+1; i != Num; ++i) {2658 if (Elem[i] == Elem[First] || isUndef(Elem[i]))2659 continue;2660 IsSplat = false;2661 break;2662 }2663 if (IsSplat) {2664 // Legalize the operand of SPLAT_VECTOR2665 SDValue S = ElemTy == MVT::f16 ? DAG.getBitcast(MVT::i16, Elem[First])2666 : Elem[First];2667 SDValue Ext = DAG.getZExtOrTrunc(S, dl, MVT::i32);2668 return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext);2669 }2670 }2671 2672 // Then try constant.2673 if (AllConst) {2674 uint64_t Val = 0;2675 unsigned W = ElemTy.getSizeInBits();2676 uint64_t Mask = (1ull << W) - 1;2677 for (unsigned i = 0; i != Num; ++i)2678 Val = (Val << W) | (Consts[Num-1-i]->getZExtValue() & Mask);2679 SDValue V0 = DAG.getConstant(Val, dl, MVT::i64);2680 return DAG.getBitcast(VecTy, V0);2681 }2682 2683 // Build two 32-bit vectors and concatenate.2684 MVT HalfTy = MVT::getVectorVT(ElemTy, Num/2);2685 SDValue L = (ElemTy == MVT::i32)2686 ? Elem[0]2687 : buildVector32(Elem.take_front(Num/2), dl, HalfTy, DAG);2688 SDValue H = (ElemTy == MVT::i32)2689 ? Elem[1]2690 : buildVector32(Elem.drop_front(Num/2), dl, HalfTy, DAG);2691 return getCombine(H, L, dl, VecTy, DAG);2692}2693 2694SDValue2695HexagonTargetLowering::extractVector(SDValue VecV, SDValue IdxV,2696 const SDLoc &dl, MVT ValTy, MVT ResTy,2697 SelectionDAG &DAG) const {2698 MVT VecTy = ty(VecV);2699 assert(!ValTy.isVector() ||2700 VecTy.getVectorElementType() == ValTy.getVectorElementType());2701 if (VecTy.getVectorElementType() == MVT::i1)2702 return extractVectorPred(VecV, IdxV, dl, ValTy, ResTy, DAG);2703 2704 unsigned VecWidth = VecTy.getSizeInBits();2705 unsigned ValWidth = ValTy.getSizeInBits();2706 unsigned ElemWidth = VecTy.getVectorElementType().getSizeInBits();2707 assert((VecWidth % ElemWidth) == 0);2708 assert(VecWidth == 32 || VecWidth == 64);2709 2710 // Cast everything to scalar integer types.2711 MVT ScalarTy = tyScalar(VecTy);2712 VecV = DAG.getBitcast(ScalarTy, VecV);2713 2714 SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32);2715 SDValue ExtV;2716 2717 if (auto *IdxN = dyn_cast<ConstantSDNode>(IdxV)) {2718 unsigned Off = IdxN->getZExtValue() * ElemWidth;2719 if (VecWidth == 64 && ValWidth == 32) {2720 assert(Off == 0 || Off == 32);2721 ExtV = Off == 0 ? LoHalf(VecV, DAG) : HiHalf(VecV, DAG);2722 } else if (Off == 0 && (ValWidth % 8) == 0) {2723 ExtV = DAG.getZeroExtendInReg(VecV, dl, tyScalar(ValTy));2724 } else {2725 SDValue OffV = DAG.getConstant(Off, dl, MVT::i32);2726 // The return type of EXTRACTU must be the same as the type of the2727 // input vector.2728 ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy,2729 {VecV, WidthV, OffV});2730 }2731 } else {2732 if (ty(IdxV) != MVT::i32)2733 IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32);2734 SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,2735 DAG.getConstant(ElemWidth, dl, MVT::i32));2736 ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy,2737 {VecV, WidthV, OffV});2738 }2739 2740 // Cast ExtV to the requested result type.2741 ExtV = DAG.getZExtOrTrunc(ExtV, dl, tyScalar(ResTy));2742 ExtV = DAG.getBitcast(ResTy, ExtV);2743 return ExtV;2744}2745 2746SDValue2747HexagonTargetLowering::extractVectorPred(SDValue VecV, SDValue IdxV,2748 const SDLoc &dl, MVT ValTy, MVT ResTy,2749 SelectionDAG &DAG) const {2750 // Special case for v{8,4,2}i1 (the only boolean vectors legal in Hexagon2751 // without any coprocessors).2752 MVT VecTy = ty(VecV);2753 unsigned VecWidth = VecTy.getSizeInBits();2754 unsigned ValWidth = ValTy.getSizeInBits();2755 assert(VecWidth == VecTy.getVectorNumElements() &&2756 "Vector elements should equal vector width size");2757 assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2);2758 2759 // Check if this is an extract of the lowest bit.2760 if (isNullConstant(IdxV) && ValTy.getSizeInBits() == 1) {2761 // Extracting the lowest bit is a no-op, but it changes the type,2762 // so it must be kept as an operation to avoid errors related to2763 // type mismatches.2764 return DAG.getNode(HexagonISD::TYPECAST, dl, MVT::i1, VecV);2765 }2766 2767 // If the value extracted is a single bit, use tstbit.2768 if (ValWidth == 1) {2769 SDValue A0 = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, {VecV}, DAG);2770 SDValue M0 = DAG.getConstant(8 / VecWidth, dl, MVT::i32);2771 SDValue I0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, M0);2772 return DAG.getNode(HexagonISD::TSTBIT, dl, MVT::i1, A0, I0);2773 }2774 2775 // Each bool vector (v2i1, v4i1, v8i1) always occupies 8 bits in2776 // a predicate register. The elements of the vector are repeated2777 // in the register (if necessary) so that the total number is 8.2778 // The extracted subvector will need to be expanded in such a way.2779 unsigned Scale = VecWidth / ValWidth;2780 2781 // Generate (p2d VecV) >> 8*Idx to move the interesting bytes to2782 // position 0.2783 assert(ty(IdxV) == MVT::i32);2784 unsigned VecRep = 8 / VecWidth;2785 SDValue S0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,2786 DAG.getConstant(8*VecRep, dl, MVT::i32));2787 SDValue T0 = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV);2788 SDValue T1 = DAG.getNode(ISD::SRL, dl, MVT::i64, T0, S0);2789 while (Scale > 1) {2790 // The longest possible subvector is at most 32 bits, so it is always2791 // contained in the low subregister.2792 T1 = LoHalf(T1, DAG);2793 T1 = expandPredicate(T1, dl, DAG);2794 Scale /= 2;2795 }2796 2797 return DAG.getNode(HexagonISD::D2P, dl, ResTy, T1);2798}2799 2800SDValue2801HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,2802 const SDLoc &dl, MVT ValTy,2803 SelectionDAG &DAG) const {2804 MVT VecTy = ty(VecV);2805 if (VecTy.getVectorElementType() == MVT::i1)2806 return insertVectorPred(VecV, ValV, IdxV, dl, ValTy, DAG);2807 2808 unsigned VecWidth = VecTy.getSizeInBits();2809 unsigned ValWidth = ValTy.getSizeInBits();2810 assert(VecWidth == 32 || VecWidth == 64);2811 assert((VecWidth % ValWidth) == 0);2812 2813 // Cast everything to scalar integer types.2814 MVT ScalarTy = MVT::getIntegerVT(VecWidth);2815 // The actual type of ValV may be different than ValTy (which is related2816 // to the vector type).2817 unsigned VW = ty(ValV).getSizeInBits();2818 ValV = DAG.getBitcast(MVT::getIntegerVT(VW), ValV);2819 VecV = DAG.getBitcast(ScalarTy, VecV);2820 if (VW != VecWidth)2821 ValV = DAG.getAnyExtOrTrunc(ValV, dl, ScalarTy);2822 2823 SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32);2824 SDValue InsV;2825 2826 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(IdxV)) {2827 unsigned W = C->getZExtValue() * ValWidth;2828 SDValue OffV = DAG.getConstant(W, dl, MVT::i32);2829 InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy,2830 {VecV, ValV, WidthV, OffV});2831 } else {2832 if (ty(IdxV) != MVT::i32)2833 IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32);2834 SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, WidthV);2835 InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy,2836 {VecV, ValV, WidthV, OffV});2837 }2838 2839 return DAG.getNode(ISD::BITCAST, dl, VecTy, InsV);2840}2841 2842SDValue2843HexagonTargetLowering::insertVectorPred(SDValue VecV, SDValue ValV,2844 SDValue IdxV, const SDLoc &dl,2845 MVT ValTy, SelectionDAG &DAG) const {2846 MVT VecTy = ty(VecV);2847 unsigned VecLen = VecTy.getVectorNumElements();2848 2849 if (ValTy == MVT::i1) {2850 SDValue ToReg = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, {VecV}, DAG);2851 SDValue Ext = DAG.getSExtOrTrunc(ValV, dl, MVT::i32);2852 SDValue Width = DAG.getConstant(8 / VecLen, dl, MVT::i32);2853 SDValue Idx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, Width);2854 SDValue Ins =2855 DAG.getNode(HexagonISD::INSERT, dl, MVT::i32, {ToReg, Ext, Width, Idx});2856 return getInstr(Hexagon::C2_tfrrp, dl, VecTy, {Ins}, DAG);2857 }2858 2859 assert(ValTy.getVectorElementType() == MVT::i1);2860 SDValue ValR = ValTy.isVector()2861 ? DAG.getNode(HexagonISD::P2D, dl, MVT::i64, ValV)2862 : DAG.getSExtOrTrunc(ValV, dl, MVT::i64);2863 2864 unsigned Scale = VecLen / ValTy.getVectorNumElements();2865 assert(Scale > 1);2866 2867 for (unsigned R = Scale; R > 1; R /= 2) {2868 ValR = contractPredicate(ValR, dl, DAG);2869 ValR = getCombine(DAG.getUNDEF(MVT::i32), ValR, dl, MVT::i64, DAG);2870 }2871 2872 SDValue Width = DAG.getConstant(64 / Scale, dl, MVT::i32);2873 SDValue Idx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, Width);2874 SDValue VecR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV);2875 SDValue Ins =2876 DAG.getNode(HexagonISD::INSERT, dl, MVT::i64, {VecR, ValR, Width, Idx});2877 return DAG.getNode(HexagonISD::D2P, dl, VecTy, Ins);2878}2879 2880SDValue2881HexagonTargetLowering::expandPredicate(SDValue Vec32, const SDLoc &dl,2882 SelectionDAG &DAG) const {2883 assert(ty(Vec32).getSizeInBits() == 32);2884 if (isUndef(Vec32))2885 return DAG.getUNDEF(MVT::i64);2886 SDValue P = DAG.getBitcast(MVT::v4i8, Vec32);2887 SDValue X = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i16, P);2888 return DAG.getBitcast(MVT::i64, X);2889}2890 2891SDValue2892HexagonTargetLowering::contractPredicate(SDValue Vec64, const SDLoc &dl,2893 SelectionDAG &DAG) const {2894 assert(ty(Vec64).getSizeInBits() == 64);2895 if (isUndef(Vec64))2896 return DAG.getUNDEF(MVT::i32);2897 // Collect even bytes:2898 SDValue A = DAG.getBitcast(MVT::v8i8, Vec64);2899 SDValue S = DAG.getVectorShuffle(MVT::v8i8, dl, A, DAG.getUNDEF(MVT::v8i8),2900 {0, 2, 4, 6, 1, 3, 5, 7});2901 return extractVector(S, DAG.getConstant(0, dl, MVT::i32), dl, MVT::v4i8,2902 MVT::i32, DAG);2903}2904 2905SDValue2906HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG)2907 const {2908 if (Ty.isVector()) {2909 unsigned W = Ty.getSizeInBits();2910 if (W <= 64)2911 return DAG.getBitcast(Ty, DAG.getConstant(0, dl, MVT::getIntegerVT(W)));2912 return DAG.getNode(ISD::SPLAT_VECTOR, dl, Ty, getZero(dl, MVT::i32, DAG));2913 }2914 2915 if (Ty.isInteger())2916 return DAG.getConstant(0, dl, Ty);2917 if (Ty.isFloatingPoint())2918 return DAG.getConstantFP(0.0, dl, Ty);2919 llvm_unreachable("Invalid type for zero");2920}2921 2922SDValue2923HexagonTargetLowering::appendUndef(SDValue Val, MVT ResTy, SelectionDAG &DAG)2924 const {2925 MVT ValTy = ty(Val);2926 assert(ValTy.getVectorElementType() == ResTy.getVectorElementType());2927 2928 unsigned ValLen = ValTy.getVectorNumElements();2929 unsigned ResLen = ResTy.getVectorNumElements();2930 if (ValLen == ResLen)2931 return Val;2932 2933 const SDLoc &dl(Val);2934 assert(ValLen < ResLen);2935 assert(ResLen % ValLen == 0);2936 2937 SmallVector<SDValue, 4> Concats = {Val};2938 for (unsigned i = 1, e = ResLen / ValLen; i < e; ++i)2939 Concats.push_back(DAG.getUNDEF(ValTy));2940 2941 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, Concats);2942}2943 2944SDValue2945HexagonTargetLowering::getCombine(SDValue Hi, SDValue Lo, const SDLoc &dl,2946 MVT ResTy, SelectionDAG &DAG) const {2947 MVT ElemTy = ty(Hi);2948 assert(ElemTy == ty(Lo));2949 2950 if (!ElemTy.isVector()) {2951 assert(ElemTy.isScalarInteger());2952 MVT PairTy = MVT::getIntegerVT(2 * ElemTy.getSizeInBits());2953 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, dl, PairTy, Lo, Hi);2954 return DAG.getBitcast(ResTy, Pair);2955 }2956 2957 unsigned Width = ElemTy.getSizeInBits();2958 MVT IntTy = MVT::getIntegerVT(Width);2959 MVT PairTy = MVT::getIntegerVT(2 * Width);2960 SDValue Pair =2961 DAG.getNode(ISD::BUILD_PAIR, dl, PairTy,2962 {DAG.getBitcast(IntTy, Lo), DAG.getBitcast(IntTy, Hi)});2963 return DAG.getBitcast(ResTy, Pair);2964}2965 2966SDValue2967HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {2968 MVT VecTy = ty(Op);2969 unsigned BW = VecTy.getSizeInBits();2970 const SDLoc &dl(Op);2971 SmallVector<SDValue,8> Ops;2972 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i)2973 Ops.push_back(Op.getOperand(i));2974 2975 if (BW == 32)2976 return buildVector32(Ops, dl, VecTy, DAG);2977 if (BW == 64)2978 return buildVector64(Ops, dl, VecTy, DAG);2979 2980 if (VecTy == MVT::v8i1 || VecTy == MVT::v4i1 || VecTy == MVT::v2i1) {2981 // Check if this is a special case or all-0 or all-1.2982 bool All0 = true, All1 = true;2983 for (SDValue P : Ops) {2984 auto *CN = dyn_cast<ConstantSDNode>(P.getNode());2985 if (CN == nullptr) {2986 All0 = All1 = false;2987 break;2988 }2989 uint32_t C = CN->getZExtValue();2990 All0 &= (C == 0);2991 All1 &= (C == 1);2992 }2993 if (All0)2994 return DAG.getNode(HexagonISD::PFALSE, dl, VecTy);2995 if (All1)2996 return DAG.getNode(HexagonISD::PTRUE, dl, VecTy);2997 2998 // For each i1 element in the resulting predicate register, put 12999 // shifted by the index of the element into a general-purpose register,3000 // then or them together and transfer it back into a predicate register.3001 SDValue Rs[8];3002 SDValue Z = getZero(dl, MVT::i32, DAG);3003 // Always produce 8 bits, repeat inputs if necessary.3004 unsigned Rep = 8 / VecTy.getVectorNumElements();3005 for (unsigned i = 0; i != 8; ++i) {3006 SDValue S = DAG.getConstant(1ull << i, dl, MVT::i32);3007 Rs[i] = DAG.getSelect(dl, MVT::i32, Ops[i/Rep], S, Z);3008 }3009 for (ArrayRef<SDValue> A(Rs); A.size() != 1; A = A.drop_back(A.size()/2)) {3010 for (unsigned i = 0, e = A.size()/2; i != e; ++i)3011 Rs[i] = DAG.getNode(ISD::OR, dl, MVT::i32, Rs[2*i], Rs[2*i+1]);3012 }3013 // Move the value directly to a predicate register.3014 return getInstr(Hexagon::C2_tfrrp, dl, VecTy, {Rs[0]}, DAG);3015 }3016 3017 return SDValue();3018}3019 3020SDValue3021HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op,3022 SelectionDAG &DAG) const {3023 MVT VecTy = ty(Op);3024 const SDLoc &dl(Op);3025 if (VecTy.getSizeInBits() == 64) {3026 assert(Op.getNumOperands() == 2);3027 return getCombine(Op.getOperand(1), Op.getOperand(0), dl, VecTy, DAG);3028 }3029 3030 MVT ElemTy = VecTy.getVectorElementType();3031 if (ElemTy == MVT::i1) {3032 assert(VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1);3033 MVT OpTy = ty(Op.getOperand(0));3034 // Scale is how many times the operands need to be contracted to match3035 // the representation in the target register.3036 unsigned Scale = VecTy.getVectorNumElements() / OpTy.getVectorNumElements();3037 assert(Scale == Op.getNumOperands() && Scale > 1);3038 3039 // First, convert all bool vectors to integers, then generate pairwise3040 // inserts to form values of doubled length. Up until there are only3041 // two values left to concatenate, all of these values will fit in a3042 // 32-bit integer, so keep them as i32 to use 32-bit inserts.3043 SmallVector<SDValue,4> Words[2];3044 unsigned IdxW = 0;3045 3046 for (SDValue P : Op.getNode()->op_values()) {3047 SDValue W = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, P);3048 for (unsigned R = Scale; R > 1; R /= 2) {3049 W = contractPredicate(W, dl, DAG);3050 W = getCombine(DAG.getUNDEF(MVT::i32), W, dl, MVT::i64, DAG);3051 }3052 W = LoHalf(W, DAG);3053 Words[IdxW].push_back(W);3054 }3055 3056 while (Scale > 2) {3057 SDValue WidthV = DAG.getConstant(64 / Scale, dl, MVT::i32);3058 Words[IdxW ^ 1].clear();3059 3060 for (unsigned i = 0, e = Words[IdxW].size(); i != e; i += 2) {3061 SDValue W0 = Words[IdxW][i], W1 = Words[IdxW][i+1];3062 // Insert W1 into W0 right next to the significant bits of W0.3063 SDValue T = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32,3064 {W0, W1, WidthV, WidthV});3065 Words[IdxW ^ 1].push_back(T);3066 }3067 IdxW ^= 1;3068 Scale /= 2;3069 }3070 3071 // At this point there should only be two words left, and Scale should be 2.3072 assert(Scale == 2 && Words[IdxW].size() == 2);3073 3074 SDValue WW = getCombine(Words[IdxW][1], Words[IdxW][0], dl, MVT::i64, DAG);3075 return DAG.getNode(HexagonISD::D2P, dl, VecTy, WW);3076 }3077 3078 return SDValue();3079}3080 3081SDValue3082HexagonTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,3083 SelectionDAG &DAG) const {3084 SDValue Vec = Op.getOperand(0);3085 MVT ElemTy = ty(Vec).getVectorElementType();3086 return extractVector(Vec, Op.getOperand(1), SDLoc(Op), ElemTy, ty(Op), DAG);3087}3088 3089SDValue3090HexagonTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,3091 SelectionDAG &DAG) const {3092 return extractVector(Op.getOperand(0), Op.getOperand(1), SDLoc(Op),3093 ty(Op), ty(Op), DAG);3094}3095 3096SDValue3097HexagonTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,3098 SelectionDAG &DAG) const {3099 return insertVector(Op.getOperand(0), Op.getOperand(1), Op.getOperand(2),3100 SDLoc(Op), ty(Op).getVectorElementType(), DAG);3101}3102 3103SDValue3104HexagonTargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,3105 SelectionDAG &DAG) const {3106 SDValue ValV = Op.getOperand(1);3107 return insertVector(Op.getOperand(0), ValV, Op.getOperand(2),3108 SDLoc(Op), ty(ValV), DAG);3109}3110 3111bool3112HexagonTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {3113 // Assuming the caller does not have either a signext or zeroext modifier, and3114 // only one value is accepted, any reasonable truncation is allowed.3115 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())3116 return false;3117 3118 // FIXME: in principle up to 64-bit could be made safe, but it would be very3119 // fragile at the moment: any support for multiple value returns would be3120 // liable to disallow tail calls involving i64 -> iN truncation in many cases.3121 return Ty1->getPrimitiveSizeInBits() <= 32;3122}3123 3124SDValue3125HexagonTargetLowering::LowerLoad(SDValue Op, SelectionDAG &DAG) const {3126 MVT Ty = ty(Op);3127 const SDLoc &dl(Op);3128 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());3129 MVT MemTy = LN->getMemoryVT().getSimpleVT();3130 ISD::LoadExtType ET = LN->getExtensionType();3131 3132 bool LoadPred = MemTy == MVT::v2i1 || MemTy == MVT::v4i1 || MemTy == MVT::v8i1;3133 if (LoadPred) {3134 SDValue NL = DAG.getLoad(3135 LN->getAddressingMode(), ISD::ZEXTLOAD, MVT::i32, dl, LN->getChain(),3136 LN->getBasePtr(), LN->getOffset(), LN->getPointerInfo(),3137 /*MemoryVT*/ MVT::i8, LN->getAlign(), LN->getMemOperand()->getFlags(),3138 LN->getAAInfo(), LN->getRanges());3139 LN = cast<LoadSDNode>(NL.getNode());3140 }3141 3142 Align ClaimAlign = LN->getAlign();3143 if (!validateConstPtrAlignment(LN->getBasePtr(), ClaimAlign, dl, DAG))3144 return replaceMemWithUndef(Op, DAG);3145 3146 // Call LowerUnalignedLoad for all loads, it recognizes loads that3147 // don't need extra aligning.3148 SDValue LU = LowerUnalignedLoad(SDValue(LN, 0), DAG);3149 if (LoadPred) {3150 SDValue TP = getInstr(Hexagon::C2_tfrrp, dl, MemTy, {LU}, DAG);3151 if (ET == ISD::SEXTLOAD) {3152 TP = DAG.getSExtOrTrunc(TP, dl, Ty);3153 } else if (ET != ISD::NON_EXTLOAD) {3154 TP = DAG.getZExtOrTrunc(TP, dl, Ty);3155 }3156 SDValue Ch = cast<LoadSDNode>(LU.getNode())->getChain();3157 return DAG.getMergeValues({TP, Ch}, dl);3158 }3159 return LU;3160}3161 3162SDValue3163HexagonTargetLowering::LowerStore(SDValue Op, SelectionDAG &DAG) const {3164 const SDLoc &dl(Op);3165 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());3166 SDValue Val = SN->getValue();3167 MVT Ty = ty(Val);3168 3169 if (Ty == MVT::v2i1 || Ty == MVT::v4i1 || Ty == MVT::v8i1) {3170 // Store the exact predicate (all bits).3171 SDValue TR = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, {Val}, DAG);3172 SDValue NS = DAG.getTruncStore(SN->getChain(), dl, TR, SN->getBasePtr(),3173 MVT::i8, SN->getMemOperand());3174 if (SN->isIndexed()) {3175 NS = DAG.getIndexedStore(NS, dl, SN->getBasePtr(), SN->getOffset(),3176 SN->getAddressingMode());3177 }3178 SN = cast<StoreSDNode>(NS.getNode());3179 }3180 3181 Align ClaimAlign = SN->getAlign();3182 if (!validateConstPtrAlignment(SN->getBasePtr(), ClaimAlign, dl, DAG))3183 return replaceMemWithUndef(Op, DAG);3184 3185 MVT StoreTy = SN->getMemoryVT().getSimpleVT();3186 Align NeedAlign = Subtarget.getTypeAlignment(StoreTy);3187 if (ClaimAlign < NeedAlign)3188 return expandUnalignedStore(SN, DAG);3189 return SDValue(SN, 0);3190}3191 3192SDValue3193HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)3194 const {3195 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());3196 MVT LoadTy = ty(Op);3197 unsigned NeedAlign = Subtarget.getTypeAlignment(LoadTy).value();3198 unsigned HaveAlign = LN->getAlign().value();3199 if (HaveAlign >= NeedAlign)3200 return Op;3201 3202 const SDLoc &dl(Op);3203 const DataLayout &DL = DAG.getDataLayout();3204 LLVMContext &Ctx = *DAG.getContext();3205 3206 // If the load aligning is disabled or the load can be broken up into two3207 // smaller legal loads, do the default (target-independent) expansion.3208 bool DoDefault = false;3209 // Handle it in the default way if this is an indexed load.3210 if (!LN->isUnindexed())3211 DoDefault = true;3212 3213 if (!AlignLoads) {3214 if (allowsMemoryAccessForAlignment(Ctx, DL, LN->getMemoryVT(),3215 *LN->getMemOperand()))3216 return Op;3217 DoDefault = true;3218 }3219 if (!DoDefault && (2 * HaveAlign) == NeedAlign) {3220 // The PartTy is the equivalent of "getLoadableTypeOfSize(HaveAlign)".3221 MVT PartTy = HaveAlign <= 8 ? MVT::getIntegerVT(8 * HaveAlign)3222 : MVT::getVectorVT(MVT::i8, HaveAlign);3223 DoDefault =3224 allowsMemoryAccessForAlignment(Ctx, DL, PartTy, *LN->getMemOperand());3225 }3226 if (DoDefault) {3227 std::pair<SDValue, SDValue> P = expandUnalignedLoad(LN, DAG);3228 return DAG.getMergeValues({P.first, P.second}, dl);3229 }3230 3231 // The code below generates two loads, both aligned as NeedAlign, and3232 // with the distance of NeedAlign between them. For that to cover the3233 // bits that need to be loaded (and without overlapping), the size of3234 // the loads should be equal to NeedAlign. This is true for all loadable3235 // types, but add an assertion in case something changes in the future.3236 assert(LoadTy.getSizeInBits() == 8*NeedAlign);3237 3238 unsigned LoadLen = NeedAlign;3239 SDValue Base = LN->getBasePtr();3240 SDValue Chain = LN->getChain();3241 auto BO = getBaseAndOffset(Base);3242 unsigned BaseOpc = BO.first.getOpcode();3243 if (BaseOpc == HexagonISD::VALIGNADDR && BO.second % LoadLen == 0)3244 return Op;3245 3246 if (BO.second % LoadLen != 0) {3247 BO.first = DAG.getNode(ISD::ADD, dl, MVT::i32, BO.first,3248 DAG.getConstant(BO.second % LoadLen, dl, MVT::i32));3249 BO.second -= BO.second % LoadLen;3250 }3251 SDValue BaseNoOff = (BaseOpc != HexagonISD::VALIGNADDR)3252 ? DAG.getNode(HexagonISD::VALIGNADDR, dl, MVT::i32, BO.first,3253 DAG.getConstant(NeedAlign, dl, MVT::i32))3254 : BO.first;3255 SDValue Base0 =3256 DAG.getMemBasePlusOffset(BaseNoOff, TypeSize::getFixed(BO.second), dl);3257 SDValue Base1 = DAG.getMemBasePlusOffset(3258 BaseNoOff, TypeSize::getFixed(BO.second + LoadLen), dl);3259 3260 MachineMemOperand *WideMMO = nullptr;3261 if (MachineMemOperand *MMO = LN->getMemOperand()) {3262 MachineFunction &MF = DAG.getMachineFunction();3263 WideMMO = MF.getMachineMemOperand(3264 MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen, Align(LoadLen),3265 MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(),3266 MMO->getSuccessOrdering(), MMO->getFailureOrdering());3267 }3268 3269 SDValue Load0 = DAG.getLoad(LoadTy, dl, Chain, Base0, WideMMO);3270 SDValue Load1 = DAG.getLoad(LoadTy, dl, Chain, Base1, WideMMO);3271 3272 SDValue Aligned = DAG.getNode(HexagonISD::VALIGN, dl, LoadTy,3273 {Load1, Load0, BaseNoOff.getOperand(0)});3274 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,3275 Load0.getValue(1), Load1.getValue(1));3276 SDValue M = DAG.getMergeValues({Aligned, NewChain}, dl);3277 return M;3278}3279 3280SDValue3281HexagonTargetLowering::LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const {3282 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);3283 auto *CY = dyn_cast<ConstantSDNode>(Y);3284 if (!CY)3285 return SDValue();3286 3287 const SDLoc &dl(Op);3288 SDVTList VTs = Op.getNode()->getVTList();3289 assert(VTs.NumVTs == 2);3290 assert(VTs.VTs[1] == MVT::i1);3291 unsigned Opc = Op.getOpcode();3292 3293 if (CY) {3294 uint64_t VY = CY->getZExtValue();3295 assert(VY != 0 && "This should have been folded");3296 // X +/- 13297 if (VY != 1)3298 return SDValue();3299 3300 if (Opc == ISD::UADDO) {3301 SDValue Op = DAG.getNode(ISD::ADD, dl, VTs.VTs[0], {X, Y});3302 SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op, getZero(dl, ty(Op), DAG),3303 ISD::SETEQ);3304 return DAG.getMergeValues({Op, Ov}, dl);3305 }3306 if (Opc == ISD::USUBO) {3307 SDValue Op = DAG.getNode(ISD::SUB, dl, VTs.VTs[0], {X, Y});3308 SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op,3309 DAG.getAllOnesConstant(dl, ty(Op)), ISD::SETEQ);3310 return DAG.getMergeValues({Op, Ov}, dl);3311 }3312 }3313 3314 return SDValue();3315}3316 3317SDValue HexagonTargetLowering::LowerUAddSubOCarry(SDValue Op,3318 SelectionDAG &DAG) const {3319 const SDLoc &dl(Op);3320 unsigned Opc = Op.getOpcode();3321 SDValue X = Op.getOperand(0), Y = Op.getOperand(1), C = Op.getOperand(2);3322 3323 if (Opc == ISD::UADDO_CARRY)3324 return DAG.getNode(HexagonISD::ADDC, dl, Op.getNode()->getVTList(),3325 { X, Y, C });3326 3327 EVT CarryTy = C.getValueType();3328 SDValue SubC = DAG.getNode(HexagonISD::SUBC, dl, Op.getNode()->getVTList(),3329 { X, Y, DAG.getLogicalNOT(dl, C, CarryTy) });3330 SDValue Out[] = { SubC.getValue(0),3331 DAG.getLogicalNOT(dl, SubC.getValue(1), CarryTy) };3332 return DAG.getMergeValues(Out, dl);3333}3334 3335SDValue3336HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {3337 SDValue Chain = Op.getOperand(0);3338 SDValue Offset = Op.getOperand(1);3339 SDValue Handler = Op.getOperand(2);3340 SDLoc dl(Op);3341 auto PtrVT = getPointerTy(DAG.getDataLayout());3342 3343 // Mark function as containing a call to EH_RETURN.3344 HexagonMachineFunctionInfo *FuncInfo =3345 DAG.getMachineFunction().getInfo<HexagonMachineFunctionInfo>();3346 FuncInfo->setHasEHReturn();3347 3348 unsigned OffsetReg = Hexagon::R28;3349 3350 SDValue StoreAddr =3351 DAG.getNode(ISD::ADD, dl, PtrVT, DAG.getRegister(Hexagon::R30, PtrVT),3352 DAG.getIntPtrConstant(4, dl));3353 Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo());3354 Chain = DAG.getCopyToReg(Chain, dl, OffsetReg, Offset);3355 3356 // Not needed we already use it as explicit input to EH_RETURN.3357 // MF.getRegInfo().addLiveOut(OffsetReg);3358 3359 return DAG.getNode(HexagonISD::EH_RETURN, dl, MVT::Other, Chain);3360}3361 3362SDValue3363HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {3364 unsigned Opc = Op.getOpcode();3365 // Handle INLINEASM first.3366 if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR)3367 return LowerINLINEASM(Op, DAG);3368 3369 if (isHvxOperation(Op.getNode(), DAG)) {3370 // If HVX lowering returns nothing, try the default lowering.3371 if (SDValue V = LowerHvxOperation(Op, DAG))3372 return V;3373 }3374 3375 switch (Opc) {3376 default:3377#ifndef NDEBUG3378 Op.getNode()->dumpr(&DAG);3379 if (Opc > HexagonISD::OP_BEGIN && Opc < HexagonISD::OP_END)3380 errs() << "Error: check for a non-legal type in this operation\n";3381#endif3382 llvm_unreachable("Should not custom lower this!");3383 3384 case ISD::FDIV:3385 return LowerFDIV(Op, DAG);3386 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);3387 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, DAG);3388 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);3389 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);3390 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);3391 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);3392 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);3393 case ISD::BITCAST: return LowerBITCAST(Op, DAG);3394 case ISD::LOAD: return LowerLoad(Op, DAG);3395 case ISD::STORE: return LowerStore(Op, DAG);3396 case ISD::UADDO:3397 case ISD::USUBO: return LowerUAddSubO(Op, DAG);3398 case ISD::UADDO_CARRY:3399 case ISD::USUBO_CARRY: return LowerUAddSubOCarry(Op, DAG);3400 case ISD::SRA:3401 case ISD::SHL:3402 case ISD::SRL: return LowerVECTOR_SHIFT(Op, DAG);3403 case ISD::ROTL: return LowerROTL(Op, DAG);3404 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);3405 case ISD::JumpTable: return LowerJumpTable(Op, DAG);3406 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);3407 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);3408 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);3409 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);3410 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG);3411 case ISD::GlobalAddress: return LowerGLOBALADDRESS(Op, DAG);3412 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);3413 case ISD::GLOBAL_OFFSET_TABLE: return LowerGLOBAL_OFFSET_TABLE(Op, DAG);3414 case ISD::VACOPY: return LowerVACOPY(Op, DAG);3415 case ISD::VASTART: return LowerVASTART(Op, DAG);3416 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);3417 case ISD::SETCC: return LowerSETCC(Op, DAG);3418 case ISD::VSELECT: return LowerVSELECT(Op, DAG);3419 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);3420 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG);3421 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG);3422 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, DAG);3423 case ISD::READSTEADYCOUNTER: return LowerREADSTEADYCOUNTER(Op, DAG);3424 break;3425 }3426 3427 return SDValue();3428}3429 3430void3431HexagonTargetLowering::LowerOperationWrapper(SDNode *N,3432 SmallVectorImpl<SDValue> &Results,3433 SelectionDAG &DAG) const {3434 if (isHvxOperation(N, DAG)) {3435 LowerHvxOperationWrapper(N, Results, DAG);3436 if (!Results.empty())3437 return;3438 }3439 3440 SDValue Op(N, 0);3441 unsigned Opc = N->getOpcode();3442 3443 switch (Opc) {3444 case HexagonISD::SSAT:3445 case HexagonISD::USAT:3446 Results.push_back(opJoin(SplitVectorOp(Op, DAG), SDLoc(Op), DAG));3447 break;3448 case ISD::STORE:3449 // We are only custom-lowering stores to verify the alignment of the3450 // address if it is a compile-time constant. Since a store can be3451 // modified during type-legalization (the value being stored may need3452 // legalization), return empty Results here to indicate that we don't3453 // really make any changes in the custom lowering.3454 return;3455 default:3456 TargetLowering::LowerOperationWrapper(N, Results, DAG);3457 break;3458 }3459}3460 3461void3462HexagonTargetLowering::ReplaceNodeResults(SDNode *N,3463 SmallVectorImpl<SDValue> &Results,3464 SelectionDAG &DAG) const {3465 if (isHvxOperation(N, DAG)) {3466 ReplaceHvxNodeResults(N, Results, DAG);3467 if (!Results.empty())3468 return;3469 }3470 3471 const SDLoc &dl(N);3472 switch (N->getOpcode()) {3473 case ISD::SRL:3474 case ISD::SRA:3475 case ISD::SHL:3476 return;3477 case ISD::BITCAST:3478 // Handle a bitcast from v8i1 to i8.3479 if (N->getValueType(0) == MVT::i8) {3480 if (N->getOperand(0).getValueType() == MVT::v8i1) {3481 SDValue P = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32,3482 N->getOperand(0), DAG);3483 SDValue T = DAG.getAnyExtOrTrunc(P, dl, MVT::i8);3484 Results.push_back(T);3485 }3486 }3487 break;3488 }3489}3490 3491SDValue3492HexagonTargetLowering::PerformDAGCombine(SDNode *N,3493 DAGCombinerInfo &DCI) const {3494 if (isHvxOperation(N, DCI.DAG)) {3495 if (SDValue V = PerformHvxDAGCombine(N, DCI))3496 return V;3497 return SDValue();3498 }3499 3500 SDValue Op(N, 0);3501 const SDLoc &dl(Op);3502 unsigned Opc = Op.getOpcode();3503 3504 if (Opc == ISD::TRUNCATE) {3505 SDValue Op0 = Op.getOperand(0);3506 // fold (truncate (build pair x, y)) -> (truncate x) or x3507 if (Op0.getOpcode() == ISD::BUILD_PAIR) {3508 EVT TruncTy = Op.getValueType();3509 SDValue Elem0 = Op0.getOperand(0);3510 // if we match the low element of the pair, just return it.3511 if (Elem0.getValueType() == TruncTy)3512 return Elem0;3513 // otherwise, if the low part is still too large, apply the truncate.3514 if (Elem0.getValueType().bitsGT(TruncTy))3515 return DCI.DAG.getNode(ISD::TRUNCATE, dl, TruncTy, Elem0);3516 }3517 }3518 3519 if (DCI.isBeforeLegalizeOps())3520 return SDValue();3521 3522 if (Opc == HexagonISD::P2D) {3523 SDValue P = Op.getOperand(0);3524 switch (P.getOpcode()) {3525 case HexagonISD::PTRUE:3526 return DCI.DAG.getAllOnesConstant(dl, ty(Op));3527 case HexagonISD::PFALSE:3528 return getZero(dl, ty(Op), DCI.DAG);3529 default:3530 break;3531 }3532 } else if (Opc == ISD::VSELECT) {3533 // This is pretty much duplicated in HexagonISelLoweringHVX...3534 //3535 // (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0)3536 SDValue Cond = Op.getOperand(0);3537 if (Cond->getOpcode() == ISD::XOR) {3538 SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1);3539 if (C1->getOpcode() == HexagonISD::PTRUE) {3540 SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0,3541 Op.getOperand(2), Op.getOperand(1));3542 return VSel;3543 }3544 }3545 } else if (Opc == ISD::TRUNCATE) {3546 SDValue Op0 = Op.getOperand(0);3547 // fold (truncate (build pair x, y)) -> (truncate x) or x3548 if (Op0.getOpcode() == ISD::BUILD_PAIR) {3549 MVT TruncTy = ty(Op);3550 SDValue Elem0 = Op0.getOperand(0);3551 // if we match the low element of the pair, just return it.3552 if (ty(Elem0) == TruncTy)3553 return Elem0;3554 // otherwise, if the low part is still too large, apply the truncate.3555 if (ty(Elem0).bitsGT(TruncTy))3556 return DCI.DAG.getNode(ISD::TRUNCATE, dl, TruncTy, Elem0);3557 }3558 } else if (Opc == ISD::OR) {3559 // fold (or (shl xx, s), (zext y)) -> (COMBINE (shl xx, s-32), y)3560 // if s >= 323561 auto fold0 = [&, this](SDValue Op) {3562 if (ty(Op) != MVT::i64)3563 return SDValue();3564 SDValue Shl = Op.getOperand(0);3565 SDValue Zxt = Op.getOperand(1);3566 if (Shl.getOpcode() != ISD::SHL)3567 std::swap(Shl, Zxt);3568 3569 if (Shl.getOpcode() != ISD::SHL || Zxt.getOpcode() != ISD::ZERO_EXTEND)3570 return SDValue();3571 3572 SDValue Z = Zxt.getOperand(0);3573 auto *Amt = dyn_cast<ConstantSDNode>(Shl.getOperand(1));3574 if (Amt && Amt->getZExtValue() >= 32 && ty(Z).getSizeInBits() <= 32) {3575 unsigned A = Amt->getZExtValue();3576 SDValue S = Shl.getOperand(0);3577 SDValue T0 = DCI.DAG.getNode(ISD::SHL, dl, ty(S), S,3578 DCI.DAG.getConstant(A - 32, dl, MVT::i32));3579 SDValue T1 = DCI.DAG.getZExtOrTrunc(T0, dl, MVT::i32);3580 SDValue T2 = DCI.DAG.getZExtOrTrunc(Z, dl, MVT::i32);3581 return DCI.DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, {T1, T2});3582 }3583 return SDValue();3584 };3585 3586 if (SDValue R = fold0(Op))3587 return R;3588 }3589 3590 return SDValue();3591}3592 3593/// Returns relocation base for the given PIC jumptable.3594SDValue3595HexagonTargetLowering::getPICJumpTableRelocBase(SDValue Table,3596 SelectionDAG &DAG) const {3597 int Idx = cast<JumpTableSDNode>(Table)->getIndex();3598 EVT VT = Table.getValueType();3599 SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL);3600 return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Table), VT, T);3601}3602 3603//===----------------------------------------------------------------------===//3604// Inline Assembly Support3605//===----------------------------------------------------------------------===//3606 3607TargetLowering::ConstraintType3608HexagonTargetLowering::getConstraintType(StringRef Constraint) const {3609 if (Constraint.size() == 1) {3610 switch (Constraint[0]) {3611 case 'q':3612 case 'v':3613 if (Subtarget.useHVXOps())3614 return C_RegisterClass;3615 break;3616 case 'a':3617 return C_RegisterClass;3618 default:3619 break;3620 }3621 }3622 return TargetLowering::getConstraintType(Constraint);3623}3624 3625std::pair<unsigned, const TargetRegisterClass*>3626HexagonTargetLowering::getRegForInlineAsmConstraint(3627 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {3628 3629 if (Constraint.size() == 1) {3630 switch (Constraint[0]) {3631 case 'r': // R0-R313632 switch (VT.SimpleTy) {3633 default:3634 return {0u, nullptr};3635 case MVT::i1:3636 case MVT::i8:3637 case MVT::i16:3638 case MVT::i32:3639 case MVT::f32:3640 return {0u, &Hexagon::IntRegsRegClass};3641 case MVT::i64:3642 case MVT::f64:3643 return {0u, &Hexagon::DoubleRegsRegClass};3644 }3645 break;3646 case 'a': // M0-M13647 if (VT != MVT::i32)3648 return {0u, nullptr};3649 return {0u, &Hexagon::ModRegsRegClass};3650 case 'q': // q0-q33651 switch (VT.getSizeInBits()) {3652 default:3653 return {0u, nullptr};3654 case 64:3655 case 128:3656 return {0u, &Hexagon::HvxQRRegClass};3657 }3658 break;3659 case 'v': // V0-V313660 switch (VT.getSizeInBits()) {3661 default:3662 return {0u, nullptr};3663 case 512:3664 return {0u, &Hexagon::HvxVRRegClass};3665 case 1024:3666 if (Subtarget.hasV60Ops() && Subtarget.useHVX128BOps())3667 return {0u, &Hexagon::HvxVRRegClass};3668 return {0u, &Hexagon::HvxWRRegClass};3669 case 2048:3670 return {0u, &Hexagon::HvxWRRegClass};3671 }3672 break;3673 default:3674 return {0u, nullptr};3675 }3676 }3677 3678 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);3679}3680 3681/// isFPImmLegal - Returns true if the target can instruction select the3682/// specified FP immediate natively. If false, the legalizer will3683/// materialize the FP immediate as a load from a constant pool.3684bool HexagonTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,3685 bool ForCodeSize) const {3686 return true;3687}3688 3689/// Returns true if it is beneficial to convert a load of a constant3690/// to just the constant itself.3691bool HexagonTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,3692 Type *Ty) const {3693 if (!ConstantLoadsToImm)3694 return false;3695 3696 assert(Ty->isIntegerTy());3697 unsigned BitSize = Ty->getPrimitiveSizeInBits();3698 return (BitSize > 0 && BitSize <= 64);3699}3700 3701/// isLegalAddressingMode - Return true if the addressing mode represented by3702/// AM is legal for this target, for a load/store of the specified type.3703bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL,3704 const AddrMode &AM, Type *Ty,3705 unsigned AS, Instruction *I) const {3706 if (Ty->isSized()) {3707 // When LSR detects uses of the same base address to access different3708 // types (e.g. unions), it will assume a conservative type for these3709 // uses:3710 // LSR Use: Kind=Address of void in addrspace(4294967295), ...3711 // The type Ty passed here would then be "void". Skip the alignment3712 // checks, but do not return false right away, since that confuses3713 // LSR into crashing.3714 Align A = DL.getABITypeAlign(Ty);3715 // The base offset must be a multiple of the alignment.3716 if (!isAligned(A, AM.BaseOffs))3717 return false;3718 // The shifted offset must fit in 11 bits.3719 if (!isInt<11>(AM.BaseOffs >> Log2(A)))3720 return false;3721 }3722 3723 // No global is ever allowed as a base.3724 if (AM.BaseGV)3725 return false;3726 3727 int Scale = AM.Scale;3728 if (Scale < 0)3729 Scale = -Scale;3730 switch (Scale) {3731 case 0: // No scale reg, "r+i", "r", or just "i".3732 break;3733 default: // No scaled addressing mode.3734 return false;3735 }3736 return true;3737}3738 3739/// Return true if folding a constant offset with the given GlobalAddress is3740/// legal. It is frequently not legal in PIC relocation models.3741bool HexagonTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA)3742 const {3743 return HTM.getRelocationModel() == Reloc::Static;3744}3745 3746/// isLegalICmpImmediate - Return true if the specified immediate is legal3747/// icmp immediate, that is the target has icmp instructions which can compare3748/// a register against the immediate without having to materialize the3749/// immediate into a register.3750bool HexagonTargetLowering::isLegalICmpImmediate(int64_t Imm) const {3751 return Imm >= -512 && Imm <= 511;3752}3753 3754/// IsEligibleForTailCallOptimization - Check whether the call is eligible3755/// for tail call optimization. Targets which want to do tail call3756/// optimization should implement this function.3757bool HexagonTargetLowering::IsEligibleForTailCallOptimization(3758 SDValue Callee,3759 CallingConv::ID CalleeCC,3760 bool IsVarArg,3761 bool IsCalleeStructRet,3762 bool IsCallerStructRet,3763 const SmallVectorImpl<ISD::OutputArg> &Outs,3764 const SmallVectorImpl<SDValue> &OutVals,3765 const SmallVectorImpl<ISD::InputArg> &Ins,3766 SelectionDAG& DAG) const {3767 const Function &CallerF = DAG.getMachineFunction().getFunction();3768 CallingConv::ID CallerCC = CallerF.getCallingConv();3769 bool CCMatch = CallerCC == CalleeCC;3770 3771 // ***************************************************************************3772 // Look for obvious safe cases to perform tail call optimization that do not3773 // require ABI changes.3774 // ***************************************************************************3775 3776 // If this is a tail call via a function pointer, then don't do it!3777 if (!isa<GlobalAddressSDNode>(Callee) &&3778 !isa<ExternalSymbolSDNode>(Callee)) {3779 return false;3780 }3781 3782 // Do not optimize if the calling conventions do not match and the conventions3783 // used are not C or Fast.3784 if (!CCMatch) {3785 bool R = (CallerCC == CallingConv::C || CallerCC == CallingConv::Fast);3786 bool E = (CalleeCC == CallingConv::C || CalleeCC == CallingConv::Fast);3787 // If R & E, then ok.3788 if (!R || !E)3789 return false;3790 }3791 3792 // Do not tail call optimize vararg calls.3793 if (IsVarArg)3794 return false;3795 3796 // Also avoid tail call optimization if either caller or callee uses struct3797 // return semantics.3798 if (IsCalleeStructRet || IsCallerStructRet)3799 return false;3800 3801 // In addition to the cases above, we also disable Tail Call Optimization if3802 // the calling convention code that at least one outgoing argument needs to3803 // go on the stack. We cannot check that here because at this point that3804 // information is not available.3805 return true;3806}3807 3808/// Returns the target specific optimal type for load and store operations as3809/// a result of memset, memcpy, and memmove lowering.3810///3811/// If DstAlign is zero that means it's safe to destination alignment can3812/// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't3813/// a need to check it against alignment requirement, probably because the3814/// source does not need to be loaded. If 'IsMemset' is true, that means it's3815/// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of3816/// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it3817/// does not need to be loaded. It returns EVT::Other if the type should be3818/// determined using generic target-independent logic.3819EVT HexagonTargetLowering::getOptimalMemOpType(3820 LLVMContext &Context, const MemOp &Op,3821 const AttributeList &FuncAttributes) const {3822 if (Op.size() >= 8 && Op.isAligned(Align(8)))3823 return MVT::i64;3824 if (Op.size() >= 4 && Op.isAligned(Align(4)))3825 return MVT::i32;3826 if (Op.size() >= 2 && Op.isAligned(Align(2)))3827 return MVT::i16;3828 return MVT::Other;3829}3830 3831bool HexagonTargetLowering::allowsMemoryAccess(3832 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,3833 Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const {3834 if (!VT.isSimple())3835 return false;3836 MVT SVT = VT.getSimpleVT();3837 if (Subtarget.isHVXVectorType(SVT, true))3838 return allowsHvxMemoryAccess(SVT, Flags, Fast);3839 return TargetLoweringBase::allowsMemoryAccess(3840 Context, DL, VT, AddrSpace, Alignment, Flags, Fast);3841}3842 3843bool HexagonTargetLowering::allowsMisalignedMemoryAccesses(3844 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,3845 unsigned *Fast) const {3846 if (!VT.isSimple())3847 return false;3848 MVT SVT = VT.getSimpleVT();3849 if (Subtarget.isHVXVectorType(SVT, true))3850 return allowsHvxMisalignedMemoryAccesses(SVT, Flags, Fast);3851 if (Fast)3852 *Fast = 0;3853 return false;3854}3855 3856std::pair<const TargetRegisterClass*, uint8_t>3857HexagonTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,3858 MVT VT) const {3859 if (Subtarget.isHVXVectorType(VT, true)) {3860 unsigned BitWidth = VT.getSizeInBits();3861 unsigned VecWidth = Subtarget.getVectorLength() * 8;3862 3863 if (VT.getVectorElementType() == MVT::i1)3864 return std::make_pair(&Hexagon::HvxQRRegClass, 1);3865 if (BitWidth == VecWidth)3866 return std::make_pair(&Hexagon::HvxVRRegClass, 1);3867 assert(BitWidth == 2 * VecWidth);3868 return std::make_pair(&Hexagon::HvxWRRegClass, 1);3869 }3870 3871 return TargetLowering::findRepresentativeClass(TRI, VT);3872}3873 3874bool HexagonTargetLowering::shouldReduceLoadWidth(3875 SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT,3876 std::optional<unsigned> ByteOffset) const {3877 // TODO: This may be worth removing. Check regression tests for diffs.3878 if (!TargetLoweringBase::shouldReduceLoadWidth(Load, ExtTy, NewVT,3879 ByteOffset))3880 return false;3881 3882 auto *L = cast<LoadSDNode>(Load);3883 std::pair<SDValue, int> BO = getBaseAndOffset(L->getBasePtr());3884 // Small-data object, do not shrink.3885 if (BO.first.getOpcode() == HexagonISD::CONST32_GP)3886 return false;3887 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(BO.first)) {3888 auto &HTM = static_cast<const HexagonTargetMachine &>(getTargetMachine());3889 const auto *GO = dyn_cast_or_null<const GlobalObject>(GA->getGlobal());3890 return !GO || !HTM.getObjFileLowering()->isGlobalInSmallSection(GO, HTM);3891 }3892 return true;3893}3894 3895void HexagonTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,3896 SDNode *Node) const {3897 AdjustHvxInstrPostInstrSelection(MI, Node);3898}3899 3900Value *HexagonTargetLowering::emitLoadLinked(IRBuilderBase &Builder,3901 Type *ValueTy, Value *Addr,3902 AtomicOrdering Ord) const {3903 unsigned SZ = ValueTy->getPrimitiveSizeInBits();3904 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported");3905 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked3906 : Intrinsic::hexagon_L4_loadd_locked;3907 3908 Value *Call =3909 Builder.CreateIntrinsic(IntID, Addr, /*FMFSource=*/nullptr, "larx");3910 3911 return Builder.CreateBitCast(Call, ValueTy);3912}3913 3914/// Perform a store-conditional operation to Addr. Return the status of the3915/// store. This should be 0 if the store succeeded, non-zero otherwise.3916Value *HexagonTargetLowering::emitStoreConditional(IRBuilderBase &Builder,3917 Value *Val, Value *Addr,3918 AtomicOrdering Ord) const {3919 BasicBlock *BB = Builder.GetInsertBlock();3920 Module *M = BB->getParent()->getParent();3921 Type *Ty = Val->getType();3922 unsigned SZ = Ty->getPrimitiveSizeInBits();3923 3924 Type *CastTy = Builder.getIntNTy(SZ);3925 assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported");3926 Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked3927 : Intrinsic::hexagon_S4_stored_locked;3928 3929 Val = Builder.CreateBitCast(Val, CastTy);3930 3931 Value *Call = Builder.CreateIntrinsic(IntID, {Addr, Val},3932 /*FMFSource=*/nullptr, "stcx");3933 Value *Cmp = Builder.CreateICmpEQ(Call, Builder.getInt32(0), "");3934 Value *Ext = Builder.CreateZExt(Cmp, Type::getInt32Ty(M->getContext()));3935 return Ext;3936}3937 3938TargetLowering::AtomicExpansionKind3939HexagonTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {3940 // Do not expand loads and stores that don't exceed 64 bits.3941 return LI->getType()->getPrimitiveSizeInBits() > 643942 ? AtomicExpansionKind::LLOnly3943 : AtomicExpansionKind::None;3944}3945 3946TargetLowering::AtomicExpansionKind3947HexagonTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {3948 // Do not expand loads and stores that don't exceed 64 bits.3949 return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 643950 ? AtomicExpansionKind::Expand3951 : AtomicExpansionKind::None;3952}3953 3954TargetLowering::AtomicExpansionKind3955HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR(3956 AtomicCmpXchgInst *AI) const {3957 return AtomicExpansionKind::LLSC;3958}3959 3960bool HexagonTargetLowering::isMaskAndCmp0FoldingBeneficial(3961 const Instruction &AndI) const {3962 // Only sink 'and' mask to cmp use block if it is masking a single bit since3963 // this will fold the and/cmp/br into a single tstbit instruction.3964 ConstantInt *Mask = dyn_cast<ConstantInt>(AndI.getOperand(1));3965 if (!Mask)3966 return false;3967 return Mask->getValue().isPowerOf2();3968}3969 3970// Check if the result of the node is only used as a return value, as3971// otherwise we can't perform a tail-call.3972bool HexagonTargetLowering::isUsedByReturnOnly(SDNode *N,3973 SDValue &Chain) const {3974 if (N->getNumValues() != 1)3975 return false;3976 if (!N->hasNUsesOfValue(1, 0))3977 return false;3978 3979 SDNode *Copy = *N->user_begin();3980 3981 if (Copy->getOpcode() == ISD::BITCAST) {3982 return isUsedByReturnOnly(Copy, Chain);3983 }3984 3985 if (Copy->getOpcode() != ISD::CopyToReg) {3986 return false;3987 }3988 3989 // If the ISD::CopyToReg has a glue operand, we conservatively assume it3990 // isn't safe to perform a tail call.3991 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == MVT::Glue)3992 return false;3993 3994 // The copy must be used by a HexagonISD::RET_GLUE, and nothing else.3995 bool HasRet = false;3996 for (SDNode *Node : Copy->users()) {3997 if (Node->getOpcode() != HexagonISD::RET_GLUE)3998 return false;3999 HasRet = true;4000 }4001 if (!HasRet)4002 return false;4003 4004 Chain = Copy->getOperand(0);4005 return true;4006}4007