344 lines · cpp
1//===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the AArch64SelectionDAGInfo class.10//11//===----------------------------------------------------------------------===//12 13#include "AArch64SelectionDAGInfo.h"14#include "AArch64MachineFunctionInfo.h"15 16#define GET_SDNODE_DESC17#include "AArch64GenSDNodeInfo.inc"18#undef GET_SDNODE_DESC19 20using namespace llvm;21 22#define DEBUG_TYPE "aarch64-selectiondag-info"23 24static cl::opt<bool>25 LowerToSMERoutines("aarch64-lower-to-sme-routines", cl::Hidden,26 cl::desc("Enable AArch64 SME memory operations "27 "to lower to librt functions"),28 cl::init(true));29 30AArch64SelectionDAGInfo::AArch64SelectionDAGInfo()31 : SelectionDAGGenTargetInfo(AArch64GenSDNodeInfo) {}32 33void AArch64SelectionDAGInfo::verifyTargetNode(const SelectionDAG &DAG,34 const SDNode *N) const {35 switch (N->getOpcode()) {36 case AArch64ISD::WrapperLarge:37 // operand #0 must have type i32, but has type i6438 return;39 }40 41 SelectionDAGGenTargetInfo::verifyTargetNode(DAG, N);42 43#ifndef NDEBUG44 // Some additional checks not yet implemented by verifyTargetNode.45 switch (N->getOpcode()) {46 case AArch64ISD::SADDWT:47 case AArch64ISD::SADDWB:48 case AArch64ISD::UADDWT:49 case AArch64ISD::UADDWB: {50 EVT VT = N->getValueType(0);51 EVT Op0VT = N->getOperand(0).getValueType();52 EVT Op1VT = N->getOperand(1).getValueType();53 assert(VT.isVector() && Op0VT.isVector() && Op1VT.isVector() &&54 VT.isInteger() && Op0VT.isInteger() && Op1VT.isInteger() &&55 "Expected integer vectors!");56 assert(VT == Op0VT &&57 "Expected result and first input to have the same type!");58 assert(Op0VT.getSizeInBits() == Op1VT.getSizeInBits() &&59 "Expected vectors of equal size!");60 assert(Op0VT.getVectorElementCount() * 2 == Op1VT.getVectorElementCount() &&61 "Expected result vector and first input vector to have half the "62 "lanes of the second input vector!");63 break;64 }65 case AArch64ISD::SUNPKLO:66 case AArch64ISD::SUNPKHI:67 case AArch64ISD::UUNPKLO:68 case AArch64ISD::UUNPKHI: {69 EVT VT = N->getValueType(0);70 EVT OpVT = N->getOperand(0).getValueType();71 assert(OpVT.isVector() && VT.isVector() && OpVT.isInteger() &&72 VT.isInteger() && "Expected integer vectors!");73 assert(OpVT.getSizeInBits() == VT.getSizeInBits() &&74 "Expected vectors of equal size!");75 assert(OpVT.getVectorElementCount() == VT.getVectorElementCount() * 2 &&76 "Expected result vector with half the lanes of its input!");77 break;78 }79 case AArch64ISD::TRN1:80 case AArch64ISD::TRN2:81 case AArch64ISD::UZP1:82 case AArch64ISD::UZP2:83 case AArch64ISD::ZIP1:84 case AArch64ISD::ZIP2: {85 EVT VT = N->getValueType(0);86 EVT Op0VT = N->getOperand(0).getValueType();87 EVT Op1VT = N->getOperand(1).getValueType();88 assert(VT.isVector() && Op0VT.isVector() && Op1VT.isVector() &&89 "Expected vectors!");90 assert(VT == Op0VT && VT == Op1VT && "Expected matching vectors!");91 break;92 }93 case AArch64ISD::RSHRNB_I: {94 EVT VT = N->getValueType(0);95 EVT Op0VT = N->getOperand(0).getValueType();96 assert(VT.isVector() && VT.isInteger() &&97 "Expected integer vector result type!");98 assert(Op0VT.isVector() && Op0VT.isInteger() &&99 "Expected first operand to be an integer vector!");100 assert(VT.getSizeInBits() == Op0VT.getSizeInBits() &&101 "Expected vectors of equal size!");102 assert(VT.getVectorElementCount() == Op0VT.getVectorElementCount() * 2 &&103 "Expected input vector with half the lanes of its result!");104 assert(isa<ConstantSDNode>(N->getOperand(1)) &&105 "Expected second operand to be a constant!");106 break;107 }108 }109#endif110}111 112SDValue AArch64SelectionDAGInfo::EmitMOPS(unsigned Opcode, SelectionDAG &DAG,113 const SDLoc &DL, SDValue Chain,114 SDValue Dst, SDValue SrcOrValue,115 SDValue Size, Align Alignment,116 bool isVolatile,117 MachinePointerInfo DstPtrInfo,118 MachinePointerInfo SrcPtrInfo) const {119 120 // Get the constant size of the copy/set.121 uint64_t ConstSize = 0;122 if (auto *C = dyn_cast<ConstantSDNode>(Size))123 ConstSize = C->getZExtValue();124 125 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||126 Opcode == AArch64::MOPSMemorySetTaggingPseudo;127 128 MachineFunction &MF = DAG.getMachineFunction();129 130 auto Vol =131 isVolatile ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;132 auto DstFlags = MachineMemOperand::MOStore | Vol;133 auto *DstOp =134 MF.getMachineMemOperand(DstPtrInfo, DstFlags, ConstSize, Alignment);135 136 if (IsSet) {137 // Extend value to i64, if required.138 if (SrcOrValue.getValueType() != MVT::i64)139 SrcOrValue = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, SrcOrValue);140 SDValue Ops[] = {Dst, Size, SrcOrValue, Chain};141 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::Other};142 MachineSDNode *Node = DAG.getMachineNode(Opcode, DL, ResultTys, Ops);143 DAG.setNodeMemRefs(Node, {DstOp});144 return SDValue(Node, 2);145 } else {146 SDValue Ops[] = {Dst, SrcOrValue, Size, Chain};147 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::i64, MVT::Other};148 MachineSDNode *Node = DAG.getMachineNode(Opcode, DL, ResultTys, Ops);149 150 auto SrcFlags = MachineMemOperand::MOLoad | Vol;151 auto *SrcOp =152 MF.getMachineMemOperand(SrcPtrInfo, SrcFlags, ConstSize, Alignment);153 DAG.setNodeMemRefs(Node, {DstOp, SrcOp});154 return SDValue(Node, 3);155 }156}157 158SDValue AArch64SelectionDAGInfo::EmitStreamingCompatibleMemLibCall(159 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,160 SDValue Size, RTLIB::Libcall LC) const {161 const AArch64Subtarget &STI =162 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();163 const AArch64TargetLowering *TLI = STI.getTargetLowering();164 TargetLowering::ArgListTy Args;165 Args.emplace_back(Dst, PointerType::getUnqual(*DAG.getContext()));166 167 RTLIB::Libcall NewLC;168 switch (LC) {169 case RTLIB::MEMCPY: {170 NewLC = RTLIB::SC_MEMCPY;171 Args.emplace_back(Src, PointerType::getUnqual(*DAG.getContext()));172 break;173 }174 case RTLIB::MEMMOVE: {175 NewLC = RTLIB::SC_MEMMOVE;176 Args.emplace_back(Src, PointerType::getUnqual(*DAG.getContext()));177 break;178 }179 case RTLIB::MEMSET: {180 NewLC = RTLIB::SC_MEMSET;181 Args.emplace_back(DAG.getZExtOrTrunc(Src, DL, MVT::i32),182 Type::getInt32Ty(*DAG.getContext()));183 break;184 }185 default:186 return SDValue();187 }188 189 EVT PointerVT = TLI->getPointerTy(DAG.getDataLayout());190 SDValue Symbol = DAG.getExternalSymbol(TLI->getLibcallName(NewLC), PointerVT);191 Args.emplace_back(Size, DAG.getDataLayout().getIntPtrType(*DAG.getContext()));192 193 TargetLowering::CallLoweringInfo CLI(DAG);194 PointerType *RetTy = PointerType::getUnqual(*DAG.getContext());195 CLI.setDebugLoc(DL).setChain(Chain).setLibCallee(196 TLI->getLibcallCallingConv(NewLC), RetTy, Symbol, std::move(Args));197 return TLI->LowerCallTo(CLI).second;198}199 200SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemcpy(201 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,202 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,203 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {204 const AArch64Subtarget &STI =205 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();206 207 if (STI.hasMOPS())208 return EmitMOPS(AArch64::MOPSMemoryCopyPseudo, DAG, DL, Chain, Dst, Src,209 Size, Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);210 211 auto *AFI = DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();212 SMEAttrs Attrs = AFI->getSMEFnAttrs();213 if (LowerToSMERoutines && !Attrs.hasNonStreamingInterfaceAndBody())214 return EmitStreamingCompatibleMemLibCall(DAG, DL, Chain, Dst, Src, Size,215 RTLIB::MEMCPY);216 return SDValue();217}218 219SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(220 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,221 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,222 MachinePointerInfo DstPtrInfo) const {223 const AArch64Subtarget &STI =224 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();225 226 if (STI.hasMOPS())227 return EmitMOPS(AArch64::MOPSMemorySetPseudo, DAG, dl, Chain, Dst, Src,228 Size, Alignment, isVolatile, DstPtrInfo,229 MachinePointerInfo{});230 231 auto *AFI = DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();232 SMEAttrs Attrs = AFI->getSMEFnAttrs();233 if (LowerToSMERoutines && !Attrs.hasNonStreamingInterfaceAndBody())234 return EmitStreamingCompatibleMemLibCall(DAG, dl, Chain, Dst, Src, Size,235 RTLIB::MEMSET);236 return SDValue();237}238 239SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemmove(240 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,241 SDValue Size, Align Alignment, bool isVolatile,242 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {243 const AArch64Subtarget &STI =244 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();245 246 if (STI.hasMOPS())247 return EmitMOPS(AArch64::MOPSMemoryMovePseudo, DAG, dl, Chain, Dst, Src,248 Size, Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);249 250 auto *AFI = DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();251 SMEAttrs Attrs = AFI->getSMEFnAttrs();252 if (LowerToSMERoutines && !Attrs.hasNonStreamingInterfaceAndBody())253 return EmitStreamingCompatibleMemLibCall(DAG, dl, Chain, Dst, Src, Size,254 RTLIB::MEMMOVE);255 return SDValue();256}257 258static const int kSetTagLoopThreshold = 176;259 260static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl,261 SDValue Chain, SDValue Ptr, uint64_t ObjSize,262 const MachineMemOperand *BaseMemOperand,263 bool ZeroData) {264 MachineFunction &MF = DAG.getMachineFunction();265 unsigned ObjSizeScaled = ObjSize / 16;266 267 SDValue TagSrc = Ptr;268 if (Ptr.getOpcode() == ISD::FrameIndex) {269 int FI = cast<FrameIndexSDNode>(Ptr)->getIndex();270 Ptr = DAG.getTargetFrameIndex(FI, MVT::i64);271 // A frame index operand may end up as [SP + offset] => it is fine to use SP272 // register as the tag source.273 TagSrc = DAG.getRegister(AArch64::SP, MVT::i64);274 }275 276 const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG;277 const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G;278 279 SmallVector<SDValue, 8> OutChains;280 unsigned OffsetScaled = 0;281 while (OffsetScaled < ObjSizeScaled) {282 if (ObjSizeScaled - OffsetScaled >= 2) {283 SDValue AddrNode = DAG.getMemBasePlusOffset(284 Ptr, TypeSize::getFixed(OffsetScaled * 16), dl);285 SDValue St = DAG.getMemIntrinsicNode(286 OpCode2, dl, DAG.getVTList(MVT::Other),287 {Chain, TagSrc, AddrNode},288 MVT::v4i64,289 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2));290 OffsetScaled += 2;291 OutChains.push_back(St);292 continue;293 }294 295 if (ObjSizeScaled - OffsetScaled > 0) {296 SDValue AddrNode = DAG.getMemBasePlusOffset(297 Ptr, TypeSize::getFixed(OffsetScaled * 16), dl);298 SDValue St = DAG.getMemIntrinsicNode(299 OpCode1, dl, DAG.getVTList(MVT::Other),300 {Chain, TagSrc, AddrNode},301 MVT::v2i64,302 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16));303 OffsetScaled += 1;304 OutChains.push_back(St);305 }306 }307 308 SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);309 return Res;310}311 312SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag(313 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,314 SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const {315 uint64_t ObjSize = Size->getAsZExtVal();316 assert(ObjSize % 16 == 0);317 318 MachineFunction &MF = DAG.getMachineFunction();319 MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(320 DstPtrInfo, MachineMemOperand::MOStore, ObjSize, Align(16));321 322 bool UseSetTagRangeLoop =323 kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold;324 if (!UseSetTagRangeLoop)325 return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand,326 ZeroData);327 328 const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};329 330 unsigned Opcode;331 if (Addr.getOpcode() == ISD::FrameIndex) {332 int FI = cast<FrameIndexSDNode>(Addr)->getIndex();333 Addr = DAG.getTargetFrameIndex(FI, MVT::i64);334 Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop;335 } else {336 Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback;337 }338 SDValue Ops[] = {DAG.getTargetConstant(ObjSize, dl, MVT::i64), Addr, Chain};339 SDNode *St = DAG.getMachineNode(Opcode, dl, ResTys, Ops);340 341 DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand});342 return SDValue(St, 2);343}344