404 lines · cpp
1//===-- ARMSelectionDAGInfo.cpp - ARM SelectionDAG Info -------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the ARMSelectionDAGInfo class.10//11//===----------------------------------------------------------------------===//12 13#include "ARMSelectionDAGInfo.h"14#include "ARMTargetTransformInfo.h"15#include "llvm/CodeGen/SelectionDAG.h"16#include "llvm/Support/CommandLine.h"17 18#define GET_SDNODE_DESC19#include "ARMGenSDNodeInfo.inc"20 21using namespace llvm;22 23#define DEBUG_TYPE "arm-selectiondag-info"24 25static cl::opt<TPLoop::MemTransfer> EnableMemtransferTPLoop(26 "arm-memtransfer-tploop", cl::Hidden,27 cl::desc("Control conversion of memcpy to "28 "Tail predicated loops (WLSTP)"),29 cl::init(TPLoop::ForceDisabled),30 cl::values(clEnumValN(TPLoop::ForceDisabled, "force-disabled",31 "Don't convert memcpy to TP loop."),32 clEnumValN(TPLoop::ForceEnabled, "force-enabled",33 "Always convert memcpy to TP loop."),34 clEnumValN(TPLoop::Allow, "allow",35 "Allow (may be subject to certain conditions) "36 "conversion of memcpy to TP loop.")));37 38ARMSelectionDAGInfo::ARMSelectionDAGInfo()39 : SelectionDAGGenTargetInfo(ARMGenSDNodeInfo) {}40 41const char *ARMSelectionDAGInfo::getTargetNodeName(unsigned Opcode) const {42#define MAKE_CASE(V) \43 case V: \44 return #V;45 46 // These nodes don't have corresponding entries in *.td files yet.47 switch (static_cast<ARMISD::NodeType>(Opcode)) {48 MAKE_CASE(ARMISD::DYN_ALLOC)49 MAKE_CASE(ARMISD::MVESEXT)50 MAKE_CASE(ARMISD::MVEZEXT)51 MAKE_CASE(ARMISD::MVETRUNC)52 MAKE_CASE(ARMISD::BUILD_VECTOR)53 MAKE_CASE(ARMISD::VLD1DUP)54 MAKE_CASE(ARMISD::VLD2DUP)55 MAKE_CASE(ARMISD::VLD3DUP)56 MAKE_CASE(ARMISD::VLD4DUP)57 MAKE_CASE(ARMISD::VLD1_UPD)58 MAKE_CASE(ARMISD::VLD2_UPD)59 MAKE_CASE(ARMISD::VLD3_UPD)60 MAKE_CASE(ARMISD::VLD4_UPD)61 MAKE_CASE(ARMISD::VLD1x2_UPD)62 MAKE_CASE(ARMISD::VLD1x3_UPD)63 MAKE_CASE(ARMISD::VLD1x4_UPD)64 MAKE_CASE(ARMISD::VLD2LN_UPD)65 MAKE_CASE(ARMISD::VLD3LN_UPD)66 MAKE_CASE(ARMISD::VLD4LN_UPD)67 MAKE_CASE(ARMISD::VLD1DUP_UPD)68 MAKE_CASE(ARMISD::VLD2DUP_UPD)69 MAKE_CASE(ARMISD::VLD3DUP_UPD)70 MAKE_CASE(ARMISD::VLD4DUP_UPD)71 MAKE_CASE(ARMISD::VST1_UPD)72 MAKE_CASE(ARMISD::VST3_UPD)73 MAKE_CASE(ARMISD::VST1x2_UPD)74 MAKE_CASE(ARMISD::VST1x3_UPD)75 MAKE_CASE(ARMISD::VST1x4_UPD)76 MAKE_CASE(ARMISD::VST2LN_UPD)77 MAKE_CASE(ARMISD::VST3LN_UPD)78 MAKE_CASE(ARMISD::VST4LN_UPD)79 }80#undef MAKE_CASE81 82 return SelectionDAGGenTargetInfo::getTargetNodeName(Opcode);83}84 85bool ARMSelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const {86 // These nodes don't have corresponding entries in *.td files yet.87 if (Opcode >= ARMISD::FIRST_MEMORY_OPCODE &&88 Opcode <= ARMISD::LAST_MEMORY_OPCODE)89 return true;90 91 return SelectionDAGGenTargetInfo::isTargetMemoryOpcode(Opcode);92}93 94void ARMSelectionDAGInfo::verifyTargetNode(const SelectionDAG &DAG,95 const SDNode *N) const {96 switch (N->getOpcode()) {97 default:98 break;99 case ARMISD::WIN__DBZCHK:100 // invalid number of results; expected 2, got 1101 case ARMISD::WIN__CHKSTK:102 // invalid number of results; expected 1, got 2103 case ARMISD::COPY_STRUCT_BYVAL:104 // invalid number of operands; expected 6, got 5105 case ARMISD::MEMCPY:106 // invalid number of operands; expected 5, got 4107 case ARMISD::VMOVRRD:108 // operand #0 must have type f64, but has type v1i64/v4f16/v8i8109 case ARMISD::VMOVIMM:110 // operand #0 must have type i32, but has type i16111 return;112 }113 114 SelectionDAGGenTargetInfo::verifyTargetNode(DAG, N);115}116 117// Emit, if possible, a specialized version of the given Libcall. Typically this118// means selecting the appropriately aligned version, but we also convert memset119// of 0 into memclr.120SDValue ARMSelectionDAGInfo::EmitSpecializedLibcall(121 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,122 SDValue Size, unsigned Align, RTLIB::Libcall LC) const {123 const ARMSubtarget &Subtarget =124 DAG.getMachineFunction().getSubtarget<ARMSubtarget>();125 const ARMTargetLowering *TLI = Subtarget.getTargetLowering();126 127 // Only use a specialized AEABI function if the default version of this128 // Libcall is an AEABI function.129 //130 // Translate RTLIB::Libcall to AEABILibcall. We only do this in order to be131 // able to translate memset to memclr and use the value to index the function132 // name array.133 enum {134 AEABI_MEMCPY = 0,135 AEABI_MEMMOVE,136 AEABI_MEMSET,137 AEABI_MEMCLR138 } AEABILibcall;139 switch (LC) {140 case RTLIB::MEMCPY:141 if (TLI->getLibcallImpl(LC) != RTLIB::impl___aeabi_memcpy)142 return SDValue();143 144 AEABILibcall = AEABI_MEMCPY;145 break;146 case RTLIB::MEMMOVE:147 if (TLI->getLibcallImpl(LC) != RTLIB::impl___aeabi_memmove)148 return SDValue();149 150 AEABILibcall = AEABI_MEMMOVE;151 break;152 case RTLIB::MEMSET:153 if (TLI->getLibcallImpl(LC) != RTLIB::impl___aeabi_memset)154 return SDValue();155 156 AEABILibcall = AEABI_MEMSET;157 if (isNullConstant(Src))158 AEABILibcall = AEABI_MEMCLR;159 break;160 default:161 return SDValue();162 }163 164 // Choose the most-aligned libcall variant that we can165 enum {166 ALIGN1 = 0,167 ALIGN4,168 ALIGN8169 } AlignVariant;170 if ((Align & 7) == 0)171 AlignVariant = ALIGN8;172 else if ((Align & 3) == 0)173 AlignVariant = ALIGN4;174 else175 AlignVariant = ALIGN1;176 177 TargetLowering::ArgListTy Args;178 Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext());179 Args.emplace_back(Dst, IntPtrTy);180 if (AEABILibcall == AEABI_MEMCLR) {181 Args.emplace_back(Size, IntPtrTy);182 } else if (AEABILibcall == AEABI_MEMSET) {183 // Adjust parameters for memset, EABI uses format (ptr, size, value),184 // GNU library uses (ptr, value, size)185 // See RTABI section 4.3.4186 Args.emplace_back(Size, IntPtrTy);187 188 // Extend or truncate the argument to be an i32 value for the call.189 if (Src.getValueType().bitsGT(MVT::i32))190 Src = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Src);191 else if (Src.getValueType().bitsLT(MVT::i32))192 Src = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src);193 194 TargetLowering::ArgListEntry Entry(Src,195 Type::getInt32Ty(*DAG.getContext()));196 Entry.IsSExt = false;197 Args.push_back(Entry);198 } else {199 Args.emplace_back(Src, IntPtrTy);200 Args.emplace_back(Size, IntPtrTy);201 }202 203 static const RTLIB::Libcall FunctionImpls[4][3] = {204 {RTLIB::MEMCPY, RTLIB::AEABI_MEMCPY4, RTLIB::AEABI_MEMCPY8},205 {RTLIB::MEMMOVE, RTLIB::AEABI_MEMMOVE4, RTLIB::AEABI_MEMMOVE8},206 {RTLIB::MEMSET, RTLIB::AEABI_MEMSET4, RTLIB::AEABI_MEMSET8},207 {RTLIB::AEABI_MEMCLR, RTLIB::AEABI_MEMCLR4, RTLIB::AEABI_MEMCLR8}};208 209 RTLIB::Libcall NewLC = FunctionImpls[AEABILibcall][AlignVariant];210 211 TargetLowering::CallLoweringInfo CLI(DAG);212 CLI.setDebugLoc(dl)213 .setChain(Chain)214 .setLibCallee(215 TLI->getLibcallCallingConv(NewLC), Type::getVoidTy(*DAG.getContext()),216 DAG.getExternalSymbol(TLI->getLibcallName(NewLC),217 TLI->getPointerTy(DAG.getDataLayout())),218 std::move(Args))219 .setDiscardResult();220 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);221 222 return CallResult.second;223}224 225static bool shouldGenerateInlineTPLoop(const ARMSubtarget &Subtarget,226 const SelectionDAG &DAG,227 ConstantSDNode *ConstantSize,228 Align Alignment, bool IsMemcpy) {229 auto &F = DAG.getMachineFunction().getFunction();230 if (!EnableMemtransferTPLoop)231 return false;232 if (EnableMemtransferTPLoop == TPLoop::ForceEnabled)233 return true;234 // Do not generate inline TP loop if optimizations is disabled,235 // or if optimization for size (-Os or -Oz) is on.236 if (F.hasOptNone() || F.hasOptSize())237 return false;238 // If cli option is unset, for memset always generate inline TP.239 // For memcpy, check some conditions240 if (!IsMemcpy)241 return true;242 if (!ConstantSize && Alignment >= Align(4))243 return true;244 if (ConstantSize &&245 ConstantSize->getZExtValue() > Subtarget.getMaxInlineSizeThreshold() &&246 ConstantSize->getZExtValue() <247 Subtarget.getMaxMemcpyTPInlineSizeThreshold())248 return true;249 return false;250}251 252SDValue ARMSelectionDAGInfo::EmitTargetCodeForMemcpy(253 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,254 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,255 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {256 const ARMSubtarget &Subtarget =257 DAG.getMachineFunction().getSubtarget<ARMSubtarget>();258 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);259 260 if (Subtarget.hasMVEIntegerOps() &&261 shouldGenerateInlineTPLoop(Subtarget, DAG, ConstantSize, Alignment, true))262 return DAG.getNode(ARMISD::MEMCPYLOOP, dl, MVT::Other, Chain, Dst, Src,263 DAG.getZExtOrTrunc(Size, dl, MVT::i32));264 265 // Do repeated 4-byte loads and stores. To be improved.266 // This requires 4-byte alignment.267 if (Alignment < Align(4))268 return SDValue();269 // This requires the copy size to be a constant, preferably270 // within a subtarget-specific limit.271 if (!ConstantSize)272 return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,273 Alignment.value(), RTLIB::MEMCPY);274 uint64_t SizeVal = ConstantSize->getZExtValue();275 if (!AlwaysInline && SizeVal > Subtarget.getMaxInlineSizeThreshold())276 return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,277 Alignment.value(), RTLIB::MEMCPY);278 279 unsigned BytesLeft = SizeVal & 3;280 unsigned NumMemOps = SizeVal >> 2;281 unsigned EmittedNumMemOps = 0;282 EVT VT = MVT::i32;283 unsigned VTSize = 4;284 unsigned i = 0;285 // Emit a maximum of 4 loads in Thumb1 since we have fewer registers286 const unsigned MaxLoadsInLDM = Subtarget.isThumb1Only() ? 4 : 6;287 SDValue TFOps[6];288 SDValue Loads[6];289 uint64_t SrcOff = 0, DstOff = 0;290 291 // FIXME: We should invent a VMEMCPY pseudo-instruction that lowers to292 // VLDM/VSTM and make this code emit it when appropriate. This would reduce293 // pressure on the general purpose registers. However this seems harder to map294 // onto the register allocator's view of the world.295 296 // The number of MEMCPY pseudo-instructions to emit. We use up to297 // MaxLoadsInLDM registers per mcopy, which will get lowered into ldm/stm298 // later on. This is a lower bound on the number of MEMCPY operations we must299 // emit.300 unsigned NumMEMCPYs = (NumMemOps + MaxLoadsInLDM - 1) / MaxLoadsInLDM;301 302 // Code size optimisation: do not inline memcpy if expansion results in303 // more instructions than the libary call.304 if (NumMEMCPYs > 1 && Subtarget.hasMinSize()) {305 return SDValue();306 }307 308 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other, MVT::Glue);309 310 for (unsigned I = 0; I != NumMEMCPYs; ++I) {311 // Evenly distribute registers among MEMCPY operations to reduce register312 // pressure.313 unsigned NextEmittedNumMemOps = NumMemOps * (I + 1) / NumMEMCPYs;314 unsigned NumRegs = NextEmittedNumMemOps - EmittedNumMemOps;315 316 Dst = DAG.getNode(ARMISD::MEMCPY, dl, VTs, Chain, Dst, Src,317 DAG.getConstant(NumRegs, dl, MVT::i32));318 Src = Dst.getValue(1);319 Chain = Dst.getValue(2);320 321 DstPtrInfo = DstPtrInfo.getWithOffset(NumRegs * VTSize);322 SrcPtrInfo = SrcPtrInfo.getWithOffset(NumRegs * VTSize);323 324 EmittedNumMemOps = NextEmittedNumMemOps;325 }326 327 if (BytesLeft == 0)328 return Chain;329 330 // Issue loads / stores for the trailing (1 - 3) bytes.331 auto getRemainingValueType = [](unsigned BytesLeft) {332 return (BytesLeft >= 2) ? MVT::i16 : MVT::i8;333 };334 auto getRemainingSize = [](unsigned BytesLeft) {335 return (BytesLeft >= 2) ? 2 : 1;336 };337 338 unsigned BytesLeftSave = BytesLeft;339 i = 0;340 while (BytesLeft) {341 VT = getRemainingValueType(BytesLeft);342 VTSize = getRemainingSize(BytesLeft);343 Loads[i] = DAG.getLoad(VT, dl, Chain,344 DAG.getNode(ISD::ADD, dl, MVT::i32, Src,345 DAG.getConstant(SrcOff, dl, MVT::i32)),346 SrcPtrInfo.getWithOffset(SrcOff));347 TFOps[i] = Loads[i].getValue(1);348 ++i;349 SrcOff += VTSize;350 BytesLeft -= VTSize;351 }352 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, ArrayRef(TFOps, i));353 354 i = 0;355 BytesLeft = BytesLeftSave;356 while (BytesLeft) {357 VT = getRemainingValueType(BytesLeft);358 VTSize = getRemainingSize(BytesLeft);359 TFOps[i] = DAG.getStore(Chain, dl, Loads[i],360 DAG.getNode(ISD::ADD, dl, MVT::i32, Dst,361 DAG.getConstant(DstOff, dl, MVT::i32)),362 DstPtrInfo.getWithOffset(DstOff));363 ++i;364 DstOff += VTSize;365 BytesLeft -= VTSize;366 }367 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, ArrayRef(TFOps, i));368}369 370SDValue ARMSelectionDAGInfo::EmitTargetCodeForMemmove(371 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,372 SDValue Size, Align Alignment, bool isVolatile,373 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {374 return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,375 Alignment.value(), RTLIB::MEMMOVE);376}377 378SDValue ARMSelectionDAGInfo::EmitTargetCodeForMemset(379 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,380 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,381 MachinePointerInfo DstPtrInfo) const {382 383 const ARMSubtarget &Subtarget =384 DAG.getMachineFunction().getSubtarget<ARMSubtarget>();385 386 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);387 388 // Generate TP loop for llvm.memset389 if (Subtarget.hasMVEIntegerOps() &&390 shouldGenerateInlineTPLoop(Subtarget, DAG, ConstantSize, Alignment,391 false)) {392 Src = DAG.getSplatBuildVector(MVT::v16i8, dl,393 DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Src));394 return DAG.getNode(ARMISD::MEMSETLOOP, dl, MVT::Other, Chain, Dst, Src,395 DAG.getZExtOrTrunc(Size, dl, MVT::i32));396 }397 398 if (!AlwaysInline)399 return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,400 Alignment.value(), RTLIB::MEMSET);401 402 return SDValue();403}404