898 lines · plain
1//===-- AMDGPUInstructions.td - Common instruction defs ---*- tablegen -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains instruction defs that are common to all hw codegen10// targets.11//12//===----------------------------------------------------------------------===//13 14def AddrSpaces {15 int Flat = 0;16 int Global = 1;17 int Region = 2;18 int Local = 3;19 int Constant = 4;20 int Private = 5;21 int Constant32Bit = 6;22}23 24 25class AMDGPUInst <dag outs, dag ins, string asm = "",26 list<dag> pattern = []> : Instruction {27 field bit isRegisterLoad = 0;28 field bit isRegisterStore = 0;29 30 let Namespace = "AMDGPU";31 let OutOperandList = outs;32 let InOperandList = ins;33 let AsmString = asm;34 let Pattern = pattern;35 let Itinerary = NullALU;36 37 let DecoderNamespace = Namespace;38 39 let TSFlags{63} = isRegisterLoad;40 let TSFlags{62} = isRegisterStore;41}42 43class AMDGPUShaderInst <dag outs, dag ins, string asm = "",44 list<dag> pattern = []> : AMDGPUInst<outs, ins, asm, pattern> {45 46 field bits<32> Inst = 0xffffffff;47}48 49//===---------------------------------------------------------------------===//50// Return instruction51//===---------------------------------------------------------------------===//52 53class ILFormat<dag outs, dag ins, string asmstr, list<dag> pattern>54: Instruction {55 56 let Namespace = "AMDGPU";57 dag OutOperandList = outs;58 dag InOperandList = ins;59 let Pattern = pattern;60 let AsmString = !strconcat(asmstr, "\n");61 let isPseudo = 1;62 let Itinerary = NullALU;63 bit hasIEEEFlag = 0;64 bit hasZeroOpFlag = 0;65 let mayLoad = 0;66 let mayStore = 0;67 let hasSideEffects = 0;68 let isCodeGenOnly = 1;69}70 71// Get the union of two Register lists72class RegListUnion<list<Register> lstA, list<Register> lstB> {73 list<Register> ret = !listconcat(lstA, !listremove(lstB, lstA));74}75 76class AMDGPUPat<dag pattern, dag result> : Pat<pattern, result>,77 PredicateControl, GISelFlags;78 79let GIIgnoreCopies = 1 in80class AMDGPUPatIgnoreCopies<dag pattern, dag result> : AMDGPUPat<pattern, result>;81 82let RecomputePerFunction = 1 in {83def FP16Denormals : Predicate<"MF->getInfo<SIMachineFunctionInfo>()->getMode().FP64FP16Denormals != DenormalMode::getPreserveSign()">;84def FP32Denormals : Predicate<"MF->getInfo<SIMachineFunctionInfo>()->getMode().FP32Denormals != DenormalMode::getPreserveSign()">;85def FP64Denormals : Predicate<"MF->getInfo<SIMachineFunctionInfo>()->getMode().FP64FP16Denormals != DenormalMode::getPreserveSign()">;86def NoFP16Denormals : Predicate<"MF->getInfo<SIMachineFunctionInfo>()->getMode().FP64FP16Denormals == DenormalMode::getPreserveSign()">;87def NoFP32Denormals : Predicate<"MF->getInfo<SIMachineFunctionInfo>()->getMode().FP32Denormals == DenormalMode::getPreserveSign()">;88def NoFP64Denormals : Predicate<"MF->getInfo<SIMachineFunctionInfo>()->getMode().FP64FP16Denormals == DenormalMode::getPreserveSign()">;89def IEEEModeEnabled : Predicate<"MF->getInfo<SIMachineFunctionInfo>()->getMode().IEEE">;90def IEEEModeDisabled : Predicate<"!MF->getInfo<SIMachineFunctionInfo>()->getMode().IEEE">;91}92 93def FMA : Predicate<"Subtarget->hasFMA()">;94 95def InstFlag : OperandWithDefaultOps <i32, (ops (i32 0))>;96 97def i1imm_0 : OperandWithDefaultOps<i1, (ops (i1 0))>;98 99class CustomOperandClass<string name, bit optional, string predicateMethod,100 string parserMethod, string defaultMethod>101 : AsmOperandClass {102 let Name = name;103 let PredicateMethod = predicateMethod;104 let ParserMethod = parserMethod;105 let RenderMethod = "addImmOperands";106 let IsOptional = optional;107 let DefaultMethod = defaultMethod;108}109 110class CustomOperandProps<bit optional = 0, string name = NAME> {111 string ImmTy = "ImmTy"#name;112 string PredicateMethod = "is"#name;113 string ParserMethod = "parse"#name;114 string DefaultValue = "0";115 string DefaultMethod = "[this]() { return "#116 "AMDGPUOperand::CreateImm(this, "#DefaultValue#", SMLoc(), "#117 "AMDGPUOperand::"#ImmTy#"); }";118 string PrintMethod = "print"#name;119 AsmOperandClass ParserMatchClass =120 CustomOperandClass<name, optional, PredicateMethod, ParserMethod,121 DefaultMethod>;122 string OperandType = "OPERAND_IMMEDIATE";123}124 125class CustomOperand<ValueType type, bit optional = 0, string name = NAME>126 : Operand<type>, CustomOperandProps<optional, name>;127 128class ImmOperand<ValueType type, string name = NAME, bit optional = 0,129 string printer = "print"#name>130 : CustomOperand<type, optional, name> {131 let ImmTy = "ImmTyNone";132 let ParserMethod = "";133 let PrintMethod = printer;134}135 136class S16ImmOperand : ImmOperand<i16, "S16Imm", 0, "printU16ImmOperand">;137 138def s16imm : S16ImmOperand;139def u16imm : ImmOperand<i16, "U16Imm", 0, "printU16ImmOperand">;140 141class ValuePredicatedOperand<CustomOperand op, string valuePredicate,142 bit optional = 0>143 : CustomOperand<op.Type, optional> {144 let ImmTy = op.ImmTy;145 defvar OpPredicate = op.ParserMatchClass.PredicateMethod;146 let PredicateMethod =147 "getPredicate([](const AMDGPUOperand &Op) -> bool { "#148 "return Op."#OpPredicate#"() && "#valuePredicate#"; })";149 let ParserMethod = op.ParserMatchClass.ParserMethod;150 let DefaultValue = op.DefaultValue;151 let DefaultMethod = op.DefaultMethod;152 let PrintMethod = op.PrintMethod;153}154 155//===--------------------------------------------------------------------===//156// Custom Operands157//===--------------------------------------------------------------------===//158def brtarget : Operand<OtherVT>;159 160//===----------------------------------------------------------------------===//161// Misc. PatFrags162//===----------------------------------------------------------------------===//163 164class HasOneUseUnaryOp<SDPatternOperator op> : PatFrag<165 (ops node:$src0),166 (op $src0)> {167 let HasOneUse = 1;168}169 170class HasOneUseBinOp<SDPatternOperator op> : PatFrag<171 (ops node:$src0, node:$src1),172 (op $src0, $src1)> {173 let HasOneUse = 1;174}175 176class HasOneUseTernaryOp<SDPatternOperator op> : PatFrag<177 (ops node:$src0, node:$src1, node:$src2),178 (op $src0, $src1, $src2)> {179 let HasOneUse = 1;180}181 182class is_canonicalized_1<SDPatternOperator op> : PatFrag<183 (ops node:$src0),184 (op $src0),185 [{186 const SITargetLowering &Lowering =187 *static_cast<const SITargetLowering *>(getTargetLowering());188 189 return Lowering.isCanonicalized(*CurDAG, N->getOperand(0));190 }]> {191 192 let GISelPredicateCode = [{193 const SITargetLowering *TLI = static_cast<const SITargetLowering *>(194 MF.getSubtarget().getTargetLowering());195 196 return TLI->isCanonicalized(MI.getOperand(1).getReg(), MF);197 }];198}199 200class is_canonicalized_2<SDPatternOperator op> : PatFrag<201 (ops node:$src0, node:$src1),202 (op $src0, $src1),203 [{204 const SITargetLowering &Lowering =205 *static_cast<const SITargetLowering *>(getTargetLowering());206 207 return Lowering.isCanonicalized(*CurDAG, N->getOperand(0)) &&208 Lowering.isCanonicalized(*CurDAG, N->getOperand(1));209 }]> {210 211 // TODO: Improve the Legalizer for g_build_vector in Global Isel to match this class212 let GISelPredicateCode = [{213 const SITargetLowering *TLI = static_cast<const SITargetLowering *>(214 MF.getSubtarget().getTargetLowering());215 216 return TLI->isCanonicalized(MI.getOperand(1).getReg(), MF) &&217 TLI->isCanonicalized(MI.getOperand(2).getReg(), MF);218 }];219}220 221class FoldTernaryOpPat<SDPatternOperator op1, SDPatternOperator op2> : PatFrag<222 (ops node:$src0, node:$src1, node:$src2),223 (op2 (op1 node:$src0, node:$src1), node:$src2)224>;225 226def imad : FoldTernaryOpPat<mul, add>;227 228let Properties = [SDNPCommutative, SDNPAssociative] in {229def smax_oneuse : HasOneUseBinOp<smax>;230def smin_oneuse : HasOneUseBinOp<smin>;231def umax_oneuse : HasOneUseBinOp<umax>;232def umin_oneuse : HasOneUseBinOp<umin>;233 234def fminnum_oneuse : HasOneUseBinOp<fminnum>;235def fmaxnum_oneuse : HasOneUseBinOp<fmaxnum>;236def fminimum_oneuse : HasOneUseBinOp<fminimum>;237def fmaximum_oneuse : HasOneUseBinOp<fmaximum>;238 239def fminnum_ieee_oneuse : HasOneUseBinOp<fminnum_ieee>;240def fmaxnum_ieee_oneuse : HasOneUseBinOp<fmaxnum_ieee>;241 242 243def and_oneuse : HasOneUseBinOp<and>;244def or_oneuse : HasOneUseBinOp<or>;245def xor_oneuse : HasOneUseBinOp<xor>;246} // Properties = [SDNPCommutative, SDNPAssociative]247 248def not_oneuse : HasOneUseUnaryOp<not>;249 250def add_oneuse : HasOneUseBinOp<add>;251def sub_oneuse : HasOneUseBinOp<sub>;252 253def srl_oneuse : HasOneUseBinOp<srl>;254def shl_oneuse : HasOneUseBinOp<shl>;255 256def select_oneuse : HasOneUseTernaryOp<select>;257 258def AMDGPUmul_u24_oneuse : HasOneUseBinOp<AMDGPUmul_u24>;259def AMDGPUmul_i24_oneuse : HasOneUseBinOp<AMDGPUmul_i24>;260 261//===----------------------------------------------------------------------===//262// PatFrags for shifts263//===----------------------------------------------------------------------===//264 265// Constrained shift PatFrags.266 267def csh_mask_16 : PatFrag<(ops node:$src0), (and node:$src0, imm),268 [{ return isUnneededShiftMask(N, 4); }]> {269 let GISelPredicateCode = [{ return isUnneededShiftMask(MI, 4); }];270 }271 272def csh_mask_32 : PatFrag<(ops node:$src0), (and node:$src0, imm),273 [{ return isUnneededShiftMask(N, 5); }]> {274 let GISelPredicateCode = [{ return isUnneededShiftMask(MI, 5); }];275 }276 277def csh_mask_64 : PatFrag<(ops node:$src0), (and node:$src0, imm),278 [{ return isUnneededShiftMask(N, 6); }]> {279 let GISelPredicateCode = [{ return isUnneededShiftMask(MI, 6); }];280 }281 282foreach width = [16, 32, 64] in {283defvar csh_mask = !cast<SDPatternOperator>("csh_mask_"#width);284 285def cshl_#width : PatFrags<(ops node:$src0, node:$src1),286 [(shl node:$src0, node:$src1), (shl node:$src0, (csh_mask node:$src1))]>;287defvar cshl = !cast<SDPatternOperator>("cshl_"#width);288def cshl_#width#_oneuse : HasOneUseBinOp<cshl>;289def clshl_rev_#width : PatFrag <(ops node:$src0, node:$src1),290 (cshl $src1, $src0)>;291 292def csrl_#width : PatFrags<(ops node:$src0, node:$src1),293 [(srl node:$src0, node:$src1), (srl node:$src0, (csh_mask node:$src1))]>;294defvar csrl = !cast<SDPatternOperator>("csrl_"#width);295def csrl_#width#_oneuse : HasOneUseBinOp<csrl>;296def clshr_rev_#width : PatFrag <(ops node:$src0, node:$src1),297 (csrl $src1, $src0)>;298 299def csra_#width : PatFrags<(ops node:$src0, node:$src1),300 [(sra node:$src0, node:$src1), (sra node:$src0, (csh_mask node:$src1))]>;301defvar csra = !cast<SDPatternOperator>("csra_"#width);302def csra_#width#_oneuse : HasOneUseBinOp<csra>;303def cashr_rev_#width : PatFrag <(ops node:$src0, node:$src1),304 (csra $src1, $src0)>;305} // end foreach width306 307def srl_16 : PatFrag<308 (ops node:$src0), (srl_oneuse node:$src0, (i32 16))309>;310 311 312def hi_i16_elt : PatFrag<313 (ops node:$src0), (i16 (trunc (i32 (srl_16 node:$src0))))314>;315 316 317def hi_f16_elt : PatLeaf<318 (vt), [{319 if (N->getOpcode() != ISD::BITCAST)320 return false;321 SDValue Tmp = N->getOperand(0);322 323 if (Tmp.getOpcode() != ISD::SRL)324 return false;325 if (const auto *RHS = dyn_cast<ConstantSDNode>(Tmp.getOperand(1))326 return RHS->getZExtValue() == 16;327 return false;328}]>;329 330//===----------------------------------------------------------------------===//331// PatLeafs for zero immediate332//===----------------------------------------------------------------------===//333 334def immzero : PatLeaf<(imm), [{ return N->isZero(); }]>;335def fpimmzero : PatLeaf<(fpimm), [{ return N->isZero(); }]>;336 337//===----------------------------------------------------------------------===//338// PatLeafs for floating-point comparisons339//===----------------------------------------------------------------------===//340 341def COND_OEQ : PatFrags<(ops), [(OtherVT SETOEQ), (OtherVT SETEQ)]>;342def COND_ONE : PatFrags<(ops), [(OtherVT SETONE), (OtherVT SETNE)]>;343def COND_OGT : PatFrags<(ops), [(OtherVT SETOGT), (OtherVT SETGT)]>;344def COND_OGE : PatFrags<(ops), [(OtherVT SETOGE), (OtherVT SETGE)]>;345def COND_OLT : PatFrags<(ops), [(OtherVT SETOLT), (OtherVT SETLT)]>;346def COND_OLE : PatFrags<(ops), [(OtherVT SETOLE), (OtherVT SETLE)]>;347def COND_O : PatFrags<(ops), [(OtherVT SETO)]>;348def COND_UO : PatFrags<(ops), [(OtherVT SETUO)]>;349 350//===----------------------------------------------------------------------===//351// PatLeafs for unsigned / unordered comparisons352//===----------------------------------------------------------------------===//353 354def COND_UEQ : PatFrag<(ops), (OtherVT SETUEQ)>;355def COND_UNE : PatFrag<(ops), (OtherVT SETUNE)>;356def COND_UGT : PatFrag<(ops), (OtherVT SETUGT)>;357def COND_UGE : PatFrag<(ops), (OtherVT SETUGE)>;358def COND_ULT : PatFrag<(ops), (OtherVT SETULT)>;359def COND_ULE : PatFrag<(ops), (OtherVT SETULE)>;360 361// XXX - For some reason R600 version is preferring to use unordered362// for setne?363def COND_UNE_NE : PatFrags<(ops), [(OtherVT SETUNE), (OtherVT SETNE)]>;364 365//===----------------------------------------------------------------------===//366// PatLeafs for signed comparisons367//===----------------------------------------------------------------------===//368 369def COND_SGT : PatFrag<(ops), (OtherVT SETGT)>;370def COND_SGE : PatFrag<(ops), (OtherVT SETGE)>;371def COND_SLT : PatFrag<(ops), (OtherVT SETLT)>;372def COND_SLE : PatFrag<(ops), (OtherVT SETLE)>;373 374//===----------------------------------------------------------------------===//375// PatLeafs for integer equality376//===----------------------------------------------------------------------===//377 378def COND_EQ : PatFrags<(ops), [(OtherVT SETEQ), (OtherVT SETUEQ)]>;379def COND_NE : PatFrags<(ops), [(OtherVT SETNE), (OtherVT SETUNE)]>;380 381// FIXME: Should not need code predicate382//def COND_NULL : PatLeaf<(OtherVT null_frag)>;383def COND_NULL : PatLeaf <384 (cond),385 [{(void)N; return false;}]386>;387 388//===----------------------------------------------------------------------===//389// PatLeafs for Texture Constants390//===----------------------------------------------------------------------===//391 392def TEX_ARRAY : PatLeaf<393 (imm),394 [{uint32_t TType = (uint32_t)N->getZExtValue();395 return TType == 9 || TType == 10 || TType == 16;396 }]397>;398 399def TEX_RECT : PatLeaf<400 (imm),401 [{uint32_t TType = (uint32_t)N->getZExtValue();402 return TType == 5;403 }]404>;405 406def TEX_SHADOW : PatLeaf<407 (imm),408 [{uint32_t TType = (uint32_t)N->getZExtValue();409 return (TType >= 6 && TType <= 8) || TType == 13;410 }]411>;412 413def TEX_SHADOW_ARRAY : PatLeaf<414 (imm),415 [{uint32_t TType = (uint32_t)N->getZExtValue();416 return TType == 11 || TType == 12 || TType == 17;417 }]418>;419 420//===----------------------------------------------------------------------===//421// Load/Store Pattern Fragments422//===----------------------------------------------------------------------===//423 424def atomic_cmp_swap_glue : SDNode <"ISD::ATOMIC_CMP_SWAP", SDTAtomic3,425 [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand, SDNPInGlue]426>;427 428class AddressSpaceList<list<int> AS> {429 list<int> AddrSpaces = AS;430}431 432class Aligned<int Bytes> {433 int MinAlignment = Bytes;434}435 436class StoreHi16<SDPatternOperator op, ValueType vt> : PatFrag <437 (ops node:$value, node:$ptr), (op (srl node:$value, (i32 16)), node:$ptr)> {438 let IsStore = 1;439 let MemoryVT = vt;440}441 442def LoadAddress_constant : AddressSpaceList<[ AddrSpaces.Constant,443 AddrSpaces.Constant32Bit ]>;444def LoadAddress_global : AddressSpaceList<[ AddrSpaces.Global,445 AddrSpaces.Constant,446 AddrSpaces.Constant32Bit ]>;447def StoreAddress_global : AddressSpaceList<[ AddrSpaces.Global,448 AddrSpaces.Constant,449 AddrSpaces.Constant32Bit ]>;450 451def LoadAddress_flat : AddressSpaceList<[ AddrSpaces.Flat,452 AddrSpaces.Global,453 AddrSpaces.Constant,454 AddrSpaces.Constant32Bit ]>;455def StoreAddress_flat : AddressSpaceList<[ AddrSpaces.Flat, AddrSpaces.Global ]>;456 457def LoadAddress_private : AddressSpaceList<[ AddrSpaces.Private ]>;458def StoreAddress_private : AddressSpaceList<[ AddrSpaces.Private ]>;459 460def LoadAddress_local : AddressSpaceList<[ AddrSpaces.Local ]>;461def StoreAddress_local : AddressSpaceList<[ AddrSpaces.Local ]>;462 463def LoadAddress_region : AddressSpaceList<[ AddrSpaces.Region ]>;464def StoreAddress_region : AddressSpaceList<[ AddrSpaces.Region ]>;465 466 467 468foreach as = [ "global", "flat", "constant", "local", "private", "region" ] in {469let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {470 471def load_#as : PatFrag<(ops node:$ptr), (unindexedload node:$ptr)> {472 let IsLoad = 1;473 let IsNonExtLoad = 1;474}475 476def extloadi8_#as : PatFrag<(ops node:$ptr), (extloadi8 node:$ptr)> {477 let IsLoad = 1;478}479 480def extloadi16_#as : PatFrag<(ops node:$ptr), (extloadi16 node:$ptr)> {481 let IsLoad = 1;482}483 484def sextloadi8_#as : PatFrag<(ops node:$ptr), (sextloadi8 node:$ptr)> {485 let IsLoad = 1;486}487 488def sextloadi16_#as : PatFrag<(ops node:$ptr), (sextloadi16 node:$ptr)> {489 let IsLoad = 1;490}491 492def zextloadi8_#as : PatFrag<(ops node:$ptr), (zextloadi8 node:$ptr)> {493 let IsLoad = 1;494}495 496def zextloadi16_#as : PatFrag<(ops node:$ptr), (zextloadi16 node:$ptr)> {497 let IsLoad = 1;498}499 500def atomic_load_nonext_16_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_16 node:$ptr)> {501 let IsAtomic = 1;502}503 504def atomic_load_nonext_32_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_32 node:$ptr)> {505 let IsAtomic = 1;506}507 508def atomic_load_nonext_64_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_64 node:$ptr)> {509 let IsAtomic = 1;510}511 512def atomic_load_nonext_128_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_128 node:$ptr)> {513 let IsAtomic = 1;514}515 516def atomic_load_zext_8_#as : PatFrag<(ops node:$ptr), (atomic_load_zext_8 node:$ptr)> {517 let IsAtomic = 1;518}519 520def atomic_load_sext_8_#as : PatFrag<(ops node:$ptr), (atomic_load_sext_8 node:$ptr)> {521 let IsAtomic = 1;522}523 524def atomic_load_aext_8_#as : PatFrag<(ops node:$ptr), (atomic_load_aext_8 node:$ptr)> {525 let IsAtomic = 1;526}527 528def atomic_load_zext_16_#as : PatFrag<(ops node:$ptr), (atomic_load_zext_16 node:$ptr)> {529 let IsAtomic = 1;530}531 532def atomic_load_sext_16_#as : PatFrag<(ops node:$ptr), (atomic_load_sext_16 node:$ptr)> {533 let IsAtomic = 1;534}535 536def atomic_load_aext_16_#as : PatFrag<(ops node:$ptr), (atomic_load_aext_16 node:$ptr)> {537 let IsAtomic = 1;538}539 540} // End let AddressSpaces541} // End foreach as542 543 544foreach as = [ "global", "flat", "local", "private", "region" ] in {545let IsStore = 1, AddressSpaces = !cast<AddressSpaceList>("StoreAddress_"#as).AddrSpaces in {546def store_#as : PatFrag<(ops node:$val, node:$ptr),547 (unindexedstore node:$val, node:$ptr)> {548 let IsTruncStore = 0;549}550 551// truncstore fragments.552def truncstore_#as : PatFrag<(ops node:$val, node:$ptr),553 (unindexedstore node:$val, node:$ptr)> {554 let IsTruncStore = 1;555}556 557// TODO: We don't really need the truncstore here. We can use558// unindexedstore with MemoryVT directly, which will save an559// unnecessary check that the memory size is less than the value type560// in the generated matcher table.561def truncstorei8_#as : PatFrag<(ops node:$val, node:$ptr),562 (truncstorei8 node:$val, node:$ptr)>;563def truncstorei16_#as : PatFrag<(ops node:$val, node:$ptr),564 (truncstorei16 node:$val, node:$ptr)>;565 566def store_hi16_#as : StoreHi16 <truncstorei16, i16>;567def truncstorei8_hi16_#as : StoreHi16<truncstorei8, i8>;568def truncstorei16_hi16_#as : StoreHi16<truncstorei16, i16>;569} // End let IsStore = 1, AddressSpaces = ...570 571let IsAtomic = 1, AddressSpaces = !cast<AddressSpaceList>("StoreAddress_"#as).AddrSpaces in {572def atomic_store_8_#as : PatFrag<(ops node:$val, node:$ptr),573 (atomic_store_8 node:$val, node:$ptr)>;574def atomic_store_16_#as : PatFrag<(ops node:$val, node:$ptr),575 (atomic_store_16 node:$val, node:$ptr)>;576def atomic_store_32_#as : PatFrag<(ops node:$val, node:$ptr),577 (atomic_store_32 node:$val, node:$ptr)>;578def atomic_store_64_#as : PatFrag<(ops node:$val, node:$ptr),579 (atomic_store_64 node:$val, node:$ptr)>;580def atomic_store_128_#as : PatFrag<(ops node:$val, node:$ptr),581 (atomic_store_128 node:$val, node:$ptr)>;582} // End let IsAtomic = 1, AddressSpaces = ...583} // End foreach as584 585multiclass noret_op {586 let HasNoUse = true in587 def "_noret" : PatFrag<(ops node:$ptr, node:$data),588 (!cast<SDPatternOperator>(NAME) node:$ptr, node:$data)>;589}590 591multiclass global_addr_space_atomic_op {592 def "_noret_global_addrspace" :593 PatFrag<(ops node:$ptr, node:$data),594 (!cast<SDPatternOperator>(NAME) node:$ptr, node:$data)>{595 let HasNoUse = true;596 let AddressSpaces = LoadAddress_global.AddrSpaces;597 let IsAtomic = 1;598 }599 def "_global_addrspace" :600 PatFrag<(ops node:$ptr, node:$data),601 (!cast<SDPatternOperator>(NAME) node:$ptr, node:$data)>{602 let AddressSpaces = LoadAddress_global.AddrSpaces;603 let IsAtomic = 1;604 }605}606 607multiclass flat_addr_space_atomic_op {608 def "_noret_flat_addrspace" :609 PatFrag<(ops node:$ptr, node:$data),610 (!cast<SDPatternOperator>(NAME) node:$ptr, node:$data)>{611 let HasNoUse = true;612 let AddressSpaces = LoadAddress_flat.AddrSpaces;613 let IsAtomic = 1;614 }615 def "_flat_addrspace" :616 PatFrag<(ops node:$ptr, node:$data),617 (!cast<SDPatternOperator>(NAME) node:$ptr, node:$data)>{618 let AddressSpaces = LoadAddress_flat.AddrSpaces;619 let IsAtomic = 1;620 }621}622 623multiclass local_addr_space_atomic_op {624 def "_noret_local_addrspace" :625 PatFrag<(ops node:$ptr, node:$data),626 (!cast<SDPatternOperator>(NAME) node:$ptr, node:$data)>{627 let HasNoUse = true;628 let AddressSpaces = LoadAddress_local.AddrSpaces;629 let IsAtomic = 1;630 }631 def "_local_addrspace" :632 PatFrag<(ops node:$ptr, node:$data),633 (!cast<SDPatternOperator>(NAME) node:$ptr, node:$data)>{634 let AddressSpaces = LoadAddress_local.AddrSpaces;635 let IsAtomic = 1;636 }637}638 639defm int_amdgcn_global_atomic_csub : noret_op;640defm int_amdgcn_global_atomic_ordered_add_b64 : noret_op;641defm int_amdgcn_flat_atomic_fmin_num : noret_op;642defm int_amdgcn_flat_atomic_fmax_num : noret_op;643defm int_amdgcn_global_atomic_fmin_num : noret_op;644defm int_amdgcn_global_atomic_fmax_num : noret_op;645defm int_amdgcn_atomic_cond_sub_u32 : local_addr_space_atomic_op;646defm int_amdgcn_atomic_cond_sub_u32 : flat_addr_space_atomic_op;647defm int_amdgcn_atomic_cond_sub_u32 : global_addr_space_atomic_op;648 649multiclass noret_binary_atomic_op<SDNode atomic_op> {650 let HasNoUse = true in651 defm "_noret" : binary_atomic_op<atomic_op>;652}653 654multiclass noret_binary_atomic_op_fp<SDNode atomic_op> {655 let HasNoUse = true in656 defm "_noret" : binary_atomic_op_fp<atomic_op>;657}658 659multiclass noret_ternary_atomic_op<SDNode atomic_op> {660 let HasNoUse = true in661 defm "_noret" : ternary_atomic_op<atomic_op>;662}663 664defvar atomic_addrspace_names = [ "global", "flat", "constant", "local", "private", "region" ];665 666multiclass binary_atomic_op_all_as<SDNode atomic_op> {667 foreach as = atomic_addrspace_names in {668 let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {669 defm "_"#as : binary_atomic_op<atomic_op>;670 defm "_"#as : noret_binary_atomic_op<atomic_op>;671 }672 }673}674multiclass binary_atomic_op_fp_all_as<SDNode atomic_op> {675 foreach as = atomic_addrspace_names in {676 let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {677 defm "_"#as : binary_atomic_op_fp<atomic_op>;678 defm "_"#as : noret_binary_atomic_op_fp<atomic_op>;679 }680 }681}682 683defm atomic_swap : binary_atomic_op_all_as<atomic_swap>;684defm atomic_load_add : binary_atomic_op_all_as<atomic_load_add>;685defm atomic_load_and : binary_atomic_op_all_as<atomic_load_and>;686defm atomic_load_max : binary_atomic_op_all_as<atomic_load_max>;687defm atomic_load_min : binary_atomic_op_all_as<atomic_load_min>;688defm atomic_load_or : binary_atomic_op_all_as<atomic_load_or>;689defm atomic_load_sub : binary_atomic_op_all_as<atomic_load_sub>;690defm atomic_load_umax : binary_atomic_op_all_as<atomic_load_umax>;691defm atomic_load_umin : binary_atomic_op_all_as<atomic_load_umin>;692defm atomic_load_xor : binary_atomic_op_all_as<atomic_load_xor>;693defm atomic_load_fadd : binary_atomic_op_fp_all_as<atomic_load_fadd>;694defm atomic_load_fmin : binary_atomic_op_fp_all_as<atomic_load_fmin>;695defm atomic_load_fmax : binary_atomic_op_fp_all_as<atomic_load_fmax>;696defm atomic_load_uinc_wrap : binary_atomic_op_all_as<atomic_load_uinc_wrap>;697defm atomic_load_udec_wrap : binary_atomic_op_all_as<atomic_load_udec_wrap>;698defm AMDGPUatomic_cmp_swap : binary_atomic_op_all_as<AMDGPUatomic_cmp_swap>;699 700def load_align8_local : PatFrag<(ops node:$ptr), (load_local node:$ptr)>,701 Aligned<8> {702 let IsLoad = 1;703}704 705def load_align16_local : PatFrag<(ops node:$ptr), (load_local node:$ptr)>,706 Aligned<16> {707 let IsLoad = 1;708}709 710def store_align8_local: PatFrag<(ops node:$val, node:$ptr),711 (store_local node:$val, node:$ptr)>, Aligned<8> {712 let IsStore = 1;713}714 715def store_align16_local: PatFrag<(ops node:$val, node:$ptr),716 (store_local node:$val, node:$ptr)>, Aligned<16> {717 let IsStore = 1;718}719 720let AddressSpaces = StoreAddress_local.AddrSpaces in {721defm atomic_cmp_swap_local : ternary_atomic_op<atomic_cmp_swap>;722defm atomic_cmp_swap_local : noret_ternary_atomic_op<atomic_cmp_swap>;723defm atomic_cmp_swap_local_m0 : noret_ternary_atomic_op<atomic_cmp_swap_glue>;724defm atomic_cmp_swap_local_m0 : ternary_atomic_op<atomic_cmp_swap_glue>;725}726 727let AddressSpaces = StoreAddress_region.AddrSpaces in {728defm atomic_cmp_swap_region : noret_ternary_atomic_op<atomic_cmp_swap>;729defm atomic_cmp_swap_region_m0 : noret_ternary_atomic_op<atomic_cmp_swap_glue>;730defm atomic_cmp_swap_region_m0 : ternary_atomic_op<atomic_cmp_swap_glue>;731}732 733//===----------------------------------------------------------------------===//734// Misc Pattern Fragments735//===----------------------------------------------------------------------===//736 737class Constants {738int TWO_PI = 0x40c90fdb;739int PI = 0x40490fdb;740int TWO_PI_INV = 0x3e22f983;741int FP_4294966784 = 0x4f7ffffe; // 4294966784 = 4294967296 - 512 = 2^32 - 2^9742int FP16_ONE = 0x3C00;743int FP16_NEG_ONE = 0xBC00;744int FP32_ONE = 0x3f800000;745int FP32_NEG_ONE = 0xbf800000;746int FP64_ONE = 0x3ff0000000000000;747int FP64_NEG_ONE = 0xbff0000000000000;748}749def CONST : Constants;750 751def FP_ZERO : PatLeaf <752 (fpimm),753 [{return N->getValueAPF().isZero();}]754>;755 756def FP_ONE : PatLeaf <757 (fpimm),758 [{return N->isExactlyValue(1.0);}]759>;760 761def FP_HALF : PatLeaf <762 (fpimm),763 [{return N->isExactlyValue(0.5);}]764>;765 766/* Generic helper patterns for intrinsics */767/* -------------------------------------- */768 769class POW_Common <AMDGPUInst log_ieee, AMDGPUInst exp_ieee, AMDGPUInst mul>770 : AMDGPUPat <771 (fpow f32:$src0, f32:$src1),772 (exp_ieee (mul f32:$src1, (log_ieee f32:$src0)))773>;774 775/* Other helper patterns */776/* --------------------- */777 778/* Extract element pattern */779class Extract_Element <ValueType sub_type, ValueType vec_type, int sub_idx,780 SubRegIndex sub_reg>781 : AMDGPUPat<782 (sub_type (extractelt vec_type:$src, sub_idx)),783 (EXTRACT_SUBREG $src, sub_reg)784>;785 786/* Insert element pattern */787class Insert_Element <ValueType elem_type, ValueType vec_type,788 int sub_idx, SubRegIndex sub_reg>789 : AMDGPUPat <790 (insertelt vec_type:$vec, elem_type:$elem, sub_idx),791 (INSERT_SUBREG $vec, $elem, sub_reg)792>;793 794// XXX: Convert to new syntax and use COPY_TO_REG, once the DFAPacketizer795// can handle COPY instructions.796// bitconvert pattern797class BitConvert <ValueType dt, ValueType st, RegisterClass rc> : AMDGPUPat <798 (dt (bitconvert (st rc:$src0))),799 (dt rc:$src0)800>;801 802// XXX: Convert to new syntax and use COPY_TO_REG, once the DFAPacketizer803// can handle COPY instructions.804class DwordAddrPat<ValueType vt, RegisterClass rc> : AMDGPUPat <805 (vt (AMDGPUdwordaddr (vt rc:$addr))),806 (vt rc:$addr)807>;808 809// rotr pattern810class ROTRPattern <Instruction BIT_ALIGN> : AMDGPUPat <811 (rotr i32:$src0, i32:$src1),812 (BIT_ALIGN $src0, $src0, $src1)813>;814 815// Special conversion patterns816 817def cvt_rpi_i32_f32 : PatFrag <818 (ops node:$src),819 (fp_to_sint (ffloor (fadd $src, FP_HALF))),820 [{ (void) N; return TM.Options.NoNaNsFPMath; }]821>;822 823def cvt_flr_i32_f32 : PatFrag <824 (ops node:$src),825 (fp_to_sint (ffloor $src)),826 [{ (void)N; return TM.Options.NoNaNsFPMath; }]827>;828 829let AddedComplexity = 2 in {830class IMad24Pat<Instruction Inst, bit HasClamp = 0> : AMDGPUPat <831 (add (AMDGPUmul_i24 i32:$src0, i32:$src1), i32:$src2),832 !if(HasClamp, (Inst $src0, $src1, $src2, (i1 0)),833 (Inst $src0, $src1, $src2))834>;835 836class UMad24Pat<Instruction Inst, bit HasClamp = 0> : AMDGPUPat <837 (add (AMDGPUmul_u24 i32:$src0, i32:$src1), i32:$src2),838 !if(HasClamp, (Inst $src0, $src1, $src2, (i1 0)),839 (Inst $src0, $src1, $src2))840>;841} // AddedComplexity.842 843class RcpPat<Instruction RcpInst, ValueType vt> : AMDGPUPat <844 (fdiv FP_ONE, vt:$src),845 (RcpInst $src)846>;847 848// Instructions which select to the same v_min_f*849def fminnum_like : PatFrags<(ops node:$src0, node:$src1),850 [(fminnum_ieee node:$src0, node:$src1),851 (fminnum node:$src0, node:$src1)]852>;853 854// Instructions which select to the same v_max_f*855def fmaxnum_like : PatFrags<(ops node:$src0, node:$src1),856 [(fmaxnum_ieee node:$src0, node:$src1),857 (fmaxnum node:$src0, node:$src1)]858>;859 860class NeverNaNPats<dag ops, list<dag> frags> : PatFrags<ops, frags> {861 let PredicateCode = [{862 return CurDAG->isKnownNeverNaN(SDValue(N,0));863 }];864 let GISelPredicateCode = [{865 return isKnownNeverNaN(MI.getOperand(0).getReg(), MRI);866 }];867}868 869def fminnum_like_nnan : NeverNaNPats<(ops node:$src0, node:$src1),870 [(fminnum_ieee node:$src0, node:$src1),871 (fminnum node:$src0, node:$src1)]872>;873 874def fmaxnum_like_nnan : NeverNaNPats<(ops node:$src0, node:$src1),875 [(fmaxnum_ieee node:$src0, node:$src1),876 (fmaxnum node:$src0, node:$src1)]877>;878 879def fminnum_like_oneuse : PatFrags<(ops node:$src0, node:$src1),880 [(fminnum_ieee_oneuse node:$src0, node:$src1),881 (fminnum_oneuse node:$src0, node:$src1)]882>;883 884def fmaxnum_like_oneuse : PatFrags<(ops node:$src0, node:$src1),885 [(fmaxnum_ieee_oneuse node:$src0, node:$src1),886 (fmaxnum_oneuse node:$src0, node:$src1)]887>;888 889def any_fmad : PatFrags<(ops node:$src0, node:$src1, node:$src2),890 [(fmad node:$src0, node:$src1, node:$src2),891 (AMDGPUfmad_ftz node:$src0, node:$src1, node:$src2)]892>;893 894// FIXME: fsqrt should not select directly895def any_amdgcn_sqrt : PatFrags<(ops node:$src0),896 [(fsqrt node:$src0), (int_amdgcn_sqrt node:$src0)]897>;898