4765 lines · plain
1//===-- SIInstructions.td - SI Instruction Definitions --------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8// This file was originally auto-generated from a GPU register header file and9// all the instruction definitions were originally commented out. Instructions10// that are not yet supported remain commented out.11//===----------------------------------------------------------------------===//12 13class GCNPat<dag pattern, dag result> : Pat<pattern, result>, PredicateControl;14 15class UniformSextInreg<ValueType VT> : PatFrag<16 (ops node:$src),17 (sext_inreg $src, VT),18 [{ return !N->isDivergent(); }]>;19 20class DivergentSextInreg<ValueType VT> : PatFrag<21 (ops node:$src),22 (sext_inreg $src, VT),23 [{ return N->isDivergent(); }]>;24 25include "SOPInstructions.td"26include "VOPInstructions.td"27include "SMInstructions.td"28include "FLATInstructions.td"29include "BUFInstructions.td"30include "EXPInstructions.td"31include "DSDIRInstructions.td"32include "VINTERPInstructions.td"33 34//===----------------------------------------------------------------------===//35// VINTRP Instructions36//===----------------------------------------------------------------------===//37 38// Used to inject printing of "_e32" suffix for VI (there are "_e64" variants for VI)39def VINTRPDst : VINTRPDstOperand <VGPR_32>;40 41let Uses = [MODE, M0, EXEC] in {42 43// FIXME: Specify SchedRW for VINTRP instructions.44 45multiclass V_INTERP_P1_F32_m : VINTRP_m <46 0x00000000,47 (outs VINTRPDst:$vdst),48 (ins VGPR_32:$vsrc, InterpAttr:$attr, InterpAttrChan:$attrchan),49 "v_interp_p1_f32$vdst, $vsrc, $attr$attrchan",50 [(set f32:$vdst, (int_amdgcn_interp_p1 f32:$vsrc,51 (i32 timm:$attrchan), (i32 timm:$attr), M0))]52>;53 54let OtherPredicates = [has32BankLDS, isNotGFX90APlus] in {55 56defm V_INTERP_P1_F32 : V_INTERP_P1_F32_m;57 58} // End OtherPredicates = [has32BankLDS, isNotGFX90APlus]59 60let OtherPredicates = [has16BankLDS, isNotGFX90APlus],61 Constraints = "@earlyclobber $vdst", isAsmParserOnly=1 in {62 63defm V_INTERP_P1_F32_16bank : V_INTERP_P1_F32_m;64 65} // End OtherPredicates = [has32BankLDS, isNotGFX90APlus],66 // Constraints = "@earlyclobber $vdst", isAsmParserOnly=167 68let OtherPredicates = [isNotGFX90APlus] in {69let Constraints = "$src0 = $vdst" in {70 71defm V_INTERP_P2_F32 : VINTRP_m <72 0x00000001,73 (outs VINTRPDst:$vdst),74 (ins VGPR_32:$src0, VGPR_32:$vsrc, InterpAttr:$attr,75 InterpAttrChan:$attrchan),76 "v_interp_p2_f32$vdst, $vsrc, $attr$attrchan",77 [(set f32:$vdst, (int_amdgcn_interp_p2 f32:$src0, f32:$vsrc,78 (i32 timm:$attrchan), (i32 timm:$attr), M0))]>;79 80} // End Constraints = "$src0 = $vdst"81 82defm V_INTERP_MOV_F32 : VINTRP_m <83 0x00000002,84 (outs VINTRPDst:$vdst),85 (ins InterpSlot:$vsrc, InterpAttr:$attr, InterpAttrChan:$attrchan),86 "v_interp_mov_f32$vdst, $vsrc, $attr$attrchan",87 [(set f32:$vdst, (int_amdgcn_interp_mov (i32 timm:$vsrc),88 (i32 timm:$attrchan), (i32 timm:$attr), M0))]>;89 90} // End OtherPredicates = [isNotGFX90APlus]91 92} // End Uses = [MODE, M0, EXEC]93 94//===----------------------------------------------------------------------===//95// Pseudo Instructions96//===----------------------------------------------------------------------===//97 98// Insert a branch to an endpgm block to use as a fallback trap.99def ENDPGM_TRAP : SPseudoInstSI<100 (outs), (ins),101 [(AMDGPUendpgm_trap)],102 "ENDPGM_TRAP"> {103 let hasSideEffects = 1;104 let usesCustomInserter = 1;105}106 107def SIMULATED_TRAP : SPseudoInstSI<(outs), (ins), [(AMDGPUsimulated_trap)],108 "SIMULATED_TRAP"> {109 let hasSideEffects = 1;110 let usesCustomInserter = 1;111}112 113def ATOMIC_FENCE : SPseudoInstSI<114 (outs), (ins i32imm:$ordering, i32imm:$scope),115 [(atomic_fence (i32 timm:$ordering), (i32 timm:$scope))],116 "ATOMIC_FENCE $ordering, $scope"> {117 let hasSideEffects = 1;118}119 120let hasSideEffects = 0, mayLoad = 0, mayStore = 0, Uses = [EXEC] in {121 122// For use in patterns123// No align needed as it will be decomposed anyway124// TODO: Remove alignment requirement from sources125def V_CNDMASK_B64_PSEUDO : VOP3Common <(outs VReg_64:$vdst),126 (ins VSrc_b64:$src0, VSrc_b64:$src1, SSrc_b64:$src2), "", []> {127 let isPseudo = 1;128 let isCodeGenOnly = 1;129 let usesCustomInserter = 1;130}131 132// 64-bit vector move instruction. This is mainly used by the133// SIFoldOperands pass to enable folding of inline immediates.134def V_MOV_B64_PSEUDO : VPseudoInstSI <(outs VReg_64_AlignTarget:$vdst),135 (ins VSrc_b64:$src0)> {136 let isReMaterializable = 1;137 let isAsCheapAsAMove = 1;138 let isMoveImm = 1;139 let SchedRW = [Write64Bit];140 let Size = 4;141 let VOP1 = 1; // Not entirely correct, but close enough.142 let UseNamedOperandTable = 1;143}144 145// 32-bit materialize immediate which supports AGPR or VGPR. Typically146// this should just expand to V_MOV_B32, unless $vdst happens to be147// allocated to an AGPR in which case it will lower to148// V_ACCVGPR_WRITE_B32. This should always use an inline immediate149// operand, as v_accvgpr_write_b32 does not support literal constants.150def AV_MOV_B32_IMM_PSEUDO151 : VPseudoInstSI<(outs AV_32:$vdst), (ins VCSrc_b32:$src0)> {152 let isReMaterializable = 1;153 let isAsCheapAsAMove = 1;154 155 // Imprecise, technically if AGPR it's VOP3 and VOP1 for VGPR. But156 // this tricks the rematerialize logic into working for it.157 let VOP3 = 1;158 let isMoveImm = 1;159 let SchedRW = [Write32Bit];160 let Size = 8;161 let FixedSize = true;162 let UseNamedOperandTable = 1;163}164 165// 64-bit materialize immediate which supports AGPR or VGPR. This has166// an unusual operand restriction which requires the two halves of the167// immediate to each be 32-bit inline immediate values.168def AV_MOV_B64_IMM_PSEUDO169 : VPseudoInstSI<(outs AV_64:$vdst), (ins AV_64_PSEUDO_IMM:$src0)> {170 let isReMaterializable = 1;171 let isAsCheapAsAMove = 1;172 173 // Imprecise, technically if AGPR it's 2 x VOP3 and 2 x VOP1 for174 // VGPR. But this tricks the rematerialize logic into working for175 // it.176 let VOP3 = 1;177 let isMoveImm = 1;178 let SchedRW = [Write32Bit, Write32Bit];179 let Size = 16; // 2 x v_accwrite_write_b32 in the worst case180 let UseNamedOperandTable = 1;181}182 183// 64-bit vector move with dpp. Expanded post-RA.184def V_MOV_B64_DPP_PSEUDO : VOP_DPP_Pseudo <"v_mov_b64_dpp", VOP_I64_I64> {185 let Size = 16; // Requires two 8-byte v_mov_b32_dpp to complete.186}187 188// 64-bit scalar move immediate instruction. This is used to avoid subregs189// initialization and allow rematerialization.190def S_MOV_B64_IMM_PSEUDO : SPseudoInstSI <(outs SReg_64:$sdst),191 (ins i64imm:$src0)> {192 let isReMaterializable = 1;193 let isAsCheapAsAMove = 1;194 let isMoveImm = 1;195 let SchedRW = [WriteSALU, Write64Bit];196 let Size = 4;197 let Uses = [];198 let UseNamedOperandTable = 1;199}200 201// Pseudoinstruction for @llvm.amdgcn.wqm. It is turned into a copy after the202// WQM pass processes it.203def WQM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>;204 205// Pseudoinstruction for @llvm.amdgcn.softwqm. Like @llvm.amdgcn.wqm it is206// turned into a copy by WQM pass, but does not seed WQM requirements.207def SOFT_WQM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>;208 209// Pseudoinstruction for @llvm.amdgcn.strict.wwm. It is turned into a copy post-RA, so210// that the @earlyclobber is respected. The @earlyclobber is to make sure that211// the instruction that defines $src0 (which is run in Whole Wave Mode) doesn't212// accidentally clobber inactive channels of $vdst.213let Constraints = "@earlyclobber $vdst" in {214def STRICT_WWM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>;215def STRICT_WQM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>;216}217 218} // End let hasSideEffects = 0, mayLoad = 0, mayStore = 0, Uses = [EXEC]219 220def WWM_COPY : SPseudoInstSI <221 (outs unknown:$dst), (ins unknown:$src)> {222 let hasSideEffects = 0;223 let isAsCheapAsAMove = 1;224 let isConvergent = 1;225}226 227def ENTER_STRICT_WWM : SPseudoInstSI <(outs SReg_1:$sdst), (ins i64imm:$src0)> {228 let Uses = [EXEC];229 let Defs = [EXEC, SCC];230 let hasSideEffects = 0;231 let mayLoad = 0;232 let mayStore = 0;233}234 235def EXIT_STRICT_WWM : SPseudoInstSI <(outs SReg_1:$sdst), (ins SReg_1:$src0)> {236 let hasSideEffects = 0;237 let mayLoad = 0;238 let mayStore = 0;239}240 241def ENTER_STRICT_WQM : SPseudoInstSI <(outs SReg_1:$sdst), (ins i64imm:$src0)> {242 let Uses = [EXEC];243 let Defs = [EXEC, SCC];244 let hasSideEffects = 0;245 let mayLoad = 0;246 let mayStore = 0;247}248 249def EXIT_STRICT_WQM : SPseudoInstSI <(outs SReg_1:$sdst), (ins SReg_1:$src0)> {250 let hasSideEffects = 0;251 let mayLoad = 0;252 let mayStore = 0;253}254 255let usesCustomInserter = 1, hasSideEffects = 0 in {256let WaveSizePredicate = isWave32 in257def S_INVERSE_BALLOT_U32 : SPseudoInstSI<258 (outs SReg_32:$sdst), (ins SSrc_b32:$mask),259 [(set i1:$sdst, (int_amdgcn_inverse_ballot i32:$mask))]260>;261 262let WaveSizePredicate = isWave64 in263def S_INVERSE_BALLOT_U64 : SPseudoInstSI<264 (outs SReg_64:$sdst), (ins SSrc_b64:$mask),265 [(set i1:$sdst, (int_amdgcn_inverse_ballot i64:$mask))]266>;267} // End usesCustomInserter = 1, hasSideEffects = 0268 269let WaveSizePredicate = isWave32 in270 def : GCNPat <271 (i1 (int_amdgcn_inverse_ballot i64:$src)),272 (S_INVERSE_BALLOT_U32 (i32 (EXTRACT_SUBREG SReg_64:$src, sub0)))273>;274 275// Pseudo instructions used for @llvm.fptrunc.round. The final codegen is done276// in the ModeRegister pass.277let Uses = [MODE, EXEC] in {278let True16Predicate = NotHasTrue16BitInsts in279def FPTRUNC_ROUND_F16_F32_PSEUDO : VPseudoInstSI <(outs VGPR_32:$vdst),280 (ins VGPR_32:$src0, i32imm:$round)>;281 282let True16Predicate = UseFakeTrue16Insts in283def FPTRUNC_ROUND_F16_F32_PSEUDO_fake16_e32 : VPseudoInstSI <(outs VGPR_32:$vdst),284 (ins VGPR_32:$src0, i32imm:$round)>;285 286let True16Predicate = UseRealTrue16Insts in287// The operands of these pseudos should match V_CVT_F16_F32_t16_e64288def FPTRUNC_ROUND_F16_F32_PSEUDO_t16_e64 : VPseudoInstSI <(outs VOPDstOperand_t16:$vdst),289 (ins FP32InputMods:$src0_modifiers, VSrc_f32:$src0, Clamp0:$clamp, omod0:$omod, op_sel0:$op_sel, i32imm:$round)> {290 let FPClamp = 1;291 let ClampLo = 1;292 let UseNamedOperandTable = 1;293}294 295def FPTRUNC_ROUND_F32_F64_PSEUDO : VPseudoInstSI <(outs VGPR_32:$vdst),296 (ins VReg_64:$src0, i32imm:$round)>;297} // End Uses = [MODE, EXEC]298 299let True16Predicate = NotHasTrue16BitInsts in300def : GCNPat <(f16 (fptrunc_round f32:$src0, (i32 SupportedRoundMode:$round))),301 (FPTRUNC_ROUND_F16_F32_PSEUDO $src0, (as_hw_round_mode $round))>;302 303let True16Predicate = UseFakeTrue16Insts in304def : GCNPat <(f16 (fptrunc_round f32:$src0, (i32 SupportedRoundMode:$round))),305 (FPTRUNC_ROUND_F16_F32_PSEUDO_fake16_e32 $src0, (as_hw_round_mode $round))>;306 307let True16Predicate = UseRealTrue16Insts in308def : GCNPat <(f16 (fptrunc_round (f32 (VOP3OpSelMods f32:$src0, i32:$src0_modifiers)), (i32 SupportedRoundMode:$round))),309 (FPTRUNC_ROUND_F16_F32_PSEUDO_t16_e64 $src0_modifiers, $src0, (as_hw_round_mode $round))>;310 311def : GCNPat <(f32 (fptrunc_round f64:$src0, (i32 SupportedRoundMode:$round))),312 (FPTRUNC_ROUND_F32_F64_PSEUDO $src0, (as_hw_round_mode $round))>;313 314// Invert the exec mask and overwrite the inactive lanes of dst with inactive,315// restoring it after we're done.316let isConvergent = 1 in317def V_SET_INACTIVE_B32 : VOP3_Pseudo<"v_set_inactive_b32", VOP2e_I32_I32_I32_I1>;318 319foreach vt = Reg32Types.types in {320def : GCNPat <(vt (int_amdgcn_set_inactive vt:$src, vt:$inactive)),321 (V_SET_INACTIVE_B32 0, VSrc_b32:$src, 0, VSrc_b32:$inactive, (IMPLICIT_DEF))>;322}323 324def : GCNPat<(i32 (int_amdgcn_set_inactive_chain_arg i32:$src, i32:$inactive)),325 (V_SET_INACTIVE_B32 0, VGPR_32:$src, 0, VGPR_32:$inactive, (IMPLICIT_DEF))>;326 327// clang-format off328 329multiclass330 AMDGPUWaveReducePseudoGenerator<string Op, string DataType, ValueType ty, RegisterClass RetReg, SrcRegOrImm9 Reg> {331 let usesCustomInserter = 1, hasSideEffects = 0, mayLoad = 0, mayStore = 0, Uses = [EXEC] in {332 def !toupper(Op) #"_PSEUDO_" #DataType333 : VPseudoInstSI<(outs RetReg : $sdst),334 (ins Reg : $src, VSrc_b32 : $strategy),335 [(set ty : $sdst, (!cast<AMDGPUWaveReduce>("int_amdgcn_wave_reduce_" #Op) ty : $src, i32 : $strategy))]> {}336 }337}338// clang-format on339 340class WaveReduceOp<string OpName, string TypeStr, ValueType Ty,341 RegisterClass ReturnRegisterClass, SrcRegOrImm9 RC> {342 string Name = OpName;343 string TypeString = TypeStr;344 ValueType VT = Ty;345 RegisterClass RetReg = ReturnRegisterClass;346 SrcRegOrImm9 Reg = RC;347}348 349// Input list : [Operation_name,350// type - Signed(I)/Unsigned(U)/Float(F)/Bitwise(B),351// input-type352// output register class,353// input register class]354defvar Operations = [355 WaveReduceOp<"umin", "U32", i32, SGPR_32, VSrc_b32>,356 WaveReduceOp<"min", "I32", i32, SGPR_32, VSrc_b32>,357 WaveReduceOp<"umax", "U32", i32, SGPR_32, VSrc_b32>,358 WaveReduceOp<"max", "I32", i32, SGPR_32, VSrc_b32>,359 WaveReduceOp<"add", "I32", i32, SGPR_32, VSrc_b32>,360 WaveReduceOp<"sub", "I32", i32, SGPR_32, VSrc_b32>,361 WaveReduceOp<"and", "B32", i32, SGPR_32, VSrc_b32>,362 WaveReduceOp<"or", "B32", i32, SGPR_32, VSrc_b32>,363 WaveReduceOp<"xor", "B32", i32, SGPR_32, VSrc_b32>,364 365 WaveReduceOp<"umin", "U64", i64, SGPR_64, VSrc_b64>,366 WaveReduceOp<"min", "I64", i64, SGPR_64, VSrc_b64>,367 WaveReduceOp<"umax", "U64", i64, SGPR_64, VSrc_b64>,368 WaveReduceOp<"max", "I64", i64, SGPR_64, VSrc_b64>,369 WaveReduceOp<"add", "U64", i64, SGPR_64, VSrc_b64>,370 WaveReduceOp<"sub", "U64", i64, SGPR_64, VSrc_b64>,371 WaveReduceOp<"and", "B64", i64, SGPR_64, VSrc_b64>,372 WaveReduceOp<"or", "B64", i64, SGPR_64, VSrc_b64>,373 WaveReduceOp<"xor", "B64", i64, SGPR_64, VSrc_b64>,374 375 WaveReduceOp<"fmin", "F32", f32, SGPR_32, VSrc_b32>,376 WaveReduceOp<"fmax", "F32", f32, SGPR_32, VSrc_b32>,377 WaveReduceOp<"fadd", "F32", f32, SGPR_32, VSrc_b32>,378 WaveReduceOp<"fsub", "F32", f32, SGPR_32, VSrc_b32>,379];380 381foreach Op = Operations in {382 defm WAVE_REDUCE_ : AMDGPUWaveReducePseudoGenerator<Op.Name, Op.TypeString,383 Op.VT, Op.RetReg, Op.Reg>;384}385 386let usesCustomInserter = 1, Defs = [VCC] in {387def V_ADD_U64_PSEUDO : VPseudoInstSI <388 (outs VReg_64_AlignTarget:$vdst), (ins VSrc_b64:$src0, VSrc_b64:$src1),389 [(set VReg_64_AlignTarget:$vdst, (DivergentBinFrag<add> i64:$src0, i64:$src1))]390>;391 392def V_SUB_U64_PSEUDO : VPseudoInstSI <393 (outs VReg_64_AlignTarget:$vdst), (ins VSrc_b64:$src0, VSrc_b64:$src1),394 [(set VReg_64_AlignTarget:$vdst, (DivergentBinFrag<sub> i64:$src0, i64:$src1))]395>;396} // End usesCustomInserter = 1, Defs = [VCC]397 398let usesCustomInserter = 1, Defs = [SCC] in {399def S_ADD_U64_PSEUDO : SPseudoInstSI <400 (outs SReg_64:$sdst), (ins SSrc_b64:$src0, SSrc_b64:$src1),401 [(set SReg_64:$sdst, (UniformBinFrag<add> i64:$src0, i64:$src1))]402>;403 404def S_SUB_U64_PSEUDO : SPseudoInstSI <405 (outs SReg_64:$sdst), (ins SSrc_b64:$src0, SSrc_b64:$src1),406 [(set SReg_64:$sdst, (UniformBinFrag<sub> i64:$src0, i64:$src1))]407>;408 409let hasSideEffects = 0 in {410 def S_ADD_CO_PSEUDO : SPseudoInstSI <411 (outs SReg_32:$sdst, SSrc_i1:$scc_out), (ins SSrc_b32:$src0, SSrc_b32:$src1, SSrc_i1:$scc_in)412 >;413 414 def S_SUB_CO_PSEUDO : SPseudoInstSI <415 (outs SReg_32:$sdst, SSrc_i1:$scc_out), (ins SSrc_b32:$src0, SSrc_b32:$src1, SSrc_i1:$scc_in)416 >;417 418 def S_UADDO_PSEUDO : SPseudoInstSI <419 (outs SReg_32:$sdst, SSrc_i1:$scc_out), (ins SSrc_b32:$src0, SSrc_b32:$src1)420 >;421 422 def S_USUBO_PSEUDO : SPseudoInstSI <423 (outs SReg_32:$sdst, SSrc_i1:$scc_out), (ins SSrc_b32:$src0, SSrc_b32:$src1)424 >;425}426 427let OtherPredicates = [HasShaderCyclesHiLoRegisters] in428def GET_SHADERCYCLESHILO : SPseudoInstSI<429 (outs SReg_64:$sdst), (ins),430 [(set SReg_64:$sdst, (i64 (readcyclecounter)))]431>;432 433} // End usesCustomInserter = 1, Defs = [SCC]434 435let usesCustomInserter = 1 in {436def GET_GROUPSTATICSIZE : SPseudoInstSI <(outs SReg_32:$sdst), (ins),437 [(set SReg_32:$sdst, (int_amdgcn_groupstaticsize))]>;438} // End let usesCustomInserter = 1, SALU = 1439 440// Wrap an instruction by duplicating it, except for setting isTerminator.441class WrapTerminatorInst<SOP_Pseudo base_inst> : SPseudoInstSI<442 base_inst.OutOperandList,443 base_inst.InOperandList> {444 let Uses = base_inst.Uses;445 let Defs = base_inst.Defs;446 let isTerminator = 1;447 let isAsCheapAsAMove = base_inst.isAsCheapAsAMove;448 let hasSideEffects = base_inst.hasSideEffects;449 let UseNamedOperandTable = base_inst.UseNamedOperandTable;450 let CodeSize = base_inst.CodeSize;451 let SchedRW = base_inst.SchedRW;452}453 454let WaveSizePredicate = isWave64 in {455def S_MOV_B64_term : WrapTerminatorInst<S_MOV_B64>;456def S_XOR_B64_term : WrapTerminatorInst<S_XOR_B64>;457def S_OR_B64_term : WrapTerminatorInst<S_OR_B64>;458def S_ANDN2_B64_term : WrapTerminatorInst<S_ANDN2_B64>;459def S_AND_B64_term : WrapTerminatorInst<S_AND_B64>;460def S_AND_SAVEEXEC_B64_term : WrapTerminatorInst<S_AND_SAVEEXEC_B64>;461}462 463let WaveSizePredicate = isWave32 in {464def S_MOV_B32_term : WrapTerminatorInst<S_MOV_B32>;465def S_XOR_B32_term : WrapTerminatorInst<S_XOR_B32>;466def S_OR_B32_term : WrapTerminatorInst<S_OR_B32>;467def S_ANDN2_B32_term : WrapTerminatorInst<S_ANDN2_B32>;468def S_AND_B32_term : WrapTerminatorInst<S_AND_B32>;469def S_AND_SAVEEXEC_B32_term : WrapTerminatorInst<S_AND_SAVEEXEC_B32>;470}471 472 473def WAVE_BARRIER : SPseudoInstSI<(outs), (ins),474 [(int_amdgcn_wave_barrier)]> {475 let SchedRW = [];476 let hasNoSchedulingInfo = 1;477 let hasSideEffects = 1;478 let mayLoad = 0;479 let mayStore = 0;480 let isConvergent = 1;481 let FixedSize = 1;482 let Size = 0;483 let isMeta = 1;484}485 486def SCHED_BARRIER : SPseudoInstSI<(outs), (ins i32imm:$mask),487 [(int_amdgcn_sched_barrier (i32 timm:$mask))]> {488 let SchedRW = [];489 let hasNoSchedulingInfo = 1;490 let hasSideEffects = 1;491 let mayLoad = 0;492 let mayStore = 0;493 let isConvergent = 1;494 let FixedSize = 1;495 let Size = 0;496 let isMeta = 1;497}498 499def SCHED_GROUP_BARRIER : SPseudoInstSI<500 (outs),501 (ins i32imm:$mask, i32imm:$size, i32imm:$syncid),502 [(int_amdgcn_sched_group_barrier (i32 timm:$mask), (i32 timm:$size), (i32 timm:$syncid))]> {503 let SchedRW = [];504 let hasNoSchedulingInfo = 1;505 let hasSideEffects = 1;506 let mayLoad = 0;507 let mayStore = 0;508 let isConvergent = 1;509 let FixedSize = 1;510 let Size = 0;511 let isMeta = 1;512}513 514def IGLP_OPT : SPseudoInstSI<(outs), (ins i32imm:$mask),515 [(int_amdgcn_iglp_opt (i32 timm:$mask))]> {516 let SchedRW = [];517 let hasNoSchedulingInfo = 1;518 let hasSideEffects = 1;519 let mayLoad = 0;520 let mayStore = 0;521 let isConvergent = 1;522 let FixedSize = 1;523 let Size = 0;524 let isMeta = 1;525}526 527// SI pseudo instructions. These are used by the CFG structurizer pass528// and should be lowered to ISA instructions prior to codegen.529 530// As we have enhanced control flow intrinsics to work under unstructured CFG,531// duplicating such intrinsics can be actually treated as legal. On the contrary,532// by making them non-duplicable, we are observing better code generation result.533// So we choose to mark them non-duplicable in hope of getting better code534// generation as well as simplied CFG during Machine IR optimization stage.535 536let isTerminator = 1, isNotDuplicable = 1 in {537 538def SI_IF: CFPseudoInstSI <539 (outs SReg_1:$dst), (ins SReg_1:$vcc, brtarget:$target),540 [(set i1:$dst, (AMDGPUif i1:$vcc, bb:$target))], 1, 1> {541 let Constraints = "";542 let Size = 12;543 let hasSideEffects = 1;544 let IsNeverUniform = 1;545}546 547def SI_ELSE : CFPseudoInstSI <548 (outs SReg_1:$dst),549 (ins SReg_1:$src, brtarget:$target), [], 1, 1> {550 let Size = 12;551 let hasSideEffects = 1;552 let IsNeverUniform = 1;553}554 555def SI_WATERFALL_LOOP : CFPseudoInstSI <556 (outs),557 (ins brtarget:$target), [], 1> {558 let Size = 8;559 let isBranch = 1;560 let Defs = [];561}562 563def SI_LOOP : CFPseudoInstSI <564 (outs), (ins SReg_1:$saved, brtarget:$target),565 [(AMDGPUloop i1:$saved, bb:$target)], 1, 1> {566 let Size = 8;567 let isBranch = 1;568 let hasSideEffects = 1;569 let IsNeverUniform = 1;570}571 572} // End isTerminator = 1573 574def SI_END_CF : CFPseudoInstSI <575 (outs), (ins SReg_1:$saved), [], 1, 1> {576 let Size = 4;577 let isAsCheapAsAMove = 1;578 let isReMaterializable = 1;579 let hasSideEffects = 1;580 let isNotDuplicable = 1; // Not a hard requirement, see long comments above for details.581 let mayLoad = 1; // FIXME: Should not need memory flags582 let mayStore = 1;583}584 585def SI_IF_BREAK : CFPseudoInstSI <586 (outs SReg_1:$dst), (ins SReg_1:$vcc, SReg_1:$src), []> {587 let Size = 4;588 let isNotDuplicable = 1; // Not a hard requirement, see long comments above for details.589 let isAsCheapAsAMove = 1;590 let isReMaterializable = 1;591}592 593// Branch to the early termination block of the shader if SCC is 0.594// This uses SCC from a previous SALU operation, i.e. the update of595// a mask of live lanes after a kill/demote operation.596// Only valid in pixel shaders.597def SI_EARLY_TERMINATE_SCC0 : SPseudoInstSI <(outs), (ins)> {598 let Uses = [EXEC,SCC];599}600 601let Uses = [EXEC] in {602 603multiclass PseudoInstKill <dag ins> {604 // Even though this pseudo can usually be expanded without an SCC def, we605 // conservatively assume that it has an SCC def, both because it is sometimes606 // required in degenerate cases (when V_CMPX cannot be used due to constant607 // bus limitations) and because it allows us to avoid having to track SCC608 // liveness across basic blocks.609 let Defs = [EXEC,SCC] in610 def _PSEUDO : PseudoInstSI <(outs), ins> {611 let isConvergent = 1;612 let usesCustomInserter = 1;613 }614 615 let Defs = [EXEC,SCC] in616 def _TERMINATOR : SPseudoInstSI <(outs), ins> {617 let isTerminator = 1;618 }619}620 621defm SI_KILL_I1 : PseudoInstKill <(ins SCSrc_i1:$src, i1imm:$killvalue)>;622let Defs = [VCC] in623defm SI_KILL_F32_COND_IMM : PseudoInstKill <(ins VSrc_b32:$src0, i32imm:$src1, i32imm:$cond)>;624 625let Defs = [EXEC,VCC] in626def SI_ILLEGAL_COPY : SPseudoInstSI <627 (outs unknown:$dst), (ins unknown:$src),628 [], " ; illegal copy $src to $dst">;629 630} // End Uses = [EXEC], Defs = [EXEC,VCC]631 632// Branch on undef scc. Used to avoid intermediate copy from633// IMPLICIT_DEF to SCC.634def SI_BR_UNDEF : SPseudoInstSI <(outs), (ins SOPPBrTarget:$simm16)> {635 let isTerminator = 1;636 let usesCustomInserter = 1;637 let isBranch = 1;638}639 640def SI_PS_LIVE : PseudoInstSI <641 (outs SReg_1:$dst), (ins),642 [(set i1:$dst, (int_amdgcn_ps_live))]> {643 let SALU = 1;644}645 646let Uses = [EXEC] in {647def SI_LIVE_MASK : PseudoInstSI <648 (outs SReg_1:$dst), (ins),649 [(set i1:$dst, (int_amdgcn_live_mask))]> {650 let SALU = 1;651}652let Defs = [EXEC,SCC] in {653// Demote: Turn a pixel shader thread into a helper lane.654def SI_DEMOTE_I1 : SPseudoInstSI <(outs), (ins SCSrc_i1:$src, i1imm:$killvalue)>;655} // End Defs = [EXEC,SCC]656} // End Uses = [EXEC]657 658def SI_MASKED_UNREACHABLE : SPseudoInstSI <(outs), (ins),659 [(int_amdgcn_unreachable)],660 "; divergent unreachable"> {661 let Size = 0;662 let hasNoSchedulingInfo = 1;663 let FixedSize = 1;664 let isMeta = 1;665 let maybeAtomic = 0;666}667 668// Used as an isel pseudo to directly emit initialization with an669// s_mov_b32 rather than a copy of another initialized670// register. MachineCSE skips copies, and we don't want to have to671// fold operands before it runs.672def SI_INIT_M0 : SPseudoInstSI <(outs), (ins SSrc_b32:$src)> {673 let Defs = [M0];674 let usesCustomInserter = 1;675 let isAsCheapAsAMove = 1;676 let isReMaterializable = 1;677}678 679def SI_INIT_EXEC : SPseudoInstSI <680 (outs), (ins i64imm:$src),681 [(int_amdgcn_init_exec (i64 timm:$src))]> {682 let Defs = [EXEC];683 let isAsCheapAsAMove = 1;684}685 686def SI_INIT_EXEC_FROM_INPUT : SPseudoInstSI <687 (outs), (ins SSrc_b32:$input, i32imm:$shift),688 [(int_amdgcn_init_exec_from_input i32:$input, (i32 timm:$shift))]> {689 let Defs = [EXEC];690}691 692// Sets EXEC to all lanes and returns the previous EXEC.693def SI_INIT_WHOLE_WAVE : SPseudoInstSI <694 (outs SReg_1:$dst), (ins),695 [(set i1:$dst, (int_amdgcn_init_whole_wave))]> {696 let Defs = [EXEC];697 let Uses = [EXEC];698 699 let isConvergent = 1;700}701 702// Sets EXEC to all lanes and returns the previous EXEC.703def SI_WHOLE_WAVE_FUNC_SETUP : SPseudoInstSI <704 (outs SReg_1:$dst), (ins), [(set i1:$dst, (AMDGPUwhole_wave_setup))]> {705 let Defs = [EXEC];706 let Uses = [EXEC];707 708 let isConvergent = 1;709}710 711// Restores the previous EXEC and otherwise behaves entirely like a SI_RETURN.712def SI_WHOLE_WAVE_FUNC_RETURN : SPseudoInstSI <713 (outs), (ins SReg_1:$orig_exec)> {714 let isTerminator = 1;715 let isBarrier = 1;716 let isReturn = 1;717 let SchedRW = [WriteBranch];718 719 // We're going to use custom handling to set the $orig_exec to the correct value.720 let usesCustomInserter = 1;721}722 723// Generate a SI_WHOLE_WAVE_FUNC_RETURN pseudo with a placeholder for its724// argument. It will be filled in by the custom inserter.725def : GCNPat<726 (AMDGPUwhole_wave_return), (SI_WHOLE_WAVE_FUNC_RETURN (i1 (IMPLICIT_DEF)))>;727 728// Restores the previous EXEC and otherwise behaves entirely like a SI_TCRETURN.729// This is used for tail calls *from* a whole wave function. Tail calls to730// a whole wave function may use the usual opcodes, depending on the calling731// convention of the caller.732def SI_TCRETURN_GFX_WholeWave : SPseudoInstSI <733 (outs),734 (ins SReg_1:$orig_exec, Gfx_CCR_SGPR_64:$src0, unknown:$callee, i32imm:$fpdiff)> {735 let isCall = 1;736 let isTerminator = 1;737 let isReturn = 1;738 let isBarrier = 1;739 let UseNamedOperandTable = 1;740 let SchedRW = [WriteBranch];741 let isConvergent = 1;742 743 // We're going to use custom handling to set the $orig_exec to the correct value.744 let usesCustomInserter = 1;745}746 747// Generate a SI_TCRETURN_GFX_WholeWave pseudo with a placeholder for its748// argument. It will be filled in by the custom inserter.749def : GCNPat<750 (AMDGPUtc_return_gfx_ww i64:$src0, tglobaladdr:$callee, i32:$fpdiff),751 (SI_TCRETURN_GFX_WholeWave (i1 (IMPLICIT_DEF)), Gfx_CCR_SGPR_64:$src0,752 tglobaladdr:$callee, i32:$fpdiff)>;753 754 755// Return for returning shaders to a shader variant epilog.756def SI_RETURN_TO_EPILOG : SPseudoInstSI <757 (outs), (ins variable_ops), [(AMDGPUreturn_to_epilog)]> {758 let isTerminator = 1;759 let isBarrier = 1;760 let isReturn = 1;761 let hasNoSchedulingInfo = 1;762 let DisableWQM = 1;763 let FixedSize = 1;764 765 // TODO: Should this be true?766 let isMeta = 0;767}768 769// Return for returning function calls.770def SI_RETURN : SPseudoInstSI <771 (outs), (ins), [(AMDGPUret_glue)],772 "; return"> {773 let isTerminator = 1;774 let isBarrier = 1;775 let isReturn = 1;776 let SchedRW = [WriteBranch];777}778 779// Return for returning function calls without output register.780//781// This version is only needed so we can fill in the output register782// in the custom inserter.783def SI_CALL_ISEL : SPseudoInstSI <784 (outs), (ins SSrc_b64:$src0, unknown:$callee),785 [(AMDGPUcall i64:$src0, tglobaladdr:$callee)]> {786 let Size = 4;787 let isCall = 1;788 let SchedRW = [WriteBranch];789 let usesCustomInserter = 1;790 // TODO: Should really base this on the call target791 let isConvergent = 1;792}793 794def : GCNPat<795 (AMDGPUcall i64:$src0, (i64 0)),796 (SI_CALL_ISEL $src0, (i64 0))797>;798 799// Funnel shift right (fshr) patterns for uniform inputs.800// These patterns implement this using scalar instructions by constructing a 64-bit801// value {a, b} and performing a single right shift.802def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, i32:$src2),803 (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), (S_AND_B32 $src2, (i32 31))), sub0))804>;805 806def : GCNPat<(UniformTernaryFrag<fshr> i32:$src0, i32:$src1, (i32 ShiftAmt32Imm:$src2)),807 (i32 (EXTRACT_SUBREG (S_LSHR_B64 (REG_SEQUENCE SReg_64, $src1, sub0, $src0, sub1), $src2), sub0))808>;809 810// Wrapper around s_swappc_b64 with extra $callee parameter to track811// the called function after regalloc.812def SI_CALL : SPseudoInstSI <813 (outs SReg_64:$dst), (ins SSrc_b64:$src0, unknown:$callee)> {814 let Size = 4;815 let FixedSize = 1;816 let isCall = 1;817 let UseNamedOperandTable = 1;818 let SchedRW = [WriteBranch];819 // TODO: Should really base this on the call target820 let isConvergent = 1;821}822 823class SI_TCRETURN_Pseudo<RegisterClass rc, SDNode sd> : SPseudoInstSI <(outs),824 (ins rc:$src0, unknown:$callee, i32imm:$fpdiff),825 [(sd i64:$src0, tglobaladdr:$callee, i32:$fpdiff)]> {826 let Size = 4;827 let FixedSize = 1;828 let isCall = 1;829 let isTerminator = 1;830 let isReturn = 1;831 let isBarrier = 1;832 let UseNamedOperandTable = 1;833 let SchedRW = [WriteBranch];834 // TODO: Should really base this on the call target835 let isConvergent = 1;836}837 838// Tail call handling pseudo839def SI_TCRETURN : SI_TCRETURN_Pseudo<CCR_SGPR_64, AMDGPUtc_return>;840def SI_TCRETURN_GFX : SI_TCRETURN_Pseudo<Gfx_CCR_SGPR_64, AMDGPUtc_return_gfx>;841 842// Handle selecting indirect tail calls843def : GCNPat<844 (AMDGPUtc_return i64:$src0, (i64 0), (i32 timm:$fpdiff)),845 (SI_TCRETURN CCR_SGPR_64:$src0, (i64 0), i32imm:$fpdiff)846>;847 848// Handle selecting indirect tail calls for AMDGPU_gfx849def : GCNPat<850 (AMDGPUtc_return_gfx i64:$src0, (i64 0), (i32 timm:$fpdiff)),851 (SI_TCRETURN_GFX Gfx_CCR_SGPR_64:$src0, (i64 0), i32imm:$fpdiff)852>;853 854// Pseudos for the llvm.amdgcn.cs.chain intrinsic.855multiclass SI_CS_CHAIN_TC<856 ValueType execvt, Predicate wavesizepred,857 RegisterOperand execrc = getSOPSrcForVT<execvt>.ret> {858 let FixedSize = 0,859 isCall = 1,860 isTerminator = 1,861 isBarrier = 1,862 isReturn = 1,863 UseNamedOperandTable = 1,864 SchedRW = [WriteBranch],865 isConvergent = 1,866 WaveSizePredicate = wavesizepred in {867 // This is essentially a tail call, but it also takes a mask to put in EXEC868 // right before jumping to the callee.869 def NAME: SPseudoInstSI <(outs),870 (ins CCR_SGPR_64:$src0, unknown:$callee, i32imm:$fpdiff, execrc:$exec)>;871 872 // Same as above, but it will first try to reallocate the VGPRs, and choose an873 // EXEC mask and a callee depending on the success of the reallocation attempt.874 def _DVGPR : SPseudoInstSI <(outs),875 (ins CCR_SGPR_64:$src0, i64imm:$callee, i32imm:$fpdiff, execrc:$exec,876 SSrc_b32:$numvgprs, execrc:$fbexec, CCR_SGPR_64:$fbcallee)>;877 } // End FixedSize = 0 etc878}879 880defm SI_CS_CHAIN_TC_W32 : SI_CS_CHAIN_TC<i32, isWave32>;881defm SI_CS_CHAIN_TC_W64 : SI_CS_CHAIN_TC<i64, isWave64>;882 883// Handle selecting direct & indirect calls via SI_CS_CHAIN_TC_W32/64884multiclass si_cs_chain_tc_pattern<885 dag callee, ValueType execvt, RegisterOperand execrc, Instruction tc> {886 def : GCNPat<887 (AMDGPUtc_return_chain i64:$src0, callee, (i32 timm:$fpdiff), execvt:$exec),888 (tc CCR_SGPR_64:$src0, callee, i32imm:$fpdiff, execrc:$exec)889 >;890}891 892multiclass si_cs_chain_tc_patterns<893 ValueType execvt,894 RegisterOperand execrc = getSOPSrcForVT<execvt>.ret,895 Instruction tc = !if(!eq(execvt, i32), SI_CS_CHAIN_TC_W32, SI_CS_CHAIN_TC_W64)896 > {897 defm direct: si_cs_chain_tc_pattern<(tglobaladdr:$callee), execvt, execrc, tc>;898 defm indirect: si_cs_chain_tc_pattern<(i64 0), execvt, execrc, tc>;899}900 901defm : si_cs_chain_tc_patterns<i32>;902defm : si_cs_chain_tc_patterns<i64>;903 904// Match dynamic VGPR case. This is always indirect since we choose the callee905// dynamically based on the result of the VGPR reallocation, so make sure to906// drop the callee info if there is any.907multiclass si_cs_chain_tc_dvgpr_patterns<908 ValueType execvt, RegisterOperand execrc = getSOPSrcForVT<execvt>.ret,909 Instruction tc = SI_CS_CHAIN_TC_W32_DVGPR> {910 foreach callee = [ (i64 0), (tglobaladdr) ] in {911 def : GCNPat<912 (AMDGPUtc_return_chain_dvgpr i64:$src0, callee, (i32 timm:$fpdiff),913 execvt:$exec, i32:$numvgprs,914 execvt:$fbexec, i64:$fbcallee),915 (tc CCR_SGPR_64:$src0, (i64 0), i32imm:$fpdiff, execrc:$exec,916 SSrc_b32:$numvgprs, execrc:$fbexec, CCR_SGPR_64:$fbcallee)917 >;918 }919}920 921defm : si_cs_chain_tc_dvgpr_patterns<i32>; // On GFX12, dVGPR mode is wave32-only.922 923def ADJCALLSTACKUP : SPseudoInstSI<924 (outs), (ins i32imm:$amt0, i32imm:$amt1),925 [(callseq_start timm:$amt0, timm:$amt1)],926 "; adjcallstackup $amt0 $amt1"> {927 let Size = 8; // Worst case. (s_add_u32 + constant)928 let FixedSize = 1;929 let hasSideEffects = 1;930 let usesCustomInserter = 1;931 let SchedRW = [WriteSALU];932 let Defs = [SCC];933}934 935def ADJCALLSTACKDOWN : SPseudoInstSI<936 (outs), (ins i32imm:$amt1, i32imm:$amt2),937 [(callseq_end timm:$amt1, timm:$amt2)],938 "; adjcallstackdown $amt1"> {939 let Size = 8; // Worst case. (s_add_u32 + constant)940 let hasSideEffects = 1;941 let usesCustomInserter = 1;942 let SchedRW = [WriteSALU];943 let Defs = [SCC];944}945 946let Defs = [M0, EXEC, SCC],947 UseNamedOperandTable = 1 in {948 949// SI_INDIRECT_SRC/DST are only used by legacy SelectionDAG indirect950// addressing implementation.951class SI_INDIRECT_SRC<RegisterClass rc> : VPseudoInstSI <952 (outs VGPR_32:$vdst),953 (ins rc:$src, VS_32:$idx, i32imm:$offset)> {954 let usesCustomInserter = 1;955}956 957class SI_INDIRECT_DST<RegisterClass rc> : VPseudoInstSI <958 (outs rc:$vdst),959 (ins rc:$src, VS_32:$idx, i32imm:$offset, VGPR_32:$val)> {960 let Constraints = "$src = $vdst";961 let usesCustomInserter = 1;962}963 964def SI_INDIRECT_SRC_V1 : SI_INDIRECT_SRC<VGPR_32>;965def SI_INDIRECT_SRC_V2 : SI_INDIRECT_SRC<VReg_64>;966def SI_INDIRECT_SRC_V4 : SI_INDIRECT_SRC<VReg_128>;967def SI_INDIRECT_SRC_V8 : SI_INDIRECT_SRC<VReg_256>;968def SI_INDIRECT_SRC_V9 : SI_INDIRECT_SRC<VReg_288>;969def SI_INDIRECT_SRC_V10 : SI_INDIRECT_SRC<VReg_320>;970def SI_INDIRECT_SRC_V11 : SI_INDIRECT_SRC<VReg_352>;971def SI_INDIRECT_SRC_V12 : SI_INDIRECT_SRC<VReg_384>;972def SI_INDIRECT_SRC_V16 : SI_INDIRECT_SRC<VReg_512>;973def SI_INDIRECT_SRC_V32 : SI_INDIRECT_SRC<VReg_1024>;974 975def SI_INDIRECT_DST_V1 : SI_INDIRECT_DST<VGPR_32>;976def SI_INDIRECT_DST_V2 : SI_INDIRECT_DST<VReg_64>;977def SI_INDIRECT_DST_V4 : SI_INDIRECT_DST<VReg_128>;978def SI_INDIRECT_DST_V8 : SI_INDIRECT_DST<VReg_256>;979def SI_INDIRECT_DST_V9 : SI_INDIRECT_DST<VReg_288>;980def SI_INDIRECT_DST_V10 : SI_INDIRECT_DST<VReg_320>;981def SI_INDIRECT_DST_V11 : SI_INDIRECT_DST<VReg_352>;982def SI_INDIRECT_DST_V12 : SI_INDIRECT_DST<VReg_384>;983def SI_INDIRECT_DST_V16 : SI_INDIRECT_DST<VReg_512>;984def SI_INDIRECT_DST_V32 : SI_INDIRECT_DST<VReg_1024>;985 986} // End Uses = [EXEC], Defs = [M0, EXEC]987 988// This is a pseudo variant of the v_movreld_b32 instruction in which the989// vector operand appears only twice, once as def and once as use. Using this990// pseudo avoids problems with the Two Address instructions pass.991class INDIRECT_REG_WRITE_MOVREL_pseudo<RegisterClass rc,992 RegisterOperand val_ty> : PseudoInstSI <993 (outs rc:$vdst), (ins rc:$vsrc, val_ty:$val, i32imm:$subreg)> {994 let Constraints = "$vsrc = $vdst";995 let Uses = [M0];996}997 998class V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<RegisterClass rc> :999 INDIRECT_REG_WRITE_MOVREL_pseudo<rc, VSrc_b32> {1000 let VALU = 1;1001 let VOP1 = 1;1002 let Uses = [M0, EXEC];1003}1004 1005class S_INDIRECT_REG_WRITE_MOVREL_pseudo<RegisterClass rc,1006 RegisterOperand val_ty> :1007 INDIRECT_REG_WRITE_MOVREL_pseudo<rc, val_ty> {1008 let SALU = 1;1009 let SOP1 = 1;1010 let Uses = [M0];1011}1012 1013class S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<RegisterClass rc> :1014 S_INDIRECT_REG_WRITE_MOVREL_pseudo<rc, SSrc_b32>;1015class S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<RegisterClass rc> :1016 S_INDIRECT_REG_WRITE_MOVREL_pseudo<rc, SSrc_b64>;1017 1018def V_INDIRECT_REG_WRITE_MOVREL_B32_V1 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VGPR_32>;1019def V_INDIRECT_REG_WRITE_MOVREL_B32_V2 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_64>;1020def V_INDIRECT_REG_WRITE_MOVREL_B32_V3 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_96>;1021def V_INDIRECT_REG_WRITE_MOVREL_B32_V4 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_128>;1022def V_INDIRECT_REG_WRITE_MOVREL_B32_V5 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_160>;1023def V_INDIRECT_REG_WRITE_MOVREL_B32_V8 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_256>;1024def V_INDIRECT_REG_WRITE_MOVREL_B32_V9 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_288>;1025def V_INDIRECT_REG_WRITE_MOVREL_B32_V10 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_320>;1026def V_INDIRECT_REG_WRITE_MOVREL_B32_V11 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_352>;1027def V_INDIRECT_REG_WRITE_MOVREL_B32_V12 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_384>;1028def V_INDIRECT_REG_WRITE_MOVREL_B32_V16 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_512>;1029def V_INDIRECT_REG_WRITE_MOVREL_B32_V32 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_1024>;1030 1031def S_INDIRECT_REG_WRITE_MOVREL_B32_V1 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_32>;1032def S_INDIRECT_REG_WRITE_MOVREL_B32_V2 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_64>;1033def S_INDIRECT_REG_WRITE_MOVREL_B32_V3 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_96>;1034def S_INDIRECT_REG_WRITE_MOVREL_B32_V4 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_128>;1035def S_INDIRECT_REG_WRITE_MOVREL_B32_V5 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_160>;1036def S_INDIRECT_REG_WRITE_MOVREL_B32_V8 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_256>;1037def S_INDIRECT_REG_WRITE_MOVREL_B32_V9 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_288>;1038def S_INDIRECT_REG_WRITE_MOVREL_B32_V10 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_320>;1039def S_INDIRECT_REG_WRITE_MOVREL_B32_V11 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_352>;1040def S_INDIRECT_REG_WRITE_MOVREL_B32_V12 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_384>;1041def S_INDIRECT_REG_WRITE_MOVREL_B32_V16 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_512>;1042def S_INDIRECT_REG_WRITE_MOVREL_B32_V32 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_1024>;1043 1044def S_INDIRECT_REG_WRITE_MOVREL_B64_V1 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_64>;1045def S_INDIRECT_REG_WRITE_MOVREL_B64_V2 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_128>;1046def S_INDIRECT_REG_WRITE_MOVREL_B64_V4 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_256>;1047def S_INDIRECT_REG_WRITE_MOVREL_B64_V8 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_512>;1048def S_INDIRECT_REG_WRITE_MOVREL_B64_V16 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_1024>;1049 1050// These variants of V_INDIRECT_REG_READ/WRITE use VGPR indexing. By using these1051// pseudos we avoid spills or copies being inserted within indirect sequences1052// that switch the VGPR indexing mode. Spills to accvgprs could be effected by1053// this mode switching.1054 1055class V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<RegisterClass rc> : PseudoInstSI <1056 (outs rc:$vdst), (ins rc:$vsrc, VSrc_b32:$val, SSrc_b32:$idx, i32imm:$subreg)> {1057 let Constraints = "$vsrc = $vdst";1058 let VALU = 1;1059 let Uses = [M0, EXEC];1060 let Defs = [M0];1061}1062 1063def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VGPR_32>;1064def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_64>;1065def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_96>;1066def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_128>;1067def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_160>;1068def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_256>;1069def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_288>;1070def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_320>;1071def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_352>;1072def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_384>;1073def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_512>;1074def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_1024>;1075 1076class V_INDIRECT_REG_READ_GPR_IDX_pseudo<RegisterClass rc> : PseudoInstSI <1077 (outs VGPR_32:$vdst), (ins rc:$vsrc, SSrc_b32:$idx, i32imm:$subreg)> {1078 let VALU = 1;1079 let Uses = [M0, EXEC];1080 let Defs = [M0];1081}1082 1083def V_INDIRECT_REG_READ_GPR_IDX_B32_V1 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VGPR_32>;1084def V_INDIRECT_REG_READ_GPR_IDX_B32_V2 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_64>;1085def V_INDIRECT_REG_READ_GPR_IDX_B32_V3 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_96>;1086def V_INDIRECT_REG_READ_GPR_IDX_B32_V4 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_128>;1087def V_INDIRECT_REG_READ_GPR_IDX_B32_V5 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_160>;1088def V_INDIRECT_REG_READ_GPR_IDX_B32_V8 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_256>;1089def V_INDIRECT_REG_READ_GPR_IDX_B32_V9 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_288>;1090def V_INDIRECT_REG_READ_GPR_IDX_B32_V10 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_320>;1091def V_INDIRECT_REG_READ_GPR_IDX_B32_V11 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_352>;1092def V_INDIRECT_REG_READ_GPR_IDX_B32_V12 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_384>;1093def V_INDIRECT_REG_READ_GPR_IDX_B32_V16 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_512>;1094def V_INDIRECT_REG_READ_GPR_IDX_B32_V32 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_1024>;1095 1096multiclass SI_SPILL_SGPR <RegisterClass sgpr_class> {1097 let UseNamedOperandTable = 1, Spill = 1, SALU = 1, Uses = [EXEC] in {1098 def _SAVE : PseudoInstSI <1099 (outs),1100 (ins sgpr_class:$data, i32imm:$addr)> {1101 let mayStore = 1;1102 let mayLoad = 0;1103 }1104 1105 def _RESTORE : PseudoInstSI <1106 (outs sgpr_class:$data),1107 (ins i32imm:$addr)> {1108 let mayStore = 0;1109 let mayLoad = 1;1110 }1111 } // End UseNamedOperandTable = 11112}1113 1114// You cannot use M0 as the output of v_readlane_b32 instructions or1115// use it in the sdata operand of SMEM instructions. We still need to1116// be able to spill the physical register m0, so allow it for1117// SI_SPILL_32_* instructions.1118defm SI_SPILL_S32 : SI_SPILL_SGPR <SReg_32>;1119defm SI_SPILL_S64 : SI_SPILL_SGPR <SReg_64>;1120defm SI_SPILL_S96 : SI_SPILL_SGPR <SReg_96>;1121defm SI_SPILL_S128 : SI_SPILL_SGPR <SReg_128>;1122defm SI_SPILL_S160 : SI_SPILL_SGPR <SReg_160>;1123defm SI_SPILL_S192 : SI_SPILL_SGPR <SReg_192>;1124defm SI_SPILL_S224 : SI_SPILL_SGPR <SReg_224>;1125defm SI_SPILL_S256 : SI_SPILL_SGPR <SReg_256>;1126defm SI_SPILL_S288 : SI_SPILL_SGPR <SReg_288>;1127defm SI_SPILL_S320 : SI_SPILL_SGPR <SReg_320>;1128defm SI_SPILL_S352 : SI_SPILL_SGPR <SReg_352>;1129defm SI_SPILL_S384 : SI_SPILL_SGPR <SReg_384>;1130defm SI_SPILL_S512 : SI_SPILL_SGPR <SReg_512>;1131defm SI_SPILL_S1024 : SI_SPILL_SGPR <SReg_1024>;1132 1133let Spill = 1, VALU = 1, isConvergent = 1 in {1134def SI_SPILL_S32_TO_VGPR : PseudoInstSI <(outs VGPR_32:$vdst),1135 (ins SReg_32:$src0, i32imm:$src1, VGPR_32:$vdst_in)> {1136 let Size = 4;1137 let FixedSize = 1;1138 let IsNeverUniform = 1;1139 let hasSideEffects = 0;1140 let mayLoad = 0;1141 let mayStore = 0;1142 let hasExtraDefRegAllocReq = 1;1143 let Constraints = "$vdst = $vdst_in";1144}1145 1146def SI_RESTORE_S32_FROM_VGPR : PseudoInstSI <(outs SReg_32:$sdst),1147 (ins VGPR_32:$src0, i32imm:$src1)> {1148 let Size = 4;1149 let FixedSize = 1;1150 let hasSideEffects = 0;1151 let mayLoad = 0;1152 let mayStore = 0;1153 let hasExtraSrcRegAllocReq = 1;1154}1155} // End Spill = 1, VALU = 1, isConvergent = 11156 1157// VGPR or AGPR spill instructions. In case of AGPR spilling a temp register1158// needs to be used and an extra instruction to move between VGPR and AGPR.1159// UsesTmp adds to the total size of an expanded spill in this case.1160multiclass SI_SPILL_VGPR <SIRegisterClassLike vgpr_class,1161 bit UsesTmp = 0, bit HasMask = 0> {1162 let UseNamedOperandTable = 1, Spill = 1, VALU = 1,1163 SchedRW = [WriteVMEM] in {1164 def _SAVE : VPseudoInstSI <1165 (outs),1166 !con(1167 (ins vgpr_class:$vdata, i32imm:$vaddr,1168 SReg_32:$soffset, i32imm:$offset),1169 !if(HasMask, (ins SReg_32:$mask), (ins)))> {1170 let mayStore = 1;1171 let mayLoad = 0;1172 // (2 * 4) + (8 * num_subregs) bytes maximum1173 int MaxSize = !add(!shl(!srl(vgpr_class.Size, 5), !add(UsesTmp, 3)), 8);1174 // Size field is unsigned char and cannot fit more.1175 let Size = !if(!le(MaxSize, 256), MaxSize, 252);1176 }1177 1178 def _RESTORE : VPseudoInstSI <1179 (outs vgpr_class:$vdata),1180 !con(1181 (ins i32imm:$vaddr,1182 SReg_32:$soffset, i32imm:$offset),1183 !if(HasMask, (ins SReg_32:$mask), (ins)))> {1184 let mayStore = 0;1185 let mayLoad = 1;1186 1187 // (2 * 4) + (8 * num_subregs) bytes maximum1188 int MaxSize = !add(!shl(!srl(vgpr_class.Size, 5), !add(UsesTmp, 3)), 8);1189 // Size field is unsigned char and cannot fit more.1190 let Size = !if(!le(MaxSize, 256), MaxSize, 252);1191 }1192 } // End UseNamedOperandTable = 1, Spill = 1, VALU = 1, SchedRW = [WriteVMEM]1193}1194 1195// TODO: Technically the AlignTarget register class constraint is1196// overly conservative for gfx90a. There is an alignment requirement,1197// but the underlying spill will be lowered to 32-bit accesses.1198 1199defm SI_SPILL_V16 : SI_SPILL_VGPR <VGPR_16>;1200defm SI_SPILL_V32 : SI_SPILL_VGPR <VGPR_32>;1201defm SI_SPILL_V64 : SI_SPILL_VGPR <VReg_64_AlignTarget>;1202defm SI_SPILL_V96 : SI_SPILL_VGPR <VReg_96_AlignTarget>;1203defm SI_SPILL_V128 : SI_SPILL_VGPR <VReg_128_AlignTarget>;1204defm SI_SPILL_V160 : SI_SPILL_VGPR <VReg_160_AlignTarget>;1205defm SI_SPILL_V192 : SI_SPILL_VGPR <VReg_192_AlignTarget>;1206defm SI_SPILL_V224 : SI_SPILL_VGPR <VReg_224_AlignTarget>;1207defm SI_SPILL_V256 : SI_SPILL_VGPR <VReg_256_AlignTarget>;1208defm SI_SPILL_V288 : SI_SPILL_VGPR <VReg_288_AlignTarget>;1209defm SI_SPILL_V320 : SI_SPILL_VGPR <VReg_320_AlignTarget>;1210defm SI_SPILL_V352 : SI_SPILL_VGPR <VReg_352_AlignTarget>;1211defm SI_SPILL_V384 : SI_SPILL_VGPR <VReg_384_AlignTarget>;1212defm SI_SPILL_V512 : SI_SPILL_VGPR <VReg_512_AlignTarget>;1213defm SI_SPILL_V1024 : SI_SPILL_VGPR <VReg_1024_AlignTarget>;1214 1215let Defs = [M0] in {1216 // Spills a block of 32 VGPRs. M0 will contain a mask describing which1217 // registers in the block need to be transferred.1218 defm SI_BLOCK_SPILL_V1024 : SI_SPILL_VGPR <VReg_1024, 0, 1>;1219}1220 1221defm SI_SPILL_A32 : SI_SPILL_VGPR <AGPR_32, 1>;1222defm SI_SPILL_A64 : SI_SPILL_VGPR <AReg_64_AlignTarget, 1>;1223defm SI_SPILL_A96 : SI_SPILL_VGPR <AReg_96_AlignTarget, 1>;1224defm SI_SPILL_A128 : SI_SPILL_VGPR <AReg_128_AlignTarget, 1>;1225defm SI_SPILL_A160 : SI_SPILL_VGPR <AReg_160_AlignTarget, 1>;1226defm SI_SPILL_A192 : SI_SPILL_VGPR <AReg_192_AlignTarget, 1>;1227defm SI_SPILL_A224 : SI_SPILL_VGPR <AReg_224_AlignTarget, 1>;1228defm SI_SPILL_A256 : SI_SPILL_VGPR <AReg_256_AlignTarget, 1>;1229defm SI_SPILL_A288 : SI_SPILL_VGPR <AReg_288_AlignTarget, 1>;1230defm SI_SPILL_A320 : SI_SPILL_VGPR <AReg_320_AlignTarget, 1>;1231defm SI_SPILL_A352 : SI_SPILL_VGPR <AReg_352_AlignTarget, 1>;1232defm SI_SPILL_A384 : SI_SPILL_VGPR <AReg_384_AlignTarget, 1>;1233defm SI_SPILL_A512 : SI_SPILL_VGPR <AReg_512_AlignTarget, 1>;1234defm SI_SPILL_A1024 : SI_SPILL_VGPR <AReg_1024_AlignTarget, 1>;1235 1236defm SI_SPILL_AV32 : SI_SPILL_VGPR <AV_32, 1>;1237defm SI_SPILL_AV64 : SI_SPILL_VGPR <AV_64_AlignTarget, 1>;1238defm SI_SPILL_AV96 : SI_SPILL_VGPR <AV_96_AlignTarget, 1>;1239defm SI_SPILL_AV128 : SI_SPILL_VGPR <AV_128_AlignTarget, 1>;1240defm SI_SPILL_AV160 : SI_SPILL_VGPR <AV_160_AlignTarget, 1>;1241defm SI_SPILL_AV192 : SI_SPILL_VGPR <AV_192_AlignTarget, 1>;1242defm SI_SPILL_AV224 : SI_SPILL_VGPR <AV_224_AlignTarget, 1>;1243defm SI_SPILL_AV256 : SI_SPILL_VGPR <AV_256_AlignTarget, 1>;1244defm SI_SPILL_AV288 : SI_SPILL_VGPR <AV_288_AlignTarget, 1>;1245defm SI_SPILL_AV320 : SI_SPILL_VGPR <AV_320_AlignTarget, 1>;1246defm SI_SPILL_AV352 : SI_SPILL_VGPR <AV_352_AlignTarget, 1>;1247defm SI_SPILL_AV384 : SI_SPILL_VGPR <AV_384_AlignTarget, 1>;1248defm SI_SPILL_AV512 : SI_SPILL_VGPR <AV_512_AlignTarget, 1>;1249defm SI_SPILL_AV1024 : SI_SPILL_VGPR <AV_1024_AlignTarget, 1>;1250 1251let isConvergent = 1 in {1252 defm SI_SPILL_WWM_V32 : SI_SPILL_VGPR <VGPR_32>;1253 defm SI_SPILL_WWM_AV32 : SI_SPILL_VGPR <AV_32, 1>;1254}1255 1256let isReMaterializable = 1, isAsCheapAsAMove = 1 in1257def SI_PC_ADD_REL_OFFSET : SPseudoInstSI <1258 (outs SReg_64:$dst),1259 (ins si_ga:$ptr_lo, si_ga:$ptr_hi),1260 [(set SReg_64:$dst,1261 (i64 (SIpc_add_rel_offset tglobaladdr:$ptr_lo, tglobaladdr:$ptr_hi)))]> {1262 let Defs = [SCC];1263}1264 1265def : GCNPat <1266 (SIpc_add_rel_offset tglobaladdr:$ptr_lo, 0),1267 (SI_PC_ADD_REL_OFFSET $ptr_lo, (i32 0))1268>;1269 1270def SI_PC_ADD_REL_OFFSET64 : SPseudoInstSI <1271 (outs SReg_64:$dst),1272 (ins si_ga:$ptr),1273 [(set SReg_64:$dst,1274 (i64 (SIpc_add_rel_offset64 tglobaladdr:$ptr)))]> {1275 let SubtargetPredicate = Has64BitLiterals;1276}1277 1278def : GCNPat<1279 (AMDGPUtrap timm:$trapid),1280 (S_TRAP $trapid)1281>;1282 1283def : GCNPat<1284 (AMDGPUelse i1:$src, bb:$target),1285 (SI_ELSE $src, $target)1286>;1287 1288def : GCNPat <1289 (int_amdgcn_kill i1:$src),1290 (SI_KILL_I1_PSEUDO SCSrc_i1:$src, 0)1291>;1292 1293def : GCNPat <1294 (int_amdgcn_kill (i1 (not i1:$src))),1295 (SI_KILL_I1_PSEUDO SCSrc_i1:$src, -1)1296>;1297 1298let SubtargetPredicate = NotHasSALUFloatInsts in1299def : GCNPat <1300 (int_amdgcn_kill (i1 (setcc f32:$src, InlineImmFP32:$imm, cond:$cond))),1301 (SI_KILL_F32_COND_IMM_PSEUDO VSrc_b32:$src, (bitcast_fpimm_to_i32 $imm), (cond_as_i32imm $cond))1302>;1303 1304def : GCNPat <1305 (int_amdgcn_wqm_demote i1:$src),1306 (SI_DEMOTE_I1 SCSrc_i1:$src, 0)1307>;1308 1309def : GCNPat <1310 (int_amdgcn_wqm_demote (i1 (not i1:$src))),1311 (SI_DEMOTE_I1 SCSrc_i1:$src, -1)1312>;1313 1314 // TODO: we could add more variants for other types of conditionals1315 1316def : GCNPat <1317 (i64 (int_amdgcn_icmp i1:$src, (i1 0), (i32 33))),1318 (COPY $src) // Return the SGPRs representing i1 src1319>;1320 1321def : GCNPat <1322 (i32 (int_amdgcn_icmp i1:$src, (i1 0), (i32 33))),1323 (COPY $src) // Return the SGPRs representing i1 src1324>;1325 1326//===----------------------------------------------------------------------===//1327// VOP1 Patterns1328//===----------------------------------------------------------------------===//1329 1330multiclass f16_to_fp_Pats<Instruction cvt_f16_f32_inst_e64, Instruction cvt_f32_f16_inst_e64> {1331 // f16_to_fp patterns1332 def : GCNPat <1333 (f32 (any_f16_to_fp i32:$src0)),1334 (cvt_f32_f16_inst_e64 SRCMODS.NONE, $src0)1335 >;1336 1337 def : GCNPat <1338 (f32 (f16_to_fp (and_oneuse i32:$src0, 0x7fff))),1339 (cvt_f32_f16_inst_e64 SRCMODS.ABS, $src0)1340 >;1341 1342 def : GCNPat <1343 (f32 (f16_to_fp (i32 (srl_oneuse (and_oneuse i32:$src0, 0x7fff0000), (i32 16))))),1344 (cvt_f32_f16_inst_e64 SRCMODS.ABS, (i32 (V_LSHRREV_B32_e64 (i32 16), i32:$src0)))1345 >;1346 1347 def : GCNPat <1348 (f32 (f16_to_fp (or_oneuse i32:$src0, 0x8000))),1349 (cvt_f32_f16_inst_e64 SRCMODS.NEG_ABS, $src0)1350 >;1351 1352 def : GCNPat <1353 (f32 (f16_to_fp (xor_oneuse i32:$src0, 0x8000))),1354 (cvt_f32_f16_inst_e64 SRCMODS.NEG, $src0)1355 >;1356 1357 // fp_to_fp16 patterns1358 def : GCNPat <1359 (i32 (AMDGPUfp_to_f16 (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))),1360 (cvt_f16_f32_inst_e64 $src0_modifiers, f32:$src0)1361 >;1362 1363 // This is only used on targets without half support1364 // TODO: Introduce strict variant of AMDGPUfp_to_f16 and share custom lowering1365 def : GCNPat <1366 (i32 (strict_fp_to_f16 (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))),1367 (cvt_f16_f32_inst_e64 $src0_modifiers, f32:$src0)1368 >;1369}1370 1371let True16Predicate = NotHasTrue16BitInsts in1372defm : f16_to_fp_Pats<V_CVT_F16_F32_e64, V_CVT_F32_F16_e64>;1373 1374let True16Predicate = UseFakeTrue16Insts in1375defm : f16_to_fp_Pats<V_CVT_F16_F32_fake16_e64, V_CVT_F32_F16_fake16_e64>;1376 1377multiclass f16_fp_Pats<Instruction cvt_f16_f32_inst_e64,1378 Instruction cvt_f32_f16_inst_e64,1379 RegOrImmOperand VSrc> {1380 def : GCNPat <1381 (f64 (any_fpextend f16:$src)),1382 (V_CVT_F64_F32_e32 (cvt_f32_f16_inst_e64 SRCMODS.NONE, $src))1383 >;1384 1385 def : GCNPat <1386 (i32 (fp_to_sint f16:$src)),1387 (V_CVT_I32_F32_e32 (cvt_f32_f16_inst_e64 SRCMODS.NONE, VSrc:$src))1388 >;1389 1390 def : GCNPat <1391 (i32 (fp_to_uint f16:$src)),1392 (V_CVT_U32_F32_e32 (cvt_f32_f16_inst_e64 SRCMODS.NONE, VSrc:$src))1393 >;1394 1395 def : GCNPat <1396 (f16 (sint_to_fp i32:$src)),1397 (cvt_f16_f32_inst_e64 SRCMODS.NONE, (V_CVT_F32_I32_e32 VSrc_b32:$src))1398 >;1399 1400 def : GCNPat <1401 (f16 (uint_to_fp i32:$src)),1402 (cvt_f16_f32_inst_e64 SRCMODS.NONE, (V_CVT_F32_U32_e32 VSrc_b32:$src))1403 >;1404}1405 1406let True16Predicate = NotHasTrue16BitInsts in1407defm : f16_fp_Pats<V_CVT_F16_F32_e64, V_CVT_F32_F16_e64, VSrc_b32>;1408 1409let True16Predicate = UseRealTrue16Insts in1410defm : f16_fp_Pats<V_CVT_F16_F32_t16_e64, V_CVT_F32_F16_t16_e64, VSrcT_b16>;1411 1412let True16Predicate = UseFakeTrue16Insts in1413defm : f16_fp_Pats<V_CVT_F16_F32_fake16_e64, V_CVT_F32_F16_fake16_e64, VSrc_b16>;1414 1415//===----------------------------------------------------------------------===//1416// VOP2 Patterns1417//===----------------------------------------------------------------------===//1418 1419// NoMods pattern used for mac. If there are any source modifiers then it's1420// better to select mad instead of mac.1421class FMADPat <ValueType vt, Instruction inst>1422 : GCNPat <(vt (any_fmad (vt (VOP3NoMods vt:$src0)),1423 (vt (VOP3NoMods vt:$src1)),1424 (vt (VOP3NoMods vt:$src2)))),1425 (inst SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,1426 SRCMODS.NONE, $src2, DSTCLAMP.NONE, DSTOMOD.NONE)1427>;1428 1429// Prefer mac form when there are no modifiers.1430let AddedComplexity = 9 in {1431let OtherPredicates = [HasMadMacF32Insts] in1432def : FMADPat <f32, V_MAC_F32_e64>;1433 1434// Don't allow source modifiers. If there are any source modifiers then it's1435// better to select mad instead of mac.1436let SubtargetPredicate = isGFX6GFX7GFX10,1437 OtherPredicates = [HasMadMacF32Insts, NoFP32Denormals] in1438def : GCNPat <1439 (f32 (fadd (AMDGPUfmul_legacy (VOP3NoMods f32:$src0),1440 (VOP3NoMods f32:$src1)),1441 (VOP3NoMods f32:$src2))),1442 (V_MAC_LEGACY_F32_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,1443 SRCMODS.NONE, $src2, DSTCLAMP.NONE, DSTOMOD.NONE)1444>;1445 1446// Don't allow source modifiers. If there are any source modifiers then it's1447// better to select fma instead of fmac.1448let SubtargetPredicate = HasFmaLegacy32 in1449def : GCNPat <1450 (f32 (int_amdgcn_fma_legacy (VOP3NoMods f32:$src0),1451 (VOP3NoMods f32:$src1),1452 (VOP3NoMods f32:$src2))),1453 (V_FMAC_LEGACY_F32_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,1454 SRCMODS.NONE, $src2, DSTCLAMP.NONE, DSTOMOD.NONE)1455>;1456 1457let SubtargetPredicate = Has16BitInsts in1458def : FMADPat <f16, V_MAC_F16_e64>;1459} // AddedComplexity = 91460 1461let OtherPredicates = [HasMadMacF32Insts, NoFP32Denormals] in1462def : GCNPat <1463 (f32 (fadd (AMDGPUfmul_legacy (VOP3Mods f32:$src0, i32:$src0_mod),1464 (VOP3Mods f32:$src1, i32:$src1_mod)),1465 (VOP3Mods f32:$src2, i32:$src2_mod))),1466 (V_MAD_LEGACY_F32_e64 $src0_mod, $src0, $src1_mod, $src1,1467 $src2_mod, $src2, DSTCLAMP.NONE, DSTOMOD.NONE)1468>;1469 1470class VOPSelectModsPat <ValueType vt> : GCNPat <1471 (vt (select i1:$src0, (VOP3ModsNonCanonicalizing vt:$src1, i32:$src1_mods),1472 (VOP3ModsNonCanonicalizing vt:$src2, i32:$src2_mods))),1473 (V_CNDMASK_B32_e64 FP32InputMods:$src2_mods, VSrc_b32:$src2,1474 FP32InputMods:$src1_mods, VSrc_b32:$src1, SSrc_i1:$src0)1475>;1476 1477class VOPSelectPat <ValueType vt> : GCNPat <1478 (vt (select i1:$src0, vt:$src1, vt:$src2)),1479 (V_CNDMASK_B32_e64 0, VSrc_b32:$src2, 0, VSrc_b32:$src1, SSrc_i1:$src0)1480>;1481class VOPSelectPat_t16 <ValueType vt> : GCNPat <1482 (vt (select i1:$src0, vt:$src1, vt:$src2)),1483 (V_CNDMASK_B16_t16_e64 0, VSrcT_b16:$src2, 0, VSrcT_b16:$src1, SSrc_i1:$src0)1484>;1485 1486def : VOPSelectModsPat <i32>;1487def : VOPSelectModsPat <f32>;1488let True16Predicate = NotUseRealTrue16Insts in {1489 def : VOPSelectPat <f16>;1490 def : VOPSelectPat <i16>;1491} // End True16Predicate = p1492let True16Predicate = UseRealTrue16Insts in {1493 def : VOPSelectPat_t16 <f16>;1494 def : VOPSelectPat_t16 <i16>;1495} // End True16Predicate = UseRealTrue16Insts1496 1497let AddedComplexity = 1 in {1498def : GCNPat <1499 (i32 (add (i32 (DivergentUnaryFrag<ctpop> i32:$popcnt)), i32:$val)),1500 (V_BCNT_U32_B32_e64 $popcnt, $val)1501>;1502}1503 1504def : GCNPat <1505 (i32 (DivergentUnaryFrag<ctpop> i32:$popcnt)),1506 (V_BCNT_U32_B32_e64 VSrc_b32:$popcnt, (i32 0))1507>;1508 1509def : GCNPat <1510 (i16 (add (i16 (trunc (i32 (DivergentUnaryFrag<ctpop> i32:$popcnt)))), i16:$val)),1511 (V_BCNT_U32_B32_e64 $popcnt, $val)1512>;1513 1514def : GCNPat <1515 (i64 (DivergentUnaryFrag<ctpop> i64:$src)),1516 (REG_SEQUENCE VReg_64,1517 (V_BCNT_U32_B32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub1)),1518 (i32 (V_BCNT_U32_B32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0)))), sub0,1519 (i32 (V_MOV_B32_e32 (i32 0))), sub1)1520>;1521 1522//===----------------------------------------------------------------------===//1523// PTRADD Patterns1524//===----------------------------------------------------------------------===//1525 1526// GlobalISel shouldn't generate 64-bit addition pseudos.1527let GISelShouldIgnore = 1 in {1528def : GCNPat<1529 (DivergentBinFrag<ptradd> i64:$src0, i64:$src1),1530 (V_ADD_U64_PSEUDO $src0, $src1)>;1531def : GCNPat<1532 (UniformBinFrag<ptradd> i64:$src0, i64:$src1),1533 (S_ADD_U64_PSEUDO $src0, $src1)>;1534}1535 1536//===----------------------------------------------------------------------===//1537// FP min/max patterns1538//===----------------------------------------------------------------------===//1539 1540 1541class FPBinOpPat <SDPatternOperator node, ValueType vt, Instruction inst>1542 : GCNPat <(vt (node (vt (VOP3Mods vt:$src0, i32:$src0_mods)),1543 (vt (VOP3Mods vt:$src1, i32:$src1_mods)))),1544 (inst $src0_mods, $src0, $src1_mods, $src1, DSTCLAMP.NONE, DSTOMOD.NONE)1545>;1546 1547class FPPkBinOpPat <SDPatternOperator node, ValueType vt, Instruction inst>1548 : GCNPat <(vt (node (VOP3PMods v2f16:$src0, i32:$src0_mods),1549 (VOP3PMods v2f16:$src1, i32:$src1_mods))),1550 (inst $src0_mods, $src0, $src1_mods, $src1, DSTCLAMP.NONE)1551>;1552 1553/// With IEEE=0, signalingness is ignored and the non-nan input will1554/// be directly returned.1555let OtherPredicates = [IEEEModeDisabled] in {1556 def : FPBinOpPat<fminimumnum, f32, V_MIN_F32_e64>;1557 def : FPBinOpPat<fmaximumnum, f32, V_MAX_F32_e64>;1558 def : FPBinOpPat<fminimumnum, f64, V_MIN_F64_e64>;1559 def : FPBinOpPat<fmaximumnum, f64, V_MAX_F64_e64>;1560 1561 let SubtargetPredicate = Has16BitInsts,1562 True16Predicate = NotHasTrue16BitInsts in {1563 def : FPBinOpPat<fminimumnum, f16, V_MIN_F16_e64>;1564 def : FPBinOpPat<fmaximumnum, f16, V_MAX_F16_e64>;1565 }1566 1567 let SubtargetPredicate = Has16BitInsts,1568 True16Predicate = UseRealTrue16Insts in {1569 def : FPBinOpPat<fminimumnum, f16, V_MIN_F16_t16_e64>;1570 def : FPBinOpPat<fmaximumnum, f16, V_MAX_F16_t16_e64>;1571 }1572 1573 let SubtargetPredicate = Has16BitInsts,1574 True16Predicate = UseFakeTrue16Insts in {1575 def : FPBinOpPat<fminimumnum, f16, V_MIN_F16_fake16_e64>;1576 def : FPBinOpPat<fmaximumnum, f16, V_MAX_F16_fake16_e64>;1577 }1578 1579 let SubtargetPredicate = HasVOP3PInsts in {1580 def : FPPkBinOpPat<fminimumnum, v2f16, V_PK_MIN_F16>;1581 def : FPPkBinOpPat<fmaximumnum, v2f16, V_PK_MAX_F16>;1582 }1583}1584 1585/********** ============================================ **********/1586/********** Extraction, Insertion, Building and Casting **********/1587/********** ============================================ **********/1588 1589// Special case for 2 element vectors. REQ_SEQUENCE produces better code1590// than an INSERT_SUBREG.1591multiclass Insert_Element_V2<RegisterClass RC, ValueType elem_type, ValueType vec_type> {1592 def : GCNPat <1593 (insertelt vec_type:$vec, elem_type:$elem, 0),1594 (REG_SEQUENCE RC, $elem, sub0, (elem_type (EXTRACT_SUBREG $vec, sub1)), sub1)1595 >;1596 1597 def : GCNPat <1598 (insertelt vec_type:$vec, elem_type:$elem, 1),1599 (REG_SEQUENCE RC, (elem_type (EXTRACT_SUBREG $vec, sub0)), sub0, $elem, sub1)1600 >;1601}1602 1603foreach Index = 0-1 in {1604 def Extract_Element_v2i32_#Index : Extract_Element <1605 i32, v2i32, Index, !cast<SubRegIndex>(sub#Index)1606 >;1607 1608 def Extract_Element_v2f32_#Index : Extract_Element <1609 f32, v2f32, Index, !cast<SubRegIndex>(sub#Index)1610 >;1611}1612 1613defm : Insert_Element_V2 <SReg_64, i32, v2i32>;1614defm : Insert_Element_V2 <SReg_64, f32, v2f32>;1615 1616foreach Index = 0-2 in {1617 def Extract_Element_v3i32_#Index : Extract_Element <1618 i32, v3i32, Index, !cast<SubRegIndex>(sub#Index)1619 >;1620 def Insert_Element_v3i32_#Index : Insert_Element <1621 i32, v3i32, Index, !cast<SubRegIndex>(sub#Index)1622 >;1623 1624 def Extract_Element_v3f32_#Index : Extract_Element <1625 f32, v3f32, Index, !cast<SubRegIndex>(sub#Index)1626 >;1627 def Insert_Element_v3f32_#Index : Insert_Element <1628 f32, v3f32, Index, !cast<SubRegIndex>(sub#Index)1629 >;1630}1631 1632foreach Index = 0-3 in {1633 def Extract_Element_v4i32_#Index : Extract_Element <1634 i32, v4i32, Index, !cast<SubRegIndex>(sub#Index)1635 >;1636 def Insert_Element_v4i32_#Index : Insert_Element <1637 i32, v4i32, Index, !cast<SubRegIndex>(sub#Index)1638 >;1639 1640 def Extract_Element_v4f32_#Index : Extract_Element <1641 f32, v4f32, Index, !cast<SubRegIndex>(sub#Index)1642 >;1643 def Insert_Element_v4f32_#Index : Insert_Element <1644 f32, v4f32, Index, !cast<SubRegIndex>(sub#Index)1645 >;1646}1647 1648foreach Index = 0-4 in {1649 def Extract_Element_v5i32_#Index : Extract_Element <1650 i32, v5i32, Index, !cast<SubRegIndex>(sub#Index)1651 >;1652 def Insert_Element_v5i32_#Index : Insert_Element <1653 i32, v5i32, Index, !cast<SubRegIndex>(sub#Index)1654 >;1655 1656 def Extract_Element_v5f32_#Index : Extract_Element <1657 f32, v5f32, Index, !cast<SubRegIndex>(sub#Index)1658 >;1659 def Insert_Element_v5f32_#Index : Insert_Element <1660 f32, v5f32, Index, !cast<SubRegIndex>(sub#Index)1661 >;1662}1663 1664foreach Index = 0-5 in {1665 def Extract_Element_v6i32_#Index : Extract_Element <1666 i32, v6i32, Index, !cast<SubRegIndex>(sub#Index)1667 >;1668 def Insert_Element_v6i32_#Index : Insert_Element <1669 i32, v6i32, Index, !cast<SubRegIndex>(sub#Index)1670 >;1671 1672 def Extract_Element_v6f32_#Index : Extract_Element <1673 f32, v6f32, Index, !cast<SubRegIndex>(sub#Index)1674 >;1675 def Insert_Element_v6f32_#Index : Insert_Element <1676 f32, v6f32, Index, !cast<SubRegIndex>(sub#Index)1677 >;1678}1679 1680foreach Index = 0-6 in {1681 def Extract_Element_v7i32_#Index : Extract_Element <1682 i32, v7i32, Index, !cast<SubRegIndex>(sub#Index)1683 >;1684 def Insert_Element_v7i32_#Index : Insert_Element <1685 i32, v7i32, Index, !cast<SubRegIndex>(sub#Index)1686 >;1687 1688 def Extract_Element_v7f32_#Index : Extract_Element <1689 f32, v7f32, Index, !cast<SubRegIndex>(sub#Index)1690 >;1691 def Insert_Element_v7f32_#Index : Insert_Element <1692 f32, v7f32, Index, !cast<SubRegIndex>(sub#Index)1693 >;1694}1695 1696foreach Index = 0-7 in {1697 def Extract_Element_v8i32_#Index : Extract_Element <1698 i32, v8i32, Index, !cast<SubRegIndex>(sub#Index)1699 >;1700 def Insert_Element_v8i32_#Index : Insert_Element <1701 i32, v8i32, Index, !cast<SubRegIndex>(sub#Index)1702 >;1703 1704 def Extract_Element_v8f32_#Index : Extract_Element <1705 f32, v8f32, Index, !cast<SubRegIndex>(sub#Index)1706 >;1707 def Insert_Element_v8f32_#Index : Insert_Element <1708 f32, v8f32, Index, !cast<SubRegIndex>(sub#Index)1709 >;1710}1711 1712foreach Index = 0-8 in {1713 def Extract_Element_v9i32_#Index : Extract_Element <1714 i32, v9i32, Index, !cast<SubRegIndex>(sub#Index)1715 >;1716 def Insert_Element_v9i32_#Index : Insert_Element <1717 i32, v9i32, Index, !cast<SubRegIndex>(sub#Index)1718 >;1719 1720 def Extract_Element_v9f32_#Index : Extract_Element <1721 f32, v9f32, Index, !cast<SubRegIndex>(sub#Index)1722 >;1723 def Insert_Element_v9f32_#Index : Insert_Element <1724 f32, v9f32, Index, !cast<SubRegIndex>(sub#Index)1725 >;1726}1727 1728foreach Index = 0-9 in {1729 def Extract_Element_v10i32_#Index : Extract_Element <1730 i32, v10i32, Index, !cast<SubRegIndex>(sub#Index)1731 >;1732 def Insert_Element_v10i32_#Index : Insert_Element <1733 i32, v10i32, Index, !cast<SubRegIndex>(sub#Index)1734 >;1735 1736 def Extract_Element_v10f32_#Index : Extract_Element <1737 f32, v10f32, Index, !cast<SubRegIndex>(sub#Index)1738 >;1739 def Insert_Element_v10f32_#Index : Insert_Element <1740 f32, v10f32, Index, !cast<SubRegIndex>(sub#Index)1741 >;1742}1743 1744foreach Index = 0-10 in {1745 def Extract_Element_v11i32_#Index : Extract_Element <1746 i32, v11i32, Index, !cast<SubRegIndex>(sub#Index)1747 >;1748 def Insert_Element_v11i32_#Index : Insert_Element <1749 i32, v11i32, Index, !cast<SubRegIndex>(sub#Index)1750 >;1751 1752 def Extract_Element_v11f32_#Index : Extract_Element <1753 f32, v11f32, Index, !cast<SubRegIndex>(sub#Index)1754 >;1755 def Insert_Element_v11f32_#Index : Insert_Element <1756 f32, v11f32, Index, !cast<SubRegIndex>(sub#Index)1757 >;1758}1759 1760foreach Index = 0-11 in {1761 def Extract_Element_v12i32_#Index : Extract_Element <1762 i32, v12i32, Index, !cast<SubRegIndex>(sub#Index)1763 >;1764 def Insert_Element_v12i32_#Index : Insert_Element <1765 i32, v12i32, Index, !cast<SubRegIndex>(sub#Index)1766 >;1767 1768 def Extract_Element_v12f32_#Index : Extract_Element <1769 f32, v12f32, Index, !cast<SubRegIndex>(sub#Index)1770 >;1771 def Insert_Element_v12f32_#Index : Insert_Element <1772 f32, v12f32, Index, !cast<SubRegIndex>(sub#Index)1773 >;1774}1775 1776foreach Index = 0-15 in {1777 def Extract_Element_v16i32_#Index : Extract_Element <1778 i32, v16i32, Index, !cast<SubRegIndex>(sub#Index)1779 >;1780 def Insert_Element_v16i32_#Index : Insert_Element <1781 i32, v16i32, Index, !cast<SubRegIndex>(sub#Index)1782 >;1783 1784 def Extract_Element_v16f32_#Index : Extract_Element <1785 f32, v16f32, Index, !cast<SubRegIndex>(sub#Index)1786 >;1787 def Insert_Element_v16f32_#Index : Insert_Element <1788 f32, v16f32, Index, !cast<SubRegIndex>(sub#Index)1789 >;1790}1791 1792 1793foreach Index = 0-31 in {1794 def Extract_Element_v32i32_#Index : Extract_Element <1795 i32, v32i32, Index, !cast<SubRegIndex>(sub#Index)1796 >;1797 1798 def Insert_Element_v32i32_#Index : Insert_Element <1799 i32, v32i32, Index, !cast<SubRegIndex>(sub#Index)1800 >;1801 1802 def Extract_Element_v32f32_#Index : Extract_Element <1803 f32, v32f32, Index, !cast<SubRegIndex>(sub#Index)1804 >;1805 1806 def Insert_Element_v32f32_#Index : Insert_Element <1807 f32, v32f32, Index, !cast<SubRegIndex>(sub#Index)1808 >;1809}1810 1811// FIXME: Why do only some of these type combinations for SReg and1812// VReg?1813// 16-bit bitcast1814foreach vt = Reg16Types.types in {1815 foreach st = Reg16Types.types in {1816 if !not(!eq (vt, st)) then {1817 def : BitConvert <vt, st, VGPR_32>;1818 def : BitConvert <vt, st, SReg_32>;1819 }1820 }1821}1822 1823// 32-bit bitcast1824foreach vt = Reg32DataTypes.types in {1825 foreach st = Reg32DataTypes.types in {1826 if !not(!eq (vt, st)) then {1827 def : BitConvert <vt, st, VGPR_32>;1828 def : BitConvert <vt, st, SReg_32>;1829 }1830 }1831}1832 1833 1834// 64-bit bitcast1835foreach vt = Reg64DataTypes.types in {1836 foreach st = Reg64DataTypes.types in {1837 if !not(!eq (vt, st)) then {1838 def : BitConvert <vt, st, VReg_64>;1839 }1840 }1841}1842 1843 1844// 96-bit bitcast1845foreach vt = Reg96Types.types in {1846 foreach st = Reg96Types.types in {1847 if !not(!eq (vt, st)) then {1848 def : BitConvert <vt, st, SGPR_96>;1849 }1850 }1851}1852 1853 1854// 128-bit bitcast1855foreach vt = Reg128Types.types in {1856 foreach st = Reg128Types.types in {1857 if !not(!eq (vt, st)) then {1858 def : BitConvert <vt, st, VReg_128>;1859 def : BitConvert <vt, st, SReg_128>;1860 }1861 }1862}1863 1864 1865// 160-bit bitcast1866foreach vt = Reg160Types.types in {1867 foreach st = Reg160Types.types in {1868 if !not(!eq (vt, st)) then {1869 def : BitConvert <vt, st, VReg_160>;1870 def : BitConvert <vt, st, SReg_160>;1871 }1872 }1873}1874 1875// 192-bit bitcast1876foreach vt = Reg192Types.types in {1877 foreach st = Reg192Types.types in {1878 if !not(!eq (vt, st)) then {1879 def : BitConvert <vt, st, VReg_192>;1880 def : BitConvert <vt, st, SReg_192>;1881 }1882 }1883}1884 1885// 224-bit bitcast1886foreach vt = Reg224Types.types in {1887 foreach st = Reg224Types.types in {1888 if !not(!eq (vt, st)) then {1889 def : BitConvert <vt, st, VReg_224>;1890 def : BitConvert <vt, st, SReg_224>;1891 }1892 }1893}1894 1895 1896// 256-bit bitcast1897foreach vt = Reg256Types.types in {1898 foreach st = Reg256Types.types in {1899 if !not(!eq (vt, st)) then {1900 def : BitConvert <vt, st, VReg_256>;1901 def : BitConvert <vt, st, SReg_256>;1902 }1903 }1904}1905 1906 1907// 288-bit bitcast1908foreach vt = Reg288Types.types in {1909 foreach st = Reg288Types.types in {1910 if !not(!eq (vt, st)) then {1911 def : BitConvert <vt, st, VReg_288>;1912 def : BitConvert <vt, st, SReg_288>;1913 }1914 }1915}1916 1917// 320-bit bitcast1918foreach vt = Reg320Types.types in {1919 foreach st = Reg320Types.types in {1920 if !not(!eq (vt, st)) then {1921 def : BitConvert <vt, st, VReg_320>;1922 def : BitConvert <vt, st, SReg_320>;1923 }1924 }1925}1926 1927// 320-bit bitcast1928foreach vt = Reg352Types.types in {1929 foreach st = Reg352Types.types in {1930 if !not(!eq (vt, st)) then {1931 def : BitConvert <vt, st, VReg_352>;1932 def : BitConvert <vt, st, SReg_352>;1933 }1934 }1935}1936 1937// 384-bit bitcast1938foreach vt = Reg384Types.types in {1939 foreach st = Reg384Types.types in {1940 if !not(!eq (vt, st)) then {1941 def : BitConvert <vt, st, VReg_384>;1942 def : BitConvert <vt, st, SReg_384>;1943 }1944 }1945}1946 1947// 512-bit bitcast1948foreach vt = Reg512Types.types in {1949 foreach st = Reg512Types.types in {1950 if !not(!eq (vt, st)) then {1951 def : BitConvert <vt, st, VReg_512>;1952 def : BitConvert <vt, st, SReg_512>;1953 }1954 }1955}1956 1957 1958// 1024-bit bitcast1959foreach vt = Reg1024Types.types in {1960 foreach st = Reg1024Types.types in {1961 if !not(!eq (vt, st)) then {1962 def : BitConvert <vt, st, VReg_1024>;1963 def : BitConvert <vt, st, SReg_1024>;1964 }1965 }1966}1967 1968 1969/********** =================== **********/1970/********** Src & Dst modifiers **********/1971/********** =================== **********/1972 1973 1974// If denormals are not enabled, it only impacts the compare of the1975// inputs. The output result is not flushed.1976class ClampPat<Instruction inst, ValueType vt> : GCNPat <1977 (vt (AMDGPUclamp (VOP3Mods vt:$src0, i32:$src0_modifiers))),1978 (inst i32:$src0_modifiers, vt:$src0,1979 i32:$src0_modifiers, vt:$src0, DSTCLAMP.ENABLE, DSTOMOD.NONE)1980>;1981 1982def : ClampPat<V_MAX_F32_e64, f32>;1983let SubtargetPredicate = isNotGFX12Plus in1984def : ClampPat<V_MAX_F64_e64, f64>;1985let SubtargetPredicate = isGFX12Plus in1986def : ClampPat<V_MAX_NUM_F64_e64, f64>;1987let SubtargetPredicate = NotHasTrue16BitInsts in1988def : ClampPat<V_MAX_F16_e64, f16>;1989let SubtargetPredicate = UseRealTrue16Insts in1990def : ClampPat<V_MAX_F16_t16_e64, f16>;1991let SubtargetPredicate = UseFakeTrue16Insts in1992def : ClampPat<V_MAX_F16_fake16_e64, f16>;1993// FIXME-TRUE16: Pseudo expansion of this won't work with True16.1994let True16Predicate = UseFakeTrue16Insts in1995def : ClampPat<V_MAX_BF16_PSEUDO_e64, bf16>;1996 1997let SubtargetPredicate = HasVOP3PInsts in {1998def : GCNPat <1999 (v2f16 (AMDGPUclamp (VOP3PMods v2f16:$src0, i32:$src0_modifiers))),2000 (V_PK_MAX_F16 $src0_modifiers, $src0,2001 $src0_modifiers, $src0, DSTCLAMP.ENABLE)2002>;2003}2004 2005let SubtargetPredicate = HasBF16PackedInsts in {2006def : GCNPat <2007 (v2bf16 (AMDGPUclamp (VOP3PMods v2bf16:$src0, i32:$src0_modifiers))),2008 (V_PK_MAX_NUM_BF16 $src0_modifiers, $src0,2009 $src0_modifiers, $src0, DSTCLAMP.ENABLE)2010>;2011} // End SubtargetPredicate = HasBF16PackedInsts2012 2013/********** ================================ **********/2014/********** Floating point absolute/negative **********/2015/********** ================================ **********/2016 2017def : GCNPat <2018 (UniformUnaryFrag<fneg> (fabs (f32 SReg_32:$src))),2019 (S_OR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80000000))) // Set sign bit2020>;2021 2022def : GCNPat <2023 (UniformUnaryFrag<fabs> (f32 SReg_32:$src)),2024 (S_AND_B32 SReg_32:$src, (S_MOV_B32 (i32 0x7fffffff)))2025>;2026 2027def : GCNPat <2028 (UniformUnaryFrag<fneg> (f32 SReg_32:$src)),2029 (S_XOR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80000000)))2030>;2031 2032foreach fp16vt = [f16, bf16] in {2033def : GCNPat <2034 (UniformUnaryFrag<fneg> (fp16vt SReg_32:$src)),2035 (S_XOR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x00008000)))2036>;2037 2038def : GCNPat <2039 (UniformUnaryFrag<fabs> (fp16vt SReg_32:$src)),2040 (S_AND_B32 SReg_32:$src, (S_MOV_B32 (i32 0x00007fff)))2041>;2042 2043def : GCNPat <2044 (UniformUnaryFrag<fneg> (fabs (fp16vt SReg_32:$src))),2045 (S_OR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x00008000))) // Set sign bit2046>;2047} // End foreach fp16vt = ...2048 2049foreach v2fp16vt = [v2f16, v2bf16] in {2050def : GCNPat <2051 (UniformUnaryFrag<fneg> (v2fp16vt SReg_32:$src)),2052 (S_XOR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80008000)))2053>;2054 2055def : GCNPat <2056 (UniformUnaryFrag<fabs> (v2fp16vt SReg_32:$src)),2057 (S_AND_B32 SReg_32:$src, (S_MOV_B32 (i32 0x7fff7fff)))2058>;2059 2060// This is really (fneg (fabs v2f16:$src))2061//2062// fabs is not reported as free because there is modifier for it in2063// VOP3P instructions, so it is turned into the bit op.2064def : GCNPat <2065 (UniformUnaryFrag<fneg> (v2fp16vt (bitconvert (and_oneuse (i32 SReg_32:$src), 0x7fff7fff)))),2066 (S_OR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80008000))) // Set sign bit2067>;2068 2069def : GCNPat <2070 (UniformUnaryFrag<fneg> (v2fp16vt (fabs SReg_32:$src))),2071 (S_OR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80008000))) // Set sign bit2072>;2073}2074 2075// COPY_TO_REGCLASS is needed to avoid using SCC from S_XOR_B32 instead2076// of the real value.2077def : GCNPat <2078 (UniformUnaryFrag<fneg> (v2f32 SReg_64:$src)),2079 (v2f32 (REG_SEQUENCE SReg_64,2080 (f32 (COPY_TO_REGCLASS (S_XOR_B32 (i32 (EXTRACT_SUBREG $src, sub0)),2081 (i32 (S_MOV_B32 (i32 0x80000000)))),2082 SReg_32)), sub0,2083 (f32 (COPY_TO_REGCLASS (S_XOR_B32 (i32 (EXTRACT_SUBREG $src, sub1)),2084 (i32 (S_MOV_B32 (i32 0x80000000)))),2085 SReg_32)), sub1))2086>;2087 2088def : GCNPat <2089 (UniformUnaryFrag<fabs> (v2f32 SReg_64:$src)),2090 (v2f32 (REG_SEQUENCE SReg_64,2091 (f32 (COPY_TO_REGCLASS (S_AND_B32 (i32 (EXTRACT_SUBREG $src, sub0)),2092 (i32 (S_MOV_B32 (i32 0x7fffffff)))),2093 SReg_32)), sub0,2094 (f32 (COPY_TO_REGCLASS (S_AND_B32 (i32 (EXTRACT_SUBREG $src, sub1)),2095 (i32 (S_MOV_B32 (i32 0x7fffffff)))),2096 SReg_32)), sub1))2097>;2098 2099def : GCNPat <2100 (UniformUnaryFrag<fneg> (fabs (v2f32 SReg_64:$src))),2101 (v2f32 (REG_SEQUENCE SReg_64,2102 (f32 (COPY_TO_REGCLASS (S_OR_B32 (i32 (EXTRACT_SUBREG $src, sub0)),2103 (i32 (S_MOV_B32 (i32 0x80000000)))),2104 SReg_32)), sub0,2105 (f32 (COPY_TO_REGCLASS (S_OR_B32 (i32 (EXTRACT_SUBREG $src, sub1)),2106 (i32 (S_MOV_B32 (i32 0x80000000)))),2107 SReg_32)), sub1))2108>;2109 2110// FIXME: Use S_BITSET0_B32/B64?2111def : GCNPat <2112 (UniformUnaryFrag<fabs> (f64 SReg_64:$src)),2113 (REG_SEQUENCE SReg_64,2114 (i32 (EXTRACT_SUBREG SReg_64:$src, sub0)),2115 sub0,2116 (i32 (COPY_TO_REGCLASS (S_AND_B32 (i32 (EXTRACT_SUBREG SReg_64:$src, sub1)),2117 (S_MOV_B32 (i32 0x7fffffff))), SReg_32)), // Set sign bit.2118 sub1)2119>;2120 2121def : GCNPat <2122 (UniformUnaryFrag<fneg> (f64 SReg_64:$src)),2123 (REG_SEQUENCE SReg_64,2124 (i32 (EXTRACT_SUBREG SReg_64:$src, sub0)),2125 sub0,2126 (i32 (COPY_TO_REGCLASS (S_XOR_B32 (i32 (EXTRACT_SUBREG SReg_64:$src, sub1)),2127 (i32 (S_MOV_B32 (i32 0x80000000)))), SReg_32)),2128 sub1)2129>;2130 2131def : GCNPat <2132 (UniformUnaryFrag<fneg> (fabs (f64 SReg_64:$src))),2133 (REG_SEQUENCE SReg_64,2134 (i32 (EXTRACT_SUBREG SReg_64:$src, sub0)),2135 sub0,2136 (i32 (COPY_TO_REGCLASS (S_OR_B32 (i32 (EXTRACT_SUBREG SReg_64:$src, sub1)),2137 (S_MOV_B32 (i32 0x80000000))), SReg_32)),// Set sign bit.2138 sub1)2139>;2140 2141 2142def : GCNPat <2143 (fneg (fabs (f32 VGPR_32:$src))),2144 (V_OR_B32_e64 (S_MOV_B32 (i32 0x80000000)), VGPR_32:$src) // Set sign bit2145>;2146 2147def : GCNPat <2148 (fabs (f32 VGPR_32:$src)),2149 (V_AND_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), VGPR_32:$src)2150>;2151 2152def : GCNPat <2153 (fneg (f32 VGPR_32:$src)),2154 (V_XOR_B32_e64 (S_MOV_B32 (i32 0x80000000)), VGPR_32:$src)2155>;2156 2157foreach fp16vt = [f16, bf16] in {2158let SubtargetPredicate = NotUseRealTrue16Insts in {2159def : GCNPat <2160 (fabs (fp16vt VGPR_32:$src)),2161 (V_AND_B32_e64 (S_MOV_B32 (i32 0x00007fff)), VGPR_32:$src)2162>;2163 2164def : GCNPat <2165 (fneg (fp16vt VGPR_32:$src)),2166 (V_XOR_B32_e64 (S_MOV_B32 (i32 0x00008000)), VGPR_32:$src)2167>;2168 2169def : GCNPat <2170 (fneg (fabs (fp16vt VGPR_32:$src))),2171 (V_OR_B32_e64 (S_MOV_B32 (i32 0x00008000)), VGPR_32:$src) // Set sign bit2172>;2173}2174 2175let SubtargetPredicate = UseRealTrue16Insts in {2176def : GCNPat <2177 (fabs (fp16vt VGPR_16:$src)),2178 (V_AND_B16_t16_e64 (i32 0), (i16 0x7fff), (i32 0), VGPR_16:$src)2179>;2180 2181def : GCNPat <2182 (fneg (fp16vt VGPR_16:$src)),2183 (V_XOR_B16_t16_e64 (i32 0), (i16 0x8000), (i32 0), VGPR_16:$src)2184>;2185 2186def : GCNPat <2187 (fneg (fabs (fp16vt VGPR_16:$src))),2188 (V_OR_B16_t16_e64 (i32 0), (i16 0x8000), (i32 0), VGPR_16:$src) // Set sign bit2189>;2190} // End SubtargetPredicate = UseRealTrue16Insts2191} // End foreach fp16vt = ...2192 2193foreach v2fp16vt = [v2f16, v2bf16] in {2194def : GCNPat <2195 (fneg (v2fp16vt VGPR_32:$src)),2196 (V_XOR_B32_e64 (S_MOV_B32 (i32 0x80008000)), VGPR_32:$src)2197>;2198 2199def : GCNPat <2200 (fabs (v2fp16vt VGPR_32:$src)),2201 (V_AND_B32_e64 (S_MOV_B32 (i32 0x7fff7fff)), VGPR_32:$src)2202>;2203 2204def : GCNPat <2205 (fneg (v2fp16vt (fabs VGPR_32:$src))),2206 (V_OR_B32_e64 (S_MOV_B32 (i32 0x80008000)), VGPR_32:$src)2207>;2208}2209 2210def : GCNPat <2211 (fabs (f64 VReg_64:$src)),2212 (REG_SEQUENCE VReg_64,2213 (i32 (EXTRACT_SUBREG VReg_64:$src, sub0)),2214 sub0,2215 (V_AND_B32_e64 (i32 (S_MOV_B32 (i32 0x7fffffff))),2216 (i32 (EXTRACT_SUBREG VReg_64:$src, sub1))),2217 sub1)2218>;2219 2220def : GCNPat <2221 (fneg (f64 VReg_64:$src)),2222 (REG_SEQUENCE VReg_64,2223 (i32 (EXTRACT_SUBREG VReg_64:$src, sub0)),2224 sub0,2225 (V_XOR_B32_e64 (i32 (S_MOV_B32 (i32 0x80000000))),2226 (i32 (EXTRACT_SUBREG VReg_64:$src, sub1))),2227 sub1)2228>;2229 2230def : GCNPat <2231 (fneg (fabs (f64 VReg_64:$src))),2232 (REG_SEQUENCE VReg_64,2233 (i32 (EXTRACT_SUBREG VReg_64:$src, sub0)),2234 sub0,2235 (V_OR_B32_e64 (i32 (S_MOV_B32 (i32 0x80000000))),2236 (i32 (EXTRACT_SUBREG VReg_64:$src, sub1))),2237 sub1)2238>;2239 2240def : GCNPat <2241 (DivergentUnaryFrag<fneg> (v2f32 VReg_64:$src)),2242 (V_PK_ADD_F32 !or(SRCMODS.OP_SEL_1, SRCMODS.NEG, SRCMODS.NEG_HI), VReg_64:$src,2243 !or(SRCMODS.OP_SEL_1, SRCMODS.NEG, SRCMODS.NEG_HI), (i64 0),2244 0, 0, 0, 0, 0)2245> {2246 let SubtargetPredicate = HasPackedFP32Ops;2247}2248 2249foreach fp16vt = [f16, bf16] in {2250let True16Predicate = NotUseRealTrue16Insts in {2251def : GCNPat <2252 (fcopysign fp16vt:$src0, fp16vt:$src1),2253 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)), $src0, $src1)2254>;2255 2256def : GCNPat <2257 (fcopysign f32:$src0, fp16vt:$src1),2258 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), $src0,2259 (V_LSHLREV_B32_e64 (i32 16), $src1))2260>;2261 2262def : GCNPat <2263 (fcopysign f64:$src0, fp16vt:$src1),2264 (REG_SEQUENCE SReg_64,2265 (i32 (EXTRACT_SUBREG $src0, sub0)), sub0,2266 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), (i32 (EXTRACT_SUBREG $src0, sub1)),2267 (V_LSHLREV_B32_e64 (i32 16), $src1)), sub1)2268>;2269 2270def : GCNPat <2271 (fcopysign fp16vt:$src0, f32:$src1),2272 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)), $src0,2273 (V_LSHRREV_B32_e64 (i32 16), $src1))2274>;2275 2276def : GCNPat <2277 (fcopysign fp16vt:$src0, f64:$src1),2278 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)), $src0,2279 (V_LSHRREV_B32_e64 (i32 16), (EXTRACT_SUBREG $src1, sub1)))2280>;2281}2282let True16Predicate = UseRealTrue16Insts in {2283def : GCNPat <2284 (fcopysign fp16vt:$src0, fp16vt:$src1),2285 (EXTRACT_SUBREG (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)),2286 (REG_SEQUENCE VGPR_32, $src0, lo16, (i16 (IMPLICIT_DEF)), hi16),2287 (REG_SEQUENCE VGPR_32, $src1, lo16, (i16 (IMPLICIT_DEF)), hi16)), lo16)2288>;2289 2290def : GCNPat <2291 (fcopysign f32:$src0, fp16vt:$src1),2292 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), $src0,2293 (REG_SEQUENCE VGPR_32, (i16 (IMPLICIT_DEF)), lo16, $src1, hi16))2294>;2295 2296def : GCNPat <2297 (fcopysign f64:$src0, fp16vt:$src1),2298 (REG_SEQUENCE VReg_64,2299 (i32 (EXTRACT_SUBREG $src0, sub0)), sub0,2300 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), (i32 (EXTRACT_SUBREG $src0, sub1)),2301 (REG_SEQUENCE VGPR_32, (i16 (IMPLICIT_DEF)), lo16, $src1, hi16)), sub1)2302>;2303 2304def : GCNPat <2305 (fcopysign fp16vt:$src0, f32:$src1),2306 (EXTRACT_SUBREG (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)),2307 (REG_SEQUENCE VGPR_32, $src0, lo16, (i16 (IMPLICIT_DEF)), hi16),2308 (V_LSHRREV_B32_e64 (i32 16), $src1)), lo16)2309>;2310 2311def : GCNPat <2312 (fcopysign fp16vt:$src0, f64:$src1),2313 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)),2314 (REG_SEQUENCE VGPR_32, $src0, lo16, (i16 (IMPLICIT_DEF)), hi16),2315 (V_LSHRREV_B32_e64 (i32 16), (EXTRACT_SUBREG $src1, sub1)))2316>;2317}2318} // End foreach fp16vt = [f16, bf16]2319 2320 2321foreach fp16vt = [v2f16, v2bf16] in {2322 2323def : GCNPat <2324 (fcopysign fp16vt:$src0, fp16vt:$src1),2325 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fff7fff)), $src0, $src1)2326>;2327 2328}2329 2330/********** ================== **********/2331/********** Immediate Patterns **********/2332/********** ================== **********/2333 2334// FIXME: Remove VGPRImm. Should be inferrable from register bank.2335 2336foreach vt = Reg32Types.types in {2337 if !and(!not(vt.isVector), !not(vt.isFP)) then {2338 def : GCNPat <2339 (VGPRImm<(vt imm)>:$imm),2340 (V_MOV_B32_e32 imm:$imm)2341 >;2342 2343 def : GCNPat <2344 (vt imm:$imm),2345 (S_MOV_B32 imm:$imm)2346 >;2347 }2348}2349 2350// FIXME: The register bank of the frame index should depend on the2351// users, and transitive users of the add. We may require an2352// unnecessary copy from SGPR to VGPR.2353def : GCNPat <2354 (VGPRImm<(p5 frameindex)>:$fi),2355 (V_MOV_B32_e32 (p5 (frameindex_to_targetframeindex $fi)))2356>;2357 2358def : GCNPat <2359 (p5 frameindex:$fi),2360 (S_MOV_B32 (p5 (frameindex_to_targetframeindex $fi)))2361>;2362 2363def : GCNPat <2364 (VGPRImm<(SIlds tglobaladdr:$ga)>),2365 (V_MOV_B32_e32 $ga)2366>;2367 2368def : GCNPat <2369 (SIlds tglobaladdr:$ga),2370 (S_MOV_B32 $ga)2371>;2372 2373let True16Predicate = NotUseRealTrue16Insts in {2374 def : GCNPat <2375 (VGPRImm<(i16 imm)>:$imm),2376 (V_MOV_B32_e32 imm:$imm)2377 >;2378 2379 // FIXME: Workaround for ordering issue with peephole optimizer where2380 // a register class copy interferes with immediate folding. Should2381 // use s_mov_b32, which can be shrunk to s_movk_i322382 2383 foreach vt = [f16, bf16] in {2384 def : GCNPat <2385 (VGPRImm<(vt fpimm)>:$imm),2386 (V_MOV_B32_e32 (vt (bitcast_fpimm_to_i32 $imm)))2387 >;2388 }2389}2390 2391let True16Predicate = UseRealTrue16Insts in {2392 def : GCNPat <2393 (VGPRImm<(i16 imm)>:$imm),2394 (V_MOV_B16_t16_e64 0, imm:$imm, 0)2395 >;2396 2397 foreach vt = [f16, bf16] in {2398 def : GCNPat <2399 (VGPRImm<(vt fpimm)>:$imm),2400 (V_MOV_B16_t16_e64 0, (vt (bitcast_fpimm_to_i16 $imm)), 0)2401 >;2402 }2403}2404 2405/// FIXME: Increasing the priority of VGPRImm over the scalar forms as2406/// a workaround for a phase ordering problem caused by overly2407/// conservative MachineCSE. If we end up with an s_mov_b64 + copy to2408/// vgpr pattern, MachineCSE will not perform the CSE which occurs2409/// after operand folding.2410let AddedComplexity = 1 in {2411 // V_MOV_B64_PSEUDO and S_MOV_B64_IMM_PSEUDO can be used with any 64-bit2412 // immediate and wil be expanded as needed, but we will only use these patterns2413 // for values which can be encoded.2414 def : GCNPat <2415 (VGPRImm<(i64 imm)>:$imm),2416 (V_MOV_B64_PSEUDO imm:$imm)>;2417 2418 def : GCNPat <2419 (VGPRImm<(f64 fpimm)>:$imm),2420 (V_MOV_B64_PSEUDO (f64 (bitcast_fpimm_to_i64 $imm)))2421 >;2422} // End let AddedComplexity = 22423 2424def : GCNPat <2425 (i64 imm:$imm),2426 (S_MOV_B64_IMM_PSEUDO imm:$imm)2427>;2428 2429def : GCNPat <2430 (f64 fpimm:$imm),2431 (S_MOV_B64_IMM_PSEUDO (i64 (bitcast_fpimm_to_i64 fpimm:$imm)))2432>;2433 2434def : GCNPat <2435 (f32 fpimm:$imm),2436 (S_MOV_B32 (f32 (bitcast_fpimm_to_i32 $imm)))2437>;2438 2439def : GCNPat <2440 (f16 fpimm:$imm),2441 (S_MOV_B32 (i32 (bitcast_fpimm_to_i32 $imm)))2442>;2443 2444def : GCNPat <2445 (VGPRImm<(bf16 fpimm)>:$imm),2446 (V_MOV_B32_e32 (bf16 (bitcast_fpimm_to_i32 $imm)))2447>;2448 2449def : GCNPat <2450 (bf16 fpimm:$imm),2451 (S_MOV_B32 (i32 (bitcast_fpimm_to_i32 $imm)))2452>;2453 2454def : GCNPat <2455 (VGPRImm<(f32 fpimm)>:$imm),2456 (V_MOV_B32_e32 (f32 (bitcast_fpimm_to_i32 $imm)))2457>;2458 2459def : GCNPat <2460 (f32 fpimm:$imm),2461 (S_MOV_B32 (f32 (bitcast_fpimm_to_i32 $imm)))2462>;2463 2464foreach vt = [i64, p1, p0, p4] in { // FIXME: Should accept arbitrary addrspace2465 def : GCNPat <2466 (VGPRImm<(vt imm)>:$imm),2467 (V_MOV_B64_PSEUDO imm:$imm)2468 >;2469 2470 def : GCNPat <2471 (vt InlineImm64:$imm),2472 (S_MOV_B64 InlineImm64:$imm)2473 >;2474 2475 def : GCNPat <2476 (vt imm:$imm),2477 (S_MOV_B64_IMM_PSEUDO imm:$imm)2478 >;2479}2480 2481def : GCNPat <2482 (VGPRImm<(f64 fpimm)>:$imm),2483 (V_MOV_B64_PSEUDO (f64 (bitcast_fpimm_to_i64 $imm)))2484>;2485 2486// V_MOV_B64_PSEUDO and S_MOV_B64_IMM_PSEUDO can be used with any 64-bit2487// immediate and wil be expanded as needed, but we will only use these patterns2488// for values which can be encoded.2489def : GCNPat <2490 (f64 InlineImmFP64:$imm),2491 (S_MOV_B64 (i64 (bitcast_fpimm_to_i64 $imm)))2492>;2493 2494def : GCNPat <2495 (f64 fpimm:$imm),2496 (S_MOV_B64_IMM_PSEUDO (i64 (bitcast_fpimm_to_i64 fpimm:$imm)))2497>;2498 2499// Set to sign-extended 64-bit value (true = -1, false = 0)2500def : GCNPat <(i1 imm:$imm),2501 (S_MOV_B64 imm:$imm)> {2502 let WaveSizePredicate = isWave64;2503}2504 2505def : GCNPat <(i1 imm:$imm),2506 (S_MOV_B32 imm:$imm)> {2507 let WaveSizePredicate = isWave32;2508}2509 2510let True16Predicate = UseRealTrue16Insts in2511def : GCNPat <2512 (i32 (DivergentBinFrag<srl> VGPR_32:$src, (i32 16))),2513 (REG_SEQUENCE VGPR_32, (i16 (EXTRACT_SUBREG $src, hi16)), lo16, (V_MOV_B16_t16_e64 0, (i16 0x0000), 0), hi16)2514>;2515 2516/********** ================== **********/2517/********** Intrinsic Patterns **********/2518/********** ================== **********/2519 2520def : GCNPat <2521 (f32 (fpow (VOP3Mods f32:$src0, i32:$src0_mods), (VOP3Mods f32:$src1, i32:$src1_mods))),2522 (V_EXP_F32_e64 SRCMODS.NONE, (V_MUL_LEGACY_F32_e64 $src1_mods, $src1, SRCMODS.NONE, (V_LOG_F32_e64 $src0_mods, $src0), 0, 0))2523>;2524 2525def : GCNPat <2526 (i32 (sext i1:$src0)),2527 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),2528 /*src1mod*/(i32 0), /*src1*/(i32 -1), i1:$src0)2529>;2530 2531class Ext32Pat <SDNode ext> : GCNPat <2532 (i32 (ext i1:$src0)),2533 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),2534 /*src1mod*/(i32 0), /*src1*/(i32 1), i1:$src0)2535>;2536 2537def : Ext32Pat <zext>;2538def : Ext32Pat <anyext>;2539 2540// The multiplication scales from [0,1) to the unsigned integer range,2541// rounding down a bit to avoid unwanted overflow.2542def : GCNPat <2543 (AMDGPUurecip i32:$src0),2544 (V_CVT_U32_F32_e322545 (V_MUL_F32_e32 (i32 CONST.FP_4294966784),2546 (V_RCP_IFLAG_F32_e32 (V_CVT_F32_U32_e32 $src0))))2547>;2548 2549//===----------------------------------------------------------------------===//2550// VOP3 Patterns2551//===----------------------------------------------------------------------===//2552 2553def : IMad24Pat<V_MAD_I32_I24_e64, 1>;2554def : UMad24Pat<V_MAD_U32_U24_e64, 1>;2555 2556// BFI patterns2557 2558def BFIImm32 : PatFrag<2559 (ops node:$x, node:$y, node:$z),2560 (i32 (DivergentBinFrag<or> (and node:$y, node:$x), (and node:$z, imm))),2561 [{2562 auto *X = dyn_cast<ConstantSDNode>(N->getOperand(0)->getOperand(1));2563 auto *NotX = dyn_cast<ConstantSDNode>(N->getOperand(1)->getOperand(1));2564 return X && NotX &&2565 ~(unsigned)X->getZExtValue() == (unsigned)NotX->getZExtValue();2566 }]2567>;2568 2569 2570// Definition from ISA doc:2571// (y & x) | (z & ~x)2572def : AMDGPUPatIgnoreCopies <2573 (DivergentBinFrag<or> (and i32:$y, i32:$x), (and i32:$z, (not i32:$x))),2574 (V_BFI_B32_e64 (COPY_TO_REGCLASS VSrc_b32:$x, VGPR_32),2575 (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32),2576 (COPY_TO_REGCLASS VSrc_b32:$z, VGPR_32))2577>;2578 2579// (y & C) | (z & ~C)2580def : AMDGPUPatIgnoreCopies <2581 (BFIImm32 i32:$x, i32:$y, i32:$z),2582 (V_BFI_B32_e64 VSrc_b32:$x, VSrc_b32:$y, VSrc_b32:$z)2583>;2584 2585// 64-bit version2586def : AMDGPUPatIgnoreCopies <2587 (DivergentBinFrag<or> (and i64:$y, i64:$x), (and i64:$z, (not i64:$x))),2588 (REG_SEQUENCE VReg_64,2589 (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub0)),2590 (i32 (EXTRACT_SUBREG VReg_64:$y, sub0)),2591 (i32 (EXTRACT_SUBREG VReg_64:$z, sub0))), sub0,2592 (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub1)),2593 (i32 (EXTRACT_SUBREG VReg_64:$y, sub1)),2594 (i32 (EXTRACT_SUBREG VReg_64:$z, sub1))), sub1)2595>;2596 2597// (z & ~x)2598def : AMDGPUPatIgnoreCopies <2599 (DivergentBinFrag<and> i32:$z, (not_oneuse i32:$x)),2600 (V_BFI_B32_e64 VSrc_b32:$x, (i32 0), VSrc_b32:$z)2601>;2602 2603// 64-bit version2604def : AMDGPUPatIgnoreCopies <2605 (DivergentBinFrag<and> i64:$z, (not_oneuse i64:$x)),2606 (REG_SEQUENCE VReg_64,2607 (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub0)), (i32 0),2608 (i32 (EXTRACT_SUBREG VReg_64:$z, sub0))), sub0,2609 (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub1)), (i32 0),2610 (i32 (EXTRACT_SUBREG VReg_64:$z, sub1))), sub1)2611>;2612 2613// (y | ~x)2614def : AMDGPUPatIgnoreCopies <2615 (DivergentBinFrag<or> i32:$y, (not_oneuse i32:$x)),2616 (V_BFI_B32_e64 VSrc_b32:$x, VSrc_b32:$y, (i32 -1))2617>;2618 2619// 64-bit version2620def : AMDGPUPatIgnoreCopies <2621 (DivergentBinFrag<or> i64:$y, (not_oneuse i64:$x)),2622 (REG_SEQUENCE VReg_64,2623 (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub0)),2624 (i32 (EXTRACT_SUBREG VReg_64:$y, sub0)), (i32 -1)), sub0,2625 (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub1)),2626 (i32 (EXTRACT_SUBREG VReg_64:$y, sub1)), (i32 -1)), sub1)2627>;2628 2629// SHA-256 Ch function2630// z ^ (x & (y ^ z))2631def : AMDGPUPatIgnoreCopies <2632 (DivergentBinFrag<xor> i32:$z, (and i32:$x, (xor i32:$y, i32:$z))),2633 (V_BFI_B32_e64 (COPY_TO_REGCLASS VSrc_b32:$x, VGPR_32),2634 (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32),2635 (COPY_TO_REGCLASS VSrc_b32:$z, VGPR_32))2636>;2637 2638foreach vt = [i64, v2i32] in {2639def : AMDGPUPatIgnoreCopies <2640 (DivergentBinFrag<xor> vt:$z, (and vt:$x, (xor vt:$y, vt:$z))),2641 (REG_SEQUENCE VReg_64,2642 (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub0)),2643 (i32 (EXTRACT_SUBREG VReg_64:$y, sub0)),2644 (i32 (EXTRACT_SUBREG VReg_64:$z, sub0))), sub0,2645 (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub1)),2646 (i32 (EXTRACT_SUBREG VReg_64:$y, sub1)),2647 (i32 (EXTRACT_SUBREG VReg_64:$z, sub1))), sub1)2648>;2649}2650 2651def : AMDGPUPat <2652 (fcopysign f32:$src0, f32:$src1),2653 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), $src0, $src1)2654>;2655 2656def : AMDGPUPat <2657 (fcopysign f32:$src0, f64:$src1),2658 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), $src0,2659 (i32 (EXTRACT_SUBREG SReg_64:$src1, sub1)))2660>;2661 2662def : AMDGPUPat <2663 (fcopysign f64:$src0, f64:$src1),2664 (REG_SEQUENCE SReg_64,2665 (i32 (EXTRACT_SUBREG $src0, sub0)), sub0,2666 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)),2667 (i32 (EXTRACT_SUBREG SReg_64:$src0, sub1)),2668 (i32 (EXTRACT_SUBREG SReg_64:$src1, sub1))), sub1)2669>;2670 2671def : AMDGPUPat <2672 (fcopysign f64:$src0, f32:$src1),2673 (REG_SEQUENCE SReg_64,2674 (i32 (EXTRACT_SUBREG $src0, sub0)), sub0,2675 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)),2676 (i32 (EXTRACT_SUBREG SReg_64:$src0, sub1)),2677 $src1), sub1)2678>;2679 2680let True16Predicate = NotHasTrue16BitInsts in {2681let SubtargetPredicate = isNotGFX9Plus in {2682def : ROTRPattern <V_ALIGNBIT_B32_e64>;2683 2684def : GCNPat<(i32 (DivergentUnaryFrag<trunc> (srl i64:$src0, (and i32:$src1, (i32 31))))),2685 (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG (i64 $src0), sub1)),2686 (i32 (EXTRACT_SUBREG (i64 $src0), sub0)), $src1)>;2687 2688def : GCNPat<(i32 (DivergentUnaryFrag<trunc> (srl i64:$src0, (i32 ShiftAmt32Imm:$src1)))),2689 (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG (i64 $src0), sub1)),2690 (i32 (EXTRACT_SUBREG (i64 $src0), sub0)), $src1)>;2691} // isNotGFX9Plus2692 2693let SubtargetPredicate = isGFX9GFX10 in {2694def : GCNPat <2695 (rotr i32:$src0, i32:$src1),2696 (V_ALIGNBIT_B32_opsel_e64 /* src0_modifiers */ 0, $src0,2697 /* src1_modifiers */ 0, $src0,2698 /* src2_modifiers */ 0,2699 $src1, /* clamp */ 0, /* op_sel */ 0)2700>;2701 2702foreach pat = [(i32 (DivergentUnaryFrag<trunc> (srl i64:$src0, (and i32:$src1, (i32 31))))),2703 (i32 (DivergentUnaryFrag<trunc> (srl i64:$src0, (i32 ShiftAmt32Imm:$src1))))] in2704def : GCNPat<pat,2705 (V_ALIGNBIT_B32_opsel_e64 0, /* src0_modifiers */2706 (i32 (EXTRACT_SUBREG (i64 $src0), sub1)),2707 0, /* src1_modifiers */2708 (i32 (EXTRACT_SUBREG (i64 $src0), sub0)),2709 0, /* src2_modifiers */2710 $src1, /* clamp */ 0, /* op_sel */ 0)2711>;2712 2713def : GCNPat<(fshr i32:$src0, i32:$src1, i32:$src2),2714 (V_ALIGNBIT_B32_opsel_e64 /* src0_modifiers */ 0, $src0,2715 /* src1_modifiers */ 0, $src1,2716 /* src2_modifiers */ 0,2717 $src2, /* clamp */ 0, /* op_sel */ 0)2718>;2719} // isGFX9GFX102720} // end True16Predicate = NotHasTrue16BitInsts2721 2722let True16Predicate = UseRealTrue16Insts in {2723def : GCNPat <2724 (rotr i32:$src0, i32:$src1),2725 (V_ALIGNBIT_B32_t16_e64 /* src0_modifiers */ 0, $src0,2726 /* src1_modifiers */ 0, $src0,2727 /* src2_modifiers */ 0,2728 (EXTRACT_SUBREG $src1, lo16),2729 /* clamp */ 0, /* op_sel */ 0)2730>;2731 2732def : GCNPat<(i32 (DivergentUnaryFrag<trunc> (srl i64:$src0, (i32 ShiftAmt32Imm:$src1)))),2733 (V_ALIGNBIT_B32_t16_e64 0, /* src0_modifiers */2734 (i32 (EXTRACT_SUBREG (i64 $src0), sub1)),2735 0, /* src1_modifiers */2736 (i32 (EXTRACT_SUBREG (i64 $src0), sub0)),2737 0, /* src2_modifiers */2738 (i16 (EXTRACT_SUBREG VGPR_32:$src1, lo16)),2739 /* clamp */ 0, /* op_sel */ 0)>;2740 2741def : GCNPat<(fshr i32:$src0, i32:$src1, i32:$src2),2742 (V_ALIGNBIT_B32_t16_e64 /* src0_modifiers */ 0, $src0,2743 /* src1_modifiers */ 0, $src1,2744 /* src2_modifiers */ 0,2745 (EXTRACT_SUBREG VGPR_32:$src2, lo16),2746 /* clamp */ 0, /* op_sel */ 0)>;2747} // end True16Predicate = UseRealTrue16Insts2748 2749let True16Predicate = UseFakeTrue16Insts in {2750def : GCNPat <2751 (rotr i32:$src0, i32:$src1),2752 (V_ALIGNBIT_B32_fake16_e64 /* src0_modifiers */ 0, $src0,2753 /* src1_modifiers */ 0, $src0,2754 /* src2_modifiers */ 0,2755 $src1, /* clamp */ 0, /* op_sel */ 0)2756>;2757 2758def : GCNPat<(i32 (DivergentUnaryFrag<trunc> (srl i64:$src0, (and i32:$src1, (i32 31))))),2759 (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */2760 (i32 (EXTRACT_SUBREG (i64 $src0), sub1)),2761 0, /* src1_modifiers */2762 (i32 (EXTRACT_SUBREG (i64 $src0), sub0)),2763 0, /* src2_modifiers */2764 $src1, /* clamp */ 0, /* op_sel */ 0)2765>;2766 2767def : GCNPat<(i32 (DivergentUnaryFrag<trunc> (srl i64:$src0, (i32 ShiftAmt32Imm:$src1)))),2768 (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */2769 (i32 (EXTRACT_SUBREG (i64 $src0), sub1)),2770 0, /* src1_modifiers */2771 (i32 (EXTRACT_SUBREG (i64 $src0), sub0)),2772 0, /* src2_modifiers */2773 $src1, /* clamp */ 0, /* op_sel */ 0)2774>;2775 2776def : GCNPat<(fshr i32:$src0, i32:$src1, i32:$src2),2777 (V_ALIGNBIT_B32_fake16_e64 /* src0_modifiers */ 0, $src0,2778 /* src1_modifiers */ 0, $src1,2779 /* src2_modifiers */ 0,2780 $src2, /* clamp */ 0, /* op_sel */ 0)2781>;2782} // end True16Predicate = UseFakeTrue16Insts2783 2784/********** ====================== **********/2785/********** Indirect addressing **********/2786/********** ====================== **********/2787 2788multiclass SI_INDIRECT_Pattern <ValueType vt, ValueType eltvt, string VecSize> {2789 // Extract with offset2790 def : GCNPat<2791 (eltvt (extractelt vt:$src, (MOVRELOffset i32:$idx, (i32 imm:$offset)))),2792 (!cast<Instruction>("SI_INDIRECT_SRC_"#VecSize) $src, $idx, imm:$offset)2793 >;2794 2795 // Insert with offset2796 def : GCNPat<2797 (insertelt vt:$src, eltvt:$val, (MOVRELOffset i32:$idx, (i32 imm:$offset))),2798 (!cast<Instruction>("SI_INDIRECT_DST_"#VecSize) $src, $idx, imm:$offset, $val)2799 >;2800}2801 2802defm : SI_INDIRECT_Pattern <v2f32, f32, "V2">;2803defm : SI_INDIRECT_Pattern <v4f32, f32, "V4">;2804defm : SI_INDIRECT_Pattern <v8f32, f32, "V8">;2805defm : SI_INDIRECT_Pattern <v9f32, f32, "V9">;2806defm : SI_INDIRECT_Pattern <v10f32, f32, "V10">;2807defm : SI_INDIRECT_Pattern <v11f32, f32, "V11">;2808defm : SI_INDIRECT_Pattern <v12f32, f32, "V12">;2809defm : SI_INDIRECT_Pattern <v16f32, f32, "V16">;2810defm : SI_INDIRECT_Pattern <v32f32, f32, "V32">;2811 2812defm : SI_INDIRECT_Pattern <v2i32, i32, "V2">;2813defm : SI_INDIRECT_Pattern <v4i32, i32, "V4">;2814defm : SI_INDIRECT_Pattern <v8i32, i32, "V8">;2815defm : SI_INDIRECT_Pattern <v9i32, i32, "V9">;2816defm : SI_INDIRECT_Pattern <v10i32, i32, "V10">;2817defm : SI_INDIRECT_Pattern <v11i32, i32, "V11">;2818defm : SI_INDIRECT_Pattern <v12i32, i32, "V12">;2819defm : SI_INDIRECT_Pattern <v16i32, i32, "V16">;2820defm : SI_INDIRECT_Pattern <v32i32, i32, "V32">;2821 2822//===----------------------------------------------------------------------===//2823// SAD Patterns2824//===----------------------------------------------------------------------===//2825 2826def : GCNPat <2827 (add (sub_oneuse (umax i32:$src0, i32:$src1),2828 (umin i32:$src0, i32:$src1)),2829 i32:$src2),2830 (V_SAD_U32_e64 $src0, $src1, $src2, (i1 0))2831>;2832 2833def : GCNPat <2834 (add (select_oneuse (i1 (setugt i32:$src0, i32:$src1)),2835 (sub i32:$src0, i32:$src1),2836 (sub i32:$src1, i32:$src0)),2837 i32:$src2),2838 (V_SAD_U32_e64 $src0, $src1, $src2, (i1 0))2839>;2840 2841//===----------------------------------------------------------------------===//2842// Conversion Patterns2843//===----------------------------------------------------------------------===//2844def : GCNPat<(i32 (UniformSextInreg<i1> i32:$src)),2845 (S_BFE_I32 i32:$src, (i32 65536))>; // 0 | 1 << 162846 2847// Handle sext_inreg in i642848def : GCNPat <2849 (i64 (UniformSextInreg<i1> i64:$src)),2850 (S_BFE_I64 i64:$src, (i32 0x10000)) // 0 | 1 << 162851>;2852 2853def : GCNPat <2854 (i16 (UniformSextInreg<i1> i16:$src)),2855 (S_BFE_I32 $src, (i32 0x00010000)) // 0 | 1 << 162856>;2857 2858def : GCNPat <2859 (i16 (UniformSextInreg<i8> i16:$src)),2860 (S_BFE_I32 $src, (i32 0x80000)) // 0 | 8 << 162861>;2862 2863def : GCNPat <2864 (i64 (UniformSextInreg<i8> i64:$src)),2865 (S_BFE_I64 i64:$src, (i32 0x80000)) // 0 | 8 << 162866>;2867 2868def : GCNPat <2869 (i64 (UniformSextInreg<i16> i64:$src)),2870 (S_BFE_I64 i64:$src, (i32 0x100000)) // 0 | 16 << 162871>;2872 2873def : GCNPat <2874 (i64 (UniformSextInreg<i32> i64:$src)),2875 (S_BFE_I64 i64:$src, (i32 0x200000)) // 0 | 32 << 162876>;2877 2878def : GCNPat<2879 (i32 (DivergentSextInreg<i1> i32:$src)),2880 (V_BFE_I32_e64 i32:$src, (i32 0), (i32 1))>;2881 2882let True16Predicate = NotUseRealTrue16Insts in {2883def : GCNPat <2884 (i16 (DivergentSextInreg<i1> i16:$src)),2885 (V_BFE_I32_e64 $src, (i32 0), (i32 1))2886>;2887 2888def : GCNPat <2889 (i16 (DivergentSextInreg<i8> i16:$src)),2890 (V_BFE_I32_e64 $src, (i32 0), (i32 8))2891>;2892}2893 2894let True16Predicate = UseRealTrue16Insts in {2895def : GCNPat <2896 (i16 (DivergentSextInreg<i1> i16:$src)),2897 (V_BFE_I32_e642898 (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (i16 (IMPLICIT_DEF)), hi16),2899 (i32 0), (i32 1))2900>;2901 2902def : GCNPat <2903 (i16 (DivergentSextInreg<i8> i16:$src)),2904 (V_BFE_I32_e642905 (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (i16 (IMPLICIT_DEF)), hi16),2906 (i32 0), (i32 8))2907>;2908}2909 2910def : GCNPat<2911 (i32 (DivergentSextInreg<i8> i32:$src)),2912 (V_BFE_I32_e64 i32:$src, (i32 0), (i32 8))2913>;2914 2915def : GCNPat <2916 (i32 (DivergentSextInreg<i16> i32:$src)),2917 (V_BFE_I32_e64 $src, (i32 0), (i32 16))2918>;2919 2920def : GCNPat <2921 (i64 (DivergentSextInreg<i1> i64:$src)),2922 (REG_SEQUENCE VReg_64,2923 (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 1)), sub0,2924 (V_ASHRREV_I32_e32 (i32 31), (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 1))), sub1)2925>;2926 2927def : GCNPat <2928 (i64 (DivergentSextInreg<i8> i64:$src)),2929 (REG_SEQUENCE VReg_64,2930 (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 8)), sub0,2931 (V_ASHRREV_I32_e32 (i32 31), (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 8))), sub1)2932>;2933 2934def : GCNPat <2935 (i64 (DivergentSextInreg<i16> i64:$src)),2936 (REG_SEQUENCE VReg_64,2937 (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 16)), sub0,2938 (V_ASHRREV_I32_e32 (i32 31), (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 16))), sub1)2939>;2940 2941def : GCNPat <2942 (i64 (DivergentSextInreg<i32> i64:$src)),2943 (REG_SEQUENCE VReg_64,2944 (i32 (EXTRACT_SUBREG i64:$src, sub0)), sub0,2945 (V_ASHRREV_I32_e32 (i32 31), (i32 (EXTRACT_SUBREG i64:$src, sub0))), sub1)2946>;2947 2948def : GCNPat <2949 (i64 (UniformUnaryFrag<zext> i32:$src)),2950 (REG_SEQUENCE SReg_64, $src, sub0, (S_MOV_B32 (i32 0)), sub1)2951>;2952 2953def : GCNPat <2954 (i64 (zext i32:$src)),2955 (REG_SEQUENCE VReg_64, $src, sub0, (V_MOV_B32_e32 (i32 0)), sub1)2956>;2957 2958def : GCNPat <2959 (i64 (UniformUnaryFrag<anyext> i32:$src)),2960 (REG_SEQUENCE SReg_64, $src, sub0, (i32 (IMPLICIT_DEF)), sub1)2961>;2962 2963def : GCNPat <2964 (i64 (anyext i32:$src)),2965 (REG_SEQUENCE VReg_64, $src, sub0, (i32 (IMPLICIT_DEF)), sub1)2966>;2967 2968class ZExt_i64_i1_Pat <SDNode ext> : GCNPat <2969 (i64 (ext i1:$src)),2970 (REG_SEQUENCE VReg_64,2971 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),2972 /*src1mod*/(i32 0), /*src1*/(i32 1), $src),2973 sub0, (S_MOV_B32 (i32 0)), sub1)2974>;2975 2976 2977def : ZExt_i64_i1_Pat<zext>;2978def : ZExt_i64_i1_Pat<anyext>;2979 2980// FIXME: We need to use COPY_TO_REGCLASS to work-around the fact that2981// REG_SEQUENCE patterns don't support instructions with multiple outputs.2982def : GCNPat <2983 (i64 (UniformUnaryFrag<sext> i32:$src)),2984 (REG_SEQUENCE SReg_64, $src, sub0,2985 (i32 (COPY_TO_REGCLASS (S_ASHR_I32 $src, (i32 31)), SReg_32_XM0)), sub1)2986>;2987 2988def : GCNPat <2989 (i64 (DivergentUnaryFrag<sext> i32:$src)),2990 (REG_SEQUENCE VReg_64, $src, sub0,2991 (i32 (COPY_TO_REGCLASS (V_ASHRREV_I32_e64 (i32 31), $src), VGPR_32)), sub1)2992>;2993 2994def : GCNPat <2995 (i64 (sext i1:$src)),2996 (REG_SEQUENCE VReg_64,2997 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),2998 /*src1mod*/(i32 0), /*src1*/(i32 -1), $src), sub0,2999 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),3000 /*src1mod*/(i32 0), /*src1*/(i32 -1), $src), sub1)3001>;3002 3003class FPToI1Pat<Instruction Inst, int KOne, ValueType kone_type, ValueType vt, SDPatternOperator fp_to_int> : GCNPat <3004 (i1 (fp_to_int (vt (VOP3Mods vt:$src0, i32:$src0_modifiers)))),3005 (i1 (Inst 0, (kone_type KOne), $src0_modifiers, $src0, DSTCLAMP.NONE))3006>;3007 3008let True16Predicate = NotHasTrue16BitInsts in {3009 def : FPToI1Pat<V_CMP_EQ_F16_e64, CONST.FP16_ONE, i16, f16, fp_to_uint>;3010 def : FPToI1Pat<V_CMP_EQ_F16_e64, CONST.FP16_NEG_ONE, i16, f16, fp_to_sint>;3011} // end True16Predicate = NotHasTrue16BitInsts3012 3013let True16Predicate = UseRealTrue16Insts in {3014 def : FPToI1Pat<V_CMP_EQ_F16_t16_e64, CONST.FP16_ONE, i16, f16, fp_to_uint>;3015 def : FPToI1Pat<V_CMP_EQ_F16_t16_e64, CONST.FP16_NEG_ONE, i16, f16, fp_to_sint>;3016} // end True16Predicate = UseRealTrue16BitInsts3017 3018let True16Predicate = UseFakeTrue16Insts in {3019 def : FPToI1Pat<V_CMP_EQ_F16_fake16_e64, CONST.FP16_ONE, i16, f16, fp_to_uint>;3020 def : FPToI1Pat<V_CMP_EQ_F16_fake16_e64, CONST.FP16_NEG_ONE, i16, f16, fp_to_sint>;3021} // end True16Predicate = UseFakeTrue16BitInsts3022 3023def : FPToI1Pat<V_CMP_EQ_F32_e64, CONST.FP32_ONE, i32, f32, fp_to_uint>;3024def : FPToI1Pat<V_CMP_EQ_F32_e64, CONST.FP32_NEG_ONE, i32, f32, fp_to_sint>;3025def : FPToI1Pat<V_CMP_EQ_F64_e64, CONST.FP64_ONE, i64, f64, fp_to_uint>;3026def : FPToI1Pat<V_CMP_EQ_F64_e64, CONST.FP64_NEG_ONE, i64, f64, fp_to_sint>;3027 3028// If we need to perform a logical operation on i1 values, we need to3029// use vector comparisons since there is only one SCC register. Vector3030// comparisons may write to a pair of SGPRs or a single SGPR, so treat3031// these as 32 or 64-bit comparisons. When legalizing SGPR copies,3032// instructions resulting in the copies from SCC to these instructions3033// will be moved to the VALU.3034 3035let WaveSizePredicate = isWave64 in {3036def : GCNPat <3037 (i1 (and i1:$src0, i1:$src1)),3038 (S_AND_B64 $src0, $src1)3039>;3040 3041def : GCNPat <3042 (i1 (or i1:$src0, i1:$src1)),3043 (S_OR_B64 $src0, $src1)3044>;3045 3046def : GCNPat <3047 (i1 (xor i1:$src0, i1:$src1)),3048 (S_XOR_B64 $src0, $src1)3049>;3050 3051def : GCNPat <3052 (i1 (add i1:$src0, i1:$src1)),3053 (S_XOR_B64 $src0, $src1)3054>;3055 3056def : GCNPat <3057 (i1 (sub i1:$src0, i1:$src1)),3058 (S_XOR_B64 $src0, $src1)3059>;3060 3061let AddedComplexity = 1 in {3062def : GCNPat <3063 (i1 (add i1:$src0, (i1 -1))),3064 (S_NOT_B64 $src0)3065>;3066 3067def : GCNPat <3068 (i1 (sub i1:$src0, (i1 -1))),3069 (S_NOT_B64 $src0)3070>;3071}3072} // end isWave643073 3074let WaveSizePredicate = isWave32 in {3075def : GCNPat <3076 (i1 (and i1:$src0, i1:$src1)),3077 (S_AND_B32 $src0, $src1)3078>;3079 3080def : GCNPat <3081 (i1 (or i1:$src0, i1:$src1)),3082 (S_OR_B32 $src0, $src1)3083>;3084 3085def : GCNPat <3086 (i1 (xor i1:$src0, i1:$src1)),3087 (S_XOR_B32 $src0, $src1)3088>;3089 3090def : GCNPat <3091 (i1 (add i1:$src0, i1:$src1)),3092 (S_XOR_B32 $src0, $src1)3093>;3094 3095def : GCNPat <3096 (i1 (sub i1:$src0, i1:$src1)),3097 (S_XOR_B32 $src0, $src1)3098>;3099 3100let AddedComplexity = 1 in {3101def : GCNPat <3102 (i1 (add i1:$src0, (i1 -1))),3103 (S_NOT_B32 $src0)3104>;3105 3106def : GCNPat <3107 (i1 (sub i1:$src0, (i1 -1))),3108 (S_NOT_B32 $src0)3109>;3110}3111} // end isWave323112 3113def : GCNPat <3114 (i32 (DivergentBinFrag<xor> i32:$src0, (i32 -1))),3115 (V_NOT_B32_e32 $src0)3116>;3117 3118def : GCNPat <3119 (i64 (DivergentBinFrag<xor> i64:$src0, (i64 -1))),3120 (REG_SEQUENCE VReg_64,3121 (V_NOT_B32_e32 (i32 (EXTRACT_SUBREG i64:$src0, sub0))), sub0,3122 (V_NOT_B32_e32 (i32 (EXTRACT_SUBREG i64:$src0, sub1))), sub13123 )3124>;3125 3126let SubtargetPredicate = NotHasTrue16BitInsts in3127def : GCNPat <3128 (f16 (sint_to_fp i1:$src)),3129 (V_CVT_F16_F32_e32 (3130 V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),3131 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_NEG_ONE),3132 SSrc_i1:$src))3133>;3134 3135let True16Predicate = UseRealTrue16Insts in3136def : GCNPat <3137 (f16 (sint_to_fp i1:$src)),3138 (V_CVT_F16_F32_t16_e64 /*src0_modifiers*/ 0,3139 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),3140 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_NEG_ONE),3141 SSrc_i1:$src),3142 /*clamp*/ 0, /*omod*/ 0, /*op_sel*/ 0)3143>;3144 3145let True16Predicate = UseFakeTrue16Insts in3146def : GCNPat <3147 (f16 (sint_to_fp i1:$src)),3148 (V_CVT_F16_F32_fake16_e64 /*src0_modifiers*/ 0,3149 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),3150 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_NEG_ONE),3151 SSrc_i1:$src),3152 /*clamp*/ 0, /*omod*/ 0)3153>;3154 3155let True16Predicate = NotHasTrue16BitInsts in3156def : GCNPat <3157 (f16 (uint_to_fp i1:$src)),3158 (V_CVT_F16_F32_e32 (3159 V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),3160 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_ONE),3161 SSrc_i1:$src))3162>;3163 3164let True16Predicate = UseRealTrue16Insts in3165def : GCNPat <3166 (f16 (uint_to_fp i1:$src)),3167 (V_CVT_F16_F32_t16_e64 /*src0_modifiers*/ 0,3168 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),3169 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_ONE),3170 SSrc_i1:$src),3171 /*clamp*/ 0, /*omod*/ 0, /*op_sel*/ 0)3172>;3173 3174let True16Predicate = UseFakeTrue16Insts in3175def : GCNPat <3176 (f16 (uint_to_fp i1:$src)),3177 (V_CVT_F16_F32_fake16_e64 /*src0_modifiers*/ 0,3178 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),3179 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_ONE),3180 SSrc_i1:$src),3181 /*clamp*/ 0, /*omod*/ 0)3182>;3183 3184def : GCNPat <3185 (f32 (sint_to_fp i1:$src)),3186 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),3187 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_NEG_ONE),3188 SSrc_i1:$src)3189>;3190 3191def : GCNPat <3192 (f32 (uint_to_fp i1:$src)),3193 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),3194 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_ONE),3195 SSrc_i1:$src)3196>;3197 3198def : GCNPat <3199 (f64 (sint_to_fp i1:$src)),3200 (V_CVT_F64_I32_e32 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),3201 /*src1mod*/(i32 0), /*src1*/(i32 -1),3202 SSrc_i1:$src))3203>;3204 3205def : GCNPat <3206 (f64 (uint_to_fp i1:$src)),3207 (V_CVT_F64_U32_e32 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),3208 /*src1mod*/(i32 0), /*src1*/(i32 1),3209 SSrc_i1:$src))3210>;3211 3212//===----------------------------------------------------------------------===//3213// Miscellaneous Patterns3214//===----------------------------------------------------------------------===//3215 3216// Eliminate a zero extension from an fp16 operation if it already3217// zeros the high bits of the 32-bit register.3218//3219// This is complicated on gfx9+. Some instructions maintain the legacy3220// zeroing behavior, but others preserve the high bits. Some have a3221// control bit to change the behavior. We can't simply say with3222// certainty what the source behavior is without more context on how3223// the src is lowered. e.g. fptrunc + fma may be lowered to a3224// v_fma_mix* instruction which does not zero, or may not.3225def : GCNPat<3226 (i32 (DivergentUnaryFrag<abs> i32:$src)),3227 (V_MAX_I32_e64 (V_SUB_CO_U32_e32 (i32 0), $src), $src)>;3228 3229let AddedComplexity = 1 in {3230def : GCNPat<3231 (i32 (DivergentUnaryFrag<abs> i32:$src)),3232 (V_MAX_I32_e64 (V_SUB_U32_e32 (i32 0), $src), $src)>{3233 let SubtargetPredicate = HasAddNoCarryInsts;3234}3235} // AddedComplexity = 13236 3237let True16Predicate = NotUseRealTrue16Insts in {3238def : GCNPat<3239 (i32 (DivergentUnaryFrag<zext> i16:$src)),3240 (V_AND_B32_e64 (S_MOV_B32 (i32 0xffff)), $src)3241>;3242 3243def : GCNPat<3244 (i64 (DivergentUnaryFrag<zext> i16:$src)),3245 (REG_SEQUENCE VReg_64,3246 (V_AND_B32_e64 (S_MOV_B32 (i32 0xffff)), $src), sub0,3247 (S_MOV_B32 (i32 0)), sub1)3248>;3249 3250def : GCNPat<3251 (i32 (zext (i16 (bitconvert fp16_zeros_high_16bits:$src)))),3252 (COPY VSrc_b16:$src)3253>;3254 3255def : GCNPat <3256 (i1 (DivergentUnaryFrag<trunc> i16:$a)),3257 (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1), $a), (i32 1))3258>;3259}3260 3261let True16Predicate = UseRealTrue16Insts in {3262def : GCNPat<3263 (i32 (DivergentUnaryFrag<zext> i16:$src)),3264 (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (V_MOV_B16_t16_e64 0, (i16 0), 0), hi16)3265>;3266 3267def : GCNPat<3268 (i64 (DivergentUnaryFrag<zext> i16:$src)),3269 (REG_SEQUENCE VReg_64, $src, lo16, (V_MOV_B16_t16_e64 0, (i16 0), 0), hi16, (V_MOV_B32_e32 (i32 0)), sub1)3270>;3271 3272def : GCNPat<3273 (i32 (zext (i16 (bitconvert fp16_zeros_high_16bits:$src)))),3274 (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (V_MOV_B16_t16_e64 0, (i16 0), 0), hi16)3275>;3276 3277def : GCNPat <3278 (i1 (DivergentUnaryFrag<trunc> i16:$a)),3279 (V_CMP_EQ_U16_t16_e64 (i32 0), (V_AND_B16_t16_e64 (i32 0), (i16 1), (i32 0), $a), (i32 0), (i16 1), (i32 0))3280>;3281}3282 3283def : GCNPat <3284 (i32 (trunc i64:$a)),3285 (EXTRACT_SUBREG $a, sub0)3286>;3287 3288def : GCNPat <3289 (i1 (UniformUnaryFrag<trunc> i32:$a)),3290 (S_CMP_EQ_U32 (S_AND_B32 (i32 1), $a), (i32 1))3291>;3292 3293def : GCNPat <3294 (i1 (UniformUnaryFrag<trunc> i16:$a)),3295 (S_CMP_EQ_U32 (S_AND_B32 (i32 1), $a), (i32 1))3296>;3297 3298def : GCNPat <3299 (i1 (UniformUnaryFrag<trunc> i64:$a)),3300 (S_CMP_EQ_U32 (S_AND_B32 (i32 1),3301 (i32 (EXTRACT_SUBREG $a, sub0))), (i32 1))3302>;3303 3304def : GCNPat <3305 (i1 (DivergentUnaryFrag<trunc> i32:$a)),3306 (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1), $a), (i32 1))3307>;3308 3309def IMMBitSelConst : SDNodeXForm<imm, [{3310 return CurDAG->getTargetConstant(1ULL << N->getZExtValue(), SDLoc(N),3311 MVT::i32);3312}]>;3313 3314// Matching separate SRL and TRUNC instructions3315// with dependent operands (SRL dest is source of TRUNC)3316// generates three instructions. However, by using bit shifts,3317// the V_LSHRREV_B32_e64 result can be directly used in the3318// operand of the V_AND_B32_e64 instruction:3319// (trunc i32 (srl i32 $a, i32 $b)) ->3320// v_and_b32_e64 $a, (1 << $b), $a3321// v_cmp_ne_u32_e64 $a, 0, $a3322 3323// Handle the VALU case.3324def : GCNPat <3325 (i1 (xor (i1 (DivergentUnaryFrag<HasOneUseUnaryOp<trunc>> i32:$a)), -1)),3326 (V_CMP_NE_U32_e64 (V_AND_B32_e64 (i32 1), i32:$a), (i32 1))3327>;3328 3329def : GCNPat <3330 (i1 (xor (i1 (DivergentUnaryFrag<HasOneUseUnaryOp<trunc>> i64:$a)), -1)),3331 (V_CMP_NE_U32_e64 (V_AND_B32_e64 (i32 1),3332 (i32 (EXTRACT_SUBREG $a, sub0))), (i32 1))3333>;3334 3335def : GCNPat <3336 (i1 (DivergentUnaryFrag<trunc> (i32 (srl i32:$a, (i32 imm:$b))))),3337 (V_CMP_NE_U32_e64 (V_AND_B32_e64 (i32 (IMMBitSelConst $b)), $a),3338 (i32 0))3339>;3340 3341// Handle the scalar case.3342def : GCNPat <3343 (i1 (UniformUnaryFrag<trunc> (i32 (srl i32:$a, (i32 imm:$b))))),3344 (S_CMP_LG_U32 (S_AND_B32 (i32 (IMMBitSelConst $b)), $a),3345 (i32 0))3346>;3347 3348def : GCNPat <3349 (i1 (DivergentUnaryFrag<trunc> i64:$a)),3350 (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1),3351 (i32 (EXTRACT_SUBREG $a, sub0))), (i32 1))3352>;3353 3354// This pattern for bswap is used for pre-GFX8. For GFX8+, bswap is mapped3355// to V_PERM_B32.3356let True16Predicate = NotHasTrue16BitInsts in3357def : GCNPat <3358 (i32 (bswap i32:$a)),3359 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),3360 (V_ALIGNBIT_B32_e64 VSrc_b32:$a, VSrc_b32:$a, (i32 24)),3361 (V_ALIGNBIT_B32_e64 VSrc_b32:$a, VSrc_b32:$a, (i32 8)))3362>;3363 3364let True16Predicate = UseFakeTrue16Insts in3365def : GCNPat <3366 (i32 (bswap i32:$a)),3367 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),3368 (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */3369 VSrc_b32:$a,3370 0, /* src1_modifiers */3371 VSrc_b32:$a,3372 0, /* src2_modifiers */3373 (i32 24), /* clamp */ 0, /* op_sel */ 0),3374 (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */3375 VSrc_b32:$a,3376 0, /* src1_modifiers */3377 VSrc_b32:$a,3378 0, /* src2_modifiers */3379 (i32 8), /* clamp */ 0, /* op_sel */ 0))3380>;3381 3382class AlignBit32Inst<dag op1, dag op2, dag op3, bit isTrue16> {3383 defvar inst = !if(isTrue16, V_ALIGNBIT_B32_fake16_e64, V_ALIGNBIT_B32_e64);3384 defvar NoMods = !if(isTrue16, (inst 0), (inst));3385 dag ret = !con(NoMods, (inst op1), NoMods, (inst op2),3386 NoMods, (inst op3), NoMods, NoMods);3387}3388 3389multiclass bswapi64ExtPat<bit hasTrue16> {3390def : GCNPat <3391 (i64 (bswap i64:$a)),3392 (REG_SEQUENCE VReg_64,3393 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),3394 AlignBit32Inst<(i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),3395 (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),3396 (i32 24), hasTrue16>.ret,3397 AlignBit32Inst<(i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),3398 (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),3399 (i32 8), hasTrue16>.ret),3400 sub0,3401 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),3402 AlignBit32Inst<(i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),3403 (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),3404 (i32 24), hasTrue16>.ret,3405 AlignBit32Inst<(i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),3406 (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),3407 (i32 8), hasTrue16>.ret),3408 sub1)3409>;3410}3411 3412// FIXME: This should have been narrowed to i32 during legalization.3413// This pattern should also be skipped for GlobalISel3414let True16Predicate = NotHasTrue16BitInsts in3415defm : bswapi64ExtPat</*hasTrue16*/0>;3416 3417let True16Predicate = UseFakeTrue16Insts in3418defm : bswapi64ExtPat</*hasTrue16*/1>;3419 3420// FIXME: The AddedComplexity should not be needed, but in GlobalISel3421// the BFI pattern ends up taking precedence without it.3422let SubtargetPredicate = isGFX8Plus, AddedComplexity = 1 in {3423// Magic number: 3 | (2 << 8) | (1 << 16) | (0 << 24)3424//3425// My reading of the manual suggests we should be using src0 for the3426// register value, but this is what seems to work.3427def : GCNPat <3428 (i32 (bswap i32:$a)),3429 (V_PERM_B32_e64 (i32 0), VSrc_b32:$a, (S_MOV_B32 (i32 0x00010203)))3430>;3431 3432// FIXME: This should have been narrowed to i32 during legalization.3433// This pattern should also be skipped for GlobalISel3434def : GCNPat <3435 (i64 (bswap i64:$a)),3436 (REG_SEQUENCE VReg_64,3437 (V_PERM_B32_e64 (i32 0), (EXTRACT_SUBREG VReg_64:$a, sub1),3438 (S_MOV_B32 (i32 0x00010203))),3439 sub0,3440 (V_PERM_B32_e64 (i32 0), (EXTRACT_SUBREG VReg_64:$a, sub0),3441 (S_MOV_B32 (i32 0x00010203))),3442 sub1)3443>;3444 3445// Magic number: 1 | (0 << 8) | (12 << 16) | (12 << 24)3446// The 12s emit 0s.3447let True16Predicate = NotUseRealTrue16Insts in {3448def : GCNPat <3449 (i16 (bswap i16:$a)),3450 (V_PERM_B32_e64 (i32 0), VSrc_b32:$a, (S_MOV_B32 (i32 0x0c0c0001)))3451>;3452 3453def : GCNPat <3454 (i32 (zext (bswap i16:$a))),3455 (V_PERM_B32_e64 (i32 0), VSrc_b32:$a, (S_MOV_B32 (i32 0x0c0c0001)))3456>;3457}3458 3459let True16Predicate = UseRealTrue16Insts in {3460def : GCNPat <3461 (i16 (bswap i16:$a)),3462 (EXTRACT_SUBREG (V_PERM_B32_e64 (i32 0), (COPY VGPR_16:$a), (S_MOV_B32 (i32 0x0c0c0001))), lo16)3463>;3464 3465def : GCNPat <3466 (i32 (zext (bswap i16:$a))),3467 (V_PERM_B32_e64 (i32 0), (COPY VGPR_16:$a), (S_MOV_B32 (i32 0x0c0c0001)))3468>;3469}3470 3471// Magic number: 1 | (0 << 8) | (3 << 16) | (2 << 24)3472def : GCNPat <3473 (v2i16 (bswap v2i16:$a)),3474 (V_PERM_B32_e64 (i32 0), VSrc_b32:$a, (S_MOV_B32 (i32 0x02030001)))3475>;3476 3477}3478 3479def : GCNPat<3480 (i64 (DivergentUnaryFrag<bitreverse> i64:$a)),3481 (REG_SEQUENCE VReg_64,3482 (V_BFREV_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub1))), sub0,3483 (V_BFREV_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub0))), sub1)>;3484 3485// If fcanonicalize's operand is implicitly canonicalized, we only need a copy.3486let AddedComplexity = 8 in {3487foreach vt = [f16, v2f16, f32, v2f32, f64] in {3488 def : GCNPat<3489 (fcanonicalize (vt is_canonicalized:$src)),3490 (COPY vt:$src)3491 >;3492}3493}3494 3495// Prefer selecting to max when legal, but using mul is always valid.3496let AddedComplexity = -5 in {3497 3498let True16Predicate = NotHasTrue16BitInsts in {3499def : GCNPat<3500 (fcanonicalize (f16 (VOP3Mods f16:$src, i32:$src_mods))),3501 (V_MUL_F16_e64 0, (i32 CONST.FP16_ONE), $src_mods, $src)3502>;3503 3504def : GCNPat<3505 (fcanonicalize (f16 (fneg (VOP3Mods f16:$src, i32:$src_mods)))),3506 (V_MUL_F16_e64 0, (i32 CONST.FP16_NEG_ONE), $src_mods, $src)3507>;3508} // End True16Predicate3509 3510def : GCNPat<3511 (fcanonicalize (v2f16 (VOP3PMods v2f16:$src, i32:$src_mods))),3512 (V_PK_MUL_F16 0, (i32 CONST.FP16_ONE), $src_mods, $src, DSTCLAMP.NONE)3513>;3514 3515def : GCNPat<3516 (fcanonicalize (f32 (VOP3Mods f32:$src, i32:$src_mods))),3517 (V_MUL_F32_e64 0, (i32 CONST.FP32_ONE), $src_mods, $src)3518>;3519 3520def : GCNPat<3521 (fcanonicalize (f32 (fneg (VOP3Mods f32:$src, i32:$src_mods)))),3522 (V_MUL_F32_e64 0, (i32 CONST.FP32_NEG_ONE), $src_mods, $src)3523>;3524 3525let SubtargetPredicate = HasPackedFP32Ops in {3526def : GCNPat<3527 (fcanonicalize (v2f32 (VOP3PMods v2f32:$src, i32:$src_mods))),3528 (V_PK_MUL_F32 0, (i64 CONST.FP32_ONE), $src_mods, $src)3529>;3530}3531 3532// TODO: Handle fneg like other types.3533let SubtargetPredicate = isNotGFX12Plus in {3534def : GCNPat<3535 (fcanonicalize (f64 (VOP3Mods f64:$src, i32:$src_mods))),3536 (V_MUL_F64_e64 0, (i64 CONST.FP64_ONE), $src_mods, $src)3537>;3538}3539} // End AddedComplexity = -53540 3541multiclass SelectCanonicalizeAsMax<3542 list<Predicate> f32_preds = [],3543 list<Predicate> f64_preds = [],3544 list<Predicate> f16_preds = []> {3545 def : GCNPat<3546 (fcanonicalize (f32 (VOP3Mods f32:$src, i32:$src_mods))),3547 (V_MAX_F32_e64 $src_mods, $src, $src_mods, $src)> {3548 let OtherPredicates = f32_preds;3549 }3550 3551 def : GCNPat<3552 (fcanonicalize (f64 (VOP3Mods f64:$src, i32:$src_mods))),3553 (V_MAX_F64_e64 $src_mods, $src, $src_mods, $src)> {3554 let OtherPredicates = !listconcat(f64_preds, [isNotGFX12Plus]);3555 }3556 3557 def : GCNPat<3558 (fcanonicalize (f64 (VOP3Mods f64:$src, i32:$src_mods))),3559 (V_MAX_NUM_F64_e64 $src_mods, $src, $src_mods, $src)> {3560 let OtherPredicates = !listconcat(f64_preds, [isGFX12Plus]);3561 }3562 3563 def : GCNPat<3564 (fcanonicalize (f16 (VOP3Mods f16:$src, i32:$src_mods))),3565 (V_MAX_F16_e64 $src_mods, $src, $src_mods, $src, 0, 0)> {3566 let OtherPredicates = !listconcat(f16_preds, [Has16BitInsts]);3567 let True16Predicate = NotHasTrue16BitInsts;3568 }3569 3570 def : GCNPat<3571 (fcanonicalize (f16 (VOP3Mods f16:$src, i32:$src_mods))),3572 (V_MAX_F16_t16_e64 $src_mods, $src, $src_mods, $src, 0, 0)> {3573 let OtherPredicates = !listconcat(f16_preds, [Has16BitInsts]);3574 let True16Predicate = UseRealTrue16Insts;3575 }3576 3577 def : GCNPat<3578 (fcanonicalize (f16 (VOP3Mods f16:$src, i32:$src_mods))),3579 (V_MAX_F16_fake16_e64 $src_mods, $src, $src_mods, $src, 0, 0)> {3580 let OtherPredicates = !listconcat(f16_preds, [Has16BitInsts]);3581 let True16Predicate = UseFakeTrue16Insts;3582 }3583 3584 def : GCNPat<3585 (fcanonicalize (v2f16 (VOP3PMods v2f16:$src, i32:$src_mods))),3586 (V_PK_MAX_F16 $src_mods, $src, $src_mods, $src, DSTCLAMP.NONE)> {3587 // FIXME: Should have VOP3P subtarget predicate3588 let OtherPredicates = f16_preds;3589 }3590}3591 3592// On pre-gfx9 targets, v_max_*/v_min_* did not respect the denormal3593// mode, and would never flush. For f64, it's faster to do implement3594// this with a max. For f16/f32 it's a wash, but prefer max when3595// valid.3596//3597// FIXME: Lowering f32/f16 with max is worse since we can use a3598// smaller encoding if the input is fneg'd. It also adds an extra3599// register use.3600let SubtargetPredicate = HasMinMaxDenormModes in {3601 defm : SelectCanonicalizeAsMax<[], [], []>;3602} // End SubtargetPredicate = HasMinMaxDenormModes3603 3604let SubtargetPredicate = NotHasMinMaxDenormModes in {3605 // Use the max lowering if we don't need to flush.3606 3607 // FIXME: We don't do use this for f32 as a workaround for the3608 // library being compiled with the default ieee mode, but3609 // potentially being called from flushing kernels. Really we should3610 // not be mixing code expecting different default FP modes, but mul3611 // works in any FP environment.3612 defm : SelectCanonicalizeAsMax<[FalsePredicate], [FP64Denormals], [FP16Denormals]>;3613} // End SubtargetPredicate = NotHasMinMaxDenormModes3614 3615 3616let OtherPredicates = [HasDLInsts] in {3617// Don't allow source modifiers. If there are any source modifiers then it's3618// better to select fma instead of fmac.3619def : GCNPat <3620 (fma (f32 (VOP3NoMods f32:$src0)),3621 (f32 (VOP3NoMods f32:$src1)),3622 (f32 (VOP3NoMods f32:$src2))),3623 (V_FMAC_F32_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,3624 SRCMODS.NONE, $src2)3625>;3626} // End OtherPredicates = [HasDLInsts]3627 3628let SubtargetPredicate = isGFX10Plus in {3629// Don't allow source modifiers. If there are any source modifiers then it's3630// better to select fma instead of fmac.3631let True16Predicate = NotHasTrue16BitInsts in3632def : GCNPat <3633 (fma (f16 (VOP3NoMods f32:$src0)),3634 (f16 (VOP3NoMods f32:$src1)),3635 (f16 (VOP3NoMods f32:$src2))),3636 (V_FMAC_F16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,3637 SRCMODS.NONE, $src2)3638>;3639let True16Predicate = UseRealTrue16Insts in3640def : GCNPat <3641 (fma (f16 (VOP3NoMods f16:$src0)),3642 (f16 (VOP3NoMods f16:$src1)),3643 (f16 (VOP3NoMods f16:$src2))),3644 (V_FMAC_F16_t16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,3645 SRCMODS.NONE, $src2)3646>;3647let True16Predicate = UseFakeTrue16Insts in3648def : GCNPat <3649 (fma (f16 (VOP3NoMods f16:$src0)),3650 (f16 (VOP3NoMods f16:$src1)),3651 (f16 (VOP3NoMods f16:$src2))),3652 (V_FMAC_F16_fake16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,3653 SRCMODS.NONE, $src2)3654>;3655}3656 3657let OtherPredicates = [HasFmacF64Inst] in3658// Don't allow source modifiers. If there are any source modifiers then it's3659// better to select fma instead of fmac.3660def : GCNPat <3661 (fma (f64 (VOP3NoMods f64:$src0)),3662 (f64 (VOP3NoMods f64:$src1)),3663 (f64 (VOP3NoMods f64:$src2))),3664 (V_FMAC_F64_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,3665 SRCMODS.NONE, $src2)3666>;3667 3668// COPY is workaround tablegen bug from multiple outputs3669// from S_LSHL_B32's multiple outputs from implicit scc def.3670let AddedComplexity = 1 in {3671def : GCNPat <3672 (v2i16 (UniformBinFrag<build_vector> (i16 0), (i16 SReg_32:$src1))),3673 (S_LSHL_B32 SReg_32:$src1, (i16 16))3674>;3675 3676let True16Predicate = NotUseRealTrue16Insts in {3677def : GCNPat <3678 (v2i16 (DivergentBinFrag<build_vector> (i16 0), (i16 VGPR_32:$src1))),3679 (v2i16 (V_LSHLREV_B32_e64 (i16 16), VGPR_32:$src1))3680>;3681 3682def : GCNPat <3683 (v2i16 (DivergentBinFrag<build_vector> (i16 VGPR_32:$src1), (i16 0))),3684 (v2i16 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), VGPR_32:$src1))3685>;3686 3687def : GCNPat <3688 (v2f16 (DivergentBinFrag<build_vector> (f16 VGPR_32:$src1), (f16 FP_ZERO))),3689 (v2f16 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), VGPR_32:$src1))3690>;3691}3692 3693def : GCNPat <3694 (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src1), (i16 0))),3695 (S_AND_B32 (S_MOV_B32 (i32 0xffff)), SReg_32:$src1)3696>;3697 3698def : GCNPat <3699 (v2f16 (UniformBinFrag<build_vector> (f16 SReg_32:$src1), (f16 FP_ZERO))),3700 (S_AND_B32 (S_MOV_B32 (i32 0xffff)), SReg_32:$src1)3701>;3702 3703foreach vecTy = [v2i16, v2f16, v2bf16] in {3704 3705defvar Ty = vecTy.ElementType;3706 3707def : GCNPat <3708 (vecTy (UniformBinFrag<build_vector> (Ty SReg_32:$src0), (Ty undef))),3709 (COPY_TO_REGCLASS SReg_32:$src0, SReg_32)3710>;3711 3712let True16Predicate = NotUseRealTrue16Insts in {3713def : GCNPat <3714 (vecTy (DivergentBinFrag<build_vector> (Ty VGPR_32:$src0), (Ty undef))),3715 (COPY_TO_REGCLASS VGPR_32:$src0, VGPR_32)3716>;3717 3718def : GCNPat <3719 (vecTy (UniformBinFrag<build_vector> (Ty undef), (Ty SReg_32:$src1))),3720 (S_LSHL_B32 SReg_32:$src1, (i32 16))3721>;3722 3723def : GCNPat <3724 (vecTy (DivergentBinFrag<build_vector> (Ty undef), (Ty VGPR_32:$src1))),3725 (vecTy (V_LSHLREV_B32_e64 (i32 16), VGPR_32:$src1))3726>;3727} // End True16Predicate = ...3728} // End foreach Ty = ...3729} // End AddedComplexity = 13730 3731let True16Predicate = UseRealTrue16Insts in {3732def : GCNPat<3733 (i32 (DivergentBinFrag<or>3734 (i32 (zext i16:$src_lo)),3735 (i32 (bitconvert (v2i16 (build_vector (i16 0), (i16 VGPR_16:$src_hi)))))3736 )),3737 (REG_SEQUENCE VGPR_32, $src_lo, lo16, $src_hi, hi16)3738>;3739def : GCNPat<3740 (i32 (DivergentBinFrag<or>3741 (i32 (bitconvert (v2i16 (build_vector (i16 0), (i16 VGPR_16:$src_hi))))),3742 (i32 (zext i16:$src_lo))3743 )),3744 (REG_SEQUENCE VGPR_32, $src_lo, lo16, $src_hi, hi16)3745>;3746}3747 3748let True16Predicate = UseRealTrue16Insts in3749def : GCNPat <3750 (v2i16 (DivergentBinFrag<build_vector> (i16 undef), (i16 (trunc i32:$src1)))),3751 (REG_SEQUENCE VGPR_32, (i16 (IMPLICIT_DEF)), lo16,3752 (i16 (EXTRACT_SUBREG VGPR_32:$src1, lo16)), hi16)3753>;3754 3755let SubtargetPredicate = HasVOP3PInsts in {3756let True16Predicate = NotUseRealTrue16Insts in3757def : GCNPat <3758 (v2i16 (DivergentBinFrag<build_vector> (i16 VGPR_32:$src0), (i16 VGPR_32:$src1))),3759 (v2i16 (V_LSHL_OR_B32_e64 $src1, (i32 16), (i32 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), $src0))))3760>;3761 3762// With multiple uses of the shift, this will duplicate the shift and3763// increase register pressure.3764def : GCNPat <3765 (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src0), (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))),3766 (v2i16 (S_PACK_LH_B32_B16 SReg_32:$src0, SReg_32:$src1))3767>;3768 3769def : GCNPat <3770 (v2i16 (UniformBinFrag<build_vector> (i16 (trunc (srl_oneuse SReg_32:$src0, (i32 16)))),3771 (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))),3772 (S_PACK_HH_B32_B16 SReg_32:$src0, SReg_32:$src1)3773>;3774 3775 3776foreach vecTy = [v2i16, v2f16, v2bf16] in {3777 3778defvar Ty = vecTy.ElementType;3779defvar immzeroTy = !if(!eq(Ty, i16), immzero, fpimmzero);3780 3781def : GCNPat <3782 (vecTy (UniformBinFrag<build_vector> (Ty SReg_32:$src0), (Ty SReg_32:$src1))),3783 (S_PACK_LL_B32_B16 SReg_32:$src0, SReg_32:$src1)3784>;3785 3786let True16Predicate = NotUseRealTrue16Insts in {3787// Take the lower 16 bits from each VGPR_32 and concat them3788def : GCNPat <3789 (vecTy (DivergentBinFrag<build_vector> (Ty VGPR_32:$a), (Ty VGPR_32:$b))),3790 (V_PERM_B32_e64 VGPR_32:$b, VGPR_32:$a, (S_MOV_B32 (i32 0x05040100)))3791>;3792 3793// Take the lower 16 bits from V[0] and the upper 16 bits from V[1]3794// Special case, can use V_BFI (0xffff literal likely more reusable than 0x70601000)3795def : GCNPat <3796 (vecTy (DivergentBinFrag<build_vector> (Ty VGPR_32:$a),3797 (Ty !if(!eq(Ty, i16),3798 (Ty (trunc (srl VGPR_32:$b, (i32 16)))),3799 (Ty (bitconvert (i16 (trunc (srl VGPR_32:$b, (i32 16)))))))))),3800 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x0000ffff)), VGPR_32:$a, VGPR_32:$b)3801>;3802}3803 3804let True16Predicate = UseRealTrue16Insts in {3805def : GCNPat <3806 (vecTy (DivergentBinFrag<build_vector> (Ty VGPR_16:$a), (Ty VGPR_16:$b))),3807 (REG_SEQUENCE VGPR_32, VGPR_16:$a, lo16, VGPR_16:$b, hi16)3808>;3809def : GCNPat <3810 (vecTy (DivergentBinFrag<build_vector> (Ty VGPR_16:$src0), (Ty undef))),3811 (REG_SEQUENCE VGPR_32, $src0, lo16, (Ty (IMPLICIT_DEF)), hi16)3812>;3813def : GCNPat <3814 (vecTy (DivergentBinFrag<build_vector> (Ty undef), (Ty VGPR_16:$src1))),3815 (REG_SEQUENCE VGPR_32, (Ty (IMPLICIT_DEF)), lo16, (Ty VGPR_16:$src1), hi16)3816>;3817}3818 3819// Take the lower 16 bits from V[0] and the upper 16 bits from V[1]3820// Special case, can use V_BFI (0xffff literal likely more reusable than 0x70601000)3821def : GCNPat <3822 (vecTy (DivergentBinFrag<build_vector> (Ty (immzeroTy)),3823 (Ty !if(!eq(Ty, i16),3824 (Ty (trunc (srl VGPR_32:$b, (i32 16)))),3825 (Ty (bitconvert (i16 (trunc (srl VGPR_32:$b, (i32 16)))))))))),3826 (V_AND_B32_e64 (S_MOV_B32 (i32 0xffff0000)), VGPR_32:$b)3827>;3828 3829// Take the upper 16 bits from V[0] and the lower 16 bits from V[1]3830// Special case, can use V_ALIGNBIT (always uses encoded literal)3831let True16Predicate = NotHasTrue16BitInsts in {3832defvar BuildVectorToAlignBitPat =3833 (vecTy (DivergentBinFrag<build_vector>3834 (Ty !if(!eq(Ty, i16),3835 (Ty (trunc (srl VGPR_32:$a, (i32 16)))),3836 (Ty (bitconvert (i16 (trunc (srl VGPR_32:$a, (i32 16)))))))),3837 (Ty VGPR_32:$b)));3838 3839let SubtargetPredicate = isNotGFX9Plus in3840def : GCNPat<BuildVectorToAlignBitPat, (V_ALIGNBIT_B32_e64 VGPR_32:$b, VGPR_32:$a, (i32 16))>;3841 3842let SubtargetPredicate = isGFX9GFX10 in3843def : GCNPat<BuildVectorToAlignBitPat, (V_ALIGNBIT_B32_opsel_e64 0, VGPR_32:$b, 0, VGPR_32:$a, 0, (i32 16), 0, 0)>;3844} //True16Predicate = NotHasTrue16BitInsts3845 3846let True16Predicate = UseFakeTrue16Insts in3847def : GCNPat <3848 (vecTy (DivergentBinFrag<build_vector>3849 (Ty !if(!eq(Ty, i16),3850 (Ty (trunc (srl VGPR_32:$a, (i32 16)))),3851 (Ty (bitconvert (i16 (trunc (srl VGPR_32:$a, (i32 16)))))))),3852 (Ty VGPR_32:$b))),3853 (V_ALIGNBIT_B32_fake16_e64 0, VGPR_32:$b, 0, VGPR_32:$a, 0, (i16 16), 0, 0)3854>;3855 3856// Take the upper 16 bits from each VGPR_32 and concat them3857let True16Predicate = NotUseRealTrue16Insts in3858def : GCNPat <3859 (vecTy (DivergentBinFrag<build_vector>3860 (Ty !if(!eq(Ty, i16),3861 (Ty (trunc (srl VGPR_32:$a, (i32 16)))),3862 (Ty (bitconvert (i16 (trunc (srl VGPR_32:$a, (i32 16)))))))),3863 (Ty !if(!eq(Ty, i16),3864 (Ty (trunc (srl VGPR_32:$b, (i32 16)))),3865 (Ty (bitconvert (i16 (trunc (srl VGPR_32:$b, (i32 16)))))))))),3866 (V_PERM_B32_e64 VGPR_32:$b, VGPR_32:$a, (S_MOV_B32 (i32 0x07060302)))3867>;3868 3869} // end foreach Ty3870 3871} // End SubtargetPredicate = HasVOP3PInsts3872 3873let AddedComplexity = 5 in {3874class PackB32Pat<Instruction inst> : GCNPat <3875 (v2f16 (is_canonicalized_2<build_vector> (f16 (VOP3Mods (f16 VGPR_32:$src0), i32:$src0_mods)),3876 (f16 (VOP3Mods (f16 VGPR_32:$src1), i32:$src1_mods)))),3877 (inst $src0_mods, VGPR_32:$src0, $src1_mods, VGPR_32:$src1)3878>;3879}3880let SubtargetPredicate = isGFX9Plus in {3881let True16Predicate = NotHasTrue16BitInsts in3882 def : PackB32Pat<V_PACK_B32_F16_e64>;3883 3884let True16Predicate = UseRealTrue16Insts in3885 def : PackB32Pat<V_PACK_B32_F16_t16_e64>;3886 3887let True16Predicate = UseFakeTrue16Insts in3888 def : PackB32Pat<V_PACK_B32_F16_fake16_e64>;3889} // End SubtargetPredicate = isGFX9Plus3890 3891// With multiple uses of the shift, this will duplicate the shift and3892// increase register pressure.3893let SubtargetPredicate = isGFX11Plus in3894def : GCNPat <3895 (v2i16 (build_vector (i16 (trunc (srl_oneuse SReg_32:$src0, (i32 16)))), (i16 SReg_32:$src1))),3896 (v2i16 (S_PACK_HL_B32_B16 SReg_32:$src0, SReg_32:$src1))3897>;3898 3899let True16Predicate = NotUseRealTrue16Insts in {3900def : GCNPat <3901 (v2f16 (scalar_to_vector f16:$src0)),3902 (COPY $src0)3903>;3904 3905def : GCNPat <3906 (v2i16 (scalar_to_vector i16:$src0)),3907 (COPY $src0)3908>;3909 3910def : GCNPat <3911 (v4i16 (scalar_to_vector i16:$src0)),3912 (INSERT_SUBREG (IMPLICIT_DEF), $src0, sub0)3913>;3914 3915def : GCNPat <3916 (v4f16 (scalar_to_vector f16:$src0)),3917 (INSERT_SUBREG (IMPLICIT_DEF), $src0, sub0)3918>;3919}3920 3921let True16Predicate = UseRealTrue16Insts in {3922def : GCNPat <3923 (v2f16 (scalar_to_vector f16:$src0)),3924 (REG_SEQUENCE VGPR_32, $src0, lo16, (i16 (IMPLICIT_DEF)), hi16)3925>;3926 3927def : GCNPat <3928 (v2i16 (scalar_to_vector i16:$src0)),3929 (REG_SEQUENCE VGPR_32, $src0, lo16, (i16 (IMPLICIT_DEF)), hi16)3930>;3931 3932def : GCNPat <3933 (v4i16 (scalar_to_vector i16:$src0)),3934 (REG_SEQUENCE VGPR_32, $src0, lo16, (i16 (IMPLICIT_DEF)), hi16, (i32 (IMPLICIT_DEF)), sub1)3935>;3936 3937def : GCNPat <3938 (v4f16 (scalar_to_vector f16:$src0)),3939 (REG_SEQUENCE VGPR_32, $src0, lo16, (i16 (IMPLICIT_DEF)), hi16, (i32 (IMPLICIT_DEF)), sub1)3940>;3941}3942 3943def : GCNPat <3944 (i64 (int_amdgcn_mov_dpp i64:$src, timm:$dpp_ctrl, timm:$row_mask,3945 timm:$bank_mask, timm:$bound_ctrl)),3946 (V_MOV_B64_DPP_PSEUDO VReg_64_Align2:$src, VReg_64_Align2:$src,3947 (as_i32timm $dpp_ctrl), (as_i32timm $row_mask),3948 (as_i32timm $bank_mask),3949 (as_i1timm $bound_ctrl))3950>;3951 3952foreach vt = Reg64Types.types in {3953def : GCNPat <3954 (vt (int_amdgcn_update_dpp vt:$old, vt:$src, timm:$dpp_ctrl, timm:$row_mask,3955 timm:$bank_mask, timm:$bound_ctrl)),3956 (V_MOV_B64_DPP_PSEUDO VReg_64_Align2:$old, VReg_64_Align2:$src, (as_i32timm $dpp_ctrl),3957 (as_i32timm $row_mask), (as_i32timm $bank_mask),3958 (as_i1timm $bound_ctrl))3959>;3960}3961 3962//===----------------------------------------------------------------------===//3963// Fract Patterns3964//===----------------------------------------------------------------------===//3965 3966let SubtargetPredicate = isGFX6 in {3967 3968// V_FRACT is buggy on SI, so the F32 version is never used and (x-floor(x)) is3969// used instead. However, SI doesn't have V_FLOOR_F64, so the most efficient3970// way to implement it is using V_FRACT_F64.3971// The workaround for the V_FRACT bug is:3972// fract(x) = isnan(x) ? x : min(V_FRACT(x), 0.99999999999999999)3973 3974// Convert floor(x) to (x - fract(x))3975 3976// Don't bother handling this for GlobalISel, it's handled during3977// lowering.3978//3979// FIXME: DAG should also custom lower this.3980def : GCNPat <3981 (f64 (ffloor (f64 (VOP3Mods f64:$x, i32:$mods)))),3982 (V_ADD_F64_e643983 $mods,3984 $x,3985 SRCMODS.NEG,3986 (V_CNDMASK_B64_PSEUDO3987 (V_MIN_F64_e643988 SRCMODS.NONE,3989 (V_FRACT_F64_e64 $mods, $x),3990 SRCMODS.NONE,3991 (V_MOV_B64_PSEUDO (i64 0x3fefffffffffffff))),3992 $x,3993 (V_CMP_CLASS_F64_e64 SRCMODS.NONE, $x, (i32 3 /*NaN*/))))3994>;3995 3996} // End SubtargetPredicates = isGFX63997 3998//============================================================================//3999// Miscellaneous Optimization Patterns4000//============================================================================//4001 4002// Undo sub x, c -> add x, -c canonicalization since c is more likely4003// an inline immediate than -c.4004// TODO: Also do for 64-bit.4005def : GCNPat<4006 (UniformBinFrag<add> i32:$src0, (i32 NegSubInlineConst32:$src1)),4007 (S_SUB_I32 SReg_32:$src0, NegSubInlineConst32:$src1)4008>;4009 4010def : GCNPat<4011 (DivergentBinFrag<add> i32:$src0, (i32 NegSubInlineConst32:$src1)),4012 (V_SUB_U32_e64 VS_32:$src0, NegSubInlineConst32:$src1)> {4013 let SubtargetPredicate = HasAddNoCarryInsts;4014}4015 4016def : GCNPat<4017 (DivergentBinFrag<add> i32:$src0, (i32 NegSubInlineConst32:$src1)),4018 (V_SUB_CO_U32_e64 VS_32:$src0, NegSubInlineConst32:$src1)> {4019 let SubtargetPredicate = NotHasAddNoCarryInsts;4020}4021 4022 4023// Avoid pointlessly materializing a constant in VGPR.4024// FIXME: Should also do this for readlane, but tablegen crashes on4025// the ignored src1.4026def : GCNPat<4027 (i32 (int_amdgcn_readfirstlane (i32 imm:$src))),4028 (S_MOV_B32 SReg_32:$src)4029>;4030 4031multiclass BFMPatterns <ValueType vt, PatFrag SHL, PatFrag ADD, InstSI BFM> {4032 def : GCNPat <4033 (vt (SHL (vt (add (vt (shl 1, vt:$a)), -1)), vt:$b)),4034 (BFM $a, $b)4035 >;4036 4037 def : GCNPat <4038 (vt (ADD (vt (shl 1, vt:$a)), -1)),4039 (BFM $a, (i32 0))4040 >;4041}4042 4043defm : BFMPatterns <i32, UniformBinFrag<shl>, UniformBinFrag<add>, S_BFM_B32>;4044// FIXME: defm : BFMPatterns <i64, UniformBinFrag<shl>, UniformBinFrag<add>, S_BFM_B64>;4045defm : BFMPatterns <i32, DivergentBinFrag<shl>, DivergentBinFrag<add>, V_BFM_B32_e64>;4046 4047// Bitfield extract patterns4048 4049def IMMZeroBasedBitfieldMask : ImmLeaf <i32, [{4050 return isMask_32(Imm);4051}]>;4052 4053def IMMPopCount : SDNodeXForm<imm, [{4054 return CurDAG->getTargetConstant(llvm::popcount(N->getZExtValue()), SDLoc(N),4055 MVT::i32);4056}]>;4057 4058def : AMDGPUPat <4059 (DivergentBinFrag<and> (i32 (srl i32:$src, i32:$rshift)),4060 IMMZeroBasedBitfieldMask:$mask),4061 (V_BFE_U32_e64 $src, $rshift, (i32 (IMMPopCount $mask)))4062>;4063 4064// x & ((1 << y) - 1)4065def : AMDGPUPat <4066 (DivergentBinFrag<and> i32:$src, (add_oneuse (shl_oneuse 1, i32:$width), -1)),4067 (V_BFE_U32_e64 $src, (i32 0), $width)4068>;4069 4070// x & ~(-1 << y)4071def : AMDGPUPat <4072 (DivergentBinFrag<and> i32:$src,4073 (xor_oneuse (shl_oneuse -1, i32:$width), -1)),4074 (V_BFE_U32_e64 $src, (i32 0), $width)4075>;4076 4077def uint5Bits : PatLeaf<(i32 VGPR_32:$width), [{4078 return CurDAG->computeKnownBits(Op).countMaxActiveBits() <= 5;4079}]>;4080 4081// x & (-1 >> (bitwidth - y))4082def : AMDGPUPat <4083 (DivergentBinFrag<and> i32:$src, (srl_oneuse -1, (sub 32, uint5Bits:$width))),4084 (V_BFE_U32_e64 $src, (i32 0), $width)4085>;4086 4087// SHA-256 Ma patterns4088 4089// ((x & z) | (y & (x | z))) -> BFI (XOR x, y), z, y4090def : AMDGPUPatIgnoreCopies <4091 (DivergentBinFrag<or> (and i32:$x, i32:$z),4092 (and i32:$y, (or i32:$x, i32:$z))),4093 (V_BFI_B32_e64 (V_XOR_B32_e64 (COPY_TO_REGCLASS VSrc_b32:$x, VGPR_32),4094 (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32)),4095 (COPY_TO_REGCLASS VSrc_b32:$z, VGPR_32),4096 (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32))4097>;4098 4099def : AMDGPUPatIgnoreCopies <4100 (DivergentBinFrag<or> (and i64:$x, i64:$z),4101 (and i64:$y, (or i64:$x, i64:$z))),4102 (REG_SEQUENCE VReg_64,4103 (V_BFI_B32_e64 (V_XOR_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub0)),4104 (i32 (EXTRACT_SUBREG VReg_64:$y, sub0))),4105 (i32 (EXTRACT_SUBREG VReg_64:$z, sub0)),4106 (i32 (EXTRACT_SUBREG VReg_64:$y, sub0))), sub0,4107 (V_BFI_B32_e64 (V_XOR_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub1)),4108 (i32 (EXTRACT_SUBREG VReg_64:$y, sub1))),4109 (i32 (EXTRACT_SUBREG VReg_64:$z, sub1)),4110 (i32 (EXTRACT_SUBREG VReg_64:$y, sub1))), sub1)4111>;4112 4113multiclass IntMed3Pat<Instruction med3Inst,4114 SDPatternOperator min,4115 SDPatternOperator max> {4116 4117 // This matches 16 permutations of4118 // min(max(a, b), max(min(a, b), c))4119 def : AMDGPUPat <4120 (min (max i32:$src0, i32:$src1),4121 (max (min i32:$src0, i32:$src1), i32:$src2)),4122 (med3Inst VSrc_b32:$src0, VSrc_b32:$src1, VSrc_b32:$src2)4123>;4124 4125 // This matches 16 permutations of4126 // max(min(x, y), min(max(x, y), z))4127 def : AMDGPUPat <4128 (max (min i32:$src0, i32:$src1),4129 (min (max i32:$src0, i32:$src1), i32:$src2)),4130 (med3Inst VSrc_b32:$src0, VSrc_b32:$src1, VSrc_b32:$src2)4131>;4132}4133 4134defm : IntMed3Pat<V_MED3_I32_e64, smin, smax>;4135defm : IntMed3Pat<V_MED3_U32_e64, umin, umax>;4136 4137multiclass FPMed3Pat<ValueType vt,4138 Instruction med3Inst> {4139 // This matches 16 permutations of max(min(x, y), min(max(x, y), z))4140 def : GCNPat<4141 (fmaxnum_like_nnan4142 (fminnum_like (VOP3Mods vt:$src0, i32:$src0_mods),4143 (VOP3Mods vt:$src1, i32:$src1_mods)),4144 (fminnum_like (fmaxnum_like (VOP3Mods vt:$src0, i32:$src0_mods),4145 (VOP3Mods vt:$src1, i32:$src1_mods)),4146 (vt (VOP3Mods vt:$src2, i32:$src2_mods)))),4147 (med3Inst $src0_mods, $src0, $src1_mods, $src1, $src2_mods, $src2,4148 DSTCLAMP.NONE, DSTOMOD.NONE)>;4149 4150 4151 // This matches 16 permutations of min(max(x, y), max(min(x, y), z))4152 def : GCNPat<4153 (fminnum_like_nnan4154 (fmaxnum_like (VOP3Mods vt:$src0, i32:$src0_mods),4155 (VOP3Mods vt:$src1, i32:$src1_mods)),4156 (fmaxnum_like (fminnum_like (VOP3Mods vt:$src0, i32:$src0_mods),4157 (VOP3Mods vt:$src1, i32:$src1_mods)),4158 (vt (VOP3Mods vt:$src2, i32:$src2_mods)))),4159 (med3Inst $src0_mods, $src0, $src1_mods, $src1, $src2_mods, $src2,4160 DSTCLAMP.NONE, DSTOMOD.NONE)>;4161}4162 4163multiclass Int16Med3Pat<Instruction med3Inst,4164 SDPatternOperator min,4165 SDPatternOperator max,4166 RegisterOperand outputSrcType> {4167 // This matches 16 permutations of4168 // max(min(x, y), min(max(x, y), z))4169 def : GCNPat <4170 (max (min i16:$src0, i16:$src1),4171 (min (max i16:$src0, i16:$src1), i16:$src2)),4172 (med3Inst SRCMODS.NONE, outputSrcType:$src0, SRCMODS.NONE, outputSrcType:$src1,4173 SRCMODS.NONE, outputSrcType:$src2, DSTCLAMP.NONE)4174>;4175 4176 // This matches 16 permutations of4177 // min(max(a, b), max(min(a, b), c))4178 def : GCNPat <4179 (min (max i16:$src0, i16:$src1),4180 (max (min i16:$src0, i16:$src1), i16:$src2)),4181 (med3Inst SRCMODS.NONE, VSrc_b16:$src0, SRCMODS.NONE, VSrc_b16:$src1, SRCMODS.NONE, VSrc_b16:$src2, DSTCLAMP.NONE)4182>;4183}4184 4185defm : FPMed3Pat<f32, V_MED3_F32_e64>;4186 4187let SubtargetPredicate = HasMed3_16 in {4188let True16Predicate = NotHasTrue16BitInsts in4189defm : FPMed3Pat<f16, V_MED3_F16_e64>;4190let True16Predicate = UseRealTrue16Insts in4191defm : FPMed3Pat<f16, V_MED3_F16_t16_e64>;4192let True16Predicate = UseFakeTrue16Insts in4193defm : FPMed3Pat<f16, V_MED3_F16_fake16_e64>;4194}4195 4196class4197IntMinMaxPat<Instruction minmaxInst, SDPatternOperator min_or_max,4198 SDPatternOperator max_or_min_oneuse> : AMDGPUPat <4199 (DivergentBinFrag<min_or_max> (max_or_min_oneuse i32:$src0, i32:$src1),4200 i32:$src2),4201 (minmaxInst VSrc_b32:$src0, VSrc_b32:$src1, VSrc_b32:$src2)4202>;4203 4204class4205FPMinMaxPat<Instruction minmaxInst, ValueType vt, SDPatternOperator min_or_max,4206 SDPatternOperator max_or_min_oneuse> : GCNPat <4207 (min_or_max (max_or_min_oneuse (VOP3Mods vt:$src0, i32:$src0_mods),4208 (VOP3Mods vt:$src1, i32:$src1_mods)),4209 (vt (VOP3Mods vt:$src2, i32:$src2_mods))),4210 (minmaxInst $src0_mods, $src0, $src1_mods, $src1, $src2_mods, $src2,4211 DSTCLAMP.NONE, DSTOMOD.NONE)4212>;4213 4214class4215FPMinCanonMaxPat<Instruction minmaxInst, ValueType vt, SDPatternOperator min_or_max,4216 SDPatternOperator max_or_min_oneuse> : GCNPat <4217 (min_or_max (is_canonicalized_1<fcanonicalize>4218 (max_or_min_oneuse (VOP3Mods vt:$src0, i32:$src0_mods),4219 (VOP3Mods vt:$src1, i32:$src1_mods))),4220 (vt (VOP3Mods vt:$src2, i32:$src2_mods))),4221 (minmaxInst $src0_mods, $src0, $src1_mods, $src1, $src2_mods, $src2,4222 DSTCLAMP.NONE, DSTOMOD.NONE)4223>;4224 4225let OtherPredicates = [isGFX11Plus] in {4226def : IntMinMaxPat<V_MAXMIN_I32_e64, smin, smax_oneuse>;4227def : IntMinMaxPat<V_MINMAX_I32_e64, smax, smin_oneuse>;4228def : IntMinMaxPat<V_MAXMIN_U32_e64, umin, umax_oneuse>;4229def : IntMinMaxPat<V_MINMAX_U32_e64, umax, umin_oneuse>;4230def : FPMinMaxPat<V_MINMAX_F32_e64, f32, fmaxnum_like, fminnum_like_oneuse>;4231def : FPMinMaxPat<V_MAXMIN_F32_e64, f32, fminnum_like, fmaxnum_like_oneuse>;4232def : FPMinCanonMaxPat<V_MINMAX_F32_e64, f32, fmaxnum_like, fminnum_like_oneuse>;4233def : FPMinCanonMaxPat<V_MAXMIN_F32_e64, f32, fminnum_like, fmaxnum_like_oneuse>;4234}4235 4236let True16Predicate = UseRealTrue16Insts in {4237def : FPMinMaxPat<V_MINMAX_F16_t16_e64, f16, fmaxnum_like, fminnum_like_oneuse>;4238def : FPMinMaxPat<V_MAXMIN_F16_t16_e64, f16, fminnum_like, fmaxnum_like_oneuse>;4239def : FPMinCanonMaxPat<V_MINMAX_F16_t16_e64, f16, fmaxnum_like, fminnum_like_oneuse>;4240def : FPMinCanonMaxPat<V_MAXMIN_F16_t16_e64, f16, fminnum_like, fmaxnum_like_oneuse>;4241}4242 4243let True16Predicate = UseFakeTrue16Insts in {4244def : FPMinMaxPat<V_MINMAX_F16_fake16_e64, f16, fmaxnum_like, fminnum_like_oneuse>;4245def : FPMinMaxPat<V_MAXMIN_F16_fake16_e64, f16, fminnum_like, fmaxnum_like_oneuse>;4246def : FPMinCanonMaxPat<V_MINMAX_F16_fake16_e64, f16, fmaxnum_like, fminnum_like_oneuse>;4247def : FPMinCanonMaxPat<V_MAXMIN_F16_fake16_e64, f16, fminnum_like, fmaxnum_like_oneuse>;4248}4249 4250let SubtargetPredicate = isGFX9Plus in {4251let True16Predicate = NotHasTrue16BitInsts in {4252 defm : Int16Med3Pat<V_MED3_I16_e64, smin, smax, VSrc_b16>;4253 defm : Int16Med3Pat<V_MED3_U16_e64, umin, umax, VSrc_b16>;4254}4255let True16Predicate = UseRealTrue16Insts in {4256 defm : Int16Med3Pat<V_MED3_I16_t16_e64, smin, smax, VSrcT_b16>;4257 defm : Int16Med3Pat<V_MED3_U16_t16_e64, umin, umax, VSrcT_b16>;4258}4259let True16Predicate = UseFakeTrue16Insts in {4260 defm : Int16Med3Pat<V_MED3_I16_fake16_e64, smin, smax, VSrc_b16>;4261 defm : Int16Med3Pat<V_MED3_U16_fake16_e64, umin, umax, VSrc_b16>;4262}4263} // End SubtargetPredicate = [isGFX9Plus]4264 4265let SubtargetPredicate = HasIEEEMinimumMaximumInsts in {4266def : FPMinMaxPat<V_MINIMUMMAXIMUM_F32_e64, f32, DivergentBinFrag<fmaximum>, fminimum_oneuse>;4267def : FPMinMaxPat<V_MAXIMUMMINIMUM_F32_e64, f32, DivergentBinFrag<fminimum>, fmaximum_oneuse>;4268def : FPMinCanonMaxPat<V_MINIMUMMAXIMUM_F32_e64, f32, DivergentBinFrag<fmaximum>, fminimum_oneuse>;4269def : FPMinCanonMaxPat<V_MAXIMUMMINIMUM_F32_e64, f32, DivergentBinFrag<fminimum>, fmaximum_oneuse>;4270}4271 4272let True16Predicate = UseRealTrue16Insts, SubtargetPredicate = HasIEEEMinimumMaximumInsts in {4273def : FPMinMaxPat<V_MINIMUMMAXIMUM_F16_t16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;4274def : FPMinMaxPat<V_MAXIMUMMINIMUM_F16_t16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;4275def : FPMinCanonMaxPat<V_MINIMUMMAXIMUM_F16_t16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;4276def : FPMinCanonMaxPat<V_MAXIMUMMINIMUM_F16_t16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;4277}4278 4279let True16Predicate = UseFakeTrue16Insts, SubtargetPredicate = HasIEEEMinimumMaximumInsts in {4280def : FPMinMaxPat<V_MINIMUMMAXIMUM_F16_fake16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;4281def : FPMinMaxPat<V_MAXIMUMMINIMUM_F16_fake16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;4282def : FPMinCanonMaxPat<V_MINIMUMMAXIMUM_F16_fake16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;4283def : FPMinCanonMaxPat<V_MAXIMUMMINIMUM_F16_fake16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;4284}4285 4286// Convert a floating-point power of 2 to the integer exponent.4287def FPPow2ToExponentXForm : SDNodeXForm<fpimm, [{4288 const auto &APF = N->getValueAPF();4289 int Log2 = APF.getExactLog2Abs();4290 assert(Log2 != INT_MIN);4291 return CurDAG->getSignedTargetConstant(Log2, SDLoc(N), MVT::i32);4292}]>;4293 4294// Check if a floating point value is a power of 2 floating-point4295// immediate where it's preferable to emit a multiply by as an4296// ldexp. We skip over 0.5 to 4.0 as those are inline immediates4297// anyway.4298def fpimm_pos_pow2_prefer_ldexp_f64 : FPImmLeaf<f64, [{4299 if (Imm.isNegative())4300 return false;4301 4302 int Exp = Imm.getExactLog2Abs();4303 // Prefer leaving the FP inline immediates as they are.4304 // 0.5, 1.0, 2.0, 4.04305 4306 // For f64 ldexp is always better than materializing a 64-bit4307 // constant.4308 return Exp != INT_MIN && (Exp < -1 || Exp > 2);4309 }], FPPow2ToExponentXForm4310>;4311 4312def fpimm_neg_pow2_prefer_ldexp_f64 : FPImmLeaf<f64, [{4313 if (!Imm.isNegative())4314 return false;4315 int Exp = Imm.getExactLog2Abs();4316 // Prefer leaving the FP inline immediates as they are.4317 // 0.5, 1.0, 2.0, 4.04318 4319 // For f64 ldexp is always better than materializing a 64-bit4320 // constant.4321 return Exp != INT_MIN && (Exp < -1 || Exp > 2);4322 }], FPPow2ToExponentXForm4323>;4324 4325// f64 is different because we also want to handle cases that may4326// require materialization of the exponent.4327// TODO: If we know f64 ops are fast, prefer add (ldexp x, N), y over fma4328// TODO: For f32/f16, it's not a clear win on code size to use ldexp4329// in place of mul since we have to use the vop3 form. Are there power4330// savings or some other reason to prefer ldexp over mul?4331def : GCNPat<4332 (any_fmul (f64 (VOP3Mods f64:$src0, i32:$src0_mods)),4333 fpimm_pos_pow2_prefer_ldexp_f64:$src1),4334 (V_LDEXP_F64_e64 i32:$src0_mods, VSrc_b64:$src0,4335 0, (S_MOV_B32 (i32 (FPPow2ToExponentXForm $src1))))4336>;4337 4338def : GCNPat<4339 (any_fmul f64:$src0, fpimm_neg_pow2_prefer_ldexp_f64:$src1),4340 (V_LDEXP_F64_e64 SRCMODS.NEG, VSrc_b64:$src0,4341 0, (S_MOV_B32 (i32 (FPPow2ToExponentXForm $src1))))4342>;4343 4344// We want to avoid using VOP3Mods which could pull in another fneg4345// which we would need to be re-negated (which should never happen in4346// practice). I don't see a way to apply an SDNodeXForm that accounts4347// for a second operand.4348def : GCNPat<4349 (any_fmul (fabs f64:$src0), fpimm_neg_pow2_prefer_ldexp_f64:$src1),4350 (V_LDEXP_F64_e64 SRCMODS.NEG_ABS, VSrc_b64:$src0,4351 0, (S_MOV_B32 (i32 (FPPow2ToExponentXForm $src1))))4352>;4353 4354class AMDGPUGenericInstruction : GenericInstruction {4355 let Namespace = "AMDGPU";4356}4357 4358// Convert a wave address to a swizzled vector address (i.e. this is4359// for copying the stack pointer to a vector address appropriate to4360// use in the offset field of mubuf instructions).4361def G_AMDGPU_WAVE_ADDRESS : AMDGPUGenericInstruction {4362 let OutOperandList = (outs type0:$dst);4363 let InOperandList = (ins type0:$src);4364 let hasSideEffects = 0;4365}4366 4367// Returns -1 if the input is zero.4368def G_AMDGPU_FFBH_U32 : AMDGPUGenericInstruction {4369 let OutOperandList = (outs type0:$dst);4370 let InOperandList = (ins type1:$src);4371 let hasSideEffects = 0;4372}4373 4374// Returns -1 if the input is zero.4375def G_AMDGPU_FFBL_B32 : AMDGPUGenericInstruction {4376 let OutOperandList = (outs type0:$dst);4377 let InOperandList = (ins type1:$src);4378 let hasSideEffects = 0;4379}4380 4381def G_AMDGPU_RCP_IFLAG : AMDGPUGenericInstruction {4382 let OutOperandList = (outs type0:$dst);4383 let InOperandList = (ins type1:$src);4384 let hasSideEffects = 0;4385}4386 4387class BufferLoadGenericInstruction : AMDGPUGenericInstruction {4388 let OutOperandList = (outs type0:$dst);4389 let InOperandList = (ins type1:$rsrc, type2:$vindex, type2:$voffset,4390 type2:$soffset, untyped_imm_0:$offset,4391 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen);4392 let hasSideEffects = 0;4393 let mayLoad = 1;4394}4395 4396class TBufferLoadGenericInstruction : AMDGPUGenericInstruction {4397 let OutOperandList = (outs type0:$dst);4398 let InOperandList = (ins type1:$rsrc, type2:$vindex, type2:$voffset,4399 type2:$soffset, untyped_imm_0:$offset, untyped_imm_0:$format,4400 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen);4401 let hasSideEffects = 0;4402 let mayLoad = 1;4403}4404 4405def G_AMDGPU_BUFFER_LOAD_UBYTE : BufferLoadGenericInstruction;4406def G_AMDGPU_BUFFER_LOAD_SBYTE : BufferLoadGenericInstruction;4407def G_AMDGPU_BUFFER_LOAD_USHORT : BufferLoadGenericInstruction;4408def G_AMDGPU_BUFFER_LOAD_SSHORT : BufferLoadGenericInstruction;4409def G_AMDGPU_BUFFER_LOAD : BufferLoadGenericInstruction;4410def G_AMDGPU_BUFFER_LOAD_UBYTE_TFE : BufferLoadGenericInstruction;4411def G_AMDGPU_BUFFER_LOAD_SBYTE_TFE : BufferLoadGenericInstruction;4412def G_AMDGPU_BUFFER_LOAD_USHORT_TFE : BufferLoadGenericInstruction;4413def G_AMDGPU_BUFFER_LOAD_SSHORT_TFE : BufferLoadGenericInstruction;4414def G_AMDGPU_BUFFER_LOAD_TFE : BufferLoadGenericInstruction;4415def G_AMDGPU_BUFFER_LOAD_FORMAT : BufferLoadGenericInstruction;4416def G_AMDGPU_BUFFER_LOAD_FORMAT_TFE : BufferLoadGenericInstruction;4417def G_AMDGPU_BUFFER_LOAD_FORMAT_D16 : BufferLoadGenericInstruction;4418def G_AMDGPU_TBUFFER_LOAD_FORMAT : TBufferLoadGenericInstruction;4419def G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 : TBufferLoadGenericInstruction;4420 4421class D16LoadGenericInstruction : AMDGPUGenericInstruction {4422 let OutOperandList = (outs type0:$dst);4423 let InOperandList = (ins ptype1:$addr, type0:$src);4424 let hasSideEffects = 0;4425 let mayLoad = 1;4426}4427 4428def G_AMDGPU_LOAD_D16_LO : D16LoadGenericInstruction;4429def G_AMDGPU_LOAD_D16_LO_U8 : D16LoadGenericInstruction;4430def G_AMDGPU_LOAD_D16_LO_I8 : D16LoadGenericInstruction;4431def G_AMDGPU_LOAD_D16_HI : D16LoadGenericInstruction;4432def G_AMDGPU_LOAD_D16_HI_U8 : D16LoadGenericInstruction;4433def G_AMDGPU_LOAD_D16_HI_I8 : D16LoadGenericInstruction;4434 4435 4436class BufferStoreGenericInstruction : AMDGPUGenericInstruction {4437 let OutOperandList = (outs);4438 let InOperandList = (ins type0:$vdata, type1:$rsrc, type2:$vindex, type2:$voffset,4439 type2:$soffset, untyped_imm_0:$offset,4440 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen);4441 let hasSideEffects = 0;4442 let mayStore = 1;4443}4444 4445class TBufferStoreGenericInstruction : AMDGPUGenericInstruction {4446 let OutOperandList = (outs);4447 let InOperandList = (ins type0:$vdata, type1:$rsrc, type2:$vindex, type2:$voffset,4448 type2:$soffset, untyped_imm_0:$offset,4449 untyped_imm_0:$format,4450 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen);4451 let hasSideEffects = 0;4452 let mayStore = 1;4453}4454 4455def G_AMDGPU_BUFFER_STORE : BufferStoreGenericInstruction;4456def G_AMDGPU_BUFFER_STORE_BYTE : BufferStoreGenericInstruction;4457def G_AMDGPU_BUFFER_STORE_SHORT : BufferStoreGenericInstruction;4458def G_AMDGPU_BUFFER_STORE_FORMAT : BufferStoreGenericInstruction;4459def G_AMDGPU_BUFFER_STORE_FORMAT_D16 : BufferStoreGenericInstruction;4460def G_AMDGPU_TBUFFER_STORE_FORMAT : TBufferStoreGenericInstruction;4461def G_AMDGPU_TBUFFER_STORE_FORMAT_D16 : TBufferStoreGenericInstruction;4462 4463def G_AMDGPU_FMIN_LEGACY : AMDGPUGenericInstruction {4464 let OutOperandList = (outs type0:$dst);4465 let InOperandList = (ins type0:$src0, type0:$src1);4466 let hasSideEffects = 0;4467}4468 4469def G_AMDGPU_FMAX_LEGACY : AMDGPUGenericInstruction {4470 let OutOperandList = (outs type0:$dst);4471 let InOperandList = (ins type0:$src0, type0:$src1);4472 let hasSideEffects = 0;4473}4474 4475foreach N = 0-3 in {4476def G_AMDGPU_CVT_F32_UBYTE#N : AMDGPUGenericInstruction {4477 let OutOperandList = (outs type0:$dst);4478 let InOperandList = (ins type0:$src0);4479 let hasSideEffects = 0;4480}4481}4482 4483def G_AMDGPU_CVT_PK_I16_I32 : AMDGPUGenericInstruction {4484 let OutOperandList = (outs type0:$dst);4485 let InOperandList = (ins type0:$src0, type0:$src1);4486 let hasSideEffects = 0;4487}4488 4489def G_AMDGPU_SMED3 : AMDGPUGenericInstruction {4490 let OutOperandList = (outs type0:$dst);4491 let InOperandList = (ins type0:$src0, type0:$src1, type0:$src2);4492 let hasSideEffects = 0;4493}4494 4495def G_AMDGPU_UMED3 : AMDGPUGenericInstruction {4496 let OutOperandList = (outs type0:$dst);4497 let InOperandList = (ins type0:$src0, type0:$src1, type0:$src2);4498 let hasSideEffects = 0;4499}4500 4501def G_AMDGPU_FMED3 : AMDGPUGenericInstruction {4502 let OutOperandList = (outs type0:$dst);4503 let InOperandList = (ins type0:$src0, type0:$src1, type0:$src2);4504 let hasSideEffects = 0;4505}4506 4507def G_AMDGPU_CLAMP : AMDGPUGenericInstruction {4508 let OutOperandList = (outs type0:$dst);4509 let InOperandList = (ins type0:$src);4510 let hasSideEffects = 0;4511}4512 4513// Integer multiply-add: arg0 * arg1 + arg2.4514//4515// arg0 and arg1 are 32-bit integers (interpreted as signed or unsigned),4516// arg2 is a 64-bit integer. Result is a 64-bit integer and a 1-bit carry-out.4517class G_AMDGPU_MAD_64_32 : AMDGPUGenericInstruction {4518 let OutOperandList = (outs type0:$dst, type1:$carry_out);4519 let InOperandList = (ins type2:$arg0, type2:$arg1, type0:$arg2);4520 let hasSideEffects = 0;4521}4522 4523def G_AMDGPU_MAD_U64_U32 : G_AMDGPU_MAD_64_32;4524def G_AMDGPU_MAD_I64_I32 : G_AMDGPU_MAD_64_32;4525 4526// Atomic cmpxchg. $cmpval ad $newval are packed in a single vector4527// operand Expects a MachineMemOperand in addition to explicit4528// operands.4529def G_AMDGPU_ATOMIC_CMPXCHG : AMDGPUGenericInstruction {4530 let OutOperandList = (outs type0:$oldval);4531 let InOperandList = (ins ptype1:$addr, type0:$cmpval_newval);4532 let hasSideEffects = 0;4533 let mayLoad = 1;4534 let mayStore = 1;4535}4536 4537class BufferAtomicGenericInstruction : AMDGPUGenericInstruction {4538 let OutOperandList = (outs type0:$dst);4539 let InOperandList = (ins type0:$vdata, type1:$rsrc, type2:$vindex, type2:$voffset,4540 type2:$soffset, untyped_imm_0:$offset,4541 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen);4542 let hasSideEffects = 0;4543 let mayLoad = 1;4544 let mayStore = 1;4545}4546 4547def G_AMDGPU_BUFFER_ATOMIC_SWAP : BufferAtomicGenericInstruction;4548def G_AMDGPU_BUFFER_ATOMIC_ADD : BufferAtomicGenericInstruction;4549def G_AMDGPU_BUFFER_ATOMIC_SUB : BufferAtomicGenericInstruction;4550def G_AMDGPU_BUFFER_ATOMIC_SMIN : BufferAtomicGenericInstruction;4551def G_AMDGPU_BUFFER_ATOMIC_UMIN : BufferAtomicGenericInstruction;4552def G_AMDGPU_BUFFER_ATOMIC_SMAX : BufferAtomicGenericInstruction;4553def G_AMDGPU_BUFFER_ATOMIC_UMAX : BufferAtomicGenericInstruction;4554def G_AMDGPU_BUFFER_ATOMIC_AND : BufferAtomicGenericInstruction;4555def G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32 : BufferAtomicGenericInstruction;4556def G_AMDGPU_BUFFER_ATOMIC_OR : BufferAtomicGenericInstruction;4557def G_AMDGPU_BUFFER_ATOMIC_XOR : BufferAtomicGenericInstruction;4558def G_AMDGPU_BUFFER_ATOMIC_INC : BufferAtomicGenericInstruction;4559def G_AMDGPU_BUFFER_ATOMIC_DEC : BufferAtomicGenericInstruction;4560def G_AMDGPU_BUFFER_ATOMIC_FADD : BufferAtomicGenericInstruction;4561def G_AMDGPU_BUFFER_ATOMIC_FMIN : BufferAtomicGenericInstruction;4562def G_AMDGPU_BUFFER_ATOMIC_FMAX : BufferAtomicGenericInstruction;4563 4564def G_AMDGPU_BUFFER_ATOMIC_CMPSWAP : AMDGPUGenericInstruction {4565 let OutOperandList = (outs type0:$dst);4566 let InOperandList = (ins type0:$vdata, type0:$cmp, type1:$rsrc, type2:$vindex,4567 type2:$voffset, type2:$soffset, untyped_imm_0:$offset,4568 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen);4569 let hasSideEffects = 0;4570 let mayLoad = 1;4571 let mayStore = 1;4572}4573 4574// Wrapper around llvm.amdgcn.s.buffer.load. This is mostly needed as4575// a workaround for the intrinsic being defined as readnone, but4576// really needs a memory operand.4577 4578class SBufferLoadInstruction : AMDGPUGenericInstruction {4579 let OutOperandList = (outs type0:$dst);4580 let InOperandList = (ins type1:$rsrc, type2:$offset, untyped_imm_0:$cachepolicy);4581 let hasSideEffects = 0;4582 let mayLoad = 1;4583 let mayStore = 0;4584}4585 4586def G_AMDGPU_S_BUFFER_LOAD : SBufferLoadInstruction;4587def G_AMDGPU_S_BUFFER_LOAD_SBYTE : SBufferLoadInstruction;4588def G_AMDGPU_S_BUFFER_LOAD_UBYTE : SBufferLoadInstruction;4589def G_AMDGPU_S_BUFFER_LOAD_SSHORT : SBufferLoadInstruction;4590def G_AMDGPU_S_BUFFER_LOAD_USHORT : SBufferLoadInstruction;4591 4592class SBufferPrefetchInstruction : AMDGPUGenericInstruction {4593 let OutOperandList = (outs);4594 let InOperandList = (ins type0:$rsrc, untyped_imm_0:$offset, type1:$len);4595 let hasSideEffects = 0;4596 let mayLoad = 1;4597 let mayStore = 1;4598}4599 4600def G_AMDGPU_S_BUFFER_PREFETCH : SBufferPrefetchInstruction;4601 4602def G_AMDGPU_S_MUL_U64_U32 : AMDGPUGenericInstruction {4603 let OutOperandList = (outs type0:$dst);4604 let InOperandList = (ins type0:$src0, type0:$src1);4605 let hasSideEffects = 0;4606}4607 4608def G_AMDGPU_S_MUL_I64_I32 : AMDGPUGenericInstruction {4609 let OutOperandList = (outs type0:$dst);4610 let InOperandList = (ins type0:$src0, type0:$src1);4611 let hasSideEffects = 0;4612}4613 4614def G_AMDGPU_WHOLE_WAVE_FUNC_SETUP : AMDGPUGenericInstruction {4615 let OutOperandList = (outs type0:$origExec);4616 let InOperandList = (ins);4617 let isConvergent = 1;4618}4619 4620def G_AMDGPU_WHOLE_WAVE_FUNC_RETURN : AMDGPUGenericInstruction {4621 let OutOperandList = (outs);4622 let InOperandList = (ins type0:$origExec);4623 let isTerminator = 1;4624 let isBarrier = 1;4625 let isReturn = 1;4626}4627 4628// This is equivalent to the G_INTRINSIC*, but the operands may have4629// been legalized depending on the subtarget requirements.4630def G_AMDGPU_INTRIN_IMAGE_LOAD : AMDGPUGenericInstruction {4631 let OutOperandList = (outs type0:$dst);4632 let InOperandList = (ins unknown:$intrin, variable_ops);4633 let hasSideEffects = 0;4634 let mayLoad = 1;4635 4636 // FIXME: Use separate opcode for atomics.4637 let mayStore = 1;4638}4639 4640def G_AMDGPU_INTRIN_IMAGE_LOAD_D16 : AMDGPUGenericInstruction {4641 let OutOperandList = (outs type0:$dst);4642 let InOperandList = (ins unknown:$intrin, variable_ops);4643 let hasSideEffects = 0;4644 let mayLoad = 1;4645 4646 // FIXME: Use separate opcode for atomics.4647 let mayStore = 1;4648}4649 4650def G_AMDGPU_INTRIN_IMAGE_LOAD_NORET : AMDGPUGenericInstruction {4651 let OutOperandList = (outs);4652 let InOperandList = (ins unknown:$intrin, variable_ops);4653 let hasSideEffects = 0;4654 let mayLoad = 1;4655 let mayStore = 1;4656}4657 4658// This is equivalent to the G_INTRINSIC*, but the operands may have4659// been legalized depending on the subtarget requirements.4660def G_AMDGPU_INTRIN_IMAGE_STORE : AMDGPUGenericInstruction {4661 let OutOperandList = (outs);4662 let InOperandList = (ins unknown:$intrin, variable_ops);4663 let hasSideEffects = 0;4664 let mayStore = 1;4665}4666 4667def G_AMDGPU_INTRIN_IMAGE_STORE_D16 : AMDGPUGenericInstruction {4668 let OutOperandList = (outs);4669 let InOperandList = (ins unknown:$intrin, variable_ops);4670 let hasSideEffects = 0;4671 let mayStore = 1;4672}4673 4674def G_AMDGPU_BVH_INTERSECT_RAY : AMDGPUGenericInstruction {4675 let OutOperandList = (outs type0:$dst);4676 let InOperandList = (ins unknown:$opcode, variable_ops);4677 let hasSideEffects = 0;4678 let mayLoad = 1;4679 let mayStore = 0;4680}4681 4682def G_AMDGPU_BVH_DUAL_INTERSECT_RAY : AMDGPUGenericInstruction {4683 let OutOperandList = (outs type0:$dst, type1:$ray_origin, type1:$ray_dir);4684 let InOperandList = (ins unknown:$opcode, variable_ops);4685 let hasSideEffects = 0;4686 let mayLoad = 1;4687 let mayStore = 0;4688}4689 4690def G_AMDGPU_BVH8_INTERSECT_RAY : AMDGPUGenericInstruction {4691 let OutOperandList = (outs type0:$dst, type1:$ray_origin, type1:$ray_dir);4692 let InOperandList = (ins unknown:$opcode, variable_ops);4693 let hasSideEffects = 0;4694 let mayLoad = 1;4695 let mayStore = 0;4696}4697 4698// Generic instruction for SI_CALL, so we can select the register bank and insert a waterfall loop4699// if necessary.4700def G_SI_CALL : AMDGPUGenericInstruction {4701 let OutOperandList = (outs SReg_64:$dst);4702 let InOperandList = (ins type0:$src0, unknown:$callee);4703 let Size = 4;4704 let isCall = 1;4705 let UseNamedOperandTable = 1;4706 let SchedRW = [WriteBranch];4707 // TODO: Should really base this on the call target4708 let isConvergent = 1;4709}4710 4711// Uniform in vgpr - vgpr with same value in all active lanes.4712 4713// $dst = $src0 != 0, selected as:4714// $dst(SCC) = s_cmp_lg $src0, 04715// src0 is either exec or 0 (same value for all active lanes),4716// for example result of comparison of two uniform in vgpr.4717def G_AMDGPU_COPY_SCC_VCC : AMDGPUGenericInstruction {4718 let OutOperandList = (outs type0:$dst);4719 let InOperandList = (ins type1:$src0);4720 let hasSideEffects = 0;4721}4722 4723// $dst = $src0 ? exec : 0, selected as:4724// SCC = COPY $src04725// $dst(SReg_32/64) = s_cselect exec, 04726def G_AMDGPU_COPY_VCC_SCC : AMDGPUGenericInstruction {4727 let OutOperandList = (outs type0:$dst);4728 let InOperandList = (ins type1:$src0);4729 let hasSideEffects = 0;4730}4731 4732// Move uniform in vgpr to sgpr. Selected as v_readfirstlane_b32.4733// Semantic difference in READ ANY instead of FIRST(active) LANE allows for4734// vgpr to sgpr back-to vgpr combine, vgpr has same value in all active lanes4735// vgprDst = COPY (G_AMDGPU_READANYLANE vgprSrc) -> vgprDst = sgprSrc4736def G_AMDGPU_READANYLANE : AMDGPUGenericInstruction {4737 let OutOperandList = (outs type0:$dst);4738 let InOperandList = (ins type0:$src0);4739 let hasSideEffects = 0;4740}4741 4742//============================================================================//4743// Dummy Instructions4744//============================================================================//4745 4746def V_ILLEGAL : Enc32, InstSI<(outs), (ins), "v_illegal"> {4747 let Inst{31-0} = 0x00000000;4748 let FixedSize = 1;4749 let Size = 4;4750 let Uses = [EXEC];4751 let hasSideEffects = 1;4752 let SubtargetPredicate = isGFX10Plus;4753}4754 4755defvar VGPR32_Ptr_Opcodes = [LOAD_STACK_GUARD];4756defvar VGPR64_Ptr_Opcodes = !listremove(PseudosWithPtrOps, VGPR32_Ptr_Opcodes);4757 4758foreach inst = VGPR32_Ptr_Opcodes in {4759 def : RemapPointerOperands<inst, VGPR_32>;4760}4761 4762foreach inst = VGPR64_Ptr_Opcodes in {4763 def : RemapPointerOperands<inst, VReg_64_AlignTarget>;4764}4765