2958 lines · plain
1//===-- VOP3Instructions.td - Vector Instruction Definitions --------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9def BITOP3_32 : ComplexPattern<i32, 4, "SelectBITOP3", [and, or, xor]>;10def BITOP3_16 : ComplexPattern<i16, 4, "SelectBITOP3", [and, or, xor]>;11 12// Matches PTRADD as a commutative operation.13def ptradd_commutative : PatFrags<(ops node:$src0, node:$src1),14 [(ptradd node:$src0, node:$src1), (ptradd node:$src1, node:$src0)]>;15 16// Special case for v_div_fmas_{f32|f64}, since it seems to be the17// only VOP instruction that implicitly reads VCC.18let Asm64 = " $vdst, $src0_modifiers, $src1_modifiers, $src2_modifiers$clamp$omod" in {19def VOP_F32_F32_F32_F32_VCC : VOPProfile<[f32, f32, f32, f32]> {20 let Outs64 = (outs DstRC.RegClass:$vdst);21 let HasExtVOP3DPP = 0;22 let HasExtDPP = 0;23 let IsSingle = 1;24}25def VOP_F64_F64_F64_F64_VCC : VOPProfile<[f64, f64, f64, f64]> {26 let Outs64 = (outs DstRC.RegClass:$vdst);27 let HasExt64BitDPP = 1;28 let IsSingle = 1;29}30}31 32class VOP3b_Profile<ValueType vt> : VOPProfile<[vt, vt, vt, vt]> {33 let Outs64 = (outs DstRC:$vdst, VOPDstS64orS32:$sdst);34 let Asm64 = "$vdst, $sdst, $src0_modifiers, $src1_modifiers, $src2_modifiers$clamp$omod";35 let IsSingle = 1;36 let HasExtVOP3DPP = 0;37 let HasExtDPP = 0;38}39 40def DIV_FIXUP_F32_PROF : VOP3_Profile<VOP_F32_F32_F32_F32> {41 let HasExtVOP3DPP = 0;42 let HasExtDPP = 0;43}44 45def VOP3b_I64_I1_I32_I32_I64 : VOPProfile<[i64, i32, i32, i64]> {46 let HasClamp = 1;47 48 let IsSingle = 1;49 let Outs64 = (outs DstRC:$vdst, VOPDstS64orS32:$sdst);50 let Asm64 = "$vdst, $sdst, $src0, $src1, $src2$clamp";51}52 53let HasExt64BitDPP = 1 in {54def VOP3b_F32_I1_F32_F32_F32 : VOP3b_Profile<f32>;55def VOP3b_F64_I1_F64_F64_F64 : VOP3b_Profile<f64> {56 let OutsVOP3DPP = Outs64;57 let AsmVOP3DPP = getAsmVOP3DPP<Asm64>.ret;58 let AsmVOP3DPP16 = getAsmVOP3DPP16<Asm64>.ret;59 let AsmVOP3DPP8 = getAsmVOP3DPP8<Asm64>.ret;60}61 62def VOP3b_I64_I1_I32_I32_I64_DPP : VOPProfile<[i64, i32, i32, i64]> {63 let HasClamp = 1;64 65 let IsSingle = 1;66 let Outs64 = (outs DstRC:$vdst, VOPDstS64orS32:$sdst);67 let OutsVOP3DPP = Outs64;68 let Asm64 = "$vdst, $sdst, $src0, $src1, $src2$clamp";69 let AsmVOP3DPP = getAsmVOP3DPP<Asm64>.ret;70 let AsmVOP3DPP16 = getAsmVOP3DPP16<Asm64>.ret;71 let AsmVOP3DPP8 = getAsmVOP3DPP8<Asm64>.ret;72}73 74class V_MUL_PROF<VOPProfile P> : VOP3_Profile<P> {75 let HasExtVOP3DPP = 0;76 let HasExtDPP = 0;77}78 79def V_LSHL_ADD_U64_PROF : VOP3_Profile<VOP_I64_I64_I32_I64>;80 81def VOP_F64_F64_F64_F64_DPP_PROF : VOP3_Profile<VOP_F64_F64_F64_F64>;82def V_MAD_U32_PROF: VOP3_Profile<VOP_I32_I32_I32_I32> {83 let HasExtVOP3DPP = 0;84 let HasExt64BitDPP = 1;85}86def VOP_I64_I64_I64_DPP : VOP3_Profile<VOP_I64_I64_I64>;87def VOP_I32_I32_I64_DPP : VOP3_Profile<VOPProfile<[i64, i32, i32, i64]>> {88 let HasClamp = 1;89}90} // End HasExt64BitDPP = 1;91 92//===----------------------------------------------------------------------===//93// VOP3 INTERP94//===----------------------------------------------------------------------===//95 96class VOP3Interp<string OpName, VOPProfile P, list<dag> pattern = []> :97 VOP3_Pseudo<OpName, P, pattern> {98 let AsmMatchConverter = "cvtVOP3Interp";99 let mayRaiseFPException = 0;100 let VOP3_OPSEL = P.HasOpSel;101}102 103def VOP3_INTERP : VOPProfile<[f32, f32, i32, untyped]> {104 let Src0Mod = FPVRegInputMods;105 let Ins64 = (ins Src0Mod:$src0_modifiers, VRegSrc_32:$src0,106 InterpAttr:$attr, InterpAttrChan:$attrchan,107 Clamp0:$clamp, omod0:$omod);108 109 let Asm64 = "$vdst, $src0_modifiers, $attr$attrchan$clamp$omod";110}111 112def VOP3_INTERP_MOV : VOPProfile<[f32, i32, i32, untyped]> {113 let Ins64 = (ins InterpSlot:$src0,114 InterpAttr:$attr, InterpAttrChan:$attrchan,115 Clamp0:$clamp, omod0:$omod);116 117 let Asm64 = "$vdst, $src0, $attr$attrchan$clamp$omod";118 119 let HasClamp = 1;120 let HasSrc0Mods = 0;121}122 123class getInterp16Asm <bit HasSrc2, bit HasOMod, bit OpSel> {124 string src2 = !if(HasSrc2, ", $src2_modifiers", "");125 string omod = !if(HasOMod, "$omod", "");126 string opsel = !if(OpSel, "$op_sel", "");127 string ret =128 " $vdst, $src0_modifiers, $attr$attrchan"#src2#"$high$clamp"#omod#opsel;129}130 131class getInterp16Ins <bit HasSrc2, bit HasOMod,132 Operand Src0Mod, Operand Src2Mod, bit OpSel> {133 dag ret1 = !if(HasSrc2,134 !if(HasOMod,135 (ins Src0Mod:$src0_modifiers, VRegSrc_32:$src0,136 InterpAttr:$attr, InterpAttrChan:$attrchan,137 Src2Mod:$src2_modifiers, VRegSrc_32:$src2,138 highmod:$high, Clamp0:$clamp, omod0:$omod),139 (ins Src0Mod:$src0_modifiers, VRegSrc_32:$src0,140 InterpAttr:$attr, InterpAttrChan:$attrchan,141 Src2Mod:$src2_modifiers, VRegSrc_32:$src2,142 highmod:$high, Clamp0:$clamp)143 ),144 (ins Src0Mod:$src0_modifiers, VRegSrc_32:$src0,145 InterpAttr:$attr, InterpAttrChan:$attrchan,146 highmod:$high, Clamp0:$clamp, omod0:$omod)147 );148 dag ret2 = !if(OpSel, (ins op_sel0:$op_sel), (ins));149 dag ret = !con(ret1, ret2);150}151 152class VOP3_INTERP16 <list<ValueType> ArgVT, bit OpSel = 0> : VOPProfile<ArgVT> {153 let IsSingle = 1;154 let HasOMod = !ne(DstVT, f16);155 let HasHigh = 1;156 let HasOpSel = OpSel;157 158 let Src0Mod = FPVRegInputMods;159 let Src2Mod = FPVRegInputMods;160 161 let Outs64 = (outs DstRC.RegClass:$vdst);162 let Ins64 = getInterp16Ins<HasSrc2, HasOMod, Src0Mod, Src2Mod, OpSel>.ret;163 let Asm64 = getInterp16Asm<HasSrc2, HasOMod, OpSel>.ret;164}165 166//===----------------------------------------------------------------------===//167// VOP3 Instructions168//===----------------------------------------------------------------------===//169 170let isCommutable = 1 in {171 172let isReMaterializable = 1 in {173let mayRaiseFPException = 0 in {174let SubtargetPredicate = HasMadMacF32Insts in {175defm V_MAD_LEGACY_F32 : VOP3Inst <"v_mad_legacy_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;176defm V_MAD_F32 : VOP3Inst <"v_mad_f32", VOP3_Profile<VOP_F32_F32_F32_F32>, any_fmad>;177} // End SubtargetPredicate = HasMadMacInsts178 179let SubtargetPredicate = HasFmaLegacy32 in180defm V_FMA_LEGACY_F32 : VOP3Inst <"v_fma_legacy_f32",181 VOP3_Profile<VOP_F32_F32_F32_F32>,182 int_amdgcn_fma_legacy>;183}184 185defm V_MAD_I32_I24 : VOP3Inst <"v_mad_i32_i24", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>>;186defm V_MAD_U32_U24 : VOP3Inst <"v_mad_u32_u24", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>>;187defm V_FMA_F32 : VOP3Inst <"v_fma_f32", VOP3_Profile<VOP_F32_F32_F32_F32>, any_fma>, VOPD_Component<0x13, "v_fma_f32">;188let SubtargetPredicate = HasLerpInst in189 defm V_LERP_U8 : VOP3Inst <"v_lerp_u8", VOP3_Profile<VOP_I32_I32_I32_I32>, int_amdgcn_lerp>;190 191let SchedRW = [WriteIntMul] in {192 let SubtargetPredicate = HasMadU32Inst in193 defm V_MAD_U32 : VOP3Inst <"v_mad_u32", V_MAD_U32_PROF>;194 let SubtargetPredicate = isGFX1250Plus in {195 defm V_MAD_NC_U64_U32 : VOP3Inst<"v_mad_nc_u64_u32", VOP_I32_I32_I64_DPP>;196 defm V_MAD_NC_I64_I32 : VOP3Inst<"v_mad_nc_i64_i32", VOP_I32_I32_I64_DPP>;197 }198}199 200let SchedRW = [WriteDoubleAdd] in {201let FPDPRounding = 1 in {202defm V_FMA_F64 : VOP3Inst <"v_fma_f64", VOP_F64_F64_F64_F64_DPP_PROF, any_fma>, VOPD_Component<0x20, "v_fma_f64">;203let SubtargetPredicate = isNotGFX12Plus in {204defm V_ADD_F64 : VOP3Inst <"v_add_f64", VOP3_Profile<VOP_F64_F64_F64>, any_fadd>;205defm V_MUL_F64 : VOP3Inst <"v_mul_f64", VOP3_Profile<VOP_F64_F64_F64>, any_fmul>;206} // End SubtargetPredicate = isNotGFX12Plus207} // End FPDPRounding = 1208let SubtargetPredicate = isNotGFX12Plus in {209defm V_MIN_F64 : VOP3Inst <"v_min_f64", VOP3_Profile<VOP_F64_F64_F64>, fminnum_like>;210defm V_MAX_F64 : VOP3Inst <"v_max_f64", VOP3_Profile<VOP_F64_F64_F64>, fmaxnum_like>;211} // End SubtargetPredicate = isNotGFX12Plus212} // End SchedRW = [WriteDoubleAdd]213 214let SchedRW = [WriteIntMul] in {215defm V_MUL_LO_U32 : VOP3Inst <"v_mul_lo_u32", V_MUL_PROF<VOP_I32_I32_I32>, DivergentBinFrag<mul>>;216defm V_MUL_HI_U32 : VOP3Inst <"v_mul_hi_u32", V_MUL_PROF<VOP_I32_I32_I32>, mulhu>;217defm V_MUL_LO_I32 : VOP3Inst <"v_mul_lo_i32", V_MUL_PROF<VOP_I32_I32_I32>>;218defm V_MUL_HI_I32 : VOP3Inst <"v_mul_hi_i32", V_MUL_PROF<VOP_I32_I32_I32>, mulhs>;219} // End SchedRW = [WriteIntMul]220 221let SubtargetPredicate = HasIEEEMinimumMaximumInsts, ReadsModeReg = 0, AddedComplexity = 1 in {222defm V_MINIMUM_F32 : VOP3Inst <"v_minimum_f32", VOP3_Profile<VOP_F32_F32_F32>, fminimum>;223defm V_MAXIMUM_F32 : VOP3Inst <"v_maximum_f32", VOP3_Profile<VOP_F32_F32_F32>, fmaximum>;224defm V_MINIMUM_F16 : VOP3Inst_t16 <"v_minimum_f16", VOP_F16_F16_F16, fminimum>;225defm V_MAXIMUM_F16 : VOP3Inst_t16 <"v_maximum_f16", VOP_F16_F16_F16, fmaximum>;226 227let SchedRW = [WriteDoubleAdd] in {228defm V_MINIMUM_F64 : VOP3Inst <"v_minimum_f64", VOP3_Profile<VOP_F64_F64_F64>, fminimum>;229defm V_MAXIMUM_F64 : VOP3Inst <"v_maximum_f64", VOP3_Profile<VOP_F64_F64_F64>, fmaximum>;230} // End SchedRW = [WriteDoubleAdd]231} // End SubtargetPredicate = HasIEEEMinimumMaximumInsts, ReadsModeReg = 0, AddedComplexity = 1232 233let SubtargetPredicate = isGFX1250Plus, SchedRW = [WriteDoubleAdd] in {234defm V_MAX_I64 : VOP3Inst <"v_max_i64", VOP_I64_I64_I64_DPP, smax>;235defm V_MAX_U64 : VOP3Inst <"v_max_u64", VOP_I64_I64_I64_DPP, umax>;236defm V_MIN_I64 : VOP3Inst <"v_min_i64", VOP_I64_I64_I64_DPP, smin>;237defm V_MIN_U64 : VOP3Inst <"v_min_u64", VOP_I64_I64_I64_DPP, umin>;238} // End SubtargetPredicate = isGFX1250Plus, SchedRW = [WriteDoubleAdd]239 240} // End isReMaterializable = 1241 242let Uses = [MODE, VCC, EXEC] in {243// v_div_fmas_f32:244// result = src0 * src1 + src2245// if (vcc)246// result *= 2^32247//248let SchedRW = [WriteFloatFMA] in249defm V_DIV_FMAS_F32 : VOP3Inst_Pseudo_Wrapper <"v_div_fmas_f32", VOP_F32_F32_F32_F32_VCC, []>;250// v_div_fmas_f64:251// result = src0 * src1 + src2252// if (vcc)253// result *= 2^64254//255let SchedRW = [WriteDouble], FPDPRounding = 1 in256defm V_DIV_FMAS_F64 : VOP3Inst <"v_div_fmas_f64", VOP_F64_F64_F64_F64_VCC>;257} // End Uses = [MODE, VCC, EXEC]258 259} // End isCommutable = 1260 261let isReMaterializable = 1 in {262let mayRaiseFPException = 0, SubtargetPredicate = HasCubeInsts in {263defm V_CUBEID_F32 : VOP3Inst <"v_cubeid_f32", VOP3_Profile<VOP_F32_F32_F32_F32>, int_amdgcn_cubeid>;264defm V_CUBESC_F32 : VOP3Inst <"v_cubesc_f32", VOP3_Profile<VOP_F32_F32_F32_F32>, int_amdgcn_cubesc>;265defm V_CUBETC_F32 : VOP3Inst <"v_cubetc_f32", VOP3_Profile<VOP_F32_F32_F32_F32>, int_amdgcn_cubetc>;266defm V_CUBEMA_F32 : VOP3Inst <"v_cubema_f32", VOP3_Profile<VOP_F32_F32_F32_F32>, int_amdgcn_cubema>;267} // mayRaiseFPException = 0, SubtargetPredicate = HasCubeInsts268 269defm V_BFE_U32 : VOP3Inst <"v_bfe_u32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUbfe_u32>;270defm V_BFE_I32 : VOP3Inst <"v_bfe_i32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUbfe_i32>;271defm V_BFI_B32 : VOP3Inst <"v_bfi_b32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUbfi>;272defm V_ALIGNBIT_B32 : VOP3Inst_t16_with_profiles <"v_alignbit_b32",273 VOP3_Profile<VOP_I32_I32_I32_I32>,274 VOP3_Profile_True16<VOP_I32_I32_I32_I16, VOP3_OPSEL>,275 VOP3_Profile_Fake16<VOP_I32_I32_I32_I16, VOP3_OPSEL>,276 fshr, null_frag>;277 278defm V_ALIGNBYTE_B32 : VOP3Inst <"v_alignbyte_b32", VOP3_Profile<VOP_I32_I32_I32_I32>, int_amdgcn_alignbyte>;279 280// In gfx9 and 10, opsel is allowed for V_ALIGNBIT_B32 and V_ALIGNBYTE_B32.281// Hardware uses opsel[1:0] to byte-select src2. Other opsel bits are ignored.282defm V_ALIGNBIT_B32_opsel : VOP3Inst <"v_alignbit_b32_opsel", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_OPSEL>>;283defm V_ALIGNBYTE_B32_opsel : VOP3Inst <"v_alignbyte_b32_opsel", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_OPSEL>>;284 285let True16Predicate = UseRealTrue16Insts in286defm V_ALIGNBYTE_B32_t16 : VOP3Inst <"v_alignbyte_b32_t16", VOP3_Profile_True16<VOP_I32_I32_I32_I16, VOP3_OPSEL>>;287let True16Predicate = UseFakeTrue16Insts in288defm V_ALIGNBYTE_B32_fake16 : VOP3Inst <"v_alignbyte_b32_fake16", VOP3_Profile_Fake16<VOP_I32_I32_I32_I16, VOP3_OPSEL>>;289 290// XXX - No FPException seems suspect but manual doesn't say it does291let mayRaiseFPException = 0 in {292 let isCommutable = 1 in {293 defm V_MIN3_I32 : VOP3Inst <"v_min3_i32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUsmin3>;294 defm V_MIN3_U32 : VOP3Inst <"v_min3_u32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUumin3>;295 defm V_MAX3_I32 : VOP3Inst <"v_max3_i32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUsmax3>;296 defm V_MAX3_U32 : VOP3Inst <"v_max3_u32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUumax3>;297 defm V_MED3_I32 : VOP3Inst <"v_med3_i32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUsmed3>;298 defm V_MED3_U32 : VOP3Inst <"v_med3_u32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUumed3>;299 } // End isCommutable = 1300 defm V_MIN3_F32 : VOP3Inst <"v_min3_f32", VOP3_Profile<VOP_F32_F32_F32_F32>, AMDGPUfmin3>;301 defm V_MAX3_F32 : VOP3Inst <"v_max3_f32", VOP3_Profile<VOP_F32_F32_F32_F32>, AMDGPUfmax3>;302 defm V_MED3_F32 : VOP3Inst <"v_med3_f32", VOP3_Profile<VOP_F32_F32_F32_F32>, AMDGPUfmed3>;303} // End mayRaiseFPException = 0304 305let SubtargetPredicate = HasMinimum3Maximum3F32, ReadsModeReg = 0 in {306 defm V_MINIMUM3_F32 : VOP3Inst <"v_minimum3_f32", VOP3_Profile<VOP_F32_F32_F32_F32>, AMDGPUfminimum3>;307 defm V_MAXIMUM3_F32 : VOP3Inst <"v_maximum3_f32", VOP3_Profile<VOP_F32_F32_F32_F32>, AMDGPUfmaximum3>;308} // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0309 310let isCommutable = 1, SubtargetPredicate = HasSadInsts in {311 defm V_SAD_U8 : VOP3Inst <"v_sad_u8", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>>;312 defm V_SAD_HI_U8 : VOP3Inst <"v_sad_hi_u8", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>>;313 defm V_SAD_U16 : VOP3Inst <"v_sad_u16", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>>;314 defm V_SAD_U32 : VOP3Inst <"v_sad_u32", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>>;315} // End isCommutable = 1, SubtargetPredicate = HasSadInsts316defm V_CVT_PK_U8_F32 : VOP3Inst<"v_cvt_pk_u8_f32", VOP3_Profile<VOP_I32_F32_I32_I32>, int_amdgcn_cvt_pk_u8_f32>;317 318defm V_DIV_FIXUP_F32 : VOP3Inst <"v_div_fixup_f32", DIV_FIXUP_F32_PROF, AMDGPUdiv_fixup>;319 320let SchedRW = [WriteDoubleAdd], FPDPRounding = 1 in {321 defm V_DIV_FIXUP_F64 : VOP3Inst <"v_div_fixup_f64", VOP_F64_F64_F64_F64_DPP_PROF, AMDGPUdiv_fixup>;322 defm V_LDEXP_F64 : VOP3Inst <"v_ldexp_f64", VOP3_Profile<VOP_F64_F64_I32>, any_fldexp>;323} // End SchedRW = [WriteDoubleAdd], FPDPRounding = 1324} // End isReMaterializable = 1325 326let SubtargetPredicate = isGFX9GFX10 in327def : GCNPat <328(i32 (int_amdgcn_alignbyte (i32 (VOP3OpSelMods i32:$src0, i32:$src0_modifiers)),329 (i32 (VOP3OpSelMods i32:$src1, i32:$src1_modifiers)),330 (i32 (VOP3OpSelMods i32:$src2, i32:$src2_modifiers)))),331(V_ALIGNBYTE_B32_opsel_e64 i32:$src0_modifiers, VSrc_b32:$src0,332 i32:$src1_modifiers, VSrc_b32:$src1,333 i32:$src2_modifiers, VGPR_32:$src2)334>;335 336let True16Predicate = UseFakeTrue16Insts in337def : GCNPat <338(i32 (int_amdgcn_alignbyte (i32 (VOP3OpSelMods i32:$src0, i32:$src0_modifiers)),339 (i32 (VOP3OpSelMods i32:$src1, i32:$src1_modifiers)),340 (i32 (VOP3OpSelMods i32:$src2, i32:$src2_modifiers)))),341(V_ALIGNBYTE_B32_fake16_e64 i32:$src0_modifiers, VSrc_b32:$src0,342 i32:$src1_modifiers, VSrc_b32:$src1,343 i32:$src2_modifiers, VGPR_32:$src2)344>;345 346let True16Predicate = UseRealTrue16Insts in347def : GCNPat <348(i32 (int_amdgcn_alignbyte (i32 (VOP3OpSelMods i32:$src0, i32:$src0_modifiers)),349 (i32 (VOP3OpSelMods i32:$src1, i32:$src1_modifiers)),350 (i32 (VOP3OpSelMods i32:$src2, i32:$src2_modifiers)))),351(V_ALIGNBYTE_B32_t16_e64 i32:$src0_modifiers, VSrc_b32:$src0,352 i32:$src1_modifiers, VSrc_b32:$src1,353 i32:$src2_modifiers, (i16 (EXTRACT_SUBREG VGPR_32:$src2, lo16)))354>;355 356let mayRaiseFPException = 0 in { // Seems suspicious but manual doesn't say it does.357 let SchedRW = [WriteFloatFMA, WriteSALU] in358 defm V_DIV_SCALE_F32 : VOP3Inst_Pseudo_Wrapper <"v_div_scale_f32", VOP3b_F32_I1_F32_F32_F32> ;359 360 // Double precision division pre-scale.361 let SchedRW = [WriteDouble, WriteSALU], FPDPRounding = 1 in362 defm V_DIV_SCALE_F64 : VOP3Inst <"v_div_scale_f64", VOP3b_F64_I1_F64_F64_F64>;363} // End mayRaiseFPException = 0364 365let isReMaterializable = 1 in366defm V_MSAD_U8 : VOP3Inst <"v_msad_u8", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>>;367 368let Constraints = "@earlyclobber $vdst" in {369defm V_MQSAD_PK_U16_U8 : VOP3Inst <"v_mqsad_pk_u16_u8", VOP3_Profile<VOP_I64_I64_I32_I64, VOP3_CLAMP>>;370} // End Constraints = "@earlyclobber $vdst"371 372 373let isReMaterializable = 1 in {374let SchedRW = [WriteDouble] in {375defm V_TRIG_PREOP_F64 : VOP3Inst <"v_trig_preop_f64", VOP3_Profile<VOP_F64_F64_I32>, int_amdgcn_trig_preop>;376} // End SchedRW = [WriteDouble]377 378let SchedRW = [Write64Bit] in {379 let SubtargetPredicate = isGFX6GFX7 in {380 defm V_LSHL_B64 : VOP3Inst <"v_lshl_b64", VOP3_Profile<VOP_I64_I64_I32>, cshl_64>;381 defm V_LSHR_B64 : VOP3Inst <"v_lshr_b64", VOP3_Profile<VOP_I64_I64_I32>, csrl_64>;382 defm V_ASHR_I64 : VOP3Inst <"v_ashr_i64", VOP3_Profile<VOP_I64_I64_I32>, csra_64>;383 } // End SubtargetPredicate = isGFX6GFX7384 385 let SubtargetPredicate = isGFX8Plus in {386 defm V_LSHRREV_B64 : VOP3Inst <"v_lshrrev_b64", VOP3_Profile<VOP_I64_I32_I64>, clshr_rev_64>;387 defm V_ASHRREV_I64 : VOP3Inst <"v_ashrrev_i64", VOP3_Profile<VOP_I64_I32_I64>, cashr_rev_64>;388 } // End SubtargetPredicate = isGFX8Plus389 390 let SubtargetPredicate = isGFX8GFX9GFX10GFX11 in {391 defm V_LSHLREV_B64 : VOP3Inst <"v_lshlrev_b64", VOP3_Profile<VOP_I64_I32_I64>, clshl_rev_64>;392 } // End SubtargetPredicate = isGFX8GFX9GFX10GFX11393} // End SchedRW = [Write64Bit]394} // End isReMaterializable = 1395 396let True16Predicate = NotUseRealTrue16Insts in397def : GCNPat<398 (i32 (DivergentUnaryFrag<sext> i16:$src)),399 (i32 (V_BFE_I32_e64 i16:$src, (i32 0), (i32 0x10)))400>;401 402let True16Predicate = UseRealTrue16Insts in403def : GCNPat<404 (i32 (DivergentUnaryFrag<sext> i16:$src)),405 (i32 (V_BFE_I32_e64406 (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (i16 (IMPLICIT_DEF)), hi16),407 (i32 0), (i32 0x10)))408>;409 410let isReMaterializable = 1 in {411let SubtargetPredicate = isGFX6GFX7GFX10Plus in {412defm V_MULLIT_F32 : VOP3Inst <"v_mullit_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;413} // End SubtargetPredicate = isGFX6GFX7GFX10Plus414 415let SchedRW = [Write32Bit] in {416let SubtargetPredicate = isGFX8Plus in {417defm V_PERM_B32 : VOP3Inst <"v_perm_b32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUperm>;418} // End SubtargetPredicate = isGFX8Plus419} // End SchedRW = [Write32Bit]420} // End isReMaterializable = 1421 422def VOPProfileMQSAD : VOP3_Profile<VOP_V4I32_I64_I32_V4I32, VOP3_CLAMP> {423 let HasModifiers = 0;424}425 426let SubtargetPredicate = isGFX7Plus in {427let Constraints = "@earlyclobber $vdst", SchedRW = [WriteQuarterRate32] in {428let SubtargetPredicate = HasQsadInsts in429 defm V_QSAD_PK_U16_U8 : VOP3Inst <"v_qsad_pk_u16_u8", VOP3_Profile<VOP_I64_I64_I32_I64, VOP3_CLAMP>>;430defm V_MQSAD_U32_U8 : VOP3Inst <"v_mqsad_u32_u8", VOPProfileMQSAD>;431} // End Constraints = "@earlyclobber $vdst", SchedRW = [WriteQuarterRate32]432} // End SubtargetPredicate = isGFX7Plus433 434let isCommutable = 1, SchedRW = [WriteIntMul, WriteSALU] in {435 let SubtargetPredicate = isGFX7Plus in {436 defm V_MAD_U64_U32 : VOP3Inst <"v_mad_u64_u32", VOP3b_I64_I1_I32_I32_I64_DPP, null_frag, [HasNotMADIntraFwdBug]>;437 defm V_MAD_I64_I32 : VOP3Inst <"v_mad_i64_i32", VOP3b_I64_I1_I32_I32_I64_DPP, null_frag, [HasNotMADIntraFwdBug]>;438 }439 let SubtargetPredicate = isGFX11Only, OtherPredicates = [HasMADIntraFwdBug],440 Constraints = "@earlyclobber $vdst" in {441 defm V_MAD_U64_U32_gfx11 : VOP3Inst <"v_mad_u64_u32", VOP3b_I64_I1_I32_I32_I64>;442 defm V_MAD_I64_I32_gfx11 : VOP3Inst <"v_mad_i64_i32", VOP3b_I64_I1_I32_I32_I64>;443 }444} // End isCommutable = 1, SchedRW = [WriteIntMul, WriteSALU]445 446 447let FPDPRounding = 1 in {448 let Predicates = [Has16BitInsts, isGFX8Only] in {449 defm V_DIV_FIXUP_F16 : VOP3Inst <"v_div_fixup_f16", VOP3_Profile<VOP_F16_F16_F16_F16>, AMDGPUdiv_fixup>;450 let isCommutable = 1 in {451 defm V_FMA_F16 : VOP3Inst <"v_fma_f16", VOP3_Profile<VOP_F16_F16_F16_F16>, any_fma>;452 } // End isCommutable = 1453 } // End Predicates = [Has16BitInsts, isGFX8Only]454 455 let SubtargetPredicate = isGFX9Plus in {456 defm V_DIV_FIXUP_F16_gfx9 : VOP3Inst_t16 <"v_div_fixup_f16_gfx9", VOP_F16_F16_F16_F16, AMDGPUdiv_fixup>;457 defm V_FMA_F16_gfx9 : VOP3Inst_t16 <"v_fma_f16_gfx9", VOP_F16_F16_F16_F16, any_fma>;458 } // End SubtargetPredicate = isGFX9Plus459} // End FPDPRounding = 1460 461let SubtargetPredicate = Has16BitInsts, isCommutable = 1 in {462 463defm V_MAD_U16 : VOP3Inst <"v_mad_u16", VOP3_Profile<VOP_I16_I16_I16_I16, VOP3_CLAMP>>;464defm V_MAD_I16 : VOP3Inst <"v_mad_i16", VOP3_Profile<VOP_I16_I16_I16_I16, VOP3_CLAMP>>;465let FPDPRounding = 1 in {466 defm V_MAD_F16 : VOP3Inst <"v_mad_f16", VOP3_Profile<VOP_F16_F16_F16_F16>, any_fmad>;467 let Uses = [MODE, M0, EXEC] in {468 let OtherPredicates = [isNotGFX90APlus] in469 // For some reason the intrinsic operands are in a different order470 // from the instruction operands.471 def V_INTERP_P2_F16 : VOP3Interp <"v_interp_p2_f16", VOP3_INTERP16<[f16, f32, i32, f32]>,472 [(set f16:$vdst,473 (int_amdgcn_interp_p2_f16 (VOP3Mods f32:$src2, i32:$src2_modifiers),474 (VOP3Mods f32:$src0, i32:$src0_modifiers),475 (i32 timm:$attrchan),476 (i32 timm:$attr),477 (i1 timm:$high),478 M0))]>;479 } // End Uses = [M0, MODE, EXEC]480} // End FPDPRounding = 1481 482let SubtargetPredicate = isGFX9Only, FPDPRounding = 1 in {483 defm V_MAD_F16_gfx9 : VOP3Inst <"v_mad_f16_gfx9", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>> ;484} // End SubtargetPredicate = isGFX9Only, FPDPRounding = 1485 486let SubtargetPredicate = isGFX9Plus in {487defm V_MAD_U16_gfx9 : VOP3Inst_t16 <"v_mad_u16_gfx9", VOP_I16_I16_I16_I16>;488defm V_MAD_I16_gfx9 : VOP3Inst_t16 <"v_mad_i16_gfx9", VOP_I16_I16_I16_I16>;489let OtherPredicates = [isNotGFX90APlus] in490def V_INTERP_P2_F16_opsel : VOP3Interp <"v_interp_p2_f16_opsel", VOP3_INTERP16<[f16, f32, i32, f32], /*OpSel*/ 1>>;491} // End SubtargetPredicate = isGFX9Plus492 493// This predicate should only apply to the selection pattern. The494// instruction still exists and should decode on subtargets with495// other bank counts.496let OtherPredicates = [isNotGFX90APlus, has32BankLDS], Uses = [MODE, M0, EXEC], FPDPRounding = 1 in {497def V_INTERP_P1LL_F16 : VOP3Interp <"v_interp_p1ll_f16", VOP3_INTERP16<[f32, f32, i32, untyped]>,498 [(set f32:$vdst, (int_amdgcn_interp_p1_f16 (VOP3Mods f32:$src0, i32:$src0_modifiers),499 (i32 timm:$attrchan),500 (i32 timm:$attr),501 (i1 timm:$high), M0))]>;502} // End OtherPredicates = [isNotGFX90APlus, has32BankLDS], Uses = [MODE, M0, EXEC], FPDPRounding = 1503 504let OtherPredicates = [isNotGFX90APlus], Uses = [MODE, M0, EXEC], FPDPRounding = 1 in {505def V_INTERP_P1LV_F16 : VOP3Interp <"v_interp_p1lv_f16", VOP3_INTERP16<[f32, f32, i32, f16]>>;506} // End OtherPredicates = [isNotGFX90APlus], Uses = [MODE, M0, EXEC], FPDPRounding = 1507 508} // End SubtargetPredicate = Has16BitInsts, isCommutable = 1509 510let True16Predicate = NotUseRealTrue16Insts in511def : GCNPat<512 (i64 (DivergentUnaryFrag<sext> i16:$src)),513 (REG_SEQUENCE VReg_64,514 (i32 (V_BFE_I32_e64 $src, (S_MOV_B32 (i32 0)), (S_MOV_B32 (i32 0x10)))), sub0,515 (i32 (COPY_TO_REGCLASS516 (V_ASHRREV_I32_e32 (S_MOV_B32 (i32 0x1f)), (i32 (V_BFE_I32_e64 $src, (S_MOV_B32 (i32 0)), (S_MOV_B32 (i32 0x10))))517 ), VGPR_32)), sub1)518>;519 520let True16Predicate = UseRealTrue16Insts in521def : GCNPat<522 (i64 (DivergentUnaryFrag<sext> i16:$src)),523 (REG_SEQUENCE VReg_64,524 (i32 (V_BFE_I32_e64525 (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (i16 (IMPLICIT_DEF)), hi16),526 (S_MOV_B32 (i32 0)), (S_MOV_B32 (i32 0x10)))), sub0,527 (i32 (COPY_TO_REGCLASS528 (V_ASHRREV_I32_e32 (S_MOV_B32 (i32 0x1f)), (i32 (V_BFE_I32_e64 $src, (S_MOV_B32 (i32 0)), (S_MOV_B32 (i32 0x10))))529 ), VGPR_32)), sub1)530>;531 532let SubtargetPredicate = isGFX8Plus, Uses = [MODE, M0, EXEC], OtherPredicates = [isNotGFX90APlus] in {533def V_INTERP_P1_F32_e64 : VOP3Interp <"v_interp_p1_f32", VOP3_INTERP>;534def V_INTERP_P2_F32_e64 : VOP3Interp <"v_interp_p2_f32", VOP3_INTERP>;535def V_INTERP_MOV_F32_e64 : VOP3Interp <"v_interp_mov_f32", VOP3_INTERP_MOV>;536} // End SubtargetPredicate = isGFX8Plus, Uses = [MODE, M0, EXEC], OtherPredicates = [isNotGFX90APlus]537 538// Note: 16-bit instructions produce a 0 result in the high 16-bits539// on GFX8 and GFX9 and preserve high 16 bits on GFX10+540multiclass Arithmetic_i16_0Hi_TernaryPats <SDPatternOperator op, Instruction inst> {541 def : GCNPat<542 (i32 (zext (op i16:$src0, i16:$src1, i16:$src2))),543 (inst VSrc_b16:$src0, VSrc_b16:$src1, VSrc_b16:$src2)544 >;545}546 547let Predicates = [Has16BitInsts, isGFX8GFX9] in {548defm : Arithmetic_i16_0Hi_TernaryPats<imad, V_MAD_U16_e64>;549}550 551let Predicates = [Has16BitInsts, isGFX6GFX7GFX8GFX9] in {552 553// FIXME: Should be able to just pass imad to the instruction554// definition pattern, but the implied clamp input interferes.555multiclass Ternary_i16_Pats <SDPatternOperator op, Instruction inst> {556 def : GCNPat <557 (op i16:$src0, i16:$src1, i16:$src2),558 (inst i16:$src0, i16:$src1, i16:$src2, (i1 0))559 >;560}561 562defm: Ternary_i16_Pats<imad, V_MAD_U16_e64>;563 564} // End Predicates = [Has16BitInsts, isGFX6GFX7GFX8GFX9]565 566multiclass Ternary_i16_Pats_gfx9<SDPatternOperator op1, SDPatternOperator op2,567 Instruction inst> {568 def : GCNPat <569 (op2 (op1 i16:$src0, i16:$src1), i16:$src2),570 (inst SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, SRCMODS.NONE, $src2, DSTCLAMP.NONE)571 >;572}573 574let True16Predicate = UseRealTrue16Insts in {575 defm: Ternary_i16_Pats_gfx9<mul, add, V_MAD_U16_gfx9_t16_e64>;576} // End True16Predicates = UseRealTrue16Insts577let True16Predicate = UseFakeTrue16Insts in {578 defm: Ternary_i16_Pats_gfx9<mul, add, V_MAD_U16_gfx9_fake16_e64>;579} // End True16Predicates = UseFakeTrue16Insts580let OtherPredicates = [isGFX10Plus, Has16BitInsts], True16Predicate = NotHasTrue16BitInsts in {581 defm: Ternary_i16_Pats_gfx9<mul, add, V_MAD_U16_gfx9_e64>;582} // End OtherPredicates = [isGFX10Plus, Has16BitInsts], True16Predicate = NotHasTrue16BitInsts583 584class ThreeOpFragSDAG<SDPatternOperator op1, SDPatternOperator op2> : PatFrag<585 (ops node:$x, node:$y, node:$z),586 // When the inner operation is used multiple times, selecting 3-op587 // instructions may still be beneficial -- if the other users can be588 // combined similarly. Let's be conservative for now.589 (op2 (HasOneUseBinOp<op1> node:$x, node:$y), node:$z),590 [{591 // Only use VALU ops when the result is divergent.592 if (!N->isDivergent())593 return false;594 595 // Check constant bus limitations.596 //597 // Note: Use !isDivergent as a conservative proxy for whether the value598 // is in an SGPR (uniform values can end up in VGPRs as well).599 unsigned ConstantBusUses = 0;600 for (unsigned i = 0; i < 3; ++i) {601 if (!Operands[i]->isDivergent() &&602 !isInlineImmediate(Operands[i].getNode())) {603 ConstantBusUses++;604 // This uses AMDGPU::V_ADD3_U32_e64, but all three operand instructions605 // have the same constant bus limit.606 if (ConstantBusUses > Subtarget->getConstantBusLimit(AMDGPU::V_ADD3_U32_e64))607 return false;608 }609 }610 611 return true;612 }]> {613 let PredicateCodeUsesOperands = 1;614}615 616class ThreeOpFrag<SDPatternOperator op1, SDPatternOperator op2> : ThreeOpFragSDAG<op1, op2> {617 // The divergence predicate is irrelevant in GlobalISel, as we have618 // proper register bank checks. We just need to verify the constant619 // bus restriction when all the sources are considered.620 //621 // FIXME: With unlucky SGPR operands, we could penalize code by622 // blocking folding SGPR->VGPR copies later.623 // FIXME: There's no register bank verifier624 let GISelPredicateCode = [{625 const int ConstantBusLimit = Subtarget->getConstantBusLimit(AMDGPU::V_ADD3_U32_e64);626 int ConstantBusUses = 0;627 for (unsigned i = 0; i < 3; ++i) {628 const RegisterBank *RegBank = RBI.getRegBank(Operands[i]->getReg(), MRI, TRI);629 if (RegBank->getID() == AMDGPU::SGPRRegBankID) {630 if (++ConstantBusUses > ConstantBusLimit)631 return false;632 }633 }634 return true;635 }];636}637 638def shl_0_to_4 : PatFrag<639 (ops node:$src0, node:$src1), (shl node:$src0, node:$src1),640 [{641 if (auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {642 return C->getZExtValue() <= 4;643 }644 return false;645 }]> {646 let GISelPredicateCode = [{647 int64_t Imm = 0;648 if (!mi_match(MI.getOperand(2).getReg(), MRI, m_ICst(Imm)) &&649 !mi_match(MI.getOperand(2).getReg(), MRI, m_Copy(m_ICst(Imm))))650 return false;651 return (uint64_t)Imm <= 4;652 }];653}654 655class VOP3_CVT_PK_F8_F32_Profile<bit _HasClamp = 0> : VOP3_Profile<VOP_I32_F32_F32, VOP3_OPSEL> {656 defvar Tail = !con(!if(_HasClamp, (ins Clamp:$clamp), (ins)),657 (ins VGPR_32:$vdst_in, op_sel0:$op_sel));658 let InsVOP3OpSel = !con(getIns64<Src0RC64, Src1RC64, Src2RC64, NumSrcArgs,659 0, HasModifiers, HasSrc2Mods,660 HasOMod, Src0Mod, Src1Mod, Src2Mod>.ret,661 Tail);662 let InsVOP3Base = !con(getInsVOP3Base<Src0VOP3DPP, Src1VOP3DPP,663 Src2VOP3DPP, NumSrcArgs, 0, HasModifiers,664 HasSrc2Mods, HasOMod, Src0ModVOP3DPP, Src1ModVOP3DPP,665 Src2ModVOP3DPP, false>.ret,666 Tail);667 let HasClamp = _HasClamp;668 let HasExtVOP3DPP = 1;669}670 671class VOP3_CVT_PK_F8_F32_Profile_fake16<bit _HasClamp = 0> : VOP3_Profile_Fake16<VOP_I16_F32_F32, VOP3_OPSEL> {672 defvar Tail = !con(!if(_HasClamp, (ins Clamp:$clamp), (ins)),673 (ins VGPR_32:$vdst_in, op_sel0:$op_sel));674 let InsVOP3OpSel = !con(getIns64<Src0RC64, Src1RC64, Src2RC64, NumSrcArgs,675 0, HasModifiers, HasSrc2Mods,676 HasOMod, Src0Mod, Src1Mod, Src2Mod>.ret,677 Tail);678 let InsVOP3Base = !con(getInsVOP3Base<Src0VOP3DPP, Src1VOP3DPP,679 Src2VOP3DPP, NumSrcArgs, 0, HasModifiers,680 HasSrc2Mods, HasOMod, Src0ModVOP3DPP, Src1ModVOP3DPP,681 Src2ModVOP3DPP, false>.ret,682 Tail);683 let HasClamp = _HasClamp;684 let HasExtVOP3DPP = 1;685}686 687// This t16 profile with vdst_in operand is for backward compatibility and is used688// for user controlled packing689class VOP3_CVT_PK_F8_F32_Profile_t16<bit _HasClamp = 0> : VOP3_Profile_True16<VOP_I16_F32_F32, VOP3_OPSEL> {690 defvar Tail = !con(!if(_HasClamp, (ins Clamp:$clamp), (ins)),691 (ins VGPR_16:$vdst_in, op_sel0:$op_sel));692 let InsVOP3OpSel = !con(getIns64<Src0RC64, Src1RC64, Src2RC64, NumSrcArgs,693 0, HasModifiers, HasSrc2Mods,694 HasOMod, Src0Mod, Src1Mod, Src2Mod>.ret,695 Tail);696 let InsVOP3Base = !con(getInsVOP3Base<Src0VOP3DPP, Src1VOP3DPP,697 Src2VOP3DPP, NumSrcArgs, 0, HasModifiers,698 HasSrc2Mods, HasOMod, Src0ModVOP3DPP, Src1ModVOP3DPP,699 Src2ModVOP3DPP, false>.ret,700 Tail);701 let HasClamp = _HasClamp;702 let HasExtVOP3DPP = 1;703}704 705def VOP3_CVT_SR_F8_F32_Profile : VOP3_Profile<VOPProfile<[i32, f32, i32, f32]>,706 VOP3_OPSEL> {707 let InsVOP3OpSel = (ins FP32InputMods:$src0_modifiers, Src0RC64:$src0,708 FP32InputMods:$src1_modifiers, Src1RC64:$src1,709 FP32InputMods:$src2_modifiers, VGPR_32:$src2,710 op_sel0:$op_sel);711 let InsVOP3DPP16 = (ins VGPR_32:$old,712 FP32InputMods:$src0_modifiers, Src0VOP3DPP:$src0,713 FP32InputMods:$src1_modifiers, Src1VOP3DPP:$src1,714 FP32InputMods:$src2_modifiers, VGPR_32:$src2,715 op_sel0:$op_sel, dpp_ctrl:$dpp_ctrl, DppRowMask:$row_mask,716 DppBankMask:$bank_mask, DppBoundCtrl:$bound_ctrl, Dpp16FI:$fi);717 let InsVOP3DPP8 = (ins VGPR_32:$old,718 FP32InputMods:$src0_modifiers, Src0VOP3DPP:$src0,719 FP32InputMods:$src1_modifiers, Src1VOP3DPP:$src1,720 FP32InputMods:$src2_modifiers, VGPR_32:$src2,721 op_sel0:$op_sel, dpp8:$dpp8, Dpp8FI:$fi);722 let HasClamp = 0;723 let HasSrc2 = 0;724 let HasSrc2Mods = 1;725 let HasExtVOP3DPP = 1;726 let HasOpSel = 1;727 let HasFP8DstByteSel = 1;728 let HasFP8ByteSel = 0; // It works as a dst-bytesel, but does not have byte_sel operand.729 let AsmVOP3Base = !subst(", $src2_modifiers", "",730 getAsmVOP3Base<NumSrcArgs, HasDst, HasClamp,731 HasOpSel, HasOMod, IsVOP3P, HasModifiers, HasModifiers, 0/*Src1Mods*/,732 HasModifiers, DstVT>.ret);733}734 735class VOP3_CVT_SR_F8_ByteSel_Profile<ValueType SrcVT, bit _HasClamp = 0> :736 VOP3_Profile<VOPProfile<[i32, SrcVT, i32, untyped]>> {737 let HasFP8DstByteSel = 1;738 let HasClamp = _HasClamp;739}740 741def IsPow2Plus1: PatLeaf<(i32 imm), [{742 uint32_t V = N->getZExtValue();743 return isPowerOf2_32(V - 1);744}]>;745 746def Log2_32: SDNodeXForm<imm, [{747 uint32_t V = N->getZExtValue();748 return CurDAG->getTargetConstant(Log2_32(V - 1), SDLoc(N), MVT::i32);749}]>;750 751let SubtargetPredicate = isGFX9Plus in {752let isCommutable = 1, isReMaterializable = 1 in {753 defm V_ADD3_U32 : VOP3Inst <"v_add3_u32", VOP3_Profile<VOP_I32_I32_I32_I32>>;754 defm V_AND_OR_B32 : VOP3Inst <"v_and_or_b32", VOP3_Profile<VOP_I32_I32_I32_I32>>;755 defm V_OR3_B32 : VOP3Inst <"v_or3_b32", VOP3_Profile<VOP_I32_I32_I32_I32>>;756 defm V_XAD_U32 : VOP3Inst <"v_xad_u32", VOP3_Profile<VOP_I32_I32_I32_I32>>;757 defm V_ADD_I32 : VOP3Inst <"v_add_i32", VOP3_Profile<VOP_I32_I32_I32_ARITH>>;758 defm V_ADD_LSHL_U32 : VOP3Inst <"v_add_lshl_u32", VOP3_Profile<VOP_I32_I32_I32_I32>>;759} // End isCommutable = 1, isReMaterializable = 1760// TODO src0 contains the opsel bit for dst, so if we commute, need to mask and swap this761// to the new src0.762defm V_MED3_F16 : VOP3Inst_t16 <"v_med3_f16", VOP_F16_F16_F16_F16, AMDGPUfmed3>;763defm V_MED3_I16 : VOP3Inst_t16 <"v_med3_i16", VOP_I16_I16_I16_I16, AMDGPUsmed3>;764defm V_MED3_U16 : VOP3Inst_t16 <"v_med3_u16", VOP_I16_I16_I16_I16, AMDGPUumed3>;765 766defm V_MIN3_F16 : VOP3Inst_t16 <"v_min3_f16", VOP_F16_F16_F16_F16, AMDGPUfmin3>;767defm V_MIN3_I16 : VOP3Inst_t16 <"v_min3_i16", VOP_I16_I16_I16_I16, AMDGPUsmin3>;768defm V_MIN3_U16 : VOP3Inst_t16 <"v_min3_u16", VOP_I16_I16_I16_I16, AMDGPUumin3>;769 770defm V_MAX3_F16 : VOP3Inst_t16 <"v_max3_f16", VOP_F16_F16_F16_F16, AMDGPUfmax3>;771defm V_MAX3_I16 : VOP3Inst_t16 <"v_max3_i16", VOP_I16_I16_I16_I16, AMDGPUsmax3>;772defm V_MAX3_U16 : VOP3Inst_t16 <"v_max3_u16", VOP_I16_I16_I16_I16, AMDGPUumax3>;773 774let SubtargetPredicate = HasMinimum3Maximum3F16, ReadsModeReg = 0 in {775 defm V_MINIMUM3_F16 : VOP3Inst_t16 <"v_minimum3_f16", VOP_F16_F16_F16_F16, AMDGPUfminimum3>;776 defm V_MAXIMUM3_F16 : VOP3Inst_t16 <"v_maximum3_f16", VOP_F16_F16_F16_F16, AMDGPUfmaximum3>;777} // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0778 779let SubtargetPredicate = HasAddMinMaxInsts, isCommutable = 1, isReMaterializable = 1 in {780 defm V_ADD_MAX_I32 : VOP3Inst <"v_add_max_i32", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>, int_amdgcn_add_max_i32>;781 defm V_ADD_MAX_U32 : VOP3Inst <"v_add_max_u32", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>, int_amdgcn_add_max_u32>;782 defm V_ADD_MIN_I32 : VOP3Inst <"v_add_min_i32", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>, int_amdgcn_add_min_i32>;783 defm V_ADD_MIN_U32 : VOP3Inst <"v_add_min_u32", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>, int_amdgcn_add_min_u32>;784}785 786defm V_ADD_I16 : VOP3Inst_t16 <"v_add_i16", VOP_I16_I16_I16>;787defm V_SUB_I16 : VOP3Inst_t16 <"v_sub_i16", VOP_I16_I16_I16>;788 789let isCommutable = 1 in {790 defm V_MAD_U32_U16 : VOP3Inst_t16 <"v_mad_u32_u16", VOP_I32_I16_I16_I32>;791 defm V_MAD_I32_I16 : VOP3Inst_t16 <"v_mad_i32_i16", VOP_I32_I16_I16_I32>;792} // End isCommutable = 1793 794defm V_PACK_B32_F16 : VOP3Inst_t16 <"v_pack_b32_f16", VOP_B32_F16_F16>;795 796let isReMaterializable = 1 in {797defm V_SUB_I32 : VOP3Inst <"v_sub_i32", VOP3_Profile<VOP_I32_I32_I32_ARITH>>;798defm V_LSHL_ADD_U32 : VOP3Inst <"v_lshl_add_u32", VOP3_Profile<VOP_I32_I32_I32_I32>>;799defm V_LSHL_OR_B32 : VOP3Inst <"v_lshl_or_b32", VOP3_Profile<VOP_I32_I32_I32_I32>>;800} // End isReMaterializable = 1801 802// V_LSHL_ADD_U64: D0.u64 = (S0.u64 << S1.u[2:0]) + S2.u64803// src0 is shifted left by 0-4 (use “0” to get ADD_U64).804let SubtargetPredicate = HasLshlAddU64Inst in805defm V_LSHL_ADD_U64 : VOP3Inst <"v_lshl_add_u64", V_LSHL_ADD_U64_PROF>;806 807let OtherPredicates = [HasFP8ConversionInsts], mayRaiseFPException = 0,808 SchedRW = [WriteFloatCvt] in {809 let Constraints = "$vdst = $vdst_in" in {810 let OtherPredicates = [HasFP8ConversionInsts, NotHasFP8E5M3Insts] in811 defm V_CVT_PK_FP8_F32 : VOP3Inst_t16_with_profiles<"v_cvt_pk_fp8_f32", VOP3_CVT_PK_F8_F32_Profile<>,812 VOP3_CVT_PK_F8_F32_Profile_t16<>,813 VOP3_CVT_PK_F8_F32_Profile_fake16<>>;814 let OtherPredicates = [HasFP8ConversionInsts, HasFP8E5M3Insts] in815 defm V_CVT_PK_FP8_F32_gfx1250 : VOP3Inst_t16_with_profiles<"v_cvt_pk_fp8_f32_gfx1250", VOP3_CVT_PK_F8_F32_Profile<true>,816 VOP3_CVT_PK_F8_F32_Profile_t16<true>,817 VOP3_CVT_PK_F8_F32_Profile_fake16<true>>;818 defm V_CVT_PK_BF8_F32 : VOP3Inst_t16_with_profiles<"v_cvt_pk_bf8_f32", VOP3_CVT_PK_F8_F32_Profile<>,819 VOP3_CVT_PK_F8_F32_Profile_t16<>,820 VOP3_CVT_PK_F8_F32_Profile_fake16<>>;821 822 let SubtargetPredicate = isGFX12Plus in {823 let OtherPredicates = [HasFP8ConversionInsts, NotHasFP8E5M3Insts] in824 defm V_CVT_SR_FP8_F32_gfx12 : VOP3Inst<"v_cvt_sr_fp8_f32_gfx12", VOP3_CVT_SR_F8_ByteSel_Profile<f32>>;825 let OtherPredicates = [HasFP8ConversionInsts, HasFP8E5M3Insts] in826 defm V_CVT_SR_FP8_F32_gfx1250 : VOP3Inst<"v_cvt_sr_fp8_f32_gfx1250", VOP3_CVT_SR_F8_ByteSel_Profile<f32, true>>;827 defm V_CVT_SR_BF8_F32_gfx12 : VOP3Inst<"v_cvt_sr_bf8_f32_gfx12", VOP3_CVT_SR_F8_ByteSel_Profile<f32>>;828 }829 }830 831 // These instructions have non-standard use of op_sel. In particular they are832 // using op_sel bits 2 and 3 while only having two sources. Therefore dummy833 // src2 is used to hold the op_sel value.834 let Constraints = "$vdst = $src2", SubtargetPredicate = isGFX940Plus in {835 defm V_CVT_SR_FP8_F32 : VOP3Inst<"v_cvt_sr_fp8_f32", VOP3_CVT_SR_F8_F32_Profile>;836 defm V_CVT_SR_BF8_F32 : VOP3Inst<"v_cvt_sr_bf8_f32", VOP3_CVT_SR_F8_F32_Profile>;837 }838}839 840class Cvt_PK_F8_F32_Pat<SDPatternOperator node, int index, VOP3_Pseudo inst> : GCNPat<841 (i32 (node f32:$src0, f32:$src1, i32:$old, index)),842 (inst !if(index, SRCMODS.DST_OP_SEL, 0), $src0, 0, $src1, $old, 0)843>;844 845class Cvt_PK_F8_F32_E5M3_Pat<SDPatternOperator node, int index, VOP3_Pseudo inst, int Clamp> : GCNPat<846 (i32 (node f32:$src0, f32:$src1, i32:$old, index)),847 (inst !if(index, SRCMODS.DST_OP_SEL, 0), $src0, 0, $src1, Clamp, $old, 0)848>;849 850multiclass Cvt_PK_F8_F32_t16_Pat<SDPatternOperator node, VOP3_Pseudo inst> {851def : GCNPat<852 (i32 (node f32:$src0, f32:$src1, i32:$old, -1)),853 (REG_SEQUENCE VGPR_32,854 (i16 (EXTRACT_SUBREG $old, lo16)), lo16,855 (i16 (inst SRCMODS.DST_OP_SEL, $src0, 0, $src1, (i16 (EXTRACT_SUBREG $old, hi16)), 0)), hi16)856>;857def : GCNPat<858 (i32 (node f32:$src0, f32:$src1, i32:$old, 0)),859 (REG_SEQUENCE VGPR_32,860 (i16 (inst 0, $src0, 0, $src1, (i16 (EXTRACT_SUBREG $old, lo16)), 0)), lo16,861 (i16 (EXTRACT_SUBREG $old, hi16)), hi16)862>;863}864 865multiclass Cvt_PK_F8_F32_E5M3_t16_Pat<SDPatternOperator node, VOP3_Pseudo inst, int Clamp> {866def : GCNPat<867 (i32 (node f32:$src0, f32:$src1, i32:$old, -1)),868 (REG_SEQUENCE VGPR_32,869 (i16 (EXTRACT_SUBREG $old, lo16)), lo16,870 (i16 (inst SRCMODS.DST_OP_SEL, $src0, 0, $src1, Clamp, (i16 (EXTRACT_SUBREG $old, hi16)), 0)), hi16)871>;872def : GCNPat<873 (i32 (node f32:$src0, f32:$src1, i32:$old, 0)),874 (REG_SEQUENCE VGPR_32,875 (i16 (inst 0, $src0, 0, $src1, Clamp, (i16 (EXTRACT_SUBREG $old, lo16)), 0)), lo16,876 (i16 (EXTRACT_SUBREG $old, hi16)), hi16)877>;878}879 880class Cvt_SR_F8_F32_Pat<SDPatternOperator node, bits<2> index, VOP3_Pseudo inst> : GCNPat<881 (i32 (node f32:$src0, i32:$src1, i32:$old, index)),882 (inst !if(index{1}, SRCMODS.DST_OP_SEL, 0), $src0, 0, $src1,883 !if(index{0}, SRCMODS.OP_SEL_0, 0), $old, 0)884>;885 886class Cvt_SR_F8_ByteSel_Pat<SDPatternOperator node, VOP3_Pseudo inst, ValueType SrcVT> : GCNPat<887 (i32 (node (VOP3Mods SrcVT:$src0, i32:$src0_modifiers), (VOP3Mods i32:$src1, i32:$src1_modifiers),888 i32:$old, timm:$byte_sel)),889 (inst $src0_modifiers, $src0, $src1_modifiers, $src1, $old, (as_i32timm $byte_sel))890>;891 892class Cvt_SR_F8_ByteSel_E5M3_Pat<SDPatternOperator node, VOP3_Pseudo inst,893 ValueType SrcVT, int Clamp> : GCNPat<894 (i32 (node (VOP3Mods SrcVT:$src0, i32:$src0_modifiers), (VOP3Mods i32:$src1, i32:$src1_modifiers),895 i32:$old, timm:$byte_sel)),896 (inst $src0_modifiers, $src0, $src1_modifiers, $src1, Clamp, $old, (as_i32timm $byte_sel))897>;898 899let OtherPredicates = [HasFP8ConversionInsts] in {900foreach Index = [0, -1] in {901let True16Predicate = NotHasTrue16BitInsts in {902 let OtherPredicates = [HasFP8ConversionInsts, NotHasFP8E5M3Insts] in903 def : Cvt_PK_F8_F32_Pat<int_amdgcn_cvt_pk_fp8_f32, Index, V_CVT_PK_FP8_F32_e64>;904 def : Cvt_PK_F8_F32_Pat<int_amdgcn_cvt_pk_bf8_f32, Index, V_CVT_PK_BF8_F32_e64>;905}906let True16Predicate = UseFakeTrue16Insts in {907 def : Cvt_PK_F8_F32_Pat<int_amdgcn_cvt_pk_fp8_f32, Index, V_CVT_PK_FP8_F32_fake16_e64>;908 def : Cvt_PK_F8_F32_Pat<int_amdgcn_cvt_pk_bf8_f32, Index, V_CVT_PK_BF8_F32_fake16_e64>;909 let OtherPredicates = [HasFP8ConversionInsts, HasFP8E5M3Insts] in {910 def : Cvt_PK_F8_F32_E5M3_Pat<int_amdgcn_cvt_pk_fp8_f32, Index, V_CVT_PK_FP8_F32_gfx1250_fake16_e64, DSTCLAMP.NONE>;911 def : Cvt_PK_F8_F32_E5M3_Pat<int_amdgcn_cvt_pk_fp8_f32_e5m3, Index, V_CVT_PK_FP8_F32_gfx1250_fake16_e64, DSTCLAMP.ENABLE>;912 }913}914}915 916let True16Predicate = UseRealTrue16Insts in {917defm : Cvt_PK_F8_F32_t16_Pat<int_amdgcn_cvt_pk_fp8_f32, V_CVT_PK_FP8_F32_t16_e64>;918defm : Cvt_PK_F8_F32_t16_Pat<int_amdgcn_cvt_pk_bf8_f32, V_CVT_PK_BF8_F32_t16_e64>;919 let OtherPredicates = [HasFP8ConversionInsts, HasFP8E5M3Insts] in {920 defm : Cvt_PK_F8_F32_E5M3_t16_Pat<int_amdgcn_cvt_pk_fp8_f32, V_CVT_PK_FP8_F32_gfx1250_t16_e64, DSTCLAMP.NONE>;921 defm : Cvt_PK_F8_F32_E5M3_t16_Pat<int_amdgcn_cvt_pk_fp8_f32_e5m3, V_CVT_PK_FP8_F32_gfx1250_t16_e64, DSTCLAMP.ENABLE>;922 }923}924 925let SubtargetPredicate = isGFX940Plus in {926 foreach Index = [0, 1, 2, 3] in {927 def : Cvt_SR_F8_F32_Pat<int_amdgcn_cvt_sr_fp8_f32, Index, V_CVT_SR_FP8_F32_e64>;928 def : Cvt_SR_F8_F32_Pat<int_amdgcn_cvt_sr_bf8_f32, Index, V_CVT_SR_BF8_F32_e64>;929 }930}931 932let SubtargetPredicate = isGFX12Plus in {933 let OtherPredicates = [HasFP8ConversionInsts, NotHasFP8E5M3Insts] in934 def : Cvt_SR_F8_ByteSel_Pat<int_amdgcn_cvt_sr_fp8_f32, V_CVT_SR_FP8_F32_gfx12_e64, f32>;935 let OtherPredicates = [HasFP8ConversionInsts, HasFP8E5M3Insts] in {936 def : Cvt_SR_F8_ByteSel_E5M3_Pat<int_amdgcn_cvt_sr_fp8_f32, V_CVT_SR_FP8_F32_gfx1250_e64, f32, DSTCLAMP.NONE>;937 def : Cvt_SR_F8_ByteSel_E5M3_Pat<int_amdgcn_cvt_sr_fp8_f32_e5m3, V_CVT_SR_FP8_F32_gfx1250_e64, f32, DSTCLAMP.ENABLE>;938 }939 def : Cvt_SR_F8_ByteSel_Pat<int_amdgcn_cvt_sr_bf8_f32, V_CVT_SR_BF8_F32_gfx12_e64, f32>;940}941}942 943class ThreeOp_i32_Pats <SDPatternOperator op1, SDPatternOperator op2, Instruction inst> : GCNPat <944 // This matches (op2 (op1 i32:$src0, i32:$src1), i32:$src2) with conditions.945 (ThreeOpFrag<op1, op2> i32:$src0, i32:$src1, i32:$src2),946 (inst VSrc_b32:$src0, VSrc_b32:$src1, VSrc_b32:$src2)947>;948 949def : ThreeOp_i32_Pats<cshl_32, add, V_LSHL_ADD_U32_e64>;950def : ThreeOp_i32_Pats<add, cshl_32, V_ADD_LSHL_U32_e64>;951def : ThreeOp_i32_Pats<add, add, V_ADD3_U32_e64>;952def : ThreeOp_i32_Pats<ptradd, ptradd, V_ADD3_U32_e64>;953def : ThreeOp_i32_Pats<cshl_32, or, V_LSHL_OR_B32_e64>;954def : ThreeOp_i32_Pats<and, or, V_AND_OR_B32_e64>;955def : ThreeOp_i32_Pats<or, or, V_OR3_B32_e64>;956def : ThreeOp_i32_Pats<xor, add, V_XAD_U32_e64>;957 958let SubtargetPredicate = HasMadU32Inst, AddedComplexity = 10 in959 def : ThreeOp_i32_Pats<mul, add, V_MAD_U32_e64>;960 961def : GCNPat<962 (DivergentBinFrag<mul> i32:$src0, IsPow2Plus1:$src1),963 (V_LSHL_ADD_U32_e64 i32:$src0, (i32 (Log2_32 imm:$src1)), i32:$src0)>;964 965let SubtargetPredicate = HasLshlAddU64Inst in {966def : GCNPat<967 (ThreeOpFrag<shl_0_to_4, add> i64:$src0, i32:$src1, i64:$src2),968 (V_LSHL_ADD_U64_e64 VSrc_b64:$src0, VSrc_b32:$src1, VSrc_b64:$src2)969>;970 971def : GCNPat <972 // (ptradd z, (shl x, y)) or (ptradd (shl x, y), z) -> ((x << y) + z)973 (ThreeOpFrag<shl_0_to_4, ptradd_commutative> i64:$src0, i32:$src1, i64:$src2),974 (V_LSHL_ADD_U64_e64 VSrc_b64:$src0, VSrc_b32:$src1, VSrc_b64:$src2)>;975} // End SubtargetPredicate = HasLshlAddU64Inst976 977let SubtargetPredicate = HasAddMinMaxInsts in {978def : ThreeOp_i32_Pats<saddsat, smax, V_ADD_MAX_I32_e64>;979def : ThreeOp_i32_Pats<uaddsat, umax, V_ADD_MAX_U32_e64>;980def : ThreeOp_i32_Pats<saddsat, smin, V_ADD_MIN_I32_e64>;981def : ThreeOp_i32_Pats<uaddsat, umin, V_ADD_MIN_U32_e64>;982}983 984def : VOPBinOpClampPat<saddsat, V_ADD_I32_e64, i32>;985def : VOPBinOpClampPat<ssubsat, V_SUB_I32_e64, i32>;986 987def : GCNPat<(DivergentBinFrag<or> (or_oneuse i64:$src0, i64:$src1), i64:$src2),988 (REG_SEQUENCE VReg_64,989 (V_OR3_B32_e64 (i32 (EXTRACT_SUBREG $src0, sub0)),990 (i32 (EXTRACT_SUBREG $src1, sub0)),991 (i32 (EXTRACT_SUBREG $src2, sub0))), sub0,992 (V_OR3_B32_e64 (i32 (EXTRACT_SUBREG $src0, sub1)),993 (i32 (EXTRACT_SUBREG $src1, sub1)),994 (i32 (EXTRACT_SUBREG $src2, sub1))), sub1)>;995 996} // End SubtargetPredicate = isGFX9Plus997 998let SubtargetPredicate = HasCvtPkNormVOP3Insts in {999 defm V_CVT_PKNORM_I16_F16 : VOP3Inst_t16 <"v_cvt_pknorm_i16_f16", VOP_B32_F16_F16>;1000 defm V_CVT_PKNORM_U16_F16 : VOP3Inst_t16 <"v_cvt_pknorm_u16_f16", VOP_B32_F16_F16>;1001} // end SubtargetPredicate = HasCvtPkNormVOP3Insts1002 1003// FIXME: Probably should hardcode clamp bit in pseudo and avoid this.1004class OpSelBinOpClampPat<SDPatternOperator node,1005 Instruction inst> : GCNPat<1006 (node (i16 (VOP3OpSel i16:$src0, i32:$src0_modifiers)),1007 (i16 (VOP3OpSel i16:$src1, i32:$src1_modifiers))),1008 (inst $src0_modifiers, $src0, $src1_modifiers, $src1, DSTCLAMP.ENABLE, 0)1009>;1010 1011let SubtargetPredicate = isGFX9Plus, True16Predicate = NotHasTrue16BitInsts in {1012 def : OpSelBinOpClampPat<saddsat, V_ADD_I16_e64>;1013 def : OpSelBinOpClampPat<ssubsat, V_SUB_I16_e64>;1014} // End SubtargetPredicate = isGFX9Plus, True16Predicate = NotHasTrue16BitInsts1015let True16Predicate = UseRealTrue16Insts in {1016 def : OpSelBinOpClampPat<saddsat, V_ADD_I16_t16_e64>;1017 def : OpSelBinOpClampPat<ssubsat, V_SUB_I16_t16_e64>;1018} // End True16Predicate = UseRealTrue16Insts1019let True16Predicate = UseFakeTrue16Insts in {1020 def : OpSelBinOpClampPat<saddsat, V_ADD_I16_fake16_e64>;1021 def : OpSelBinOpClampPat<ssubsat, V_SUB_I16_fake16_e64>;1022} // End True16Predicate = UseFakeTrue16Insts1023 1024multiclass IMAD32_Pats <VOP3_Pseudo inst> {1025 def : GCNPat <1026 (ThreeOpFrag<mul, add> i32:$src0, i32:$src1, i32:$src2),1027 (EXTRACT_SUBREG (inst i32:$src0, i32:$src1,1028 (REG_SEQUENCE SReg_64, // Use scalar and let it be legalized1029 $src2, sub0,1030 (i32 (IMPLICIT_DEF)), sub1),1031 0 /* clamp */),1032 sub0)1033 >;1034 1035 // GISel-specific pattern that avoids creating a SGPR->VGPR copy if1036 // $src2 is a VGPR.1037 def : GCNPat <1038 (ThreeOpFrag<mul, add> i32:$src0, i32:$src1, VGPR_32:$src2),1039 (EXTRACT_SUBREG (inst i32:$src0, i32:$src1,1040 (REG_SEQUENCE VReg_64,1041 $src2, sub0,1042 (i32 (IMPLICIT_DEF)), sub1),1043 0 /* clamp */),1044 sub0)1045 >;1046 1047 // Immediate src2 in the pattern above will not fold because it would be partially1048 // undef. Hence define specialized pattern for this case.1049 def : GCNPat <1050 (ThreeOpFrag<mul, add> i32:$src0, i32:$src1, (i32 imm:$src2)),1051 (EXTRACT_SUBREG (inst i32:$src0, i32:$src1, (i64 (as_i64imm $src2)), 0 /* clamp */), sub0)1052 >;1053}1054 1055// Handle cases where amdgpu-codegenprepare-mul24 made a mul24 instead of a normal mul.1056// We need to separate this because otherwise OtherPredicates would be overriden.1057class IMAD32_Mul24_Pats_Impl<VOP3_Pseudo inst, SDPatternOperator AddOp> : GCNPat <1058 (i64 (AddOp (i64 (AMDGPUmul_u24 i32:$src0, i32:$src1)), i64:$src2)),1059 (inst $src0, $src1, $src2, 0 /* clamp */)1060 >;1061 1062multiclass IMAD32_Mul24_Pats<VOP3_Pseudo inst> {1063 def : IMAD32_Mul24_Pats_Impl<inst, add>;1064 def : IMAD32_Mul24_Pats_Impl<inst, ptradd_commutative>;1065}1066 1067// exclude pre-GFX9 where it was slow1068let OtherPredicates = [HasNotMADIntraFwdBug], SubtargetPredicate = isGFX9Plus in {1069 defm : IMAD32_Pats<V_MAD_U64_U32_e64>;1070 defm : IMAD32_Mul24_Pats<V_MAD_U64_U32_e64>;1071}1072let OtherPredicates = [HasMADIntraFwdBug], SubtargetPredicate = isGFX11Only in {1073 defm : IMAD32_Pats<V_MAD_U64_U32_gfx11_e64>;1074 defm : IMAD32_Mul24_Pats<V_MAD_U64_U32_gfx11_e64>;1075}1076 1077def VOP3_PERMLANE_Profile : VOP3_Profile<VOPProfile <[i32, i32, i32, i32]>, VOP3_OPSEL> {1078 let InsVOP3OpSel = (ins IntOpSelMods:$src0_modifiers, VRegSrc_32:$src0,1079 IntOpSelMods:$src1_modifiers, SSrc_b32:$src1,1080 IntOpSelMods:$src2_modifiers, SSrc_b32:$src2,1081 VGPR_32:$vdst_in, op_sel0:$op_sel);1082 let HasClamp = 0;1083 let HasExtVOP3DPP = 0;1084 let HasExtDPP = 0;1085}1086 1087def VOP3_PERMLANE_VAR_Profile : VOP3_Profile<VOPProfile <[i32, i32, i32, untyped]>, VOP3_OPSEL> {1088 let InsVOP3OpSel = (ins IntOpSelMods:$src0_modifiers, VRegSrc_32:$src0,1089 IntOpSelMods:$src1_modifiers, VRegSrc_32:$src1,1090 VGPR_32:$vdst_in, op_sel0:$op_sel);1091 let HasClamp = 0;1092 let HasExtVOP3DPP = 0;1093 let HasExtDPP = 0;1094}1095 1096class VOP3_PERMLANE_NOOPSEL_Profile<VOPProfile P> : VOP3_Profile<P> {1097 let Ins64 = !con((ins VRegSrc_32:$src0, SSrc_b32:$src1),1098 !if(P.HasSrc2, (ins SSrc_b32:$src2), (ins)));1099 let HasClamp = 0;1100 let HasExtVOP3DPP = 0;1101 let HasExtDPP = 0;1102}1103 1104def opsel_i1timm : SDNodeXForm<timm, [{1105 return CurDAG->getTargetConstant(1106 N->getZExtValue() ? SISrcMods::OP_SEL_0 : SISrcMods::NONE,1107 SDLoc(N), MVT::i32);1108}]>;1109def gi_opsel_i1timm : GICustomOperandRenderer<"renderOpSelTImm">,1110 GISDNodeXFormEquiv<opsel_i1timm>;1111 1112class SrcAndDstSelToOpSelXForm<int modifier_idx, bit dest_sel> : SDNodeXForm<timm, [{1113 unsigned Val = N->getZExtValue();1114 unsigned New = 0;1115 if (}] # modifier_idx # [{ == 0) {1116 New = (}] # dest_sel # [{ == 1) ? ((Val & 0x1) ? (SISrcMods::OP_SEL_0 | SISrcMods::DST_OP_SEL) : SISrcMods::DST_OP_SEL)1117 : ((Val & 0x1) ? SISrcMods::OP_SEL_0 : SISrcMods::NONE);1118 } else if (}] # modifier_idx # [{== 1) {1119 New = (Val & 0x2) ? SISrcMods::OP_SEL_0 : SISrcMods::NONE;1120 } if (}] # modifier_idx # [{== 2) {1121 New = (Val & 0x1) ? SISrcMods::OP_SEL_0 : SISrcMods::NONE;1122 }1123 return CurDAG->getTargetConstant(New, SDLoc(N), MVT::i32);1124}]>;1125 1126def SrcAndDstSelToOpSelXForm_0_0 : SrcAndDstSelToOpSelXForm<0,0>;1127def SrcAndDstSelToOpSelXForm_0_1 : SrcAndDstSelToOpSelXForm<0,1>;1128def SrcAndDstSelToOpSelXForm_1_0 : SrcAndDstSelToOpSelXForm<1,0>;1129def SrcAndDstSelToOpSelXForm_1_1 : SrcAndDstSelToOpSelXForm<1,1>;1130def SrcAndDstSelToOpSelXForm_2_0 : SrcAndDstSelToOpSelXForm<2,0>;1131 1132// The global isel renderer has no way to access the templatized args of (SrcAndDstSelToOpSelXForm) in1133// renderer C++ APIs. Therefore, combinations of modifier_idx & dest_sel are embedded in renderer name itself.1134// FixMe: Avoid combinations of modifier_idx & dest_sel for global isel cases.1135def gi_SrcAndDstSelToOpSelXForm_0_0 : GICustomOperandRenderer<"renderSrcAndDstSelToOpSelXForm_0_0">,1136 GISDNodeXFormEquiv<SrcAndDstSelToOpSelXForm_0_0>;1137def gi_SrcAndDstSelToOpSelXForm_0_1 : GICustomOperandRenderer<"renderSrcAndDstSelToOpSelXForm_0_1">,1138 GISDNodeXFormEquiv<SrcAndDstSelToOpSelXForm_0_1>;1139def gi_SrcAndDstSelToOpSelXForm_1_0 : GICustomOperandRenderer<"renderSrcAndDstSelToOpSelXForm_1_0">,1140 GISDNodeXFormEquiv<SrcAndDstSelToOpSelXForm_1_0>;1141def gi_SrcAndDstSelToOpSelXForm_1_1 : GICustomOperandRenderer<"renderSrcAndDstSelToOpSelXForm_1_1">,1142 GISDNodeXFormEquiv<SrcAndDstSelToOpSelXForm_1_1>;1143def gi_SrcAndDstSelToOpSelXForm_2_0 : GICustomOperandRenderer<"renderSrcAndDstSelToOpSelXForm_2_0">,1144 GISDNodeXFormEquiv<SrcAndDstSelToOpSelXForm_2_0>;1145 1146def DstSelToOpSelXForm : SDNodeXForm<timm, [{1147 return CurDAG->getTargetConstant(1148 N->getZExtValue() ? SISrcMods::DST_OP_SEL : SISrcMods::NONE,1149 SDLoc(N), MVT::i32);1150}]>;1151def gi_DstSelToOpSelXForm : GICustomOperandRenderer<"renderDstSelToOpSelXForm">,1152 GISDNodeXFormEquiv<DstSelToOpSelXForm>;1153 1154def SrcSelToOpSelXForm : SDNodeXForm<timm, [{1155 return CurDAG->getTargetConstant(1156 N->getZExtValue() ? SISrcMods::OP_SEL_0 : SISrcMods::NONE,1157 SDLoc(N), MVT::i32);1158}]>;1159def gi_SrcSelToOpSelXForm : GICustomOperandRenderer<"renderSrcSelToOpSelXForm">,1160 GISDNodeXFormEquiv<SrcSelToOpSelXForm>;1161 1162def DstSelToOpSel3XForm : SDNodeXForm<timm, [{1163 uint32_t V = N->getZExtValue();1164 return CurDAG->getTargetConstant(1165 (V & 0x2) ? SISrcMods::DST_OP_SEL : SISrcMods::NONE,1166 SDLoc(N), MVT::i32);1167}]>;1168def gi_DstSelToOpSel3XForm : GICustomOperandRenderer<"renderDstSelToOpSel3XFormXForm">,1169 GISDNodeXFormEquiv<DstSelToOpSel3XForm>;1170 1171class PermlanePat<SDPatternOperator permlane,1172 Instruction inst, ValueType vt> : GCNPat<1173 (vt (permlane vt:$vdst_in, vt:$src0, i32:$src1, i32:$src2,1174 timm:$fi, timm:$bc)),1175 (inst (opsel_i1timm $fi), VGPR_32:$src0, (opsel_i1timm $bc),1176 SCSrc_b32:$src1, 0, SCSrc_b32:$src2, VGPR_32:$vdst_in)1177>;1178 1179class PermlaneVarPat<SDPatternOperator permlane,1180 Instruction inst> : GCNPat<1181 (permlane i32:$vdst_in, i32:$src0, i32:$src1,1182 timm:$fi, timm:$bc),1183 (inst (opsel_i1timm $fi), VGPR_32:$src0, (opsel_i1timm $bc),1184 VGPR_32:$src1, VGPR_32:$vdst_in)1185>;1186 1187class PermlaneNoDppPat3Src<SDPatternOperator permlane,1188 Instruction inst> : GCNPat<1189 (permlane i32:$src0, i32:$src1, i32:$src2),1190 (inst VGPR_32:$src0, SCSrc_b32:$src1, SCSrc_b32:$src2)1191>;1192 1193class PermlaneNoDppPat2Src<SDPatternOperator permlane,1194 Instruction inst> : GCNPat<1195 (permlane i32:$src0, i32:$src1),1196 (inst VGPR_32:$src0, SCSrc_b32:$src1)1197>;1198 1199class VOP3_BITOP3_Profile<VOPProfile pfl, VOP3Features f> : VOP3_Profile<pfl, f> {1200 let HasClamp = 0;1201 let HasOMod = 0;1202 let HasModifiers = 0;1203 let HasVOPD3Src2 = 0;1204 let HasBitOp3 = 1;1205 1206 let InsVOPD3Y = (ins Src0VOPD3:$src0Y, Src1VOPD3:$vsrc1Y, bitop3_0:$bitop3);1207 let AsmVOPD3Y = getAsmVOPDPart<NumSrcArgs, "Y", HasVOPD3Src2, HasModifiers>.ret # "$bitop3";1208}1209 1210class VOP3_CVT_SCALE_F1632_FP8BF8_Profile<ValueType DstTy> : VOP3_Profile<VOPProfile<[DstTy, i32, f32, untyped]>,1211 VOP3_OPSEL> {1212 let InsVOP3OpSel = (ins FP32InputMods:$src0_modifiers, Src0RC64:$src0,1213 FP32InputMods:$src1_modifiers, Src1RC64:$src1,1214 op_sel0:$op_sel);1215 let HasClamp = 0;1216 let HasSrc2 = 0;1217 let HasSrc2Mods = 0;1218 let HasExtVOP3DPP = 0;1219 let HasOpSel = 1;1220 let HasOMod = 0;1221}1222 1223class VOP3_CVT_SCALE_F1632_FP8BF8_TiedInput_Profile<VOPProfile P> : VOP3_Profile<P, VOP3_OPSEL> {1224 let InsVOP3OpSel = (ins FP32InputMods:$src0_modifiers, Src0RC64:$src0,1225 FP32InputMods:$src1_modifiers, Src1RC64:$src1,1226 VGPR_32:$vdst_in, op_sel0:$op_sel);1227 let HasClamp = 0;1228 let HasSrc2 = 0;1229 let HasSrc2Mods = 0;1230 let HasExtVOP3DPP = 0;1231 let HasOpSel = 1;1232 let HasOMod = 0;1233}1234 1235class VOP3_CVT_SCALE_FP4FP8BF8_F32_TiedInput_Profile<VOPProfile P> : VOP3_Profile<P, VOP3_OPSEL> {1236 let InsVOP3OpSel = (ins FP32InputMods:$src0_modifiers, Src0RC64:$src0,1237 FP32InputMods:$src1_modifiers, Src1RC64:$src1,1238 FP32InputMods:$src2_modifiers, Src2RC64:$src2,1239 VGPR_32:$vdst_in, op_sel0:$op_sel);1240 let HasClamp = 0;1241 let HasExtVOP3DPP = 0;1242 let HasOpSel = 1;1243 let HasOMod = 0;1244}1245 1246class VOP3_CVT_SCALE_FP4_F32_TiedInput_Profile<VOPProfile P> : VOP3_CVT_SCALE_FP4FP8BF8_F32_TiedInput_Profile<P> {1247 let HasFP8DstByteSel = 1;1248 let HasFP8ByteSel = 0; // It works as a dst-bytesel, but does not have byte_sel operand.1249}1250 1251class VOP3_CVT_SCALE_SR_F8BF8_F16BF16F32_TiedInput_Profile<VOPProfile P> : VOP3_CVT_SCALE_FP4FP8BF8_F32_TiedInput_Profile<P> {1252 let InsVOP3OpSel = (ins FP32InputMods:$src0_modifiers, Src0RC64:$src0,1253 Int32InputMods:$src1_modifiers, Src1RC64:$src1,1254 FP32InputMods:$src2_modifiers, Src2RC64:$src2,1255 VGPR_32:$vdst_in, op_sel0:$op_sel);1256 let HasFP8DstByteSel = 1;1257 let HasFP8ByteSel = 0; // It works as a dst-bytesel, but does not have byte_sel operand.1258}1259 1260 1261class VOP3_CVT_SCALE_FP4_F16BF16_TiedInput_Profile<VOPProfile P> : VOP3_Profile<P, VOP3_OPSEL> {1262 let InsVOP3OpSel = (ins FP32InputMods:$src0_modifiers, Src0RC64:$src0,1263 FP32InputMods:$src1_modifiers, Src1RC64:$src1,1264 FP32InputMods:$src2_modifiers, VGPR_32:$src2,1265 op_sel0:$op_sel);1266 let HasClamp = 0;1267 let HasSrc2 = 0;1268 let HasSrc2Mods = 1;1269 let HasOpSel = 1;1270 let Asm64 = !subst(", $src2_modifiers", "", AsmVOP3Base);1271 let HasExtVOP3DPP = 0;1272 let HasFP8DstByteSel = 1;1273 let HasFP8ByteSel = 0;1274}1275 1276class VOP3_CVT_SCALE_SR_PK_F4_F16BF16_TiedInput_Profile<ValueType Src0Ty> :1277 VOP3_Profile<VOPProfile<[i32, Src0Ty, i32, f32]>, VOP3_OPSEL> {1278 let InsVOP3OpSel = (ins PackedF16InputMods: $src0_modifiers, Src0RC64:$src0,1279 Int32InputMods: $src1_modifiers, Src1RC64:$src1,1280 FP32InputMods: $src2_modifiers, Src2RC64:$src2,1281 VGPR_32:$vdst_in, op_sel0:$op_sel);1282 let HasClamp = 0;1283 let HasExtVOP3DPP = 0;1284 let HasOpSel = 1;1285 let HasOMod = 0;1286 let HasFP4DstByteSel = 1;1287}1288 1289class VOP3_CVT_SCALE_SR_PK_F4_F32_TiedInput_Profile<VOPProfile P>1290 : VOP3_Profile<P, VOP3_OPSEL> {1291 1292 let Src0RC64 = !if(!gt(P.Src0VT.Size, 32), getVOP3VRegSrcForVT<P.Src0VT>.ret,1293 getVOP3SrcForVT<P.Src0VT>.ret);1294 let InsVOP3OpSel = (ins PackedVGPRF32InputMods: $src0_modifiers, Src0RC64:$src0,1295 Int32InputMods: $src1_modifiers, Src1RC64:$src1,1296 FP32InputMods: $src2_modifiers, Src2RC64:$src2,1297 VGPR_32:$vdst_in, op_sel0:$op_sel);1298 let HasClamp = 0;1299 let HasExtVOP3DPP = 0;1300 let HasOpSel = 1;1301 let HasOMod = 0;1302 let HasFP4DstByteSel = 1;1303}1304 1305class VOP3_CVT_SCALE_PK_F16BF16F32_FP4FP8BF8_Profile<ValueType DstTy> : VOP3_Profile<VOPProfile<[DstTy, i32, f32, untyped]>,1306 VOP3_OPSEL> {1307 let InsVOP3OpSel = (ins FP32InputMods:$src0_modifiers, Src0RC64:$src0,1308 FP32InputMods:$src1_modifiers, Src1RC64:$src1,1309 op_sel0:$op_sel);1310 let HasClamp = 0;1311 let HasSrc2 = 0;1312 let HasSrc2Mods = 0;1313 let HasExtVOP3DPP = 0;1314 let HasOpSel = 1;1315 let HasOMod = 0;1316}1317 1318class VOP3_CVT_SCALE_PK_FP8BF8_F16BF16_TiedInput_Profile<VOPProfile P> : VOP3_Profile<P,VOP3_OPSEL> {1319 let InsVOP3OpSel = (ins FP32InputMods:$src0_modifiers, Src0RC64:$src0,1320 FP32InputMods:$src1_modifiers, Src1RC64:$src1,1321 VGPR_32:$vdst_in, op_sel0:$op_sel);1322 let HasClamp = 0;1323 let HasSrc2 = 0;1324 let HasSrc2Mods = 0;1325 let HasExtVOP3DPP = 0;1326 let HasOpSel = 1;1327 let HasOMod = 0;1328}1329 1330class VOP3_CVT_SCALEF32_PK_F864_Profile<VOPProfile P> : VOP3_Profile<P> {1331 let HasModifiers = 0;1332 let HasSrc0IntMods = 0;1333 let HasSrc1IntMods = 0;1334 let HasSrc0FloatMods = 0;1335 let HasSrc1FloatMods = 0;1336 let HasSrc2FloatMods = 0;1337 let HasOMod = 0;1338 let HasOpSel = 0;1339 let HasClamp = 0;1340 let HasExtDPP = 0;1341 let HasExt32BitDPP = 0;1342 let HasExtVOP3DPP = 0;1343 let HasExt64BitDPP = 0;1344 1345 // All convert opcodes operating on FP6/BF6/FP4 data must use VGPR sources for1346 // any operand slots > 32 bit.1347 let Src0RC64 = !if(!gt(P.Src0VT.Size, 32), getVOP3VRegSrcForVT<P.Src0VT>.ret,1348 getVOP3SrcForVT<P.Src0VT>.ret);1349}1350 1351let SubtargetPredicate = HasFP8ConversionScaleInsts, mayRaiseFPException = 0 in {1352 let Constraints = "$vdst = $vdst_in" in {1353 defm V_CVT_SCALEF32_SR_FP8_BF16 : VOP3Inst<"v_cvt_scalef32_sr_fp8_bf16", VOP3_CVT_SCALE_SR_F8BF8_F16BF16F32_TiedInput_Profile<VOP_I32_BF16_I32_F32>>;1354 defm V_CVT_SCALEF32_SR_FP8_F16 : VOP3Inst<"v_cvt_scalef32_sr_fp8_f16", VOP3_CVT_SCALE_SR_F8BF8_F16BF16F32_TiedInput_Profile<VOP_I32_F16_I32_F32>>;1355 defm V_CVT_SCALEF32_SR_FP8_F32 : VOP3Inst<"v_cvt_scalef32_sr_fp8_f32", VOP3_CVT_SCALE_SR_F8BF8_F16BF16F32_TiedInput_Profile<VOP_I32_F32_I32_F32>>;1356 defm V_CVT_SCALEF32_F16_FP8 : VOP3Inst<"v_cvt_scalef32_f16_fp8", VOP3_CVT_SCALE_F1632_FP8BF8_TiedInput_Profile<VOP_V2F16_I32_F32>>;1357 defm V_CVT_SCALEF32_PK_FP8_F32 : VOP3Inst<"v_cvt_scalef32_pk_fp8_f32", VOP3_CVT_SCALE_FP4FP8BF8_F32_TiedInput_Profile<VOP_V2I16_F32_F32_F32>>;1358 defm V_CVT_SCALEF32_PK_FP8_F16 : VOP3Inst<"v_cvt_scalef32_pk_fp8_f16", VOP3_CVT_SCALE_PK_FP8BF8_F16BF16_TiedInput_Profile<VOP_V2I16_V2F16_F32>>;1359 defm V_CVT_SCALEF32_PK_FP8_BF16 : VOP3Inst<"v_cvt_scalef32_pk_fp8_bf16", VOP3_CVT_SCALE_PK_FP8BF8_F16BF16_TiedInput_Profile<VOP_V2I16_V2BF16_F32>>;1360 }1361 defm V_CVT_SCALEF32_F32_FP8 : VOP3Inst<"v_cvt_scalef32_f32_fp8", VOP3_CVT_SCALE_F1632_FP8BF8_Profile<f32>>;1362 defm V_CVT_SCALEF32_PK_F32_FP8 : VOP3Inst<"v_cvt_scalef32_pk_f32_fp8", VOP3_CVT_SCALE_PK_F16BF16F32_FP4FP8BF8_Profile<v2f32>>;1363 defm V_CVT_SCALEF32_PK_F16_FP8 : VOP3Inst<"v_cvt_scalef32_pk_f16_fp8", VOP3_CVT_SCALE_PK_F16BF16F32_FP4FP8BF8_Profile<v2f16>>;1364 defm V_CVT_SCALEF32_PK_BF16_FP8 : VOP3Inst<"v_cvt_scalef32_pk_bf16_fp8", VOP3_CVT_SCALE_PK_F16BF16F32_FP4FP8BF8_Profile<v2f16>>;1365}1366 1367let SubtargetPredicate = HasBF8ConversionScaleInsts, mayRaiseFPException = 0 in {1368 let Constraints = "$vdst = $vdst_in" in {1369 defm V_CVT_SCALEF32_SR_BF8_BF16 : VOP3Inst<"v_cvt_scalef32_sr_bf8_bf16", VOP3_CVT_SCALE_SR_F8BF8_F16BF16F32_TiedInput_Profile<VOP_I32_BF16_I32_F32>>;1370 defm V_CVT_SCALEF32_SR_BF8_F16 : VOP3Inst<"v_cvt_scalef32_sr_bf8_f16", VOP3_CVT_SCALE_SR_F8BF8_F16BF16F32_TiedInput_Profile<VOP_I32_F16_I32_F32>>;1371 defm V_CVT_SCALEF32_SR_BF8_F32 : VOP3Inst<"v_cvt_scalef32_sr_bf8_f32", VOP3_CVT_SCALE_SR_F8BF8_F16BF16F32_TiedInput_Profile<VOP_I32_F32_I32_F32>>;1372 defm V_CVT_SCALEF32_F16_BF8 : VOP3Inst<"v_cvt_scalef32_f16_bf8", VOP3_CVT_SCALE_F1632_FP8BF8_TiedInput_Profile<VOP_V2F16_I32_F32>>;1373 defm V_CVT_SCALEF32_PK_BF8_F32 : VOP3Inst<"v_cvt_scalef32_pk_bf8_f32", VOP3_CVT_SCALE_FP4FP8BF8_F32_TiedInput_Profile<VOP_V2I16_F32_F32_F32>>;1374 defm V_CVT_SCALEF32_PK_BF8_F16 : VOP3Inst<"v_cvt_scalef32_pk_bf8_f16", VOP3_CVT_SCALE_PK_FP8BF8_F16BF16_TiedInput_Profile<VOP_V2I16_V2F16_F32>>;1375 defm V_CVT_SCALEF32_PK_BF8_BF16 : VOP3Inst<"v_cvt_scalef32_pk_bf8_bf16", VOP3_CVT_SCALE_PK_FP8BF8_F16BF16_TiedInput_Profile<VOP_V2I16_V2BF16_F32>>;1376 }1377 defm V_CVT_SCALEF32_F32_BF8 : VOP3Inst<"v_cvt_scalef32_f32_bf8", VOP3_CVT_SCALE_F1632_FP8BF8_Profile<f32>>;1378 defm V_CVT_SCALEF32_PK_F32_BF8 : VOP3Inst<"v_cvt_scalef32_pk_f32_bf8", VOP3_CVT_SCALE_PK_F16BF16F32_FP4FP8BF8_Profile<v2f32>>;1379 defm V_CVT_SCALEF32_PK_F16_BF8 : VOP3Inst<"v_cvt_scalef32_pk_f16_bf8", VOP3_CVT_SCALE_PK_F16BF16F32_FP4FP8BF8_Profile<v2bf16>>;1380 defm V_CVT_SCALEF32_PK_BF16_BF8 : VOP3Inst<"v_cvt_scalef32_pk_bf16_bf8", VOP3_CVT_SCALE_PK_F16BF16F32_FP4FP8BF8_Profile<v2bf16>>;1381}1382 1383let SubtargetPredicate = HasFP4ConversionScaleInsts, mayRaiseFPException = 0 in {1384 defm V_CVT_SCALEF32_PK_F32_FP4 : VOP3Inst<"v_cvt_scalef32_pk_f32_fp4", VOP3_CVT_SCALE_PK_F16BF16F32_FP4FP8BF8_Profile<v2f32>>;1385 let Constraints = "$vdst = $vdst_in" in {1386 defm V_CVT_SCALEF32_PK_FP4_F32 : VOP3Inst<"v_cvt_scalef32_pk_fp4_f32", VOP3_CVT_SCALE_FP4_F32_TiedInput_Profile<VOP_I32_F32_F32_F32>>;1387 let Constraints = "@earlyclobber $vdst" in {1388 defm V_CVT_SCALEF32_SR_PK_FP4_F16: VOP3Inst<"v_cvt_scalef32_sr_pk_fp4_f16", VOP3_CVT_SCALE_SR_PK_F4_F16BF16_TiedInput_Profile<v2f16>>;1389 defm V_CVT_SCALEF32_SR_PK_FP4_BF16: VOP3Inst<"v_cvt_scalef32_sr_pk_fp4_bf16", VOP3_CVT_SCALE_SR_PK_F4_F16BF16_TiedInput_Profile<v2bf16>>;1390 defm V_CVT_SCALEF32_SR_PK_FP4_F321391 : VOP3Inst<"v_cvt_scalef32_sr_pk_fp4_f32",1392 VOP3_CVT_SCALE_SR_PK_F4_F32_TiedInput_Profile<1393 VOP_I32_V2F32_I32_F32>>;1394 }1395 }1396 defm V_CVT_SCALEF32_PK_F16_FP4 : VOP3Inst<"v_cvt_scalef32_pk_f16_fp4", VOP3_CVT_SCALE_PK_F16BF16F32_FP4FP8BF8_Profile<v2f16>>;1397 defm V_CVT_SCALEF32_PK_BF16_FP4 : VOP3Inst<"v_cvt_scalef32_pk_bf16_fp4", VOP3_CVT_SCALE_PK_F16BF16F32_FP4FP8BF8_Profile<v2bf16>>;1398 1399 // These instructions have non-standard use of op_sel. In particular they are1400 // using op_sel bits 2 and 3 while only having two sources.1401 let Constraints = "$vdst = $src2" in {1402 defm V_CVT_SCALEF32_PK_FP4_F16 : VOP3Inst<"v_cvt_scalef32_pk_fp4_f16", VOP3_CVT_SCALE_FP4_F16BF16_TiedInput_Profile<VOP_I32_V2F16_F32_F32>>;1403 defm V_CVT_SCALEF32_PK_FP4_BF16 : VOP3Inst<"v_cvt_scalef32_pk_fp4_bf16", VOP3_CVT_SCALE_FP4_F16BF16_TiedInput_Profile<VOP_I32_V2BF16_F32_F32>>;1404 }1405}1406 1407let SubtargetPredicate = HasFP6BF6ConversionScaleInsts, mayRaiseFPException = 0, Constraints = "@earlyclobber $vdst" in {1408 defm V_CVT_SCALEF32_PK32_F32_FP6 : VOP3Inst<"v_cvt_scalef32_pk32_f32_fp6", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V32F32_V6I32_F32>, int_amdgcn_cvt_scalef32_pk32_f32_fp6>;1409 defm V_CVT_SCALEF32_PK32_F32_BF6 : VOP3Inst<"v_cvt_scalef32_pk32_f32_bf6", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V32F32_V6I32_F32>, int_amdgcn_cvt_scalef32_pk32_f32_bf6>;1410 defm V_CVT_SCALEF32_PK32_F16_FP6 : VOP3Inst<"v_cvt_scalef32_pk32_f16_fp6", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V32F16_V6I32_F32>, int_amdgcn_cvt_scalef32_pk32_f16_fp6>;1411 defm V_CVT_SCALEF32_PK32_BF16_FP6 : VOP3Inst<"v_cvt_scalef32_pk32_bf16_fp6", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V32BF16_V6I32_F32>, int_amdgcn_cvt_scalef32_pk32_bf16_fp6>;1412 defm V_CVT_SCALEF32_PK32_F16_BF6 : VOP3Inst<"v_cvt_scalef32_pk32_f16_bf6", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V32F16_V6I32_F32>, int_amdgcn_cvt_scalef32_pk32_f16_bf6>;1413 defm V_CVT_SCALEF32_PK32_BF16_BF6 : VOP3Inst<"v_cvt_scalef32_pk32_bf16_bf6", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V32BF16_V6I32_F32>, int_amdgcn_cvt_scalef32_pk32_bf16_bf6>;1414}1415 1416let SubtargetPredicate = HasF16BF16ToFP6BF6ConversionScaleInsts, mayRaiseFPException = 0, Constraints = "@earlyclobber $vdst" in {1417 defm V_CVT_SCALEF32_PK32_FP6_F16 : VOP3Inst<"v_cvt_scalef32_pk32_fp6_f16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V6I32_V32F16_F32>, int_amdgcn_cvt_scalef32_pk32_fp6_f16>;1418 defm V_CVT_SCALEF32_PK32_BF6_F16 : VOP3Inst<"v_cvt_scalef32_pk32_bf6_f16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V6I32_V32F16_F32>, int_amdgcn_cvt_scalef32_pk32_bf6_f16>;1419 defm V_CVT_SCALEF32_PK32_FP6_BF16 : VOP3Inst<"v_cvt_scalef32_pk32_fp6_bf16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V6I32_V32BF16_F32>, int_amdgcn_cvt_scalef32_pk32_fp6_bf16>;1420 defm V_CVT_SCALEF32_PK32_BF6_BF16 : VOP3Inst<"v_cvt_scalef32_pk32_bf6_bf16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V6I32_V32BF16_F32>, int_amdgcn_cvt_scalef32_pk32_bf6_bf16>;1421 defm V_CVT_SCALEF32_SR_PK32_BF6_BF16 : VOP3Inst<"v_cvt_scalef32_sr_pk32_bf6_bf16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V6I32_V32BF16_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk32_bf6_bf16>;1422 defm V_CVT_SCALEF32_SR_PK32_BF6_F16 : VOP3Inst<"v_cvt_scalef32_sr_pk32_bf6_f16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V6I32_V32F16_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk32_bf6_f16>;1423 defm V_CVT_SCALEF32_SR_PK32_BF6_F32 : VOP3Inst<"v_cvt_scalef32_sr_pk32_bf6_f32", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V6I32_V32F32_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk32_bf6_f32>;1424 defm V_CVT_SCALEF32_SR_PK32_FP6_BF16 : VOP3Inst<"v_cvt_scalef32_sr_pk32_fp6_bf16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V6I32_V32BF16_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk32_fp6_bf16>;1425 defm V_CVT_SCALEF32_SR_PK32_FP6_F16 : VOP3Inst<"v_cvt_scalef32_sr_pk32_fp6_f16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V6I32_V32F16_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk32_fp6_f16>;1426 defm V_CVT_SCALEF32_SR_PK32_FP6_F32 : VOP3Inst<"v_cvt_scalef32_sr_pk32_fp6_f32", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V6I32_V32F32_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk32_fp6_f32>;1427}1428 1429let SubtargetPredicate = HasGFX950Insts, mayRaiseFPException = 0 in {1430 defm V_CVT_SCALEF32_2XPK16_FP6_F32 : VOP3Inst<"v_cvt_scalef32_2xpk16_fp6_f32", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V6I32_V16F32_V16F32_F32>, int_amdgcn_cvt_scalef32_2xpk16_fp6_f32>;1431 defm V_CVT_SCALEF32_2XPK16_BF6_F32 : VOP3Inst<"v_cvt_scalef32_2xpk16_bf6_f32", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V6I32_V16F32_V16F32_F32>, int_amdgcn_cvt_scalef32_2xpk16_bf6_f32>;1432}1433 1434let SubtargetPredicate = HasCvtPkF16F32Inst in {1435 let ReadsModeReg = 0 in {1436 defm V_CVT_PK_F16_F32 : VOP3Inst<"v_cvt_pk_f16_f32", VOP3_Profile<VOP_V2F16_F32_F32>>;1437 }1438 1439 def : GCNPat<(v2f16 (fpround v2f32:$src)),1440 (V_CVT_PK_F16_F32_e64 0, (EXTRACT_SUBREG VReg_64:$src, sub0), 0, (EXTRACT_SUBREG VReg_64:$src, sub1))>;1441 def : GCNPat<(v2f16 (fpround v2f64:$src)),1442 (V_CVT_PK_F16_F32_e64 0, (V_CVT_F32_F64_e64 0, (EXTRACT_SUBREG VReg_128:$src, sub0_sub1)),1443 0, (V_CVT_F32_F64_e64 0, (EXTRACT_SUBREG VReg_128:$src, sub2_sub3)))>;1444 def : GCNPat<(v2f16 (build_vector (f16 (fpround (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))),1445 (f16 (fpround (f32 (VOP3Mods f32:$src1, i32:$src1_modifiers)))))),1446 (V_CVT_PK_F16_F32_e64 $src0_modifiers, $src0, $src1_modifiers, $src1)>;1447}1448 1449class Cvt_Scale_FP4FP8BF8ToF16F32_Pat<SDPatternOperator node, VOP3_Pseudo inst, ValueType DstTy> : GCNPat<1450 (DstTy (node i32:$src0, f32:$src1, timm:$index)),1451 (inst (SrcAndDstSelToOpSelXForm_0_0 $index), $src0, (SrcAndDstSelToOpSelXForm_1_0 $index), $src1)1452>;1453def : Cvt_Scale_FP4FP8BF8ToF16F32_Pat<int_amdgcn_cvt_scalef32_f32_fp8, V_CVT_SCALEF32_F32_FP8_e64, f32>;1454def : Cvt_Scale_FP4FP8BF8ToF16F32_Pat<int_amdgcn_cvt_scalef32_f32_bf8, V_CVT_SCALEF32_F32_BF8_e64, f32>;1455def : Cvt_Scale_FP4FP8BF8ToF16F32_Pat<int_amdgcn_cvt_scalef32_pk_f16_fp4, V_CVT_SCALEF32_PK_F16_FP4_e64, v2f16>;1456def : Cvt_Scale_FP4FP8BF8ToF16F32_Pat<int_amdgcn_cvt_scalef32_pk_bf16_fp4, V_CVT_SCALEF32_PK_BF16_FP4_e64, v2bf16>;1457 1458class Cvt_Scale_FP8BF8ToF16_Pat<SDPatternOperator node, VOP3_Pseudo inst, int dst_sel> : GCNPat<1459 (v2f16 (node v2f16:$vdst_in, i32:$src0, f32:$src1, timm:$src_sel, dst_sel)),1460 (inst !if(!eq(dst_sel, 0), (SrcAndDstSelToOpSelXForm_0_0 $src_sel), (SrcAndDstSelToOpSelXForm_0_1 $src_sel)), $src0,1461 !if(!eq(dst_sel, 0), (SrcAndDstSelToOpSelXForm_1_0 $src_sel), (SrcAndDstSelToOpSelXForm_1_1 $src_sel)), $src1, VGPR_32:$vdst_in)1462>;1463foreach DstSel = [0, -1] in {1464 def : Cvt_Scale_FP8BF8ToF16_Pat<int_amdgcn_cvt_scalef32_f16_fp8, V_CVT_SCALEF32_F16_FP8_e64, DstSel>;1465 def : Cvt_Scale_FP8BF8ToF16_Pat<int_amdgcn_cvt_scalef32_f16_bf8, V_CVT_SCALEF32_F16_BF8_e64, DstSel>;1466}1467 1468class Cvt_Scale_PK_F32ToFP8BF8_Pat<SDPatternOperator node, VOP3_Pseudo inst> : GCNPat<1469 (v2i16 (node v2i16:$vdst_in, f32:$src0, f32:$src1, f32:$src2, timm:$word_sel)),1470 (inst (DstSelToOpSelXForm $word_sel), $src0, 0, $src1, 0, $src2, VGPR_32:$vdst_in)1471>;1472def : Cvt_Scale_PK_F32ToFP8BF8_Pat<int_amdgcn_cvt_scalef32_pk_fp8_f32, V_CVT_SCALEF32_PK_FP8_F32_e64>;1473def : Cvt_Scale_PK_F32ToFP8BF8_Pat<int_amdgcn_cvt_scalef32_pk_bf8_f32, V_CVT_SCALEF32_PK_BF8_F32_e64>;1474 1475class Cvt_Scale_PK_FP8BF8ToF16F32_Pat<SDPatternOperator node, VOP3_Pseudo inst, ValueType DstTy> : GCNPat<1476 (DstTy (node i32:$src0, f32:$src1, timm:$word_sel)),1477 (inst (SrcSelToOpSelXForm $word_sel), $src0, 0, $src1)1478>;1479def : Cvt_Scale_PK_FP8BF8ToF16F32_Pat<int_amdgcn_cvt_scalef32_pk_f32_fp8, V_CVT_SCALEF32_PK_F32_FP8_e64, v2f32>;1480def : Cvt_Scale_PK_FP8BF8ToF16F32_Pat<int_amdgcn_cvt_scalef32_pk_f32_bf8, V_CVT_SCALEF32_PK_F32_BF8_e64, v2f32>;1481def : Cvt_Scale_PK_FP8BF8ToF16F32_Pat<int_amdgcn_cvt_scalef32_pk_f16_bf8, V_CVT_SCALEF32_PK_F16_BF8_e64, v2f16>;1482def : Cvt_Scale_PK_FP8BF8ToF16F32_Pat<int_amdgcn_cvt_scalef32_pk_bf16_bf8, V_CVT_SCALEF32_PK_BF16_BF8_e64, v2bf16>;1483def : Cvt_Scale_PK_FP8BF8ToF16F32_Pat<int_amdgcn_cvt_scalef32_pk_f16_fp8, V_CVT_SCALEF32_PK_F16_FP8_e64, v2f16>;1484def : Cvt_Scale_PK_FP8BF8ToF16F32_Pat<int_amdgcn_cvt_scalef32_pk_bf16_fp8, V_CVT_SCALEF32_PK_BF16_FP8_e64, v2bf16>;1485 1486class Cvt_Scale_PK_F16BF16ToFP8BF8_Pat<SDPatternOperator node, VOP3_Pseudo inst, ValueType SrcTy> : GCNPat<1487 (v2i16 (node v2i16:$vdst_in, SrcTy:$src0, f32:$src1, timm:$word_sel)),1488 (inst (DstSelToOpSelXForm $word_sel), $src0, 0, $src1, VGPR_32:$vdst_in)1489>;1490def : Cvt_Scale_PK_F16BF16ToFP8BF8_Pat<int_amdgcn_cvt_scalef32_pk_fp8_f16, V_CVT_SCALEF32_PK_FP8_F16_e64, v2f16>;1491def : Cvt_Scale_PK_F16BF16ToFP8BF8_Pat<int_amdgcn_cvt_scalef32_pk_fp8_bf16, V_CVT_SCALEF32_PK_FP8_BF16_e64, v2bf16>;1492def : Cvt_Scale_PK_F16BF16ToFP8BF8_Pat<int_amdgcn_cvt_scalef32_pk_bf8_f16, V_CVT_SCALEF32_PK_BF8_F16_e64, v2f16>;1493def : Cvt_Scale_PK_F16BF16ToFP8BF8_Pat<int_amdgcn_cvt_scalef32_pk_bf8_bf16, V_CVT_SCALEF32_PK_BF8_BF16_e64, v2bf16>;1494 1495class Cvt_Scale_PK_F32ToFP4_Pat<SDPatternOperator node, VOP3_Pseudo inst> : GCNPat<1496 (i32 (node i32:$vdst_in, f32:$src0, f32:$src1, f32:$src2, timm:$index)),1497 (inst (DstSelToOpSel3XForm $index), $src0, 0, $src1, (SrcAndDstSelToOpSelXForm_2_0 $index), $src2, VGPR_32:$vdst_in)1498>;1499def : Cvt_Scale_FP4FP8BF8ToF16F32_Pat<int_amdgcn_cvt_scalef32_pk_f32_fp4, V_CVT_SCALEF32_PK_F32_FP4_e64, v2f32>;1500def : Cvt_Scale_PK_F32ToFP4_Pat<int_amdgcn_cvt_scalef32_pk_fp4_f32, V_CVT_SCALEF32_PK_FP4_F32_e64>;1501 1502class Cvt_Scale_PK_F16ToFP4_Pat<SDPatternOperator node, VOP3_Pseudo inst, ValueType SrcTy> : GCNPat<1503 (i32 (node i32:$src2, SrcTy:$src0, f32:$src1, timm:$index)),1504 (inst (DstSelToOpSel3XForm $index), $src0, 0, $src1, (SrcAndDstSelToOpSelXForm_2_0 $index), $src2)1505>;1506def : Cvt_Scale_PK_F16ToFP4_Pat<int_amdgcn_cvt_scalef32_pk_fp4_f16, V_CVT_SCALEF32_PK_FP4_F16_e64, v2f16>;1507def : Cvt_Scale_PK_F16ToFP4_Pat<int_amdgcn_cvt_scalef32_pk_fp4_bf16, V_CVT_SCALEF32_PK_FP4_BF16_e64, v2bf16>;1508 1509class Cvt_Scale_SR_PK_BF16F16F32ToFP4BF8FP8_Pat<SDPatternOperator node, VOP3_Pseudo inst, ValueType SrcTy> : GCNPat<1510 (i32 (node i32:$vdst_in, SrcTy:$src0, i32:$src1, f32:$src2, timm:$index)),1511 (inst (DstSelToOpSel3XForm $index), $src0, 0, $src1, (SrcAndDstSelToOpSelXForm_2_0 $index), $src2, VGPR_32:$vdst_in)1512>;1513def : Cvt_Scale_SR_PK_BF16F16F32ToFP4BF8FP8_Pat<int_amdgcn_cvt_scalef32_sr_pk_fp4_f16, V_CVT_SCALEF32_SR_PK_FP4_F16_e64, v2f16>;1514def : Cvt_Scale_SR_PK_BF16F16F32ToFP4BF8FP8_Pat<int_amdgcn_cvt_scalef32_sr_pk_fp4_bf16, V_CVT_SCALEF32_SR_PK_FP4_BF16_e64, v2bf16>;1515def : Cvt_Scale_SR_PK_BF16F16F32ToFP4BF8FP8_Pat<int_amdgcn_cvt_scalef32_sr_pk_fp4_f32, V_CVT_SCALEF32_SR_PK_FP4_F32_e64, v2f32>;1516def : Cvt_Scale_SR_PK_BF16F16F32ToFP4BF8FP8_Pat<int_amdgcn_cvt_scalef32_sr_bf8_bf16, V_CVT_SCALEF32_SR_BF8_BF16_e64, bf16>;1517def : Cvt_Scale_SR_PK_BF16F16F32ToFP4BF8FP8_Pat<int_amdgcn_cvt_scalef32_sr_bf8_f16, V_CVT_SCALEF32_SR_BF8_F16_e64, f16>;1518def : Cvt_Scale_SR_PK_BF16F16F32ToFP4BF8FP8_Pat<int_amdgcn_cvt_scalef32_sr_bf8_f32, V_CVT_SCALEF32_SR_BF8_F32_e64, f32>;1519def : Cvt_Scale_SR_PK_BF16F16F32ToFP4BF8FP8_Pat<int_amdgcn_cvt_scalef32_sr_fp8_bf16, V_CVT_SCALEF32_SR_FP8_BF16_e64, bf16>;1520def : Cvt_Scale_SR_PK_BF16F16F32ToFP4BF8FP8_Pat<int_amdgcn_cvt_scalef32_sr_fp8_f16, V_CVT_SCALEF32_SR_FP8_F16_e64, f16>;1521def : Cvt_Scale_SR_PK_BF16F16F32ToFP4BF8FP8_Pat<int_amdgcn_cvt_scalef32_sr_fp8_f32, V_CVT_SCALEF32_SR_FP8_F32_e64, f32>;1522 1523let SubtargetPredicate = isGFX10Plus in {1524 let isCommutable = 1, isReMaterializable = 1 in {1525 defm V_XOR3_B32 : VOP3Inst <"v_xor3_b32", VOP3_Profile<VOP_I32_I32_I32_I32>>;1526 } // End isCommutable = 1, isReMaterializable = 11527 def : ThreeOp_i32_Pats<xor, xor, V_XOR3_B32_e64>;1528 1529 let Constraints = "$vdst = $vdst_in", isConvergent = 1 in {1530 defm V_PERMLANE16_B32 : VOP3Inst<"v_permlane16_b32", VOP3_PERMLANE_Profile>;1531 defm V_PERMLANEX16_B32 : VOP3Inst<"v_permlanex16_b32", VOP3_PERMLANE_Profile>;1532 } // End $vdst = $vdst_in, isConvergent = 11533 1534 foreach vt = Reg32Types.types in {1535 def : PermlanePat<int_amdgcn_permlane16, V_PERMLANE16_B32_e64, vt>;1536 def : PermlanePat<int_amdgcn_permlanex16, V_PERMLANEX16_B32_e64, vt>;1537 }1538 let isCommutable = 1 in {1539 defm V_ADD_NC_U16 : VOP3Inst_t16 <"v_add_nc_u16", VOP_I16_I16_I16, add>;1540 } // End isCommutable = 11541 defm V_SUB_NC_U16 : VOP3Inst_t16 <"v_sub_nc_u16", VOP_I16_I16_I16, sub>;1542 1543} // End SubtargetPredicate = isGFX10Plus1544 1545let True16Predicate = NotHasTrue16BitInsts, SubtargetPredicate = isGFX10Plus in {1546 def : OpSelBinOpClampPat<uaddsat, V_ADD_NC_U16_e64>;1547 def : OpSelBinOpClampPat<usubsat, V_SUB_NC_U16_e64>;1548 // Undo sub x, c -> add x, -c canonicalization since c is more likely1549 // an inline immediate than -c.1550 def : GCNPat<1551 (add i16:$src0, (i16 NegSubInlineIntConst16:$src1)),1552 (V_SUB_NC_U16_e64 0, VSrc_b16:$src0, 0, NegSubInlineIntConst16:$src1, 0, 0)1553 >;1554} // End True16Predicate = NotHasTrue16BitInsts, SubtargetPredicate = isGFX10Plus1555 1556let True16Predicate = UseRealTrue16Insts in {1557 def : OpSelBinOpClampPat<uaddsat, V_ADD_NC_U16_t16_e64>;1558 def : OpSelBinOpClampPat<usubsat, V_SUB_NC_U16_t16_e64>;1559 def : GCNPat<1560 (add i16:$src0, (i16 NegSubInlineIntConst16:$src1)),1561 (V_SUB_NC_U16_t16_e64 0, VSrc_b16:$src0, 0, NegSubInlineIntConst16:$src1, 0, 0)1562 >;1563} // End True16Predicate = UseRealTrue16Insts1564 1565let True16Predicate = UseFakeTrue16Insts in {1566 def : OpSelBinOpClampPat<uaddsat, V_ADD_NC_U16_fake16_e64>;1567 def : OpSelBinOpClampPat<usubsat, V_SUB_NC_U16_fake16_e64>;1568 def : GCNPat<1569 (add i16:$src0, (i16 NegSubInlineIntConst16:$src1)),1570 (V_SUB_NC_U16_fake16_e64 0, VSrc_b16:$src0, 0, NegSubInlineIntConst16:$src1, 0, 0)1571 >;1572} // End True16Predicate = UseFakeTrue16Insts1573 1574let SubtargetPredicate = isGFX12Plus in {1575 let Constraints = "$vdst = $vdst_in" in {1576 defm V_PERMLANE16_VAR_B32 : VOP3Inst<"v_permlane16_var_b32", VOP3_PERMLANE_VAR_Profile>;1577 defm V_PERMLANEX16_VAR_B32 : VOP3Inst<"v_permlanex16_var_b32", VOP3_PERMLANE_VAR_Profile>;1578 } // End $vdst = $vdst_in1579 1580 def : PermlaneVarPat<int_amdgcn_permlane16_var, V_PERMLANE16_VAR_B32_e64>;1581 def : PermlaneVarPat<int_amdgcn_permlanex16_var, V_PERMLANEX16_VAR_B32_e64>;1582 1583} // End SubtargetPredicate = isGFX12Plus1584 1585let SubtargetPredicate = isGFX1250Plus, WaveSizePredicate = isWave32 in {1586 defm V_PERMLANE_BCAST_B32 : VOP3Inst<"v_permlane_bcast_b32", VOP3_PERMLANE_NOOPSEL_Profile<VOP_I32_I32_I32_I32>>;1587 defm V_PERMLANE_UP_B32 : VOP3Inst<"v_permlane_up_b32", VOP3_PERMLANE_NOOPSEL_Profile<VOP_I32_I32_I32_I32>>;1588 defm V_PERMLANE_DOWN_B32 : VOP3Inst<"v_permlane_down_b32", VOP3_PERMLANE_NOOPSEL_Profile<VOP_I32_I32_I32_I32>>;1589 defm V_PERMLANE_XOR_B32 : VOP3Inst<"v_permlane_xor_b32", VOP3_PERMLANE_NOOPSEL_Profile<VOP_I32_I32_I32_I32>>;1590 defm V_PERMLANE_IDX_GEN_B32 : VOP3Inst<"v_permlane_idx_gen_b32", VOP3_PERMLANE_NOOPSEL_Profile<VOP_I32_I32_I32>>;1591 1592 def : PermlaneNoDppPat3Src<int_amdgcn_permlane_bcast, V_PERMLANE_BCAST_B32_e64>;1593 def : PermlaneNoDppPat3Src<int_amdgcn_permlane_up, V_PERMLANE_UP_B32_e64>;1594 def : PermlaneNoDppPat3Src<int_amdgcn_permlane_down, V_PERMLANE_DOWN_B32_e64>;1595 def : PermlaneNoDppPat3Src<int_amdgcn_permlane_xor, V_PERMLANE_XOR_B32_e64>;1596 def : PermlaneNoDppPat2Src<int_amdgcn_permlane_idx_gen, V_PERMLANE_IDX_GEN_B32_e64>;1597} // End SubtargetPredicate = isGFX1250Plus, WaveSizePredicate = isWave321598 1599let HasClamp = 0, HasModifiers = 1 in {1600def BitOp3_B16_Profile : VOP3_BITOP3_Profile<VOPProfile <[i16, i16, i16, i16, i32]>, VOP3_OPSEL>;1601def BitOp3_B16_t16_Profile : VOP3_Profile_True16<BitOp3_B16_Profile>;1602def BitOp3_B16_fake16_Profile : VOP3_Profile_Fake16<BitOp3_B16_Profile>;1603}1604 1605let OtherPredicates = [HasBitOp3Insts] in {1606 let isReMaterializable = 1 in {1607 let SubtargetPredicate = isGFX940Plus in1608 defm V_BITOP3_B16 : VOP3Inst <"v_bitop3_b16", BitOp3_B16_Profile>;1609 let SubtargetPredicate = isGFX1250Plus in1610 defm V_BITOP3_B16_gfx1250 : VOP3Inst_t16_with_profiles <"v_bitop3_b16_gfx1250", BitOp3_B16_Profile,1611 BitOp3_B16_t16_Profile, BitOp3_B16_fake16_Profile>;1612 defm V_BITOP3_B32 : VOP3Inst <"v_bitop3_b32",1613 VOP3_BITOP3_Profile<VOPProfile <[i32, i32, i32, i32, i32]>, VOP3_REGULAR>>,1614 VOPD_Component<0x12, "v_bitop2_b32">;1615 }1616 1617 def : GCNPat<1618 (i32 (int_amdgcn_bitop3 i32:$src0, i32:$src1, i32:$src2, i32:$bitop3)),1619 (i32 (V_BITOP3_B32_e64 VSrc_b32:$src0, VSrc_b32:$src1, VSrc_b32:$src2, timm:$bitop3))1620 >;1621 1622 def : GCNPat<1623 (i32 (BITOP3_32 i32:$src0, i32:$src1, i32:$src2, i32:$bitop3)),1624 (i32 (V_BITOP3_B32_e64 VSrc_b32:$src0, VSrc_b32:$src1, VSrc_b32:$src2, timm:$bitop3))1625 >;1626 1627 let SubtargetPredicate = isGFX940Plus in {1628 def : GCNPat<1629 (i16 (int_amdgcn_bitop3 i16:$src0, i16:$src1, i16:$src2, i32:$bitop3)),1630 (i16 (V_BITOP3_B16_e64 0, VSrc_b16:$src0, 0, VSrc_b16:$src1, 0, VSrc_b16:$src2, timm:$bitop3, 0))1631 >;1632 1633 def : GCNPat<1634 (i16 (BITOP3_16 i16:$src0, i16:$src1, i16:$src2, i32:$bitop3)),1635 (i16 (V_BITOP3_B16_e64 0, VSrc_b16:$src0, 0, VSrc_b16:$src1, 0, VSrc_b16:$src2, timm:$bitop3, 0))1636 >;1637 } // End SubtargetPredicate = isGFX940Plus1638 1639 let SubtargetPredicate = isGFX1250Plus in {1640 let True16Predicate = UseFakeTrue16Insts in {1641 def : GCNPat<1642 (i16 (int_amdgcn_bitop3 i16:$src0, i16:$src1, i16:$src2, i32:$bitop3)),1643 (i16 (V_BITOP3_B16_gfx1250_fake16_e64 0, VSrc_b16:$src0, 0, VSrc_b16:$src1, 0, VSrc_b16:$src2, timm:$bitop3, 0))1644 >;1645 1646 def : GCNPat<1647 (i16 (BITOP3_16 i16:$src0, i16:$src1, i16:$src2, i32:$bitop3)),1648 (i16 (V_BITOP3_B16_gfx1250_fake16_e64 0, VSrc_b16:$src0, 0, VSrc_b16:$src1, 0, VSrc_b16:$src2, timm:$bitop3, 0))1649 >;1650 }1651 let True16Predicate = UseRealTrue16Insts in {1652 def : GCNPat<1653 (i16 (int_amdgcn_bitop3 i16:$src0, i16:$src1, i16:$src2, i32:$bitop3)),1654 (i16 (V_BITOP3_B16_gfx1250_t16_e64 0, VSrcT_b16:$src0, 0, VSrcT_b16:$src1, 0, VSrcT_b16:$src2, timm:$bitop3, 0))1655 >;1656 1657 def : GCNPat<1658 (i16 (BITOP3_16 i16:$src0, i16:$src1, i16:$src2, i32:$bitop3)),1659 (i16 (V_BITOP3_B16_gfx1250_t16_e64 0, VSrcT_b16:$src0, 0, VSrcT_b16:$src1, 0, VSrcT_b16:$src2, timm:$bitop3, 0))1660 >;1661 }1662 } // End SubtargetPredicate = isGFX1250Plus1663 1664} // End OtherPredicates = [HasBitOp3Insts]1665 1666class DivFmasPat<ValueType vt, Instruction inst, Register CondReg> : GCNPat<1667 (AMDGPUdiv_fmas (vt (VOP3Mods vt:$src0, i32:$src0_modifiers)),1668 (vt (VOP3Mods vt:$src1, i32:$src1_modifiers)),1669 (vt (VOP3Mods vt:$src2, i32:$src2_modifiers)),1670 (i1 CondReg)),1671 (inst $src0_modifiers, $src0, $src1_modifiers, $src1, $src2_modifiers, $src2)1672>;1673 1674let WaveSizePredicate = isWave64 in {1675def : DivFmasPat<f32, V_DIV_FMAS_F32_e64, VCC>;1676def : DivFmasPat<f64, V_DIV_FMAS_F64_e64, VCC>;1677}1678 1679let WaveSizePredicate = isWave32 in {1680def : DivFmasPat<f32, V_DIV_FMAS_F32_e64, VCC_LO>;1681def : DivFmasPat<f64, V_DIV_FMAS_F64_e64, VCC_LO>;1682}1683 1684class VOP3_DOT_Profile<VOPProfile P> : VOP3_Profile<P, VOP3_OPSEL> {1685 let HasClamp = 0;1686 let HasOMod = 0;1687}1688 1689class VOP3_DOT_Profile_t16<VOPProfile P, VOP3Features Features = VOP3_REGULAR> : VOP3_Profile_True16<P, Features> {1690 let HasClamp = 0;1691 let HasOMod = 0;1692 // Override modifiers for bf16(i16) (same as float modifiers).1693 let HasSrc0Mods = 1;1694 let HasSrc1Mods = 1;1695 let HasSrc2Mods = 1;1696 let Src0ModVOP3DPP = FPVRegInputMods;1697 let Src1ModVOP3DPP = FP32VCSrcInputMods;1698 let Src2ModVOP3DPP = FPT16VCSrcInputMods</*IsFake16*/0>;1699}1700 1701class VOP3_DOT_Profile_fake16<VOPProfile P, VOP3Features Features = VOP3_REGULAR> : VOP3_Profile_Fake16<P, Features> {1702 let HasClamp = 0;1703 let HasOMod = 0;1704 // Override modifiers for bf16(i16) (same as float modifiers).1705 let HasSrc0Mods = 1;1706 let HasSrc1Mods = 1;1707 let HasSrc2Mods = 1;1708 let AsmVOP3Base = getAsmVOP3Base<NumSrcArgs, HasDst, HasClamp,1709 HasOpSel, HasOMod, IsVOP3P, HasModifiers, 1/*HasSrc0Mods*/, 1/*HasSrc1Mods*/,1710 1/*HasSrc2Mods*/, DstVT>.ret;1711}1712 1713let SubtargetPredicate = isGFX11Plus in {1714 defm V_MAXMIN_F32 : VOP3Inst<"v_maxmin_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;1715 defm V_MINMAX_F32 : VOP3Inst<"v_minmax_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;1716 defm V_MAXMIN_F16 : VOP3Inst_t16<"v_maxmin_f16", VOP_F16_F16_F16_F16>;1717 defm V_MINMAX_F16 : VOP3Inst_t16<"v_minmax_f16", VOP_F16_F16_F16_F16>;1718 defm V_MAXMIN_U32 : VOP3Inst<"v_maxmin_u32", VOP3_Profile<VOP_I32_I32_I32_I32>>;1719 defm V_MINMAX_U32 : VOP3Inst<"v_minmax_u32", VOP3_Profile<VOP_I32_I32_I32_I32>>;1720 defm V_MAXMIN_I32 : VOP3Inst<"v_maxmin_i32", VOP3_Profile<VOP_I32_I32_I32_I32>>;1721 defm V_MINMAX_I32 : VOP3Inst<"v_minmax_i32", VOP3_Profile<VOP_I32_I32_I32_I32>>;1722 defm V_CVT_PK_I16_F32 : VOP3Inst<"v_cvt_pk_i16_f32", VOP3_Profile<VOP_V2I16_F32_F32>>;1723 defm V_CVT_PK_U16_F32 : VOP3Inst<"v_cvt_pk_u16_f32", VOP3_Profile<VOP_V2I16_F32_F32>>;1724} // End SubtargetPredicate = isGFX11Plus1725 1726class VOP3_CVT_SR_FP16_TiedInput_Profile<VOPProfile P> : VOP3_CVT_SCALE_F1632_FP8BF8_TiedInput_Profile<P> {1727 let InsVOP3OpSel = (ins FP32InputMods:$src0_modifiers, Src0RC64:$src0,1728 Int32InputMods:$src1_modifiers, Src1RC64:$src1,1729 VGPR_32:$vdst_in, op_sel0:$op_sel);1730}1731 1732// FIXME: GlobalISel cannot distinguish f16 and bf16 and may start using bf16 patterns1733// instead of less complex f16. Disable GlobalISel for these for now.1734def bf16_fpround : PatFrag <(ops node:$src0), (fpround $src0), [{ return true; }]> {1735 let GISelPredicateCode = [{return false;}];1736}1737 1738let SubtargetPredicate = HasBF16ConversionInsts in {1739 let ReadsModeReg = 0 in {1740 defm V_CVT_PK_BF16_F32 : VOP3Inst<"v_cvt_pk_bf16_f32", VOP3_Profile<VOP_V2BF16_F32_F32>>;1741 defm V_CVT_SR_PK_BF16_F32 : VOP3Inst<"v_cvt_sr_pk_bf16_f32", VOP3_Profile<VOP_V2BF16_F32_F32_I32>, int_amdgcn_cvt_sr_pk_bf16_f32>;1742 }1743 def : GCNPat<(v2bf16 (bf16_fpround v2f32:$src)),1744 (V_CVT_PK_BF16_F32_e64 0, (EXTRACT_SUBREG VReg_64:$src, sub0), 0, (EXTRACT_SUBREG VReg_64:$src, sub1))>;1745 def : GCNPat<(v2bf16 (build_vector (bf16 (bf16_fpround (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))),1746 (bf16 (bf16_fpround (f32 (VOP3Mods f32:$src1, i32:$src1_modifiers)))))),1747 (V_CVT_PK_BF16_F32_e64 $src0_modifiers, $src0, $src1_modifiers, $src1)>;1748 def : GCNPat<(bf16 (bf16_fpround (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))),1749 (V_CVT_PK_BF16_F32_e64 $src0_modifiers, $src0, 0, (f32 (IMPLICIT_DEF)))>;1750}1751 1752class VOP3_CVT_SCALE_PK_F16_F864_Profile<VOPProfile P> : VOP3_CVT_SCALEF32_PK_F864_Profile<P> {1753 let Src0RC64 = getVOP3VRegSrcForVT<Src0VT>.ret;1754 let Ins64 = !con(getIns64<Src0RC64, Src1RC64, Src2RC64, NumSrcArgs,1755 HasClamp, HasModifiers, HasSrc2Mods,1756 HasOMod, Src0Mod, Src1Mod, Src2Mod>.ret,1757 (ins ScaleSel:$scale_sel));1758 let Asm64 = getAsmVOP3Base<NumSrcArgs, HasDst, HasClamp,1759 HasOpSel, HasOMod, IsVOP3P, HasNeg, HasSrc0Mods, HasSrc1Mods,1760 HasSrc2Mods, DstVT>.ret # "$scale_sel";1761}1762 1763multiclass VOP3CvtScaleSelInst<string OpName, VOPProfile P, SDPatternOperator node> {1764 def _e64 : VOP3InstBase<OpName, VOP3_CVT_SCALE_PK_F16_F864_Profile<P>> {1765 let Pattern = [(set P.DstVT:$vdst, (node (P.Src0VT (VOP3Mods0 P.Src0VT:$src0)), i32:$src1, i32:$scale_sel))];1766 }1767}1768 1769let HasExtVOP3DPP = 0, HasModifiers = 0 in {1770def VOP3_V2I32_I32_I32_V2I32 : VOP3_Profile<VOPProfile<[v2i32, i32, i32, v2i32]>>;1771def VOP3_V3I32_I32_I64_V2I32 : VOP3_Profile<VOPProfile<[v3i32, i32, i64, v2i32]>>;1772def VOP3_V4I32_I64_I64_V2I32 : VOP3_Profile<VOPProfile<[v4i32, i64, i64, v2i32]>>;1773}1774 1775let Src0RC64 = VSrc_NoInline_v2f16 in {1776def VOP3_CVT_PK_F8_F16_Profile : VOP3_Profile<VOP_I16_V2F16>;1777def VOP3_CVT_PK_F8_F16_True16_Profile : VOP3_Profile_True16<VOP3_CVT_PK_F8_F16_Profile>;1778def VOP3_CVT_PK_F8_F16_Fake16_Profile : VOP3_Profile_Fake16<VOP3_CVT_PK_F8_F16_Profile>;1779}1780 1781let ReadsModeReg = 0, IsPacked = 0, SubtargetPredicate = isGFX125xOnly in {1782 defm V_CVT_PK_FP8_F16_gfx1250 : VOP3Inst_t16_with_profiles<"v_cvt_pk_fp8_f16_gfx1250",1783 VOP3_CVT_PK_F8_F16_Profile,1784 VOP3_CVT_PK_F8_F16_True16_Profile,1785 VOP3_CVT_PK_F8_F16_Fake16_Profile,1786 int_amdgcn_cvt_pk_fp8_f16>;1787 defm V_CVT_PK_BF8_F16_gfx1250 : VOP3Inst_t16_with_profiles<"v_cvt_pk_bf8_f16_gfx1250",1788 VOP3_CVT_PK_F8_F16_Profile,1789 VOP3_CVT_PK_F8_F16_True16_Profile,1790 VOP3_CVT_PK_F8_F16_Fake16_Profile,1791 int_amdgcn_cvt_pk_bf8_f16>;1792}1793 1794let HasClamp = 0, HasOpSel = 1 in {1795def VOP3_CVT_SR_F8_F16_Profile : VOP3_CVT_SR_F8_ByteSel_Profile<f16>;1796def VOP3_CVT_SR_F8_F16_True16_Profile : VOP3_Profile_True16<VOP3_CVT_SR_F8_F16_Profile>;1797def VOP3_CVT_SR_F8_F16_Fake16_Profile : VOP3_Profile_Fake16<VOP3_CVT_SR_F8_F16_Profile>;1798}1799 1800let SubtargetPredicate = isGFX1250Plus in {1801 let ReadsModeReg = 0 in {1802 defm V_CVT_SR_PK_F16_F32 : VOP3Inst<"v_cvt_sr_pk_f16_f32", VOP3_Profile<VOP_V2F16_F32_F32_I32>, int_amdgcn_cvt_sr_pk_f16_f32>;1803 1804 // These instructions have non-standard use of op_sel. They are using bits 2 and 3 of opsel1805 // to select a byte in the vdst. Bits 0 and 1 are unused.1806 let Constraints = "$vdst = $vdst_in" in {1807 defm V_CVT_SR_FP8_F16 : VOP3Inst_t16_with_profiles<"v_cvt_sr_fp8_f16", VOP3_CVT_SR_F8_F16_Profile,1808 VOP3_CVT_SR_F8_F16_True16_Profile, VOP3_CVT_SR_F8_F16_Fake16_Profile>;1809 defm V_CVT_SR_BF8_F16 : VOP3Inst_t16_with_profiles<"v_cvt_sr_bf8_f16", VOP3_CVT_SR_F8_F16_Profile,1810 VOP3_CVT_SR_F8_F16_True16_Profile, VOP3_CVT_SR_F8_F16_Fake16_Profile>;1811 }1812 1813 let Constraints = "@earlyclobber $vdst" in {1814 defm V_CVT_SCALE_PK8_F16_FP8 : VOP3CvtScaleSelInst<"v_cvt_scale_pk8_f16_fp8", VOP_V8F16_V2I32_I32, int_amdgcn_cvt_scale_pk8_f16_fp8>;1815 defm V_CVT_SCALE_PK8_BF16_FP8 : VOP3CvtScaleSelInst<"v_cvt_scale_pk8_bf16_fp8", VOP_V8BF16_V2I32_I32, int_amdgcn_cvt_scale_pk8_bf16_fp8>;1816 defm V_CVT_SCALE_PK8_F16_BF8 : VOP3CvtScaleSelInst<"v_cvt_scale_pk8_f16_bf8", VOP_V8F16_V2I32_I32, int_amdgcn_cvt_scale_pk8_f16_bf8>;1817 defm V_CVT_SCALE_PK8_BF16_BF8 : VOP3CvtScaleSelInst<"v_cvt_scale_pk8_bf16_bf8", VOP_V8BF16_V2I32_I32, int_amdgcn_cvt_scale_pk8_bf16_bf8>;1818 defm V_CVT_SCALE_PK8_F32_FP8 : VOP3CvtScaleSelInst<"v_cvt_scale_pk8_f32_fp8", VOP_V8F32_V2I32_I32, int_amdgcn_cvt_scale_pk8_f32_fp8>;1819 defm V_CVT_SCALE_PK8_F32_BF8 : VOP3CvtScaleSelInst<"v_cvt_scale_pk8_f32_bf8", VOP_V8F32_V2I32_I32, int_amdgcn_cvt_scale_pk8_f32_bf8>;1820 defm V_CVT_SCALE_PK16_F16_FP6 : VOP3CvtScaleSelInst<"v_cvt_scale_pk16_f16_fp6", VOP_V16F16_V3I32_I32, int_amdgcn_cvt_scale_pk16_f16_fp6>;1821 defm V_CVT_SCALE_PK16_BF16_FP6 : VOP3CvtScaleSelInst<"v_cvt_scale_pk16_bf16_fp6", VOP_V16BF16_V3I32_I32, int_amdgcn_cvt_scale_pk16_bf16_fp6>;1822 defm V_CVT_SCALE_PK16_F16_BF6 : VOP3CvtScaleSelInst<"v_cvt_scale_pk16_f16_bf6", VOP_V16F16_V3I32_I32, int_amdgcn_cvt_scale_pk16_f16_bf6>;1823 defm V_CVT_SCALE_PK16_BF16_BF6 : VOP3CvtScaleSelInst<"v_cvt_scale_pk16_bf16_bf6", VOP_V16BF16_V3I32_I32, int_amdgcn_cvt_scale_pk16_bf16_bf6>;1824 defm V_CVT_SCALE_PK16_F32_FP6 : VOP3CvtScaleSelInst<"v_cvt_scale_pk16_f32_fp6", VOP_V16F32_V3I32_I32, int_amdgcn_cvt_scale_pk16_f32_fp6>;1825 defm V_CVT_SCALE_PK16_F32_BF6 : VOP3CvtScaleSelInst<"v_cvt_scale_pk16_f32_bf6", VOP_V16F32_V3I32_I32, int_amdgcn_cvt_scale_pk16_f32_bf6>;1826 } // End Constraints = "@earlyclobber $vdst"1827 1828 defm V_CVT_SCALE_PK8_F16_FP4 : VOP3CvtScaleSelInst<"v_cvt_scale_pk8_f16_fp4", VOP_V8F16_I32_I32, int_amdgcn_cvt_scale_pk8_f16_fp4>;1829 defm V_CVT_SCALE_PK8_BF16_FP4 : VOP3CvtScaleSelInst<"v_cvt_scale_pk8_bf16_fp4", VOP_V8BF16_I32_I32, int_amdgcn_cvt_scale_pk8_bf16_fp4>;1830 defm V_CVT_SCALE_PK8_F32_FP4 : VOP3CvtScaleSelInst<"v_cvt_scale_pk8_f32_fp4", VOP_V8F32_I32_I32, int_amdgcn_cvt_scale_pk8_f32_fp4>;1831 } // End ReadsModeReg = 01832 1833 let Constraints = "@earlyclobber $vdst" in {1834 let WaveSizePredicate = isWave32 in {1835 defm V_CVT_SCALEF32_PK8_FP8_BF16 : VOP3Inst<"v_cvt_scalef32_pk8_fp8_bf16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V2I32_V8BF16_F32>, int_amdgcn_cvt_scalef32_pk8_fp8_bf16>;1836 defm V_CVT_SCALEF32_PK8_BF8_BF16 : VOP3Inst<"v_cvt_scalef32_pk8_bf8_bf16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V2I32_V8BF16_F32>, int_amdgcn_cvt_scalef32_pk8_bf8_bf16>;1837 defm V_CVT_SCALEF32_PK8_FP8_F16 : VOP3Inst<"v_cvt_scalef32_pk8_fp8_f16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V2I32_V8F16_F32>, int_amdgcn_cvt_scalef32_pk8_fp8_f16>;1838 defm V_CVT_SCALEF32_PK8_BF8_F16 : VOP3Inst<"v_cvt_scalef32_pk8_bf8_f16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V2I32_V8F16_F32>, int_amdgcn_cvt_scalef32_pk8_bf8_f16>;1839 defm V_CVT_SCALEF32_PK8_FP8_F32 : VOP3Inst<"v_cvt_scalef32_pk8_fp8_f32", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V2I32_V8F32_F32>, int_amdgcn_cvt_scalef32_pk8_fp8_f32>;1840 defm V_CVT_SCALEF32_PK8_BF8_F32 : VOP3Inst<"v_cvt_scalef32_pk8_bf8_f32", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V2I32_V8F32_F32>, int_amdgcn_cvt_scalef32_pk8_bf8_f32>;1841 defm V_CVT_SCALEF32_PK8_FP4_F32 : VOP3Inst<"v_cvt_scalef32_pk8_fp4_f32", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_I32_V8F32_F32>, int_amdgcn_cvt_scalef32_pk8_fp4_f32>;1842 defm V_CVT_SCALEF32_PK8_FP4_F16 : VOP3Inst<"v_cvt_scalef32_pk8_fp4_f16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_I32_V8F16_F32>, int_amdgcn_cvt_scalef32_pk8_fp4_f16>;1843 defm V_CVT_SCALEF32_PK8_FP4_BF16 : VOP3Inst<"v_cvt_scalef32_pk8_fp4_bf16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_I32_V8BF16_F32>, int_amdgcn_cvt_scalef32_pk8_fp4_bf16>;1844 } // End WaveSizePredicate = isWave321845 defm V_CVT_SCALEF32_PK16_FP6_F32 : VOP3Inst<"v_cvt_scalef32_pk16_fp6_f32", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V3I32_V16F32_F32>, int_amdgcn_cvt_scalef32_pk16_fp6_f32>;1846 defm V_CVT_SCALEF32_PK16_BF6_F32 : VOP3Inst<"v_cvt_scalef32_pk16_bf6_f32", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V3I32_V16F32_F32>, int_amdgcn_cvt_scalef32_pk16_bf6_f32>;1847 defm V_CVT_SCALEF32_PK16_FP6_F16 : VOP3Inst<"v_cvt_scalef32_pk16_fp6_f16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V3I32_V16F16_F32>, int_amdgcn_cvt_scalef32_pk16_fp6_f16>;1848 defm V_CVT_SCALEF32_PK16_BF6_F16 : VOP3Inst<"v_cvt_scalef32_pk16_bf6_f16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V3I32_V16F16_F32>, int_amdgcn_cvt_scalef32_pk16_bf6_f16>;1849 defm V_CVT_SCALEF32_PK16_FP6_BF16 : VOP3Inst<"v_cvt_scalef32_pk16_fp6_bf16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V3I32_V16BF16_F32>, int_amdgcn_cvt_scalef32_pk16_fp6_bf16>;1850 defm V_CVT_SCALEF32_PK16_BF6_BF16 : VOP3Inst<"v_cvt_scalef32_pk16_bf6_bf16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V3I32_V16BF16_F32>, int_amdgcn_cvt_scalef32_pk16_bf6_bf16>;1851 1852 let WaveSizePredicate = isWave32 in {1853 defm V_CVT_SCALEF32_SR_PK8_FP8_BF16 : VOP3Inst<"v_cvt_scalef32_sr_pk8_fp8_bf16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V2I32_V8BF16_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk8_fp8_bf16>;1854 defm V_CVT_SCALEF32_SR_PK8_BF8_BF16 : VOP3Inst<"v_cvt_scalef32_sr_pk8_bf8_bf16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V2I32_V8BF16_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk8_bf8_bf16>;1855 defm V_CVT_SCALEF32_SR_PK8_FP8_F16 : VOP3Inst<"v_cvt_scalef32_sr_pk8_fp8_f16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V2I32_V8F16_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk8_fp8_f16>;1856 defm V_CVT_SCALEF32_SR_PK8_BF8_F16 : VOP3Inst<"v_cvt_scalef32_sr_pk8_bf8_f16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V2I32_V8F16_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk8_bf8_f16>;1857 defm V_CVT_SCALEF32_SR_PK8_FP8_F32 : VOP3Inst<"v_cvt_scalef32_sr_pk8_fp8_f32", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V2I32_V8F32_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk8_fp8_f32>;1858 defm V_CVT_SCALEF32_SR_PK8_BF8_F32 : VOP3Inst<"v_cvt_scalef32_sr_pk8_bf8_f32", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V2I32_V8F32_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk8_bf8_f32>;1859 defm V_CVT_SCALEF32_SR_PK8_FP4_F32 : VOP3Inst<"v_cvt_scalef32_sr_pk8_fp4_f32", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_I32_V8F32_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk8_fp4_f32>;1860 defm V_CVT_SCALEF32_SR_PK8_FP4_F16 : VOP3Inst<"v_cvt_scalef32_sr_pk8_fp4_f16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_I32_V8F16_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk8_fp4_f16>;1861 defm V_CVT_SCALEF32_SR_PK8_FP4_BF16 : VOP3Inst<"v_cvt_scalef32_sr_pk8_fp4_bf16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_I32_V8BF16_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk8_fp4_bf16>;1862 } // End WaveSizePredicate = isWave321863 defm V_CVT_SCALEF32_SR_PK16_BF6_BF16 : VOP3Inst<"v_cvt_scalef32_sr_pk16_bf6_bf16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V3I32_V16BF16_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk16_bf6_bf16>;1864 defm V_CVT_SCALEF32_SR_PK16_BF6_F16 : VOP3Inst<"v_cvt_scalef32_sr_pk16_bf6_f16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V3I32_V16F16_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk16_bf6_f16>;1865 defm V_CVT_SCALEF32_SR_PK16_BF6_F32 : VOP3Inst<"v_cvt_scalef32_sr_pk16_bf6_f32", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V3I32_V16F32_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk16_bf6_f32>;1866 defm V_CVT_SCALEF32_SR_PK16_FP6_BF16 : VOP3Inst<"v_cvt_scalef32_sr_pk16_fp6_bf16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V3I32_V16BF16_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk16_fp6_bf16>;1867 defm V_CVT_SCALEF32_SR_PK16_FP6_F16 : VOP3Inst<"v_cvt_scalef32_sr_pk16_fp6_f16", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V3I32_V16F16_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk16_fp6_f16>;1868 defm V_CVT_SCALEF32_SR_PK16_FP6_F32 : VOP3Inst<"v_cvt_scalef32_sr_pk16_fp6_f32", VOP3_CVT_SCALEF32_PK_F864_Profile<VOP_V3I32_V16F32_I32_F32>, int_amdgcn_cvt_scalef32_sr_pk16_fp6_f32>;1869 } // End Constraints = "@earlyclobber $vdst"1870 1871 let True16Predicate = UseRealTrue16Insts in {1872 def : Cvt_SR_F8_ByteSel_Pat<int_amdgcn_cvt_sr_fp8_f16, V_CVT_SR_FP8_F16_t16_e64, f16>;1873 def : Cvt_SR_F8_ByteSel_Pat<int_amdgcn_cvt_sr_bf8_f16, V_CVT_SR_BF8_F16_t16_e64, f16>;1874 }1875 let True16Predicate = UseFakeTrue16Insts in {1876 def : Cvt_SR_F8_ByteSel_Pat<int_amdgcn_cvt_sr_fp8_f16, V_CVT_SR_FP8_F16_fake16_e64, f16>;1877 def : Cvt_SR_F8_ByteSel_Pat<int_amdgcn_cvt_sr_bf8_f16, V_CVT_SR_BF8_F16_fake16_e64, f16>;1878 }1879} // End SubtargetPredicate = isGFX1250Plus1880 1881let SubtargetPredicate = HasTensorCvtLutInsts in {1882 defm V_PERM_PK16_B4_U4 : VOP3Inst<"v_perm_pk16_b4_u4", VOP3_V2I32_I32_I32_V2I32, int_amdgcn_perm_pk16_b4_u4>;1883 defm V_PERM_PK16_B6_U4 : VOP3Inst<"v_perm_pk16_b6_u4", VOP3_V3I32_I32_I64_V2I32, int_amdgcn_perm_pk16_b6_u4>;1884 defm V_PERM_PK16_B8_U4 : VOP3Inst<"v_perm_pk16_b8_u4", VOP3_V4I32_I64_I64_V2I32, int_amdgcn_perm_pk16_b8_u4>;1885} // End SubtargetPredicate = HasTensorCvtLutInsts1886 1887class Cvt_Scale_Sr_F32ToBF16F16_Pat<SDPatternOperator node, VOP3_Pseudo inst, ValueType DstTy> : GCNPat<1888 (DstTy (node DstTy:$vdst_in, f32:$src0, i32:$src1, timm:$word_sel)),1889 (inst (DstSelToOpSelXForm $word_sel), $src0, 0, $src1, VGPR_32:$vdst_in)1890>;1891 1892let SubtargetPredicate = HasF32ToF16BF16ConversionSRInsts in {1893 let Constraints = "$vdst = $vdst_in" in {1894 defm V_CVT_SR_F16_F32 : VOP3Inst<"v_cvt_sr_f16_f32", VOP3_CVT_SR_FP16_TiedInput_Profile<VOP_F16_F32_I32>>;1895 defm V_CVT_SR_BF16_F32 : VOP3Inst<"v_cvt_sr_bf16_f32", VOP3_CVT_SR_FP16_TiedInput_Profile<VOP_BF16_F32_I32>>;1896 }1897 def : Cvt_Scale_Sr_F32ToBF16F16_Pat<int_amdgcn_cvt_sr_bf16_f32, V_CVT_SR_BF16_F32_e64, v2bf16>;1898 def : Cvt_Scale_Sr_F32ToBF16F16_Pat<int_amdgcn_cvt_sr_f16_f32, V_CVT_SR_F16_F32_e64, v2f16>;1899}1900 1901let SubtargetPredicate = HasIEEEMinimumMaximumInsts, ReadsModeReg = 0 in {1902 defm V_MAXIMUMMINIMUM_F32 : VOP3Inst<"v_maximumminimum_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;1903 defm V_MINIMUMMAXIMUM_F32 : VOP3Inst<"v_minimummaximum_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;1904 defm V_MAXIMUMMINIMUM_F16 : VOP3Inst_t16<"v_maximumminimum_f16", VOP_F16_F16_F16_F16>;1905 defm V_MINIMUMMAXIMUM_F16 : VOP3Inst_t16<"v_minimummaximum_f16", VOP_F16_F16_F16_F16>;1906} // End SubtargetPredicate = HasIEEEMinimumMaximumInsts, ReadsModeReg = 01907 1908let SubtargetPredicate = HasDot9Insts, IsDOT=1 in {1909 defm V_DOT2_F16_F16 : VOP3Inst_t16_with_profiles<"v_dot2_f16_f16", VOP3_DOT_Profile<VOP_F16_V2F16_V2F16_F16>,1910 VOP3_DOT_Profile_t16<VOP_F16_V2F16_V2F16_F16>,1911 VOP3_DOT_Profile_fake16<VOP_F16_V2F16_V2F16_F16>,1912 int_amdgcn_fdot2_f16_f16>;1913 defm V_DOT2_BF16_BF16 : VOP3Inst_t16_with_profiles<"v_dot2_bf16_bf16", VOP3_DOT_Profile<VOP_BF16_V2BF16_V2BF16_BF16>,1914 VOP3_DOT_Profile_t16<VOP_BF16_V2BF16_V2BF16_BF16>,1915 VOP3_DOT_Profile_fake16<VOP_BF16_V2BF16_V2BF16_BF16>,1916 int_amdgcn_fdot2_bf16_bf16>;1917}1918 1919class VOP_Pseudo_Scalar<RegisterClass Dst, RegisterOperand SrcOp,1920 ValueType dstVt, ValueType srcVt = dstVt>1921 : VOPProfile<[dstVt, srcVt, untyped, untyped]> {1922 let DstRC = VOPDstOperand<Dst>;1923 let Src0RC64 = SrcOp;1924 1925 let HasOMod = 1;1926 let HasModifiers = 1;1927}1928 1929def VOP_Pseudo_Scalar_F32 : VOP_Pseudo_Scalar<SReg_32_XEXEC, SSrc_f32, f32>;1930def VOP_Pseudo_Scalar_F16 : VOP_Pseudo_Scalar<SReg_32_XEXEC, SSrc_f16, f32, f16>;1931 1932let SubtargetPredicate = HasPseudoScalarTrans, TRANS = 1,1933 isReMaterializable = 1, SchedRW = [WritePseudoScalarTrans] in {1934 defm V_S_EXP_F32 : VOP3PseudoScalarInst<"v_s_exp_f32", VOP_Pseudo_Scalar_F32, AMDGPUexp>;1935 defm V_S_EXP_F16 : VOP3PseudoScalarInst<"v_s_exp_f16", VOP_Pseudo_Scalar_F16>;1936 defm V_S_LOG_F32 : VOP3PseudoScalarInst<"v_s_log_f32", VOP_Pseudo_Scalar_F32, AMDGPUlog>;1937 defm V_S_LOG_F16 : VOP3PseudoScalarInst<"v_s_log_f16", VOP_Pseudo_Scalar_F16>;1938 defm V_S_RCP_F32 : VOP3PseudoScalarInst<"v_s_rcp_f32", VOP_Pseudo_Scalar_F32, AMDGPUrcp>;1939 defm V_S_RCP_F16 : VOP3PseudoScalarInst<"v_s_rcp_f16", VOP_Pseudo_Scalar_F16>;1940 defm V_S_RSQ_F32 : VOP3PseudoScalarInst<"v_s_rsq_f32", VOP_Pseudo_Scalar_F32, AMDGPUrsq>;1941 defm V_S_RSQ_F16 : VOP3PseudoScalarInst<"v_s_rsq_f16", VOP_Pseudo_Scalar_F16>;1942 defm V_S_SQRT_F32 : VOP3PseudoScalarInst<"v_s_sqrt_f32", VOP_Pseudo_Scalar_F32, any_amdgcn_sqrt>;1943 defm V_S_SQRT_F16 : VOP3PseudoScalarInst<"v_s_sqrt_f16", VOP_Pseudo_Scalar_F16>;1944}1945 1946class PseudoScalarPatF16<SDPatternOperator node, VOP3_Pseudo inst> : GCNPat <1947 (f16 (UniformUnaryFrag<node> (f16 (VOP3Mods0 f16:$src0, i32:$src0_modifiers,1948 i1:$clamp, i32:$omod)))),1949 (f16 (COPY_TO_REGCLASS (f32 (inst i32:$src0_modifiers, f16:$src0, i1:$clamp,1950 i32:$omod)),1951 SReg_32_XEXEC))1952>;1953 1954let SubtargetPredicate = HasPseudoScalarTrans in {1955 def : PseudoScalarPatF16<AMDGPUexpf16, V_S_EXP_F16_e64>;1956 def : PseudoScalarPatF16<AMDGPUlogf16, V_S_LOG_F16_e64>;1957 def : PseudoScalarPatF16<AMDGPUrcp, V_S_RCP_F16_e64>;1958 def : PseudoScalarPatF16<AMDGPUrsq, V_S_RSQ_F16_e64>;1959 def : PseudoScalarPatF16<any_amdgcn_sqrt, V_S_SQRT_F16_e64>;1960}1961 1962let HasModifiers = 1 in1963def ASHR_PK_I8_Profile : VOP3_Profile<VOP_I16_I32_I32_I32, VOP3_OPSEL_ONLY>;1964 1965let SubtargetPredicate = HasAshrPkInsts, isReMaterializable = 1 in {1966 defm V_ASHR_PK_I8_I32 : VOP3Inst<"v_ashr_pk_i8_i32", ASHR_PK_I8_Profile, int_amdgcn_ashr_pk_i8_i32>;1967 defm V_ASHR_PK_U8_I32 : VOP3Inst<"v_ashr_pk_u8_i32", ASHR_PK_I8_Profile, int_amdgcn_ashr_pk_u8_i32>;1968} // End SubtargetPredicate = HasAshrPkInsts, isReMaterializable = 11969 1970class AshrPkI8Pat<VOP3_Pseudo inst, int lo, int hi>: GCNPat<1971 (i32 (or (i32 (shl (i32 (AMDGPUsmed3 (i32 (sra i32:$src1, i32:$src2)), (i32 lo), (i32 hi))), (i32 8))),1972 (i32 (and (i32 (AMDGPUsmed3 (i32 (sra i32:$src0, i32:$src2)), (i32 lo), (i32 hi))), (i32 255))))),1973 (inst 0, VSrc_b32:$src0, 0, VSrc_b32:$src1, 0, VSrc_b32:$src2, 0 )1974>;1975 1976class AshrPkU8Pat<VOP3_Pseudo inst, int lo, int hi>: GCNPat<1977 (i32 (or (i32 (shl (i32 (AMDGPUsmed3 (i32 (sra i32:$src1, i32:$src2)), (i32 lo), (i32 hi))), (i32 8))),1978 (i32 (AMDGPUsmed3 (i32 (sra i32:$src0, i32:$src2)), (i32 lo), (i32 hi))))),1979 (inst 0, VSrc_b32:$src0, 0, VSrc_b32:$src1, 0, VSrc_b32:$src2, 0 )1980>;1981 1982let SubtargetPredicate = HasAshrPkInsts in {1983 def : AshrPkI8Pat<V_ASHR_PK_I8_I32_e64, -128, 127>;1984 def : AshrPkU8Pat<V_ASHR_PK_U8_I32_e64, 0, 255>;1985}1986 1987//===----------------------------------------------------------------------===//1988// Integer Clamp Patterns1989//===----------------------------------------------------------------------===//1990 1991class getClampPat<VOPProfile P, SDPatternOperator node> {1992 dag ret3 = (P.DstVT (node P.Src0VT:$src0, P.Src1VT:$src1, P.Src2VT:$src2));1993 dag ret2 = (P.DstVT (node P.Src0VT:$src0, P.Src1VT:$src1));1994 dag ret1 = (P.DstVT (node P.Src0VT:$src0));1995 dag ret = !if(!eq(P.NumSrcArgs, 3), ret3,1996 !if(!eq(P.NumSrcArgs, 2), ret2,1997 ret1));1998}1999 2000class getClampRes<VOPProfile P, Instruction inst> {2001 dag ret3 = (inst P.Src0VT:$src0, P.Src1VT:$src1, P.Src2VT:$src2, (i1 0));2002 dag ret2 = (inst P.Src0VT:$src0, P.Src1VT:$src1, (i1 0));2003 dag ret1 = (inst P.Src0VT:$src0, (i1 0));2004 dag ret = !if(!eq(P.NumSrcArgs, 3), ret3,2005 !if(!eq(P.NumSrcArgs, 2), ret2,2006 ret1));2007}2008 2009class IntClampPat<VOP3InstBase inst, SDPatternOperator node> : GCNPat<2010 getClampPat<inst.Pfl, node>.ret,2011 getClampRes<inst.Pfl, inst>.ret2012>;2013 2014def : IntClampPat<V_MAD_I32_I24_e64, AMDGPUmad_i24>;2015def : IntClampPat<V_MAD_U32_U24_e64, AMDGPUmad_u24>;2016 2017def : IntClampPat<V_SAD_U8_e64, int_amdgcn_sad_u8>;2018def : IntClampPat<V_SAD_HI_U8_e64, int_amdgcn_sad_hi_u8>;2019def : IntClampPat<V_SAD_U16_e64, int_amdgcn_sad_u16>;2020 2021def : IntClampPat<V_MSAD_U8_e64, int_amdgcn_msad_u8>;2022def : IntClampPat<V_MQSAD_PK_U16_U8_e64, int_amdgcn_mqsad_pk_u16_u8>;2023 2024def : IntClampPat<V_QSAD_PK_U16_U8_e64, int_amdgcn_qsad_pk_u16_u8>;2025def : IntClampPat<V_MQSAD_U32_U8_e64, int_amdgcn_mqsad_u32_u8>;2026 2027//===----------------------------------------------------------------------===//2028// Floating-point operation Patterns2029//===----------------------------------------------------------------------===//2030 2031// Implement fminimum(x, y) by using minimum3(x, y, y)2032class MinimumMaximumByMinimum3Maximum3<SDPatternOperator node, ValueType vt,2033 Instruction inst> : GCNPat<2034 (vt (node (VOP3Mods vt:$src0, i32:$src0_mods), (VOP3Mods vt:$src1, i32:$src1_mods))),2035 (inst $src0_mods, $src0, $src1_mods, $src1, $src1_mods, $src1)2036>;2037 2038// Prefer the real 2 operand form if legal2039let SubtargetPredicate = HasMinimum3Maximum3F32 in {2040def : MinimumMaximumByMinimum3Maximum3<fminimum, f32, V_MINIMUM3_F32_e64>;2041def : MinimumMaximumByMinimum3Maximum3<fmaximum, f32, V_MAXIMUM3_F32_e64>;2042}2043 2044//===----------------------------------------------------------------------===//2045// Target-specific instruction encodings.2046//===----------------------------------------------------------------------===//2047 2048//===----------------------------------------------------------------------===//2049// GFX12.2050//===----------------------------------------------------------------------===//2051 2052defm V_MIN3_NUM_F32 : VOP3_Realtriple_with_name_gfx12<0x229, "V_MIN3_F32", "v_min3_num_f32">;2053defm V_MAX3_NUM_F32 : VOP3_Realtriple_with_name_gfx12<0x22a, "V_MAX3_F32", "v_max3_num_f32">;2054defm V_MIN3_NUM_F16 : VOP3_Realtriple_t16_and_fake16_gfx12<0x22b, "v_min3_num_f16", "V_MIN3_F16", "v_min3_f16">;2055defm V_MAX3_NUM_F16 : VOP3_Realtriple_t16_and_fake16_gfx12<0x22c, "v_max3_num_f16", "V_MAX3_F16", "v_max3_f16">;2056defm V_MINIMUM3_F32 : VOP3Only_Realtriple_gfx12<0x22d>;2057defm V_MAXIMUM3_F32 : VOP3Only_Realtriple_gfx12<0x22e>;2058defm V_MINIMUM3_F16 : VOP3Only_Realtriple_t16_and_fake16_gfx12<0x22f, "v_minimum3_f16">;2059defm V_MAXIMUM3_F16 : VOP3Only_Realtriple_t16_and_fake16_gfx12<0x230, "v_maximum3_f16">;2060defm V_MED3_NUM_F32 : VOP3_Realtriple_with_name_gfx12<0x231, "V_MED3_F32", "v_med3_num_f32">;2061defm V_MED3_NUM_F16 : VOP3_Realtriple_t16_and_fake16_gfx12<0x232, "v_med3_num_f16", "V_MED3_F16", "v_med3_f16">;2062defm V_MINMAX_NUM_F32 : VOP3_Realtriple_with_name_gfx12<0x268, "V_MINMAX_F32", "v_minmax_num_f32">;2063defm V_MAXMIN_NUM_F32 : VOP3_Realtriple_with_name_gfx12<0x269, "V_MAXMIN_F32", "v_maxmin_num_f32">;2064defm V_MINMAX_NUM_F16 : VOP3_Realtriple_t16_and_fake16_gfx12<0x26a, "v_minmax_num_f16", "V_MINMAX_F16", "v_minmax_f16">;2065defm V_MAXMIN_NUM_F16 : VOP3_Realtriple_t16_and_fake16_gfx12<0x26b, "v_maxmin_num_f16", "V_MAXMIN_F16", "v_maxmin_f16">;2066defm V_MINIMUMMAXIMUM_F32 : VOP3Only_Realtriple_gfx12<0x26c>;2067defm V_MAXIMUMMINIMUM_F32 : VOP3Only_Realtriple_gfx12<0x26d>;2068defm V_MINIMUMMAXIMUM_F16 : VOP3Only_Realtriple_t16_and_fake16_gfx12<0x26e, "v_minimummaximum_f16">;2069defm V_MAXIMUMMINIMUM_F16 : VOP3Only_Realtriple_t16_and_fake16_gfx12<0x26f, "v_maximumminimum_f16">;2070defm V_S_EXP_F32 : VOP3Only_Real_Base_gfx12<0x280>;2071defm V_S_EXP_F16 : VOP3Only_Real_Base_gfx12<0x281>;2072defm V_S_LOG_F32 : VOP3Only_Real_Base_gfx12<0x282>;2073defm V_S_LOG_F16 : VOP3Only_Real_Base_gfx12<0x283>;2074defm V_S_RCP_F32 : VOP3Only_Real_Base_gfx12<0x284>;2075defm V_S_RCP_F16 : VOP3Only_Real_Base_gfx12<0x285>;2076defm V_S_RSQ_F32 : VOP3Only_Real_Base_gfx12<0x286>;2077defm V_S_RSQ_F16 : VOP3Only_Real_Base_gfx12<0x287>;2078defm V_S_SQRT_F32 : VOP3Only_Real_Base_gfx12<0x288>;2079defm V_S_SQRT_F16 : VOP3Only_Real_Base_gfx12<0x289>;2080defm V_MAD_CO_U64_U32 : VOP3be_Real_with_name_gfx12<0x2fe, "V_MAD_U64_U32", "v_mad_co_u64_u32">;2081defm V_MAD_CO_I64_I32 : VOP3be_Real_with_name_gfx12<0x2ff, "V_MAD_I64_I32", "v_mad_co_i64_i32">;2082defm V_MINIMUM_F64 : VOP3Only_Realtriple_gfx11_gfx12<0x341>;2083defm V_MAXIMUM_F64 : VOP3Only_Realtriple_gfx11_gfx12<0x342>;2084defm V_MINIMUM_F32 : VOP3Only_Realtriple_gfx12<0x365>;2085defm V_MAXIMUM_F32 : VOP3Only_Realtriple_gfx12<0x366>;2086defm V_MINIMUM_F16 : VOP3Only_Realtriple_t16_and_fake16_gfx12<0x367, "v_minimum_f16">;2087defm V_MAXIMUM_F16 : VOP3Only_Realtriple_t16_and_fake16_gfx12<0x368, "v_maximum_f16">;2088 2089defm V_PERMLANE16_VAR_B32 : VOP3Only_Real_Base_gfx12<0x30f>;2090defm V_PERMLANEX16_VAR_B32 : VOP3Only_Real_Base_gfx12<0x310>;2091 2092defm V_BITOP3_B16_gfx1250 : VOP3_Real_BITOP3_t16_and_fake16_gfx1250<0x233, "v_bitop3_b16">;2093defm V_BITOP3_B32 : VOP3_Real_BITOP3_gfx1250<0x234>;2094 2095defm V_MAD_U32 : VOP3Only_Realtriple_gfx1250<0x235>;2096defm V_MAD_NC_U64_U32 : VOP3Only_Realtriple_gfx1250<0x2fa>;2097defm V_MAD_NC_I64_I32 : VOP3Only_Realtriple_gfx1250<0x2fb>;2098defm V_MIN_U64 : VOP3Only_Realtriple_gfx1250<0x318>;2099defm V_MAX_U64 : VOP3Only_Realtriple_gfx1250<0x319>;2100defm V_MIN_I64 : VOP3Only_Realtriple_gfx1250<0x31a>;2101defm V_MAX_I64 : VOP3Only_Realtriple_gfx1250<0x31b>;2102defm V_ADD_MAX_I32 : VOP3Only_Realtriple_gfx1250<0x25e>;2103defm V_ADD_MAX_U32 : VOP3Only_Realtriple_gfx1250<0x25f>;2104defm V_ADD_MIN_I32 : VOP3Only_Realtriple_gfx1250<0x260>;2105defm V_ADD_MIN_U32 : VOP3Only_Realtriple_gfx1250<0x261>;2106defm V_PERMLANE_BCAST_B32 : VOP3Only_Real_Base_gfx12<0x270>;2107defm V_PERMLANE_UP_B32 : VOP3Only_Real_Base_gfx12<0x271>;2108defm V_PERMLANE_DOWN_B32 : VOP3Only_Real_Base_gfx12<0x272>;2109defm V_PERMLANE_XOR_B32 : VOP3Only_Real_Base_gfx12<0x273>;2110defm V_PERMLANE_IDX_GEN_B32 : VOP3Only_Real_Base_gfx12<0x314>;2111 2112//===----------------------------------------------------------------------===//2113// GFX11, GFX122114//===----------------------------------------------------------------------===//2115 2116multiclass VOP3_Real_with_name_gfx11_gfx12<bits<10> op, string opName,2117 string asmName> :2118 VOP3_Real_with_name<GFX11Gen, op, opName, asmName>,2119 VOP3_Real_with_name<GFX12Gen, op, opName, asmName>;2120 2121multiclass VOP3_Realtriple_gfx11_gfx12<bits<10> op> :2122 VOP3_Realtriple<GFX11Gen, op>, VOP3_Realtriple<GFX12Gen, op>;2123 2124multiclass VOP3_Real_Base_gfx11_gfx12<bits<10> op> :2125 VOP3_Real_Base<GFX11Gen, op>, VOP3_Real_Base<GFX12Gen, op>;2126 2127multiclass VOP3_Real_Base_gfx11_gfx12_not_gfx1250<bits<10> op> :2128 VOP3_Real_Base<GFX11Gen, op>, VOP3_Real_Base<GFX12Not12_50Gen, op>;2129 2130multiclass VOP3_Realtriple_with_name_gfx11_gfx12<bits<10> op, string opName,2131 string asmName> :2132 VOP3_Realtriple_with_name<GFX11Gen, op, opName, asmName>,2133 VOP3_Realtriple_with_name<GFX12Gen, op, opName, asmName>;2134 2135multiclass VOP3Dot_Realtriple_t16_and_fake16_gfx11_gfx12<bits<10> op, string asmName, string opName = NAME> {2136 defm _t16: VOP3Dot_Realtriple_gfx11_gfx12<op, asmName, 0, opName#"_t16">;2137 defm _fake16: VOP3Dot_Realtriple_gfx11_gfx12<op, asmName, 0, opName#"_fake16">;2138}2139 2140multiclass VOP3_Realtriple_t16_gfx11_gfx12<bits<10> op, string asmName, string opName = NAME,2141 string pseudo_mnemonic = "", bit isSingle = 0> :2142 VOP3_Realtriple_with_name<GFX11Gen, op, opName, asmName, pseudo_mnemonic, isSingle>,2143 VOP3_Realtriple_with_name<GFX12Gen, op, opName, asmName, pseudo_mnemonic, isSingle>;2144 2145multiclass VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<bits<10> op, string asmName, string opName = NAME,2146 string pseudo_mnemonic = "", bit isSingle = 0> {2147 defm opName#"_t16": VOP3_Realtriple_t16_gfx11_gfx12<op, asmName, opName#"_t16", pseudo_mnemonic, isSingle>;2148 defm opName#"_fake16": VOP3_Realtriple_t16_gfx11_gfx12<op, asmName, opName#"_fake16", pseudo_mnemonic, isSingle>;2149}2150 2151multiclass VOP3be_Real_gfx11_gfx12<bits<10> op, string opName, string asmName> :2152 VOP3be_Real<GFX11Gen, op, opName, asmName>,2153 VOP3be_Real<GFX12Gen, op, opName, asmName>;2154 2155multiclass VOP3be_Real_gfx11_gfx12_not_gfx1250<bits<10> op, string opName, string asmName> :2156 VOP3be_Real<GFX11Gen, op, opName, asmName>,2157 VOP3be_Real<GFX12Not12_50Gen, op, opName, asmName>;2158 2159multiclass VOP3be_Realtriple_gfx1250<bits<10> op> :2160 VOP3be_Realtriple<GFX1250Gen, op>;2161 2162multiclass VOP3_Real_No_Suffix_gfx11_gfx12<bits<10> op> :2163 VOP3_Real_No_Suffix<GFX11Gen, op>, VOP3_Real_No_Suffix<GFX12Gen, op>;2164 2165defm V_FMA_DX9_ZERO_F32 : VOP3_Real_with_name_gfx11_gfx12<0x209, "V_FMA_LEGACY_F32", "v_fma_dx9_zero_f32">;2166defm V_MAD_I32_I24 : VOP3_Realtriple_gfx11_gfx12<0x20a>;2167defm V_MAD_U32_U24 : VOP3_Realtriple_gfx11_gfx12<0x20b>;2168defm V_CUBEID_F32 : VOP3_Realtriple_gfx11_gfx12<0x20c>;2169defm V_CUBESC_F32 : VOP3_Realtriple_gfx11_gfx12<0x20d>;2170defm V_CUBETC_F32 : VOP3_Realtriple_gfx11_gfx12<0x20e>;2171defm V_CUBEMA_F32 : VOP3_Realtriple_gfx11_gfx12<0x20f>;2172defm V_BFE_U32 : VOP3_Realtriple_gfx11_gfx12<0x210>;2173defm V_BFE_I32 : VOP3_Realtriple_gfx11_gfx12<0x211>;2174defm V_BFI_B32 : VOP3_Realtriple_gfx11_gfx12<0x212>;2175defm V_FMA_F32 : VOP3_Realtriple_gfx11_gfx12<0x213>;2176defm V_FMA_F64 : VOP3_Real_Base_gfx11_gfx12_not_gfx1250<0x214>;2177defm V_LERP_U8 : VOP3_Realtriple_gfx11_gfx12<0x215>;2178defm V_ALIGNBIT_B32 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x216, "v_alignbit_b32">;2179defm V_ALIGNBYTE_B32 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x217, "v_alignbyte_b32">;2180defm V_MULLIT_F32 : VOP3_Realtriple_gfx11_gfx12<0x218>;2181defm V_MIN3_F32 : VOP3_Realtriple_gfx11<0x219>;2182defm V_MIN3_I32 : VOP3_Realtriple_gfx11_gfx12<0x21a>;2183defm V_MIN3_U32 : VOP3_Realtriple_gfx11_gfx12<0x21b>;2184defm V_MAX3_F32 : VOP3_Realtriple_gfx11<0x21c>;2185defm V_MAX3_I32 : VOP3_Realtriple_gfx11_gfx12<0x21d>;2186defm V_MAX3_U32 : VOP3_Realtriple_gfx11_gfx12<0x21e>;2187defm V_MED3_F32 : VOP3_Realtriple_gfx11<0x21f>;2188defm V_MED3_I32 : VOP3_Realtriple_gfx11_gfx12<0x220>;2189defm V_MED3_U32 : VOP3_Realtriple_gfx11_gfx12<0x221>;2190defm V_SAD_U8 : VOP3_Realtriple_gfx11_gfx12<0x222>;2191defm V_SAD_HI_U8 : VOP3_Realtriple_gfx11_gfx12<0x223>;2192defm V_SAD_U16 : VOP3_Realtriple_gfx11_gfx12<0x224>;2193defm V_SAD_U32 : VOP3_Realtriple_gfx11_gfx12<0x225>;2194defm V_CVT_PK_U8_F32 : VOP3_Realtriple_gfx11_gfx12<0x226>;2195defm V_DIV_FIXUP_F32 : VOP3_Real_Base_gfx11_gfx12<0x227>;2196defm V_DIV_FIXUP_F64 : VOP3_Real_Base_gfx11_gfx12_not_gfx1250<0x228>;2197defm V_DIV_FMAS_F32 : VOP3_Real_Base_gfx11_gfx12<0x237>;2198defm V_DIV_FMAS_F64 : VOP3_Real_Base_gfx11_gfx12_not_gfx1250<0x238>;2199defm V_MSAD_U8 : VOP3_Realtriple_gfx11_gfx12<0x239>;2200defm V_QSAD_PK_U16_U8 : VOP3_Real_Base_gfx11_gfx12<0x23a>;2201defm V_MQSAD_PK_U16_U8 : VOP3_Real_Base_gfx11_gfx12<0x23b>;2202defm V_MQSAD_U32_U8 : VOP3_Real_Base_gfx11_gfx12<0x23d>;2203defm V_XOR3_B32 : VOP3_Realtriple_gfx11_gfx12<0x240>;2204defm V_MAD_U16 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x241, "v_mad_u16", "V_MAD_U16_gfx9">;2205defm V_PERM_B32 : VOP3_Realtriple_gfx11_gfx12<0x244>;2206defm V_XAD_U32 : VOP3_Realtriple_gfx11_gfx12<0x245>;2207defm V_LSHL_ADD_U32 : VOP3_Realtriple_gfx11_gfx12<0x246>;2208defm V_ADD_LSHL_U32 : VOP3_Realtriple_gfx11_gfx12<0x247>;2209defm V_FMA_F16 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x248, "v_fma_f16", "V_FMA_F16_gfx9">;2210defm V_MIN3_F16 : VOP3Only_Realtriple_t16_and_fake16_gfx11<0x249, "v_min3_f16">;2211defm V_MIN3_I16 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x24a, "v_min3_i16">;2212defm V_MIN3_U16 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x24b, "v_min3_u16">;2213defm V_MAX3_F16 : VOP3Only_Realtriple_t16_and_fake16_gfx11<0x24c, "v_max3_f16">;2214defm V_MAX3_I16 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x24d, "v_max3_i16">;2215defm V_MAX3_U16 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x24e, "v_max3_u16">;2216defm V_MED3_F16 : VOP3Only_Realtriple_t16_and_fake16_gfx11<0x24f, "v_med3_f16">;2217defm V_MED3_I16 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x250, "v_med3_i16">;2218defm V_MED3_U16 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x251, "v_med3_u16">;2219defm V_MAD_I16 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x253, "v_mad_i16", "V_MAD_I16_gfx9">;2220defm V_DIV_FIXUP_F16 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x254, "v_div_fixup_f16", "V_DIV_FIXUP_F16_gfx9">;2221defm V_ADD3_U32 : VOP3_Realtriple_gfx11_gfx12<0x255>;2222defm V_LSHL_OR_B32 : VOP3_Realtriple_gfx11_gfx12<0x256>;2223defm V_AND_OR_B32 : VOP3_Realtriple_gfx11_gfx12<0x257>;2224defm V_OR3_B32 : VOP3_Realtriple_gfx11_gfx12<0x258>;2225defm V_MAD_U32_U16 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x259, "v_mad_u32_u16">;2226defm V_MAD_I32_I16 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x25a, "v_mad_i32_i16">;2227defm V_PERMLANE16_B32 : VOP3_Real_Base_gfx11_gfx12<0x25b>;2228defm V_PERMLANEX16_B32 : VOP3_Real_Base_gfx11_gfx12<0x25c>;2229defm V_MAXMIN_F32 : VOP3_Realtriple_gfx11<0x25e>;2230defm V_MINMAX_F32 : VOP3_Realtriple_gfx11<0x25f>;2231defm V_MAXMIN_F16 : VOP3_Realtriple_t16_and_fake16_gfx11<0x260, "v_maxmin_f16">;2232defm V_MINMAX_F16 : VOP3_Realtriple_t16_and_fake16_gfx11<0x261, "v_minmax_f16">;2233defm V_MAXMIN_U32 : VOP3_Realtriple_gfx11_gfx12<0x262>;2234defm V_MINMAX_U32 : VOP3_Realtriple_gfx11_gfx12<0x263>;2235defm V_MAXMIN_I32 : VOP3_Realtriple_gfx11_gfx12<0x264>;2236defm V_MINMAX_I32 : VOP3_Realtriple_gfx11_gfx12<0x265>;2237defm V_DOT2_F16_F16 : VOP3Dot_Realtriple_t16_and_fake16_gfx11_gfx12<0x266, "v_dot2_f16_f16">;2238defm V_DOT2_BF16_BF16 : VOP3Dot_Realtriple_t16_and_fake16_gfx11_gfx12<0x267, "v_dot2_bf16_bf16">;2239defm V_DIV_SCALE_F32 : VOP3be_Real_gfx11_gfx12<0x2fc, "V_DIV_SCALE_F32", "v_div_scale_f32">;2240defm V_DIV_SCALE_F64 : VOP3be_Real_gfx11_gfx12_not_gfx1250<0x2fd, "V_DIV_SCALE_F64", "v_div_scale_f64">;2241defm V_MAD_U64_U32_gfx11 : VOP3be_Real_gfx11<0x2fe, "V_MAD_U64_U32_gfx11", "v_mad_u64_u32">;2242defm V_MAD_I64_I32_gfx11 : VOP3be_Real_gfx11<0x2ff, "V_MAD_I64_I32_gfx11", "v_mad_i64_i32">;2243defm V_ADD_NC_U16 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x303, "v_add_nc_u16">;2244defm V_SUB_NC_U16 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x304, "v_sub_nc_u16">;2245defm V_MUL_LO_U16 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x305, "v_mul_lo_u16">;2246defm V_CVT_PK_I16_F32 : VOP3_Realtriple_gfx11_gfx12<0x306>;2247defm V_CVT_PK_U16_F32 : VOP3_Realtriple_gfx11_gfx12<0x307>;2248defm V_MAX_U16 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x309, "v_max_u16">;2249defm V_MAX_I16 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x30a, "v_max_i16">;2250defm V_MIN_U16 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x30b, "v_min_u16">;2251defm V_MIN_I16 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x30c, "v_min_i16">;2252defm V_ADD_NC_I16 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x30d, "v_add_nc_i16", "V_ADD_I16">;2253defm V_SUB_NC_I16 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x30e, "v_sub_nc_i16", "V_SUB_I16">;2254defm V_PACK_B32_F16 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x311, "v_pack_b32_f16">;2255defm V_CVT_PK_NORM_I16_F16 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x312, "v_cvt_pk_norm_i16_f16", "V_CVT_PKNORM_I16_F16", "v_cvt_pknorm_i16_f16">;2256defm V_CVT_PK_NORM_U16_F16 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x313, "v_cvt_pk_norm_u16_f16", "V_CVT_PKNORM_U16_F16", "v_cvt_pknorm_u16_f16">;2257defm V_SUB_NC_I32 : VOP3_Realtriple_with_name_gfx11_gfx12<0x325, "V_SUB_I32", "v_sub_nc_i32">;2258defm V_ADD_NC_I32 : VOP3_Realtriple_with_name_gfx11_gfx12<0x326, "V_ADD_I32", "v_add_nc_i32">;2259defm V_ADD_F64 : VOP3_Real_Base_gfx11<0x327>;2260defm V_MUL_F64 : VOP3_Real_Base_gfx11<0x328>;2261defm V_MIN_F64 : VOP3_Real_Base_gfx11<0x329>;2262defm V_MAX_F64 : VOP3_Real_Base_gfx11<0x32a>;2263defm V_LDEXP_F64 : VOP3_Real_Base_gfx11_gfx12_not_gfx1250<0x32b>;2264defm V_MUL_LO_U32 : VOP3_Real_Base_gfx11_gfx12_not_gfx1250<0x32c>;2265defm V_MUL_HI_U32 : VOP3_Real_Base_gfx11_gfx12_not_gfx1250<0x32d>;2266defm V_MUL_HI_I32 : VOP3_Real_Base_gfx11_gfx12_not_gfx1250<0x32e>;2267defm V_TRIG_PREOP_F64 : VOP3_Real_Base_gfx11_gfx12<0x32f>;2268defm V_LSHLREV_B16 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x338, "v_lshlrev_b16">;2269defm V_LSHRREV_B16 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x339, "v_lshrrev_b16">;2270defm V_ASHRREV_I16 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x33a, "v_ashrrev_i16">;2271defm V_LSHLREV_B64 : VOP3_Real_Base_gfx11<0x33c>;2272defm V_LSHRREV_B64 : VOP3_Real_Base_gfx11_gfx12_not_gfx1250<0x33d>;2273defm V_ASHRREV_I64 : VOP3_Real_Base_gfx11_gfx12_not_gfx1250<0x33e>;2274defm V_READLANE_B32 : VOP3_Real_No_Suffix_gfx11_gfx12<0x360>; // Pseudo in VOP22275let InOperandList = (ins SSrcOrLds_b32:$src0, SCSrc_b32:$src1, VGPR_32:$vdst_in) in {2276 defm V_WRITELANE_B32 : VOP3_Real_No_Suffix_gfx11_gfx12<0x361>; // Pseudo in VOP22277} // End InOperandList = (ins SSrcOrLds_b32:$src0, SCSrc_b32:$src1, VGPR_32:$vdst_in)2278defm V_AND_B16 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x362, "v_and_b16">;2279defm V_OR_B16 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x363, "v_or_b16">;2280defm V_XOR_B16 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x364, "v_xor_b16">;2281 2282defm V_CVT_PK_FP8_F32 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12_not_gfx1250<0x369, "v_cvt_pk_fp8_f32">;2283defm V_CVT_PK_FP8_F32_gfx1250 : VOP3Only_Realtriple_t16_and_fake16_gfx1250<0x369, "v_cvt_pk_fp8_f32">;2284defm V_CVT_PK_BF8_F32 : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x36a, "v_cvt_pk_bf8_f32">;2285defm V_CVT_SR_FP8_F32_gfx12 : VOP3_Realtriple_with_name_gfx11_gfx12_not_gfx1250<0x36b, "V_CVT_SR_FP8_F32_gfx12", "v_cvt_sr_fp8_f32">;2286defm V_CVT_SR_FP8_F32_gfx1250 : VOP3Only_Realtriple_with_name_gfx1250<0x36b, "V_CVT_SR_FP8_F32_gfx1250", "v_cvt_sr_fp8_f32">;2287defm V_CVT_SR_BF8_F32_gfx12 : VOP3_Realtriple_with_name_gfx11_gfx12<0x36c, "V_CVT_SR_BF8_F32_gfx12", "v_cvt_sr_bf8_f32">;2288 2289let AssemblerPredicate = isGFX11Plus in {2290 def : AMDGPUMnemonicAlias<"v_add3_nc_u32", "v_add3_u32">;2291 def : AMDGPUMnemonicAlias<"v_xor_add_u32", "v_xad_u32">;2292}2293 2294// These instructions differ from GFX12 variant by supporting DPP:2295defm V_FMA_F64 : VOP3Only_Realtriple_gfx1250<0x214>;2296defm V_DIV_FIXUP_F64 : VOP3Only_Realtriple_gfx1250<0x228>;2297defm V_DIV_FMAS_F64 : VOP3Only_Realtriple_gfx1250<0x238>;2298defm V_DIV_SCALE_F64 : VOP3be_Realtriple_gfx1250<0x2fd>;2299defm V_LDEXP_F64 : VOP3Only_Realtriple_gfx1250<0x32b>;2300defm V_MUL_LO_U32 : VOP3Only_Realtriple_gfx1250<0x32c>;2301defm V_MUL_HI_U32 : VOP3Only_Realtriple_gfx1250<0x32d>;2302defm V_MUL_HI_I32 : VOP3Only_Realtriple_gfx1250<0x32e>;2303defm V_LSHRREV_B64 : VOP3Only_Realtriple_gfx1250<0x33d>;2304defm V_ASHRREV_I64 : VOP3Only_Realtriple_gfx1250<0x33e>;2305 2306defm V_PERM_PK16_B4_U4 : VOP3Only_Real_Base_gfx1250<0x23f>;2307defm V_PERM_PK16_B6_U4 : VOP3Only_Real_Base_gfx1250<0x242>;2308defm V_PERM_PK16_B8_U4 : VOP3Only_Real_Base_gfx1250<0x243>;2309defm V_LSHL_ADD_U64 : VOP3Only_Realtriple_gfx1250<0x252>;2310defm V_ASHR_PK_I8_I32 : VOP3Only_Realtriple_gfx1250<0x290>;2311defm V_ASHR_PK_U8_I32 : VOP3Only_Realtriple_gfx1250<0x291>;2312defm V_CVT_SCALE_PK8_F16_FP4 : VOP3Only_ScaleSel_Real_gfx1250<0x29f>;2313defm V_CVT_SCALE_PK8_BF16_FP4 : VOP3Only_ScaleSel_Real_gfx1250<0x2a0>;2314defm V_CVT_SCALE_PK8_F32_FP4 : VOP3Only_ScaleSel_Real_gfx1250<0x2a1>;2315defm V_CVT_SCALE_PK8_F16_FP8 : VOP3Only_ScaleSel_Real_gfx1250<0x2a8>;2316defm V_CVT_SCALE_PK8_BF16_FP8 : VOP3Only_ScaleSel_Real_gfx1250<0x2a9>;2317defm V_CVT_SCALE_PK8_F32_FP8 : VOP3Only_ScaleSel_Real_gfx1250<0x2aa>;2318defm V_CVT_SCALE_PK8_F16_BF8 : VOP3Only_ScaleSel_Real_gfx1250<0x2ab>;2319defm V_CVT_SCALE_PK8_BF16_BF8 : VOP3Only_ScaleSel_Real_gfx1250<0x2ac>;2320defm V_CVT_SCALE_PK8_F32_BF8 : VOP3Only_ScaleSel_Real_gfx1250<0x2ad>;2321defm V_CVT_SCALEF32_PK8_FP4_F32 : VOP3Only_Real_Base_gfx1250<0x2b0>;2322defm V_CVT_SCALEF32_PK8_FP4_F16 : VOP3Only_Real_Base_gfx1250<0x2b3>;2323defm V_CVT_SCALEF32_PK8_FP8_BF16 : VOP3Only_Real_Base_gfx1250<0x2b4>;2324defm V_CVT_SCALEF32_PK8_BF8_BF16 : VOP3Only_Real_Base_gfx1250<0x2b5>;2325defm V_CVT_SCALEF32_PK8_FP4_BF16 : VOP3Only_Real_Base_gfx1250<0x2b8>;2326defm V_CVT_SCALEF32_PK8_FP8_F32 : VOP3Only_Real_Base_gfx1250<0x2c3>;2327defm V_CVT_SCALEF32_PK8_FP8_F16 : VOP3Only_Real_Base_gfx1250<0x2c4>;2328defm V_CVT_SCALEF32_PK8_BF8_F32 : VOP3Only_Real_Base_gfx1250<0x2c5>;2329defm V_CVT_SCALEF32_PK8_BF8_F16 : VOP3Only_Real_Base_gfx1250<0x2c6>;2330defm V_CVT_SCALE_PK16_F16_FP6 : VOP3Only_ScaleSel_Real_gfx1250<0x2c7>;2331defm V_CVT_SCALE_PK16_BF16_FP6 : VOP3Only_ScaleSel_Real_gfx1250<0x2c8>;2332defm V_CVT_SCALE_PK16_F32_FP6 : VOP3Only_ScaleSel_Real_gfx1250<0x2c9>;2333defm V_CVT_SCALE_PK16_F16_BF6 : VOP3Only_ScaleSel_Real_gfx1250<0x2ca>;2334defm V_CVT_SCALE_PK16_BF16_BF6 : VOP3Only_ScaleSel_Real_gfx1250<0x2cb>;2335defm V_CVT_SCALE_PK16_F32_BF6 : VOP3Only_ScaleSel_Real_gfx1250<0x2cc>;2336defm V_CVT_SCALEF32_PK16_FP6_F32 : VOP3Only_Real_Base_gfx1250<0x2cd>;2337defm V_CVT_SCALEF32_PK16_BF6_F32 : VOP3Only_Real_Base_gfx1250<0x2ce>;2338defm V_CVT_SCALEF32_PK16_FP6_F16 : VOP3Only_Real_Base_gfx1250<0x2cf>;2339defm V_CVT_SCALEF32_PK16_BF6_F16 : VOP3Only_Real_Base_gfx1250<0x2d0>;2340defm V_CVT_SCALEF32_PK16_FP6_BF16 : VOP3Only_Real_Base_gfx1250<0x2d1>;2341defm V_CVT_SCALEF32_PK16_BF6_BF16 : VOP3Only_Real_Base_gfx1250<0x2d2>;2342defm V_CVT_SCALEF32_SR_PK16_FP6_F32 : VOP3Only_Real_Base_gfx1250<0x2d3>;2343defm V_CVT_SCALEF32_SR_PK16_BF6_F32 : VOP3Only_Real_Base_gfx1250<0x2d4>;2344defm V_CVT_SCALEF32_SR_PK16_FP6_F16 : VOP3Only_Real_Base_gfx1250<0x2d5>;2345defm V_CVT_SCALEF32_SR_PK16_BF6_F16 : VOP3Only_Real_Base_gfx1250<0x2d6>;2346defm V_CVT_SCALEF32_SR_PK16_FP6_BF16 : VOP3Only_Real_Base_gfx1250<0x2d7>;2347defm V_CVT_SCALEF32_SR_PK16_BF6_BF16 : VOP3Only_Real_Base_gfx1250<0x2d8>;2348defm V_CVT_SCALEF32_SR_PK8_FP4_F32 : VOP3Only_Real_Base_gfx1250<0x297>;2349defm V_CVT_SCALEF32_SR_PK8_FP8_F32 : VOP3Only_Real_Base_gfx1250<0x298>;2350defm V_CVT_SCALEF32_SR_PK8_BF8_F32 : VOP3Only_Real_Base_gfx1250<0x299>;2351defm V_CVT_SCALEF32_SR_PK8_FP4_F16 : VOP3Only_Real_Base_gfx1250<0x2b9>;2352defm V_CVT_SCALEF32_SR_PK8_FP4_BF16 : VOP3Only_Real_Base_gfx1250<0x2bc>;2353defm V_CVT_SCALEF32_SR_PK8_FP8_F16 : VOP3Only_Real_Base_gfx1250<0x2bf>;2354defm V_CVT_SCALEF32_SR_PK8_FP8_BF16 : VOP3Only_Real_Base_gfx1250<0x2c0>;2355defm V_CVT_SCALEF32_SR_PK8_BF8_F16 : VOP3Only_Real_Base_gfx1250<0x2c1>;2356defm V_CVT_SCALEF32_SR_PK8_BF8_BF16 : VOP3Only_Real_Base_gfx1250<0x2c2>;2357defm V_CVT_PK_BF16_F32 : VOP3Only_Realtriple_gfx1250<0x36d>;2358defm V_CVT_SR_PK_BF16_F32 : VOP3Only_Realtriple_gfx1250<0x36e>;2359defm V_CVT_PK_F16_F32 : VOP3Only_Realtriple_gfx1250<0x36f>;2360defm V_CVT_SR_PK_F16_F32 : VOP3Only_Realtriple_gfx1250<0x370>;2361defm V_CVT_PK_FP8_F16_gfx1250 : VOP3Only_Realtriple_t16_and_fake16_gfx1250<0x372, "v_cvt_pk_fp8_f16">;2362defm V_CVT_PK_BF8_F16_gfx1250 : VOP3Only_Realtriple_t16_and_fake16_gfx1250<0x373, "v_cvt_pk_bf8_f16">;2363defm V_CVT_SR_FP8_F16 : VOP3Only_Realtriple_t16_and_fake16_gfx1250<0x374>;2364defm V_CVT_SR_BF8_F16 : VOP3Only_Realtriple_t16_and_fake16_gfx1250<0x375>;2365 2366//===----------------------------------------------------------------------===//2367// GFX10.2368//===----------------------------------------------------------------------===//2369 2370let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in {2371 multiclass VOP3_Real_gfx10<bits<10> op> {2372 def _gfx10 :2373 VOP3_Real<!cast<VOP_Pseudo>(NAME#"_e64"), SIEncodingFamily.GFX10>,2374 VOP3e_gfx10<op, !cast<VOP_Pseudo>(NAME#"_e64").Pfl>;2375 }2376 multiclass VOP3_Real_No_Suffix_gfx10<bits<10> op> {2377 def _gfx10 :2378 VOP3_Real<!cast<VOP_Pseudo>(NAME), SIEncodingFamily.GFX10>,2379 VOP3e_gfx10<op, !cast<VOP_Pseudo>(NAME).Pfl>;2380 }2381 multiclass VOP3_Real_gfx10_with_name<bits<10> op, string opName,2382 string asmName> {2383 def _gfx10 :2384 VOP3_Real<!cast<VOP3_Pseudo>(opName#"_e64"), SIEncodingFamily.GFX10>,2385 VOP3e_gfx10<op, !cast<VOP3_Pseudo>(opName#"_e64").Pfl> {2386 VOP3_Pseudo ps = !cast<VOP3_Pseudo>(opName#"_e64");2387 let AsmString = asmName # ps.AsmOperands;2388 let IsSingle = 1;2389 }2390 }2391 multiclass VOP3be_Real_gfx10<bits<10> op> {2392 def _gfx10 :2393 VOP3_Real<!cast<VOP3_Pseudo>(NAME#"_e64"), SIEncodingFamily.GFX10>,2394 VOP3be_gfx10<op, !cast<VOP3_Pseudo>(NAME#"_e64").Pfl>;2395 }2396 multiclass VOP3Interp_Real_gfx10<bits<10> op> {2397 def _gfx10 :2398 VOP3_Real<!cast<VOP3_Pseudo>(NAME), SIEncodingFamily.GFX10>,2399 VOP3Interp_gfx10<op, !cast<VOP3_Pseudo>(NAME).Pfl>;2400 }2401 multiclass VOP3OpSel_Real_gfx10<bits<10> op> {2402 def _gfx10 :2403 VOP3_Real<!cast<VOP3_Pseudo>(NAME#"_e64"), SIEncodingFamily.GFX10>,2404 VOP3OpSel_gfx10<op, !cast<VOP3_Pseudo>(NAME#"_e64").Pfl>;2405 }2406 multiclass VOP3OpSel_Real_gfx10_with_name<bits<10> op, string opName,2407 string asmName> {2408 def _gfx10 :2409 VOP3_Real<!cast<VOP3_Pseudo>(opName#"_e64"), SIEncodingFamily.GFX10>,2410 VOP3OpSel_gfx10<op, !cast<VOP3_Pseudo>(opName#"_e64").Pfl> {2411 VOP3_Pseudo ps = !cast<VOP3_Pseudo>(opName#"_e64");2412 let AsmString = asmName # ps.AsmOperands;2413 }2414 }2415} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10"2416 2417defm V_ALIGNBIT_B32_opsel : VOP3OpSel_Real_gfx10_with_name<0x14e, "V_ALIGNBIT_B32_opsel", "v_alignbit_b32">;2418defm V_ALIGNBYTE_B32_opsel : VOP3OpSel_Real_gfx10_with_name<0x14f, "V_ALIGNBYTE_B32_opsel", "v_alignbyte_b32">;2419 2420defm V_READLANE_B32 : VOP3_Real_No_Suffix_gfx10<0x360>;2421 2422let InOperandList = (ins SSrcOrLds_b32:$src0, SCSrc_b32:$src1, VGPR_32:$vdst_in) in {2423 defm V_WRITELANE_B32 : VOP3_Real_No_Suffix_gfx10<0x361>;2424} // End InOperandList = (ins SSrcOrLds_b32:$src0, SCSrc_b32:$src1, VGPR_32:$vdst_in)2425 2426let SubtargetPredicate = isGFX10Before1030 in {2427 defm V_MUL_LO_I32 : VOP3_Real_gfx10<0x16b>;2428}2429 2430defm V_XOR3_B32 : VOP3_Real_gfx10<0x178>;2431defm V_LSHLREV_B64 : VOP3_Real_gfx10<0x2ff>;2432defm V_LSHRREV_B64 : VOP3_Real_gfx10<0x300>;2433defm V_ASHRREV_I64 : VOP3_Real_gfx10<0x301>;2434defm V_PERM_B32 : VOP3_Real_gfx10<0x344>;2435defm V_XAD_U32 : VOP3_Real_gfx10<0x345>;2436defm V_LSHL_ADD_U32 : VOP3_Real_gfx10<0x346>;2437defm V_ADD_LSHL_U32 : VOP3_Real_gfx10<0x347>;2438defm V_ADD3_U32 : VOP3_Real_gfx10<0x36d>;2439defm V_LSHL_OR_B32 : VOP3_Real_gfx10<0x36f>;2440defm V_AND_OR_B32 : VOP3_Real_gfx10<0x371>;2441defm V_OR3_B32 : VOP3_Real_gfx10<0x372>;2442 2443// TODO-GFX10: add MC tests for v_add/sub_nc_i162444defm V_ADD_NC_I16 :2445 VOP3OpSel_Real_gfx10_with_name<0x30d, "V_ADD_I16", "v_add_nc_i16">;2446defm V_SUB_NC_I16 :2447 VOP3OpSel_Real_gfx10_with_name<0x30e, "V_SUB_I16", "v_sub_nc_i16">;2448defm V_SUB_NC_I32 :2449 VOP3_Real_gfx10_with_name<0x376, "V_SUB_I32", "v_sub_nc_i32">;2450defm V_ADD_NC_I32 :2451 VOP3_Real_gfx10_with_name<0x37f, "V_ADD_I32", "v_add_nc_i32">;2452 2453defm V_INTERP_P1_F32_e64 : VOP3Interp_Real_gfx10<0x200>;2454defm V_INTERP_P2_F32_e64 : VOP3Interp_Real_gfx10<0x201>;2455defm V_INTERP_MOV_F32_e64 : VOP3Interp_Real_gfx10<0x202>;2456 2457defm V_INTERP_P1LL_F16 : VOP3Interp_Real_gfx10<0x342>;2458defm V_INTERP_P1LV_F16 : VOP3Interp_Real_gfx10<0x343>;2459defm V_INTERP_P2_F16 : VOP3Interp_Real_gfx10<0x35a>;2460 2461defm V_PACK_B32_F16 : VOP3OpSel_Real_gfx10<0x311>;2462defm V_CVT_PKNORM_I16_F16 : VOP3OpSel_Real_gfx10<0x312>;2463defm V_CVT_PKNORM_U16_F16 : VOP3OpSel_Real_gfx10<0x313>;2464 2465defm V_MIN3_F16 : VOP3OpSel_Real_gfx10<0x351>;2466defm V_MIN3_I16 : VOP3OpSel_Real_gfx10<0x352>;2467defm V_MIN3_U16 : VOP3OpSel_Real_gfx10<0x353>;2468defm V_MAX3_F16 : VOP3OpSel_Real_gfx10<0x354>;2469defm V_MAX3_I16 : VOP3OpSel_Real_gfx10<0x355>;2470defm V_MAX3_U16 : VOP3OpSel_Real_gfx10<0x356>;2471defm V_MED3_F16 : VOP3OpSel_Real_gfx10<0x357>;2472defm V_MED3_I16 : VOP3OpSel_Real_gfx10<0x358>;2473defm V_MED3_U16 : VOP3OpSel_Real_gfx10<0x359>;2474defm V_MAD_U32_U16 : VOP3OpSel_Real_gfx10<0x373>;2475defm V_MAD_I32_I16 : VOP3OpSel_Real_gfx10<0x375>;2476 2477defm V_MAD_U16 :2478 VOP3OpSel_Real_gfx10_with_name<0x340, "V_MAD_U16_gfx9", "v_mad_u16">;2479defm V_FMA_F16 :2480 VOP3OpSel_Real_gfx10_with_name<0x34b, "V_FMA_F16_gfx9", "v_fma_f16">;2481defm V_MAD_I16 :2482 VOP3OpSel_Real_gfx10_with_name<0x35e, "V_MAD_I16_gfx9", "v_mad_i16">;2483defm V_DIV_FIXUP_F16 :2484 VOP3OpSel_Real_gfx10_with_name<0x35f, "V_DIV_FIXUP_F16_gfx9", "v_div_fixup_f16">;2485 2486defm V_ADD_NC_U16 : VOP3OpSel_Real_gfx10<0x303>;2487defm V_SUB_NC_U16 : VOP3OpSel_Real_gfx10<0x304>;2488 2489defm V_MUL_LO_U16 : VOP3OpSel_Real_gfx10_with_name<0x305, "V_MUL_LO_U16_opsel", "v_mul_lo_u16">;2490defm V_LSHRREV_B16 : VOP3OpSel_Real_gfx10_with_name<0x307, "V_LSHRREV_B16_opsel", "v_lshrrev_b16">;2491defm V_ASHRREV_I16 : VOP3OpSel_Real_gfx10_with_name<0x308, "V_ASHRREV_I16_opsel", "v_ashrrev_i16">;2492defm V_MAX_U16 : VOP3OpSel_Real_gfx10_with_name<0x309, "V_MAX_U16_opsel", "v_max_u16">;2493defm V_MAX_I16 : VOP3OpSel_Real_gfx10_with_name<0x30a, "V_MAX_I16_opsel", "v_max_i16">;2494defm V_MIN_U16 : VOP3OpSel_Real_gfx10_with_name<0x30b, "V_MIN_U16_opsel", "v_min_u16">;2495defm V_MIN_I16 : VOP3OpSel_Real_gfx10_with_name<0x30c, "V_MIN_I16_opsel", "v_min_i16">;2496defm V_LSHLREV_B16 : VOP3OpSel_Real_gfx10_with_name<0x314, "V_LSHLREV_B16_opsel", "v_lshlrev_b16">;2497defm V_PERMLANE16_B32 : VOP3OpSel_Real_gfx10<0x377>;2498defm V_PERMLANEX16_B32 : VOP3OpSel_Real_gfx10<0x378>;2499 2500//===----------------------------------------------------------------------===//2501// GFX7, GFX10.2502//===----------------------------------------------------------------------===//2503 2504let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in {2505 multiclass VOP3_Real_gfx7<bits<10> op> {2506 def _gfx7 :2507 VOP3_Real<!cast<VOP3_Pseudo>(NAME#"_e64"), SIEncodingFamily.SI>,2508 VOP3e_gfx6_gfx7<op{8-0}, !cast<VOP3_Pseudo>(NAME#"_e64").Pfl>;2509 }2510 multiclass VOP3be_Real_gfx7<bits<10> op> {2511 def _gfx7 :2512 VOP3_Real<!cast<VOP3_Pseudo>(NAME#"_e64"), SIEncodingFamily.SI>,2513 VOP3be_gfx6_gfx7<op{8-0}, !cast<VOP3_Pseudo>(NAME#"_e64").Pfl>;2514 }2515} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7"2516 2517multiclass VOP3_Real_gfx7_gfx10<bits<10> op> :2518 VOP3_Real_gfx7<op>, VOP3_Real_gfx10<op>;2519 2520multiclass VOP3be_Real_gfx7_gfx10<bits<10> op> :2521 VOP3be_Real_gfx7<op>, VOP3be_Real_gfx10<op>;2522 2523defm V_QSAD_PK_U16_U8 : VOP3_Real_gfx7_gfx10<0x172>;2524defm V_MQSAD_U32_U8 : VOP3_Real_gfx7_gfx10<0x175>;2525defm V_MAD_U64_U32 : VOP3be_Real_gfx7_gfx10<0x176>;2526defm V_MAD_I64_I32 : VOP3be_Real_gfx7_gfx10<0x177>;2527 2528//===----------------------------------------------------------------------===//2529// GFX6, GFX7, GFX10.2530//===----------------------------------------------------------------------===//2531 2532let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {2533 multiclass VOP3_Real_gfx6_gfx7<bits<10> op> {2534 def _gfx6_gfx7 :2535 VOP3_Real<!cast<VOP3_Pseudo>(NAME#"_e64"), SIEncodingFamily.SI>,2536 VOP3e_gfx6_gfx7<op{8-0}, !cast<VOP3_Pseudo>(NAME#"_e64").Pfl>;2537 }2538 multiclass VOP3be_Real_gfx6_gfx7<bits<10> op> {2539 def _gfx6_gfx7 :2540 VOP3_Real<!cast<VOP3_Pseudo>(NAME#"_e64"), SIEncodingFamily.SI>,2541 VOP3be_gfx6_gfx7<op{8-0}, !cast<VOP3_Pseudo>(NAME#"_e64").Pfl>;2542 }2543} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"2544 2545multiclass VOP3_Real_gfx6_gfx7_gfx10<bits<10> op> :2546 VOP3_Real_gfx6_gfx7<op>, VOP3_Real_gfx10<op>;2547 2548multiclass VOP3be_Real_gfx6_gfx7_gfx10<bits<10> op> :2549 VOP3be_Real_gfx6_gfx7<op>, VOP3be_Real_gfx10<op>;2550 2551defm V_LSHL_B64 : VOP3_Real_gfx6_gfx7<0x161>;2552defm V_LSHR_B64 : VOP3_Real_gfx6_gfx7<0x162>;2553defm V_ASHR_I64 : VOP3_Real_gfx6_gfx7<0x163>;2554defm V_MUL_LO_I32 : VOP3_Real_gfx6_gfx7<0x16b>;2555 2556defm V_MAD_LEGACY_F32 : VOP3_Real_gfx6_gfx7_gfx10<0x140>;2557defm V_MAD_F32 : VOP3_Real_gfx6_gfx7_gfx10<0x141>;2558defm V_MAD_I32_I24 : VOP3_Real_gfx6_gfx7_gfx10<0x142>;2559defm V_MAD_U32_U24 : VOP3_Real_gfx6_gfx7_gfx10<0x143>;2560defm V_CUBEID_F32 : VOP3_Real_gfx6_gfx7_gfx10<0x144>;2561defm V_CUBESC_F32 : VOP3_Real_gfx6_gfx7_gfx10<0x145>;2562defm V_CUBETC_F32 : VOP3_Real_gfx6_gfx7_gfx10<0x146>;2563defm V_CUBEMA_F32 : VOP3_Real_gfx6_gfx7_gfx10<0x147>;2564defm V_BFE_U32 : VOP3_Real_gfx6_gfx7_gfx10<0x148>;2565defm V_BFE_I32 : VOP3_Real_gfx6_gfx7_gfx10<0x149>;2566defm V_BFI_B32 : VOP3_Real_gfx6_gfx7_gfx10<0x14a>;2567defm V_FMA_F32 : VOP3_Real_gfx6_gfx7_gfx10<0x14b>;2568defm V_FMA_F64 : VOP3_Real_gfx6_gfx7_gfx10<0x14c>;2569defm V_LERP_U8 : VOP3_Real_gfx6_gfx7_gfx10<0x14d>;2570defm V_ALIGNBIT_B32 : VOP3_Real_gfx6_gfx7<0x14e>;2571defm V_ALIGNBYTE_B32 : VOP3_Real_gfx6_gfx7<0x14f>;2572defm V_MULLIT_F32 : VOP3_Real_gfx6_gfx7_gfx10<0x150>;2573defm V_MIN3_F32 : VOP3_Real_gfx6_gfx7_gfx10<0x151>;2574defm V_MIN3_I32 : VOP3_Real_gfx6_gfx7_gfx10<0x152>;2575defm V_MIN3_U32 : VOP3_Real_gfx6_gfx7_gfx10<0x153>;2576defm V_MAX3_F32 : VOP3_Real_gfx6_gfx7_gfx10<0x154>;2577defm V_MAX3_I32 : VOP3_Real_gfx6_gfx7_gfx10<0x155>;2578defm V_MAX3_U32 : VOP3_Real_gfx6_gfx7_gfx10<0x156>;2579defm V_MED3_F32 : VOP3_Real_gfx6_gfx7_gfx10<0x157>;2580defm V_MED3_I32 : VOP3_Real_gfx6_gfx7_gfx10<0x158>;2581defm V_MED3_U32 : VOP3_Real_gfx6_gfx7_gfx10<0x159>;2582defm V_SAD_U8 : VOP3_Real_gfx6_gfx7_gfx10<0x15a>;2583defm V_SAD_HI_U8 : VOP3_Real_gfx6_gfx7_gfx10<0x15b>;2584defm V_SAD_U16 : VOP3_Real_gfx6_gfx7_gfx10<0x15c>;2585defm V_SAD_U32 : VOP3_Real_gfx6_gfx7_gfx10<0x15d>;2586defm V_CVT_PK_U8_F32 : VOP3_Real_gfx6_gfx7_gfx10<0x15e>;2587defm V_DIV_FIXUP_F32 : VOP3_Real_gfx6_gfx7_gfx10<0x15f>;2588defm V_DIV_FIXUP_F64 : VOP3_Real_gfx6_gfx7_gfx10<0x160>;2589defm V_ADD_F64 : VOP3_Real_gfx6_gfx7_gfx10<0x164>;2590defm V_MUL_F64 : VOP3_Real_gfx6_gfx7_gfx10<0x165>;2591defm V_MIN_F64 : VOP3_Real_gfx6_gfx7_gfx10<0x166>;2592defm V_MAX_F64 : VOP3_Real_gfx6_gfx7_gfx10<0x167>;2593defm V_LDEXP_F64 : VOP3_Real_gfx6_gfx7_gfx10<0x168>;2594defm V_MUL_LO_U32 : VOP3_Real_gfx6_gfx7_gfx10<0x169>;2595defm V_MUL_HI_U32 : VOP3_Real_gfx6_gfx7_gfx10<0x16a>;2596defm V_MUL_HI_I32 : VOP3_Real_gfx6_gfx7_gfx10<0x16c>;2597defm V_DIV_FMAS_F32 : VOP3_Real_gfx6_gfx7_gfx10<0x16f>;2598defm V_DIV_FMAS_F64 : VOP3_Real_gfx6_gfx7_gfx10<0x170>;2599defm V_MSAD_U8 : VOP3_Real_gfx6_gfx7_gfx10<0x171>;2600defm V_MQSAD_PK_U16_U8 : VOP3_Real_gfx6_gfx7_gfx10<0x173>;2601defm V_TRIG_PREOP_F64 : VOP3_Real_gfx6_gfx7_gfx10<0x174>;2602defm V_DIV_SCALE_F32 : VOP3be_Real_gfx6_gfx7_gfx10<0x16d>;2603defm V_DIV_SCALE_F64 : VOP3be_Real_gfx6_gfx7_gfx10<0x16e>;2604 2605// NB: Same opcode as v_mad_legacy_f322606let DecoderNamespace = "GFX10_B" in2607defm V_FMA_LEGACY_F32 : VOP3_Real_gfx10<0x140>;2608 2609//===----------------------------------------------------------------------===//2610// GFX8, GFX9 (VI).2611//===----------------------------------------------------------------------===//2612 2613let AssemblerPredicate = isGFX8GFX9, DecoderNamespace = "GFX8" in {2614 2615multiclass VOP3_Real_vi<bits<10> op> {2616 def _vi : VOP3_Real<!cast<VOP_Pseudo>(NAME#"_e64"), SIEncodingFamily.VI>,2617 VOP3e_vi <op, !cast<VOP_Pseudo>(NAME#"_e64").Pfl>;2618}2619multiclass VOP3_Real_No_Suffix_vi<bits<10> op> {2620 def _vi : VOP3_Real<!cast<VOP_Pseudo>(NAME), SIEncodingFamily.VI>,2621 VOP3e_vi <op, !cast<VOP_Pseudo>(NAME).Pfl>;2622}2623 2624multiclass VOP3be_Real_vi<bits<10> op> {2625 def _vi : VOP3_Real<!cast<VOP_Pseudo>(NAME#"_e64"), SIEncodingFamily.VI>,2626 VOP3be_vi <op, !cast<VOP_Pseudo>(NAME#"_e64").Pfl>;2627}2628 2629multiclass VOP3OpSel_Real_gfx9<bits<10> op> {2630 def _vi : VOP3_Real<!cast<VOP_Pseudo>(NAME#"_e64"), SIEncodingFamily.VI>,2631 VOP3OpSel_gfx9 <op, !cast<VOP_Pseudo>(NAME#"_e64").Pfl>;2632}2633 2634multiclass VOP3OpSel_Real_gfx9_forced_opsel2<bits<10> op> {2635 def _vi : VOP3_Real<!cast<VOP_Pseudo>(NAME#"_e64"), SIEncodingFamily.VI>,2636 VOP3OpSel_gfx9 <op, !cast<VOP_Pseudo>(NAME#"_e64").Pfl> {2637 let Inst{13} = src2_modifiers{2}; // op_sel(2)2638 }2639}2640 2641multiclass VOP3Interp_Real_vi<bits<10> op> {2642 def _vi : VOP3_Real<!cast<VOP_Pseudo>(NAME), SIEncodingFamily.VI>,2643 VOP3Interp_vi <op, !cast<VOP_Pseudo>(NAME).Pfl>;2644}2645 2646} // End AssemblerPredicate = isGFX8GFX9, DecoderNamespace = "GFX8"2647 2648let AssemblerPredicate = isGFX8Only, DecoderNamespace = "GFX8" in {2649 2650multiclass VOP3_F16_Real_vi<bits<10> op> {2651 def _vi : VOP3_Real<!cast<VOP3_Pseudo>(NAME#"_e64"), SIEncodingFamily.VI>,2652 VOP3e_vi <op, !cast<VOP3_Pseudo>(NAME#"_e64").Pfl>;2653}2654 2655multiclass VOP3Interp_F16_Real_vi<bits<10> op> {2656 def _vi : VOP3_Real<!cast<VOP3_Pseudo>(NAME), SIEncodingFamily.VI>,2657 VOP3Interp_vi <op, !cast<VOP3_Pseudo>(NAME).Pfl>;2658}2659 2660} // End AssemblerPredicate = isGFX8Only, DecoderNamespace = "GFX8"2661 2662let AssemblerPredicate = isGFX9Only, DecoderNamespace = "GFX9" in {2663 2664multiclass VOP3_F16_Real_gfx9<bits<10> op, string OpName, string AsmName> {2665 def _gfx9 : VOP3_Real<!cast<VOP3_Pseudo>(OpName#"_e64"), SIEncodingFamily.GFX9>,2666 VOP3e_vi <op, !cast<VOP3_Pseudo>(OpName#"_e64").Pfl> {2667 VOP3_Pseudo ps = !cast<VOP3_Pseudo>(OpName#"_e64");2668 let AsmString = AsmName # ps.AsmOperands;2669 }2670}2671 2672multiclass VOP3OpSel_F16_Real_gfx9<bits<10> op, string AsmName> {2673 def _gfx9 : VOP3_Real<!cast<VOP3_Pseudo>(NAME#"_e64"), SIEncodingFamily.GFX9>,2674 VOP3OpSel_gfx9 <op, !cast<VOP3_Pseudo>(NAME#"_e64").Pfl> {2675 VOP3_Pseudo ps = !cast<VOP3_Pseudo>(NAME#"_e64");2676 let AsmString = AsmName # ps.AsmOperands;2677 }2678}2679 2680multiclass VOP3Interp_F16_Real_gfx9<bits<10> op, string OpName, string AsmName> {2681 def _gfx9 : VOP3_Real<!cast<VOP3_Pseudo>(OpName), SIEncodingFamily.GFX9>,2682 VOP3Interp_vi <op, !cast<VOP3_Pseudo>(OpName).Pfl> {2683 VOP3_Pseudo ps = !cast<VOP3_Pseudo>(OpName);2684 let AsmString = AsmName # ps.AsmOperands;2685 }2686}2687 2688multiclass VOP3Interp_F16_OpSel_Real_gfx9<bits<10> op, string OpName, string AsmName> {2689 def _gfx9 : VOP3_Real<!cast<VOP3_Pseudo>(OpName), SIEncodingFamily.GFX9>,2690 VOP3Interp_OpSel_gfx9 <op, !cast<VOP3_Pseudo>(OpName).Pfl> {2691 VOP3_Pseudo ps = !cast<VOP3_Pseudo>(OpName);2692 let AsmString = AsmName # ps.AsmOperands;2693 }2694}2695 2696multiclass VOP3_Real_gfx9<bits<10> op, string AsmName> {2697 def _gfx9 : VOP3_Real<!cast<VOP_Pseudo>(NAME#"_e64"), SIEncodingFamily.GFX9>,2698 VOP3e_vi <op, !cast<VOP_Pseudo>(NAME#"_e64").Pfl> {2699 VOP_Pseudo ps = !cast<VOP_Pseudo>(NAME#"_e64");2700 let AsmString = AsmName # ps.AsmOperands;2701 }2702}2703 2704multiclass VOP3_Real_BITOP3_gfx9<bits<10> op, string AsmName, bit isSingle = 0> {2705 defvar ps = !cast<VOP_Pseudo>(NAME#"_e64");2706 let IsSingle = !or(isSingle, ps.Pfl.IsSingle) in {2707 def _gfx9 : VOP3_Real<!cast<VOP_Pseudo>(NAME#"_e64"), SIEncodingFamily.GFX9>,2708 VOP3e_vi <op, !cast<VOP_Pseudo>(NAME#"_e64").Pfl> {2709 let AsmString = AsmName # ps.AsmOperands;2710 bits<8> bitop3;2711 let Inst{60-59} = bitop3{7-6};2712 let Inst{10-8} = bitop3{5-3};2713 let Inst{63-61} = bitop3{2-0};2714 let Inst{11} = !if(ps.Pfl.HasOpSel, src0_modifiers{2}, 0);2715 let Inst{12} = !if(ps.Pfl.HasOpSel, src1_modifiers{2}, 0);2716 let Inst{13} = !if(ps.Pfl.HasOpSel, src2_modifiers{2}, 0);2717 let Inst{14} = !if(ps.Pfl.HasOpSel, src0_modifiers{3}, 0);2718 }2719 }2720}2721 2722// Instructions such as v_alignbyte_b32 allows op_sel in gfx9, but not in vi.2723// The following is created to support that.2724multiclass VOP3OpSel_Real_gfx9_with_name<bits<10> op, string opName, string AsmName> {2725 defvar psName = opName#"_e64";2726 def _gfx9 : VOP3_Real<!cast<VOP3_Pseudo>(psName), SIEncodingFamily.VI>, // note: encoding family is VI2727 VOP3OpSel_gfx9 <op, !cast<VOP3_Pseudo>(psName).Pfl> {2728 VOP3_Pseudo ps = !cast<VOP3_Pseudo>(psName);2729 let AsmString = AsmName # ps.AsmOperands;2730 }2731}2732 2733} // End AssemblerPredicate = isGFX9Only, DecoderNamespace = "GFX9"2734 2735defm V_MAD_U64_U32 : VOP3be_Real_vi <0x1E8>;2736defm V_MAD_I64_I32 : VOP3be_Real_vi <0x1E9>;2737 2738defm V_MAD_LEGACY_F32 : VOP3_Real_vi <0x1c0>;2739defm V_MAD_F32 : VOP3_Real_vi <0x1c1>;2740defm V_MAD_I32_I24 : VOP3_Real_vi <0x1c2>;2741defm V_MAD_U32_U24 : VOP3_Real_vi <0x1c3>;2742defm V_CUBEID_F32 : VOP3_Real_vi <0x1c4>;2743defm V_CUBESC_F32 : VOP3_Real_vi <0x1c5>;2744defm V_CUBETC_F32 : VOP3_Real_vi <0x1c6>;2745defm V_CUBEMA_F32 : VOP3_Real_vi <0x1c7>;2746defm V_BFE_U32 : VOP3_Real_vi <0x1c8>;2747defm V_BFE_I32 : VOP3_Real_vi <0x1c9>;2748defm V_BFI_B32 : VOP3_Real_vi <0x1ca>;2749defm V_FMA_F32 : VOP3_Real_vi <0x1cb>;2750defm V_FMA_F64 : VOP3_Real_vi <0x1cc>;2751defm V_LERP_U8 : VOP3_Real_vi <0x1cd>;2752let SubtargetPredicate = isGFX8Only in {2753defm V_ALIGNBIT_B32 : VOP3_Real_vi <0x1ce>;2754defm V_ALIGNBYTE_B32 : VOP3_Real_vi <0x1cf>;2755}2756defm V_MIN3_F32 : VOP3_Real_vi <0x1d0>;2757defm V_MIN3_I32 : VOP3_Real_vi <0x1d1>;2758defm V_MIN3_U32 : VOP3_Real_vi <0x1d2>;2759defm V_MAX3_F32 : VOP3_Real_vi <0x1d3>;2760defm V_MAX3_I32 : VOP3_Real_vi <0x1d4>;2761defm V_MAX3_U32 : VOP3_Real_vi <0x1d5>;2762defm V_MED3_F32 : VOP3_Real_vi <0x1d6>;2763defm V_MED3_I32 : VOP3_Real_vi <0x1d7>;2764defm V_MED3_U32 : VOP3_Real_vi <0x1d8>;2765defm V_SAD_U8 : VOP3_Real_vi <0x1d9>;2766defm V_SAD_HI_U8 : VOP3_Real_vi <0x1da>;2767defm V_SAD_U16 : VOP3_Real_vi <0x1db>;2768defm V_SAD_U32 : VOP3_Real_vi <0x1dc>;2769defm V_CVT_PK_U8_F32 : VOP3_Real_vi <0x1dd>;2770defm V_DIV_FIXUP_F32 : VOP3_Real_vi <0x1de>;2771defm V_DIV_FIXUP_F64 : VOP3_Real_vi <0x1df>;2772defm V_DIV_SCALE_F32 : VOP3be_Real_vi <0x1e0>;2773defm V_DIV_SCALE_F64 : VOP3be_Real_vi <0x1e1>;2774defm V_DIV_FMAS_F32 : VOP3_Real_vi <0x1e2>;2775defm V_DIV_FMAS_F64 : VOP3_Real_vi <0x1e3>;2776defm V_MSAD_U8 : VOP3_Real_vi <0x1e4>;2777defm V_QSAD_PK_U16_U8 : VOP3_Real_vi <0x1e5>;2778defm V_MQSAD_PK_U16_U8 : VOP3_Real_vi <0x1e6>;2779defm V_MQSAD_U32_U8 : VOP3_Real_vi <0x1e7>;2780 2781defm V_PERM_B32 : VOP3_Real_vi <0x1ed>;2782 2783defm V_MAD_F16 : VOP3_F16_Real_vi <0x1ea>;2784defm V_MAD_U16 : VOP3_F16_Real_vi <0x1eb>;2785defm V_MAD_I16 : VOP3_F16_Real_vi <0x1ec>;2786defm V_FMA_F16 : VOP3_F16_Real_vi <0x1ee>;2787defm V_DIV_FIXUP_F16 : VOP3_F16_Real_vi <0x1ef>;2788defm V_INTERP_P2_F16 : VOP3Interp_F16_Real_vi <0x276>;2789 2790let FPDPRounding = 1 in {2791defm V_MAD_LEGACY_F16 : VOP3_F16_Real_gfx9 <0x1ea, "V_MAD_F16", "v_mad_legacy_f16">;2792defm V_FMA_LEGACY_F16 : VOP3_F16_Real_gfx9 <0x1ee, "V_FMA_F16", "v_fma_legacy_f16">;2793defm V_DIV_FIXUP_LEGACY_F16 : VOP3_F16_Real_gfx9 <0x1ef, "V_DIV_FIXUP_F16", "v_div_fixup_legacy_f16">;2794defm V_INTERP_P2_LEGACY_F16 : VOP3Interp_F16_Real_gfx9 <0x276, "V_INTERP_P2_F16", "v_interp_p2_legacy_f16">;2795} // End FPDPRounding = 12796 2797defm V_MAD_LEGACY_U16 : VOP3_F16_Real_gfx9 <0x1eb, "V_MAD_U16", "v_mad_legacy_u16">;2798defm V_MAD_LEGACY_I16 : VOP3_F16_Real_gfx9 <0x1ec, "V_MAD_I16", "v_mad_legacy_i16">;2799 2800defm V_ALIGNBIT_B32_opsel : VOP3OpSel_Real_gfx9_with_name <0x1ce, "V_ALIGNBIT_B32_opsel", "v_alignbit_b32">;2801defm V_ALIGNBYTE_B32_opsel : VOP3OpSel_Real_gfx9_with_name <0x1cf, "V_ALIGNBYTE_B32_opsel", "v_alignbyte_b32">;2802 2803defm V_MAD_F16_gfx9 : VOP3OpSel_F16_Real_gfx9 <0x203, "v_mad_f16">;2804defm V_MAD_U16_gfx9 : VOP3OpSel_F16_Real_gfx9 <0x204, "v_mad_u16">;2805defm V_MAD_I16_gfx9 : VOP3OpSel_F16_Real_gfx9 <0x205, "v_mad_i16">;2806defm V_FMA_F16_gfx9 : VOP3OpSel_F16_Real_gfx9 <0x206, "v_fma_f16">;2807defm V_DIV_FIXUP_F16_gfx9 : VOP3OpSel_F16_Real_gfx9 <0x207, "v_div_fixup_f16">;2808defm V_INTERP_P2_F16_opsel : VOP3Interp_F16_OpSel_Real_gfx9 <0x277, "V_INTERP_P2_F16_opsel", "v_interp_p2_f16">;2809 2810defm V_ADD_I32 : VOP3_Real_vi <0x29c>;2811defm V_SUB_I32 : VOP3_Real_vi <0x29d>;2812 2813defm V_INTERP_P1_F32_e64 : VOP3Interp_Real_vi <0x270>;2814defm V_INTERP_P2_F32_e64 : VOP3Interp_Real_vi <0x271>;2815defm V_INTERP_MOV_F32_e64 : VOP3Interp_Real_vi <0x272>;2816 2817defm V_INTERP_P1LL_F16 : VOP3Interp_Real_vi <0x274>;2818defm V_INTERP_P1LV_F16 : VOP3Interp_Real_vi <0x275>;2819defm V_ADD_F64 : VOP3_Real_vi <0x280>;2820defm V_MUL_F64 : VOP3_Real_vi <0x281>;2821defm V_MIN_F64 : VOP3_Real_vi <0x282>;2822defm V_MAX_F64 : VOP3_Real_vi <0x283>;2823defm V_LDEXP_F64 : VOP3_Real_vi <0x284>;2824defm V_MUL_LO_U32 : VOP3_Real_vi <0x285>;2825 2826// removed from VI as identical to V_MUL_LO_U322827let isAsmParserOnly = 1 in {2828defm V_MUL_LO_I32 : VOP3_Real_vi <0x285>;2829}2830 2831defm V_MUL_HI_U32 : VOP3_Real_vi <0x286>;2832defm V_MUL_HI_I32 : VOP3_Real_vi <0x287>;2833 2834defm V_READLANE_B32 : VOP3_Real_No_Suffix_vi <0x289>;2835defm V_WRITELANE_B32 : VOP3_Real_No_Suffix_vi <0x28a>;2836 2837defm V_LSHLREV_B64 : VOP3_Real_vi <0x28f>;2838defm V_LSHRREV_B64 : VOP3_Real_vi <0x290>;2839defm V_ASHRREV_I64 : VOP3_Real_vi <0x291>;2840defm V_TRIG_PREOP_F64 : VOP3_Real_vi <0x292>;2841 2842defm V_LSHL_ADD_U32 : VOP3_Real_vi <0x1fd>;2843defm V_ADD_LSHL_U32 : VOP3_Real_vi <0x1fe>;2844defm V_ADD3_U32 : VOP3_Real_vi <0x1ff>;2845defm V_LSHL_OR_B32 : VOP3_Real_vi <0x200>;2846defm V_AND_OR_B32 : VOP3_Real_vi <0x201>;2847defm V_OR3_B32 : VOP3_Real_vi <0x202>;2848defm V_PACK_B32_F16 : VOP3OpSel_Real_gfx9 <0x2a0>;2849 2850defm V_XAD_U32 : VOP3_Real_vi <0x1f3>;2851 2852defm V_MIN3_F16 : VOP3OpSel_Real_gfx9 <0x1f4>;2853defm V_MIN3_I16 : VOP3OpSel_Real_gfx9 <0x1f5>;2854defm V_MIN3_U16 : VOP3OpSel_Real_gfx9 <0x1f6>;2855 2856defm V_MAX3_F16 : VOP3OpSel_Real_gfx9 <0x1f7>;2857defm V_MAX3_I16 : VOP3OpSel_Real_gfx9 <0x1f8>;2858defm V_MAX3_U16 : VOP3OpSel_Real_gfx9 <0x1f9>;2859 2860defm V_MED3_F16 : VOP3OpSel_Real_gfx9 <0x1fa>;2861defm V_MED3_I16 : VOP3OpSel_Real_gfx9 <0x1fb>;2862defm V_MED3_U16 : VOP3OpSel_Real_gfx9 <0x1fc>;2863 2864defm V_ADD_I16 : VOP3OpSel_Real_gfx9 <0x29e>;2865defm V_SUB_I16 : VOP3OpSel_Real_gfx9 <0x29f>;2866 2867defm V_MAD_U32_U16 : VOP3OpSel_Real_gfx9 <0x1f1>;2868defm V_MAD_I32_I16 : VOP3OpSel_Real_gfx9 <0x1f2>;2869 2870defm V_CVT_PKNORM_I16_F16 : VOP3OpSel_Real_gfx9 <0x299>;2871defm V_CVT_PKNORM_U16_F16 : VOP3OpSel_Real_gfx9 <0x29a>;2872 2873defm V_LSHL_ADD_U64 : VOP3_Real_vi <0x208>;2874 2875defm V_CVT_PK_FP8_F32 : VOP3OpSel_Real_gfx9 <0x2a2>;2876defm V_CVT_PK_BF8_F32 : VOP3OpSel_Real_gfx9 <0x2a3>;2877defm V_CVT_PK_BF16_F32: VOP3OpSel_Real_gfx9 <0x268>;2878defm V_CVT_SR_FP8_F32 : VOP3OpSel_Real_gfx9_forced_opsel2 <0x2a4>;2879defm V_CVT_SR_BF8_F32 : VOP3OpSel_Real_gfx9_forced_opsel2 <0x2a5>;2880 2881defm V_MINIMUM3_F32 : VOP3_Real_vi <0x2a8>;2882defm V_MAXIMUM3_F32 : VOP3_Real_vi <0x2a9>;2883 2884defm V_BITOP3_B16 : VOP3_Real_BITOP3_gfx9<0x233, "v_bitop3_b16">;2885defm V_BITOP3_B32 : VOP3_Real_BITOP3_gfx9<0x234, "v_bitop3_b32">;2886let OtherPredicates = [HasFP8ConversionScaleInsts] in {2887defm V_CVT_SCALEF32_SR_FP8_BF16 : VOP3OpSel_Real_gfx9<0x246>;2888defm V_CVT_SCALEF32_SR_FP8_F16 : VOP3OpSel_Real_gfx9<0x242>;2889defm V_CVT_SCALEF32_SR_FP8_F32 : VOP3OpSel_Real_gfx9<0x237>;2890defm V_CVT_SCALEF32_F16_FP8 : VOP3OpSel_Real_gfx9 <0x24a>;2891defm V_CVT_SCALEF32_F32_FP8 : VOP3OpSel_Real_gfx9 <0x23b>;2892defm V_CVT_SCALEF32_PK_FP8_F32 : VOP3OpSel_Real_gfx9 <0x235>;2893defm V_CVT_SCALEF32_PK_F32_FP8 : VOP3OpSel_Real_gfx9 <0x239>;2894defm V_CVT_SCALEF32_PK_FP8_F16 : VOP3OpSel_Real_gfx9 <0x240>;2895defm V_CVT_SCALEF32_PK_FP8_BF16: VOP3OpSel_Real_gfx9 <0x244>;2896defm V_CVT_SCALEF32_PK_F16_FP8 : VOP3OpSel_Real_gfx9<0x248>;2897defm V_CVT_SCALEF32_PK_BF16_FP8 : VOP3OpSel_Real_gfx9<0x269>;2898}2899let OtherPredicates = [HasBF8ConversionScaleInsts] in {2900defm V_CVT_SCALEF32_SR_BF8_BF16 : VOP3OpSel_Real_gfx9<0x247>;2901defm V_CVT_SCALEF32_SR_BF8_F16 : VOP3OpSel_Real_gfx9<0x243>;2902defm V_CVT_SCALEF32_SR_BF8_F32 : VOP3OpSel_Real_gfx9<0x238>;2903defm V_CVT_SCALEF32_F16_BF8 : VOP3OpSel_Real_gfx9 <0x24b>;2904defm V_CVT_SCALEF32_F32_BF8 : VOP3OpSel_Real_gfx9 <0x23c>;2905defm V_CVT_SCALEF32_PK_BF8_F32 : VOP3OpSel_Real_gfx9 <0x236>;2906defm V_CVT_SCALEF32_PK_F32_BF8 : VOP3OpSel_Real_gfx9 <0x23a>;2907defm V_CVT_SCALEF32_PK_BF8_F16 : VOP3OpSel_Real_gfx9 <0x241>;2908defm V_CVT_SCALEF32_PK_BF8_BF16: VOP3OpSel_Real_gfx9 <0x245>;2909defm V_CVT_SCALEF32_PK_F16_BF8 : VOP3OpSel_Real_gfx9<0x249>;2910defm V_CVT_SCALEF32_PK_BF16_BF8 : VOP3OpSel_Real_gfx9<0x26a>;2911}2912let OtherPredicates = [HasFP4ConversionScaleInsts] in {2913defm V_CVT_SCALEF32_PK_F32_FP4 : VOP3OpSel_Real_gfx9 <0x23f>;2914defm V_CVT_SCALEF32_PK_FP4_F32 : VOP3OpSel_Real_gfx9 <0x23d>;2915defm V_CVT_SCALEF32_PK_F16_FP4 : VOP3OpSel_Real_gfx9 <0x250>;2916defm V_CVT_SCALEF32_PK_BF16_FP4 : VOP3OpSel_Real_gfx9 <0x251>;2917defm V_CVT_SCALEF32_PK_FP4_F16 : VOP3OpSel_Real_gfx9_forced_opsel2 <0x24c>;2918defm V_CVT_SCALEF32_PK_FP4_BF16: VOP3OpSel_Real_gfx9_forced_opsel2 <0x24d>;2919defm V_CVT_SCALEF32_SR_PK_FP4_F16: VOP3OpSel_Real_gfx9 <0x24e>;2920defm V_CVT_SCALEF32_SR_PK_FP4_BF16: VOP3OpSel_Real_gfx9 <0x24f>;2921defm V_CVT_SCALEF32_SR_PK_FP4_F32: VOP3OpSel_Real_gfx9 <0x23e>;2922}2923let OtherPredicates = [HasFP6BF6ConversionScaleInsts] in {2924defm V_CVT_SCALEF32_PK32_F32_FP6 : VOP3_Real_gfx9<0x256, "v_cvt_scalef32_pk32_f32_fp6">;2925defm V_CVT_SCALEF32_PK32_F32_BF6 : VOP3_Real_gfx9<0x257, "v_cvt_scalef32_pk32_f32_bf6">;2926defm V_CVT_SCALEF32_PK32_F16_FP6 : VOP3_Real_gfx9<0x260, "v_cvt_scalef32_pk32_f16_fp6">;2927defm V_CVT_SCALEF32_PK32_BF16_FP6 : VOP3_Real_gfx9<0x261, "v_cvt_scalef32_pk32_bf16_fp6">;2928defm V_CVT_SCALEF32_PK32_F16_BF6 : VOP3_Real_gfx9<0x262, "v_cvt_scalef32_pk32_f16_bf6">;2929defm V_CVT_SCALEF32_PK32_BF16_BF6 : VOP3_Real_gfx9<0x263, "v_cvt_scalef32_pk32_bf16_bf6">;2930}2931 2932let OtherPredicates = [HasF16BF16ToFP6BF6ConversionScaleInsts] in {2933defm V_CVT_SCALEF32_PK32_FP6_F16 : VOP3_Real_gfx9<0x258, "v_cvt_scalef32_pk32_fp6_f16">;2934defm V_CVT_SCALEF32_PK32_FP6_BF16 : VOP3_Real_gfx9<0x259, "v_cvt_scalef32_pk32_fp6_bf16">;2935defm V_CVT_SCALEF32_PK32_BF6_F16 : VOP3_Real_gfx9<0x25a, "v_cvt_scalef32_pk32_bf6_f16">;2936defm V_CVT_SCALEF32_PK32_BF6_BF16 : VOP3_Real_gfx9<0x25b, "v_cvt_scalef32_pk32_bf6_bf16">;2937defm V_CVT_SCALEF32_SR_PK32_BF6_BF16 : VOP3_Real_gfx9<0x25f, "v_cvt_scalef32_sr_pk32_bf6_bf16">;2938defm V_CVT_SCALEF32_SR_PK32_BF6_F16 : VOP3_Real_gfx9<0x25e, "v_cvt_scalef32_sr_pk32_bf6_f16">;2939defm V_CVT_SCALEF32_SR_PK32_BF6_F32 : VOP3_Real_gfx9<0x255, "v_cvt_scalef32_sr_pk32_bf6_f32">;2940defm V_CVT_SCALEF32_SR_PK32_FP6_BF16 : VOP3_Real_gfx9<0x25d, "v_cvt_scalef32_sr_pk32_fp6_bf16">;2941defm V_CVT_SCALEF32_SR_PK32_FP6_F16 : VOP3_Real_gfx9<0x25c, "v_cvt_scalef32_sr_pk32_fp6_f16">;2942defm V_CVT_SCALEF32_SR_PK32_FP6_F32 : VOP3_Real_gfx9<0x254, "v_cvt_scalef32_sr_pk32_fp6_f32">;2943}2944 2945let OtherPredicates = [HasF32ToF16BF16ConversionSRInsts] in {2946defm V_CVT_SR_F16_F32 : VOP3OpSel_Real_gfx9 <0x2a6>;2947defm V_CVT_SR_BF16_F32: VOP3OpSel_Real_gfx9 <0x2a7>;2948}2949 2950defm V_ASHR_PK_I8_I32 : VOP3OpSel_Real_gfx9 <0x265>;2951defm V_ASHR_PK_U8_I32 : VOP3OpSel_Real_gfx9 <0x266>;2952let OtherPredicates = [HasCvtPkF16F32Inst] in {2953defm V_CVT_PK_F16_F32 : VOP3_Real_gfx9<0x267, "v_cvt_pk_f16_f32">;2954}2955 2956defm V_CVT_SCALEF32_2XPK16_FP6_F32 : VOP3_Real_gfx9<0x252, "v_cvt_scalef32_2xpk16_fp6_f32">;2957defm V_CVT_SCALEF32_2XPK16_BF6_F32 : VOP3_Real_gfx9<0x253, "v_cvt_scalef32_2xpk16_bf6_f32">;2958