2055 lines · plain
1//===-- DSInstructions.td - DS Instruction Definitions --------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9class DS_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> :10 InstSI <outs, ins, "", pattern>,11 SIMCInstr <NAME, SIEncodingFamily.NONE> {12 13 let LGKM_CNT = 1;14 let DS = 1;15 let GWS = 0;16 let Size = 8;17 let UseNamedOperandTable = 1;18 19 // Most instruction load and store data, so set this as the default.20 let mayLoad = 1;21 let mayStore = 1;22 let FixedSize = true;23 24 let hasSideEffects = 0;25 let SchedRW = [WriteLDS];26 27 let isPseudo = 1;28 let isCodeGenOnly = 1;29 30 string Mnemonic = opName;31 string AsmOperands = asmOps;32 33 // Well these bits a kind of hack because it would be more natural34 // to test "outs" and "ins" dags for the presence of particular operands35 bits<1> has_vdst = 1;36 bits<1> has_addr = 1;37 bits<1> has_data0 = 1;38 bits<1> has_data1 = 1;39 40 bits<1> has_gws_data0 = 0; // data0 is encoded as addr41 42 bits<1> has_offset = 1; // has "offset" that should be split to offset0,143 bits<1> has_offset0 = 1;44 bits<1> has_offset1 = 1;45 46 bits<1> has_gds = 1;47 bits<1> gdsValue = 0; // if has_gds == 0 set gds to this value48 49 bits<1> has_m0_read = 1;50 51 let Uses = !if(has_m0_read, [M0, EXEC], [EXEC]);52}53 54class DS_Real <DS_Pseudo ps, string opName = ps.Mnemonic> :55 InstSI <ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands>,56 Enc64 {57 58 let isPseudo = 0;59 let isCodeGenOnly = 0;60 let LGKM_CNT = 1;61 let DS = 1;62 let UseNamedOperandTable = 1;63 64 // copy relevant pseudo op flags65 let GWS = ps.GWS;66 let SubtargetPredicate = ps.SubtargetPredicate;67 let WaveSizePredicate = ps.WaveSizePredicate;68 let OtherPredicates = ps.OtherPredicates;69 let TSFlags = ps.TSFlags;70 let SchedRW = ps.SchedRW;71 let mayLoad = ps.mayLoad;72 let mayStore = ps.mayStore;73 let IsAtomicRet = ps.IsAtomicRet;74 let IsAtomicNoRet = ps.IsAtomicNoRet;75 let Uses = ps.Uses;76 let Defs = ps.Defs;77 let isConvergent = ps.isConvergent;78 79 let Constraints = ps.Constraints;80 81 // encoding fields82 bits<10> vdst;83 bits<1> gds;84 bits<8> addr;85 bits<10> data0;86 bits<10> data1;87 bits<8> offset0;88 bits<8> offset1;89 90 bits<16> offset;91 let offset0 = !if(ps.has_offset, offset{7-0}, ?);92 let offset1 = !if(ps.has_offset, offset{15-8}, ?);93 94 // Figure out if we should set the acc bit. Simple load and store95 // instructions with a single data operand can use AV_* classes, in96 // which case the encoding comes from the assigned register field.97 98 // For more compliated cases with multiple data operands, since the99 // register fields are only 8-bit, so data operands must all be AGPR100 // or VGPR.101 defvar DstOpIsAV = !if(ps.has_vdst,102 VDstOperandIsAV<ps.OutOperandList>.ret, 0);103 defvar DstOpIsAGPR = !if(ps.has_vdst,104 VDstOperandIsAGPR<ps.OutOperandList>.ret, 0);105 defvar DataOpIsAV = !if(!or(ps.has_data0, ps.has_gws_data0),106 Data0OperandIsAV<ps.InOperandList>.ret, 0);107 defvar DataOpIsAGPR = !if(!or(ps.has_data0, ps.has_gws_data0),108 Data0OperandIsAGPR<ps.InOperandList>.ret, 0);109 110 bits<1> acc = !if(ps.has_vdst,111 !if(DstOpIsAV, vdst{9}, DstOpIsAGPR),112 !if(DataOpIsAV, data0{9}, DataOpIsAGPR));113}114 115// DS Pseudo instructions116 117class DS_0A1D_NORET<string opName, RegisterOperand rc = AVLdSt_32>118: DS_Pseudo<opName,119 (outs),120 (ins rc:$data0, Offset:$offset, gds:$gds),121 " $data0$offset$gds"> {122 123 let has_addr = 0;124 let has_data1 = 0;125 let has_vdst = 0;126}127 128class DS_1A1D_NORET<string opName, RegisterOperand rc = AVLdSt_32>129: DS_Pseudo<opName,130 (outs),131 (ins VGPR_32:$addr, rc:$data0, Offset:$offset, gds:$gds),132 " $addr, $data0$offset$gds"> {133 134 let has_data1 = 0;135 let has_vdst = 0;136 let IsAtomicNoRet = 1;137}138 139multiclass DS_1A1D_NORET_mc<string opName, RegisterOperand rc = AVLdSt_32> {140 def "" : DS_1A1D_NORET<opName, rc>;141 142 let has_m0_read = 0 in {143 def _gfx9 : DS_1A1D_NORET<opName, rc>;144 }145}146 147multiclass DS_1A1D_NORET_t16<string opName, RegisterOperand rc = AVLdSt_32>148: DS_1A1D_NORET_mc<opName, rc> {149 let has_m0_read = 0 in {150 let True16Predicate = UseRealTrue16Insts in {151 def "_t16" : DS_1A1D_NORET<opName#"_t16", VGPROp_16>,152 True16D16Table<NAME#"_D16_HI", NAME#"_gfx9">;153 }154 }155}156 157multiclass DS_1A1D_NORET_mc_gfx9<string opName, RegisterOperand rc = AVLdSt_32> {158 let has_m0_read = 0 in {159 def "" : DS_1A1D_NORET<opName, rc>;160 }161}162 163class DS_1A2D_NORET<string opName, RegisterOperand data_op = VGPROp_32>164: DS_Pseudo<opName,165 (outs),166 (ins VGPR_32:$addr, data_op:$data0, data_op:$data1, Offset:$offset, gds:$gds),167 " $addr, $data0, $data1$offset$gds"> {168 169 let has_vdst = 0;170 let IsAtomicNoRet = 1;171}172 173// DS_xx2D cases should only be instantiated with VGPR operand classes.174multiclass DS_1A2D_NORET_mc<string opName, RegisterOperand rc = VGPROp_32> {175 assert OperandIsVGPR<rc>.ret,176 "DS with 2 data operands should be declared with VGPRs";177 178 def "" : DS_1A2D_NORET<opName, rc>;179 180 let has_m0_read = 0 in {181 def _gfx9 : DS_1A2D_NORET<opName, rc>;182 183 // All data operands are replaced with AGPRs in this form.184 let SubtargetPredicate = isGFX90APlus in {185 def _agpr : DS_1A2D_NORET<opName, getEquivalentAGPROperand<rc>.ret>;186 }187 }188}189 190class DS_1A2D_Off8_NORET <string opName, RegisterOperand data_op = VGPROp_32>191: DS_Pseudo<opName,192 (outs),193 (ins VGPR_32:$addr, data_op:$data0, data_op:$data1,194 Offset0:$offset0, Offset1:$offset1, gds:$gds),195 " $addr, $data0, $data1$offset0$offset1$gds"> {196 197 let has_vdst = 0;198 let has_offset = 0;199}200 201multiclass DS_1A2D_Off8_NORET_mc <string opName, RegisterOperand rc = VGPROp_32> {202 assert OperandIsVGPR<rc>.ret,203 "DS with 2 data operands should be declared with VGPRs";204 205 def "" : DS_1A2D_Off8_NORET<opName, rc>;206 207 let has_m0_read = 0 in {208 def _gfx9 : DS_1A2D_Off8_NORET<opName, rc>;209 210 let SubtargetPredicate = isGFX90APlus in {211 def _agpr : DS_1A2D_Off8_NORET<opName, getEquivalentAGPROperand<rc>.ret>;212 }213 }214}215 216class DS_0A1D_RET_GDS<string opName, RegisterOperand dst_op = AVLdSt_32,217 RegisterOperand src_op = dst_op>218: DS_Pseudo<opName,219 (outs dst_op:$vdst),220 (ins src_op:$data0, Offset:$offset),221 " $vdst, $data0$offset gds"> {222 223 let has_addr = 0;224 let has_data1 = 0;225 let has_gds = 0;226 let gdsValue = 1;227 let hasSideEffects = 1;228}229 230class DS_1A1D_RET <string opName, RegisterOperand data_op = AVLdSt_32>231: DS_Pseudo<opName,232 (outs data_op:$vdst),233 (ins VGPR_32:$addr, data_op:$data0, Offset:$offset, gds:$gds),234 " $vdst, $addr, $data0$offset$gds"> {235 236 let has_data1 = 0;237 let IsAtomicRet = 1;238}239 240multiclass DS_1A1D_RET_mc <string opName, RegisterOperand rc = VGPROp_32> {241 assert OperandIsVGPR<rc>.ret,242 "DS with 2 data operands should be declared with VGPRs";243 244 def "" : DS_1A1D_RET<opName, rc>;245 246 let has_m0_read = 0 in {247 def _gfx9 : DS_1A1D_RET<opName, rc>;248 def _agpr : DS_1A1D_RET<opName, getEquivalentAGPROperand<rc>.ret>;249 }250}251 252multiclass DS_1A1D_RET_mc_gfx9 <string opName, RegisterOperand rc = VGPROp_32> {253 let has_m0_read = 0 in {254 def "" : DS_1A1D_RET<opName, rc>;255 def _agpr : DS_1A1D_RET<opName, getEquivalentAGPROperand<rc>.ret>;256 }257}258 259class DS_1A2D_RET<string opName,260 RegisterOperand dst_rc = VGPROp_32,261 RegisterOperand src_rc = dst_rc>: DS_Pseudo<opName,262 (outs dst_rc:$vdst),263 (ins VGPR_32:$addr, src_rc:$data0, src_rc:$data1, Offset:$offset, gds:$gds),264 " $vdst, $addr, $data0, $data1$offset$gds"> {265 266 let IsAtomicRet = 1;267}268 269multiclass DS_1A2D_RET_mc<string opName,270 RegisterOperand dst_rc = VGPROp_32,271 RegisterOperand src_rc = dst_rc> {272 assert !and(OperandIsVGPR<dst_rc>.ret, OperandIsVGPR<src_rc>.ret),273 "DS with 2 data operands should be declared with VGPRs";274 275 def "" : DS_1A2D_RET<opName, dst_rc, src_rc>;276 277 let has_m0_read = 0 in {278 def _gfx9 : DS_1A2D_RET<opName, dst_rc, src_rc>;279 def _agpr : DS_1A2D_RET<opName, getEquivalentAGPROperand<dst_rc>.ret,280 getEquivalentAGPROperand<src_rc>.ret>;281 }282}283 284class DS_1A2D_Off8_RET<string opName,285 RegisterOperand dst_rc = VGPROp_32,286 RegisterOperand src_rc = dst_rc>287: DS_Pseudo<opName,288 (outs dst_rc:$vdst),289 (ins VGPR_32:$addr, src_rc:$data0, src_rc:$data1, Offset0:$offset0, Offset1:$offset1, gds:$gds),290 " $vdst, $addr, $data0, $data1$offset0$offset1$gds"> {291 292 let has_offset = 0;293}294 295multiclass DS_1A2D_Off8_RET_mc<string opName,296 RegisterOperand dst_rc = VGPROp_32,297 RegisterOperand src_rc = dst_rc> {298 assert !and(OperandIsVGPR<dst_rc>.ret, OperandIsVGPR<src_rc>.ret) ,299 "DS with 2 data operands should be declared with VGPRs";300 301 def "" : DS_1A2D_Off8_RET<opName, dst_rc, src_rc>;302 303 let has_m0_read = 0 in {304 def _gfx9 : DS_1A2D_Off8_RET<opName, dst_rc, src_rc>;305 def _agpr : DS_1A2D_Off8_RET<opName, getEquivalentAGPROperand<dst_rc>.ret,306 getEquivalentAGPROperand<src_rc>.ret>;307 }308}309 310class DS_BVH_STACK<string opName,311 RegisterOperand vdst_rc,312 RegisterOperand data1_rc>313: DS_Pseudo<opName,314 (outs vdst_rc:$vdst, VGPR_32:$addr),315 (ins VGPR_32:$addr_in, VGPR_32:$data0, data1_rc:$data1, Offset:$offset),316 " $vdst, $addr, $data0, $data1$offset"> {317 let Constraints = "$addr = $addr_in";318 let has_gds = 0;319 let gdsValue = 0;320 // TODO: Use MMOs in the LDS address space instead of hasSideEffects = 1.321 let hasSideEffects = 1;322 let SchedRW = [WriteLDS, WriteLDS];323}324 325class DS_1A_RET<string opName, RegisterOperand data_op = AVLdSt_32,326 bit HasTiedOutput = 0, Operand ofs = Offset>327: DS_Pseudo<opName,328 (outs data_op:$vdst),329 !if(HasTiedOutput,330 (ins VGPR_32:$addr, ofs:$offset, gds:$gds, data_op:$vdst_in),331 (ins VGPR_32:$addr, ofs:$offset, gds:$gds)),332 " $vdst, $addr$offset$gds"> {333 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", "");334 let has_data0 = 0;335 let has_data1 = 0;336}337 338multiclass DS_1A_RET_mc<string opName, RegisterOperand rc = AVLdSt_32,339 bit HasTiedOutput = 0, Operand ofs = Offset> {340 def "" : DS_1A_RET<opName, rc, HasTiedOutput, ofs>;341 342 let has_m0_read = 0 in {343 def _gfx9 : DS_1A_RET<opName, rc, HasTiedOutput, ofs>;344 }345}346 347multiclass DS_1A_RET_t16<string opName, RegisterOperand rc = AVLdSt_32,348 bit HasTiedOutput = 0, Operand ofs = Offset>349: DS_1A_RET_mc<opName, rc, HasTiedOutput, ofs> {350 let has_m0_read = 0 in {351 let True16Predicate = UseRealTrue16Insts in {352 def "_t16" : DS_1A_RET<opName#"_t16", VGPROp_16, HasTiedOutput, ofs>, True16D16Table<NAME#"_D16_HI", NAME#"_D16">;353 }354 }355}356 357multiclass DS_1A_RET_NoM0<string opName, RegisterOperand rc = VGPROp_32> {358 let has_m0_read = 0 in {359 def "" : DS_1A_RET<opName, rc>;360 }361}362 363class DS_1A_RET_Tied<string opName, RegisterOperand rc = AVLdSt_32> :364 DS_1A_RET<opName, rc, 1>;365 366class DS_1A_Off8_RET <string opName, RegisterOperand rc = AVLdSt_32>367: DS_Pseudo<opName,368 (outs rc:$vdst),369 (ins VGPR_32:$addr, Offset0:$offset0, Offset1:$offset1, gds:$gds),370 " $vdst, $addr$offset0$offset1$gds"> {371 372 let has_offset = 0;373 let has_data0 = 0;374 let has_data1 = 0;375}376 377multiclass DS_1A_Off8_RET_mc <string opName, RegisterOperand rc = VGPROp_32> {378 def "" : DS_1A_Off8_RET<opName, rc>;379 380 let has_m0_read = 0 in {381 def _gfx9 : DS_1A_Off8_RET<opName, rc>;382 }383}384 385class DS_1A_RET_GDS <string opName> : DS_Pseudo<opName,386 (outs AVLdSt_32:$vdst),387 (ins VGPR_32:$addr, Offset:$offset),388 " $vdst, $addr$offset gds"> {389 390 let has_data0 = 0;391 let has_data1 = 0;392 let has_gds = 0;393 let gdsValue = 1;394}395 396class DS_1A_Off16_NORET <string opName>397: DS_Pseudo<opName,398 (outs),399 (ins VGPR_32:$addr, Offset:$offset, gds:$gds),400 " $addr$offset$gds"> {401 402 let has_vdst = 0;403 let has_offset = 1;404 let has_data0 = 0;405 let has_data1 = 0;406 let has_m0_read = 0;407 let IsAtomicNoRet = 1;408}409 410class DS_0A_RET <string opName> : DS_Pseudo<opName,411 (outs AVLdSt_32:$vdst),412 (ins Offset:$offset, gds:$gds),413 " $vdst$offset$gds"> {414 415 let mayLoad = 1;416 let mayStore = 1;417 418 let has_addr = 0;419 let has_data0 = 0;420 let has_data1 = 0;421}422 423class DS_1A <string opName> : DS_Pseudo<opName,424 (outs),425 (ins VGPR_32:$addr, Offset:$offset, gds:$gds),426 " $addr$offset$gds"> {427 428 let mayLoad = 1;429 let mayStore = 1;430 431 let has_vdst = 0;432 let has_data0 = 0;433 let has_data1 = 0;434}435 436multiclass DS_1A_mc <string opName> {437 def "" : DS_1A<opName>;438 439 let has_m0_read = 0 in {440 def _gfx9 : DS_1A<opName>;441 }442}443 444 445class DS_GWS <string opName, dag ins, string asmOps>446: DS_Pseudo<opName, (outs), ins, asmOps> {447 let GWS = 1;448 449 let has_vdst = 0;450 let has_addr = 0;451 let has_data0 = 0;452 let has_data1 = 0;453 454 let has_gds = 0;455 let gdsValue = 1;456}457 458class DS_GWS_0D <string opName>459: DS_GWS<opName,460 (ins Offset:$offset), "$offset gds"> {461 let hasSideEffects = 1;462}463 464class DS_GWS_1D <string opName>465: DS_GWS<opName,466 (ins AV_LdSt_32_Align2_RegOp:$data0, Offset:$offset),467 " $data0$offset gds"> {468 469 let has_gws_data0 = 1;470 let hasSideEffects = 1;471}472 473class DS_VOID <string opName> : DS_Pseudo<opName,474 (outs), (ins), ""> {475 let mayLoad = 0;476 let mayStore = 0;477 let hasSideEffects = 1;478 let UseNamedOperandTable = 0;479 480 let has_vdst = 0;481 let has_addr = 0;482 let has_data0 = 0;483 let has_data1 = 0;484 let has_offset = 0;485 let has_offset0 = 0;486 let has_offset1 = 0;487 let has_gds = 0;488}489 490class DS_1A1D_PERMUTE <string opName, SDPatternOperator node = null_frag,491 RegisterOperand data_op = AVLdSt_32>492: DS_Pseudo<opName,493 (outs data_op:$vdst),494 (ins VGPR_32:$addr, data_op:$data0, Offset:$offset),495 " $vdst, $addr, $data0$offset",496 [(set i32:$vdst,497 (node (DS1Addr1Offset i32:$addr, i32:$offset), i32:$data0))] > {498 499 let mayLoad = 0;500 let mayStore = 0;501 let isConvergent = 1;502 503 let has_data1 = 0;504 let has_gds = 0;505}506 507multiclass DS_1A1D_PERMUTE_mc <string opName, SDPatternOperator node = null_frag,508 RegisterOperand data_op = VGPROp_32> {509 assert OperandIsVGPR<data_op>.ret,510 "DS with 2 data operands should be declared with VGPRs";511 def "" : DS_1A1D_PERMUTE<opName, node, data_op>;512 513 let SubtargetPredicate = isGFX90APlus in {514 def _agpr : DS_1A1D_PERMUTE<opName, null_frag,515 getEquivalentAGPROperand<data_op>.ret>;516 }517}518 519 520class DSAtomicRetPat<DS_Pseudo inst, ValueType vt, PatFrag frag, int complexity = 0,521 bit gds=0> : GCNPat <(frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$value),522 (inst $ptr, getVregSrcForVT<vt>.ret:$value, Offset:$offset, (i1 gds))> {523 let AddedComplexity = complexity;524}525 526multiclass DSAtomicRetPat_mc<DS_Pseudo inst, ValueType vt, string frag> {527 let OtherPredicates = [LDSRequiresM0Init] in {528 def : DSAtomicRetPat<inst, vt, !cast<PatFrag>(frag#"_local_m0_"#vt)>;529 }530 531 let OtherPredicates = [NotLDSRequiresM0Init] in {532 def : DSAtomicRetPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,533 !cast<PatFrag>(frag#"_local_"#vt)>;534 }535 536 let OtherPredicates = [HasGDS] in {537 def : DSAtomicRetPat<inst, vt, !cast<PatFrag>(frag#"_region_m0_"#vt),538 /* complexity */ 0, /* gds */ 1>;539 }540}541 542multiclass DSAtomicRetNoRetPat_NoM0_mc<DS_Pseudo inst, DS_Pseudo noRetInst,543 ValueType vt, string frag> {544 def : DSAtomicRetPat<inst, vt,545 !cast<PatFrag>(frag#"_local_"#vt)>;546 def : DSAtomicRetPat<noRetInst, vt,547 !cast<PatFrag>(frag#"_local_noret_"#vt), /* complexity */ 1>;548}549 550multiclass DSAtomicRetNoRetPat_mc<DS_Pseudo inst, DS_Pseudo noRetInst,551 ValueType vt, string frag> {552 let OtherPredicates = [LDSRequiresM0Init] in {553 def : DSAtomicRetPat<inst, vt,554 !cast<PatFrag>(frag#"_local_m0_"#vt)>;555 def : DSAtomicRetPat<noRetInst, vt,556 !cast<PatFrag>(frag#"_local_m0_noret_"#vt), /* complexity */ 1>;557 }558 559 let OtherPredicates = [NotLDSRequiresM0Init] in {560 defm : DSAtomicRetNoRetPat_NoM0_mc<561 !cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"),562 !cast<DS_Pseudo>(!cast<string>(noRetInst)#"_gfx9"),563 vt, frag>;564 }565 566 let OtherPredicates = [HasGDS] in {567 def : DSAtomicRetPat<inst, vt,568 !cast<PatFrag>(frag#"_region_m0_"#vt),569 /* complexity */ 0, /* gds */ 1>;570 def : DSAtomicRetPat<noRetInst, vt,571 !cast<PatFrag>(frag#"_region_m0_noret_"#vt),572 /* complexity */ 1, /* gds */ 1>;573 }574}575 576defm DS_ADD_U32 : DS_1A1D_NORET_mc<"ds_add_u32">;577defm DS_SUB_U32 : DS_1A1D_NORET_mc<"ds_sub_u32">;578defm DS_RSUB_U32 : DS_1A1D_NORET_mc<"ds_rsub_u32">;579defm DS_INC_U32 : DS_1A1D_NORET_mc<"ds_inc_u32">;580defm DS_DEC_U32 : DS_1A1D_NORET_mc<"ds_dec_u32">;581defm DS_MIN_I32 : DS_1A1D_NORET_mc<"ds_min_i32">;582defm DS_MAX_I32 : DS_1A1D_NORET_mc<"ds_max_i32">;583defm DS_MIN_U32 : DS_1A1D_NORET_mc<"ds_min_u32">;584defm DS_MAX_U32 : DS_1A1D_NORET_mc<"ds_max_u32">;585defm DS_AND_B32 : DS_1A1D_NORET_mc<"ds_and_b32">;586defm DS_OR_B32 : DS_1A1D_NORET_mc<"ds_or_b32">;587defm DS_XOR_B32 : DS_1A1D_NORET_mc<"ds_xor_b32">;588 589let SubtargetPredicate = HasLDSFPAtomicAddF32 in {590defm DS_ADD_F32 : DS_1A1D_NORET_mc<"ds_add_f32">;591}592 593defm DS_MIN_F32 : DS_1A1D_NORET_mc<"ds_min_f32">;594defm DS_MAX_F32 : DS_1A1D_NORET_mc<"ds_max_f32">;595 596let mayLoad = 0 in {597defm DS_WRITE_B32 : DS_1A1D_NORET_mc<"ds_write_b32">;598defm DS_WRITE2_B32 : DS_1A2D_Off8_NORET_mc<"ds_write2_b32">;599defm DS_WRITE2ST64_B32: DS_1A2D_Off8_NORET_mc<"ds_write2st64_b32">;600 601 602let has_m0_read = 0 in {603 604let SubtargetPredicate = HasD16LoadStore in {605def DS_WRITE_B8_D16_HI : DS_1A1D_NORET<"ds_write_b8_d16_hi">;606def DS_WRITE_B16_D16_HI : DS_1A1D_NORET<"ds_write_b16_d16_hi">;607}608 609} // End has_m0_read = 0610 611defm DS_WRITE_B8 : DS_1A1D_NORET_t16<"ds_write_b8">;612defm DS_WRITE_B16 : DS_1A1D_NORET_t16<"ds_write_b16">;613 614let SubtargetPredicate = HasDSAddTid in {615def DS_WRITE_ADDTID_B32 : DS_0A1D_NORET<"ds_write_addtid_b32">;616}617 618} // End mayLoad = 0619 620let SubtargetPredicate = HasLdsAtomicAddF64 in {621 defm DS_ADD_F64 : DS_1A1D_NORET_mc_gfx9<"ds_add_f64", AVLdSt_64>;622 defm DS_ADD_RTN_F64 : DS_1A1D_RET_mc_gfx9<"ds_add_rtn_f64", VGPROp_64>;623} // End SubtargetPredicate = HasLdsAtomicAddF64624 625let SubtargetPredicate = HasAtomicDsPkAdd16Insts in {626 defm DS_PK_ADD_F16 : DS_1A1D_NORET_mc_gfx9<"ds_pk_add_f16">;627 defm DS_PK_ADD_RTN_F16 : DS_1A1D_RET_mc_gfx9<"ds_pk_add_rtn_f16">;628 defm DS_PK_ADD_BF16 : DS_1A1D_NORET_mc_gfx9<"ds_pk_add_bf16">;629 defm DS_PK_ADD_RTN_BF16 : DS_1A1D_RET_mc_gfx9<"ds_pk_add_rtn_bf16">;630} // End SubtargetPredicate = HasAtomicDsPkAdd16Insts631 632defm DS_CMPSTORE_B32 : DS_1A2D_NORET_mc<"ds_cmpstore_b32">;633defm DS_CMPSTORE_F32 : DS_1A2D_NORET_mc<"ds_cmpstore_f32">;634defm DS_CMPSTORE_B64 : DS_1A2D_NORET_mc<"ds_cmpstore_b64", VGPROp_64>;635defm DS_CMPSTORE_F64 : DS_1A2D_NORET_mc<"ds_cmpstore_f64", VGPROp_64>;636defm DS_CMPSTORE_RTN_B32 : DS_1A2D_RET_mc<"ds_cmpstore_rtn_b32">;637defm DS_CMPSTORE_RTN_F32 : DS_1A2D_RET_mc<"ds_cmpstore_rtn_f32">;638defm DS_CMPSTORE_RTN_B64 : DS_1A2D_RET_mc<"ds_cmpstore_rtn_b64", VGPROp_64>;639defm DS_CMPSTORE_RTN_F64 : DS_1A2D_RET_mc<"ds_cmpstore_rtn_f64", VGPROp_64>;640 641defm DS_MSKOR_B32 : DS_1A2D_NORET_mc<"ds_mskor_b32">;642defm DS_CMPST_B32 : DS_1A2D_NORET_mc<"ds_cmpst_b32">;643defm DS_CMPST_F32 : DS_1A2D_NORET_mc<"ds_cmpst_f32">;644 645defm DS_ADD_U64 : DS_1A1D_NORET_mc<"ds_add_u64", AVLdSt_64>;646defm DS_SUB_U64 : DS_1A1D_NORET_mc<"ds_sub_u64", AVLdSt_64>;647defm DS_RSUB_U64 : DS_1A1D_NORET_mc<"ds_rsub_u64", AVLdSt_64>;648defm DS_INC_U64 : DS_1A1D_NORET_mc<"ds_inc_u64", AVLdSt_64>;649defm DS_DEC_U64 : DS_1A1D_NORET_mc<"ds_dec_u64", AVLdSt_64>;650defm DS_MIN_I64 : DS_1A1D_NORET_mc<"ds_min_i64", AVLdSt_64>;651defm DS_MAX_I64 : DS_1A1D_NORET_mc<"ds_max_i64", AVLdSt_64>;652defm DS_MIN_U64 : DS_1A1D_NORET_mc<"ds_min_u64", AVLdSt_64>;653defm DS_MAX_U64 : DS_1A1D_NORET_mc<"ds_max_u64", AVLdSt_64>;654defm DS_AND_B64 : DS_1A1D_NORET_mc<"ds_and_b64", AVLdSt_64>;655defm DS_OR_B64 : DS_1A1D_NORET_mc<"ds_or_b64", AVLdSt_64>;656defm DS_XOR_B64 : DS_1A1D_NORET_mc<"ds_xor_b64", AVLdSt_64>;657defm DS_MSKOR_B64 : DS_1A2D_NORET_mc<"ds_mskor_b64", VGPROp_64>;658let mayLoad = 0 in {659defm DS_WRITE_B64 : DS_1A1D_NORET_mc<"ds_write_b64", AVLdSt_64>;660defm DS_WRITE2_B64 : DS_1A2D_Off8_NORET_mc<"ds_write2_b64", VGPROp_64>;661defm DS_WRITE2ST64_B64: DS_1A2D_Off8_NORET_mc<"ds_write2st64_b64", VGPROp_64>;662}663defm DS_CMPST_B64 : DS_1A2D_NORET_mc<"ds_cmpst_b64", VGPROp_64>;664defm DS_CMPST_F64 : DS_1A2D_NORET_mc<"ds_cmpst_f64", VGPROp_64>;665defm DS_MIN_F64 : DS_1A1D_NORET_mc<"ds_min_f64", AVLdSt_64>;666defm DS_MAX_F64 : DS_1A1D_NORET_mc<"ds_max_f64", AVLdSt_64>;667 668defm DS_ADD_RTN_U32 : DS_1A1D_RET_mc<"ds_add_rtn_u32">;669 670let SubtargetPredicate = HasLDSFPAtomicAddF32 in {671defm DS_ADD_RTN_F32 : DS_1A1D_RET_mc<"ds_add_rtn_f32">;672}673defm DS_SUB_RTN_U32 : DS_1A1D_RET_mc<"ds_sub_rtn_u32">;674defm DS_RSUB_RTN_U32 : DS_1A1D_RET_mc<"ds_rsub_rtn_u32">;675defm DS_INC_RTN_U32 : DS_1A1D_RET_mc<"ds_inc_rtn_u32">;676defm DS_DEC_RTN_U32 : DS_1A1D_RET_mc<"ds_dec_rtn_u32">;677defm DS_MIN_RTN_I32 : DS_1A1D_RET_mc<"ds_min_rtn_i32">;678defm DS_MAX_RTN_I32 : DS_1A1D_RET_mc<"ds_max_rtn_i32">;679defm DS_MIN_RTN_U32 : DS_1A1D_RET_mc<"ds_min_rtn_u32">;680defm DS_MAX_RTN_U32 : DS_1A1D_RET_mc<"ds_max_rtn_u32">;681defm DS_AND_RTN_B32 : DS_1A1D_RET_mc<"ds_and_rtn_b32">;682defm DS_OR_RTN_B32 : DS_1A1D_RET_mc<"ds_or_rtn_b32">;683defm DS_XOR_RTN_B32 : DS_1A1D_RET_mc<"ds_xor_rtn_b32">;684defm DS_MSKOR_RTN_B32 : DS_1A2D_RET_mc<"ds_mskor_rtn_b32", VGPROp_32>;685defm DS_CMPST_RTN_B32 : DS_1A2D_RET_mc<"ds_cmpst_rtn_b32", VGPROp_32>;686defm DS_CMPST_RTN_F32 : DS_1A2D_RET_mc<"ds_cmpst_rtn_f32", VGPROp_32>;687defm DS_MIN_RTN_F32 : DS_1A1D_RET_mc<"ds_min_rtn_f32">;688defm DS_MAX_RTN_F32 : DS_1A1D_RET_mc<"ds_max_rtn_f32">;689 690defm DS_WRXCHG_RTN_B32 : DS_1A1D_RET_mc<"ds_wrxchg_rtn_b32">;691defm DS_WRXCHG2_RTN_B32 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2_rtn_b32", VGPROp_64, VGPROp_32>;692defm DS_WRXCHG2ST64_RTN_B32 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2st64_rtn_b32", VGPROp_64, VGPROp_32>;693 694defm DS_ADD_RTN_U64 : DS_1A1D_RET_mc<"ds_add_rtn_u64", VGPROp_64>;695defm DS_SUB_RTN_U64 : DS_1A1D_RET_mc<"ds_sub_rtn_u64", VGPROp_64>;696defm DS_RSUB_RTN_U64 : DS_1A1D_RET_mc<"ds_rsub_rtn_u64", VGPROp_64>;697defm DS_INC_RTN_U64 : DS_1A1D_RET_mc<"ds_inc_rtn_u64", VGPROp_64>;698defm DS_DEC_RTN_U64 : DS_1A1D_RET_mc<"ds_dec_rtn_u64", VGPROp_64>;699defm DS_MIN_RTN_I64 : DS_1A1D_RET_mc<"ds_min_rtn_i64", VGPROp_64>;700defm DS_MAX_RTN_I64 : DS_1A1D_RET_mc<"ds_max_rtn_i64", VGPROp_64>;701defm DS_MIN_RTN_U64 : DS_1A1D_RET_mc<"ds_min_rtn_u64", VGPROp_64>;702defm DS_MAX_RTN_U64 : DS_1A1D_RET_mc<"ds_max_rtn_u64", VGPROp_64>;703defm DS_AND_RTN_B64 : DS_1A1D_RET_mc<"ds_and_rtn_b64", VGPROp_64>;704defm DS_OR_RTN_B64 : DS_1A1D_RET_mc<"ds_or_rtn_b64", VGPROp_64>;705defm DS_XOR_RTN_B64 : DS_1A1D_RET_mc<"ds_xor_rtn_b64", VGPROp_64>;706defm DS_MSKOR_RTN_B64 : DS_1A2D_RET_mc<"ds_mskor_rtn_b64", VGPROp_64>;707defm DS_CMPST_RTN_B64 : DS_1A2D_RET_mc<"ds_cmpst_rtn_b64", VGPROp_64>;708defm DS_CMPST_RTN_F64 : DS_1A2D_RET_mc<"ds_cmpst_rtn_f64", VGPROp_64>;709defm DS_MIN_RTN_F64 : DS_1A1D_RET_mc<"ds_min_rtn_f64", VGPROp_64>;710defm DS_MAX_RTN_F64 : DS_1A1D_RET_mc<"ds_max_rtn_f64", VGPROp_64>;711 712defm DS_WRXCHG_RTN_B64 : DS_1A1D_RET_mc<"ds_wrxchg_rtn_b64", VGPROp_64>;713defm DS_WRXCHG2_RTN_B64 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2_rtn_b64", VGPROp_128, VGPROp_64>;714defm DS_WRXCHG2ST64_RTN_B64 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2st64_rtn_b64", VGPROp_128, VGPROp_64>;715 716let isConvergent = 1, usesCustomInserter = 1 in {717def DS_GWS_INIT : DS_GWS_1D<"ds_gws_init"> {718 let mayLoad = 0;719}720def DS_GWS_SEMA_V : DS_GWS_0D<"ds_gws_sema_v">;721def DS_GWS_SEMA_BR : DS_GWS_1D<"ds_gws_sema_br">;722def DS_GWS_SEMA_P : DS_GWS_0D<"ds_gws_sema_p">;723def DS_GWS_BARRIER : DS_GWS_1D<"ds_gws_barrier">;724}725 726let SubtargetPredicate = HasDsSrc2Insts in {727def DS_ADD_SRC2_U32 : DS_1A<"ds_add_src2_u32">;728def DS_SUB_SRC2_U32 : DS_1A<"ds_sub_src2_u32">;729def DS_RSUB_SRC2_U32 : DS_1A<"ds_rsub_src2_u32">;730def DS_INC_SRC2_U32 : DS_1A<"ds_inc_src2_u32">;731def DS_DEC_SRC2_U32 : DS_1A<"ds_dec_src2_u32">;732def DS_MIN_SRC2_I32 : DS_1A<"ds_min_src2_i32">;733def DS_MAX_SRC2_I32 : DS_1A<"ds_max_src2_i32">;734def DS_MIN_SRC2_U32 : DS_1A<"ds_min_src2_u32">;735def DS_MAX_SRC2_U32 : DS_1A<"ds_max_src2_u32">;736def DS_AND_SRC2_B32 : DS_1A<"ds_and_src2_b32">;737def DS_OR_SRC2_B32 : DS_1A<"ds_or_src2_b32">;738def DS_XOR_SRC2_B32 : DS_1A<"ds_xor_src2_b32">;739def DS_MIN_SRC2_F32 : DS_1A<"ds_min_src2_f32">;740def DS_MAX_SRC2_F32 : DS_1A<"ds_max_src2_f32">;741 742def DS_ADD_SRC2_U64 : DS_1A<"ds_add_src2_u64">;743def DS_SUB_SRC2_U64 : DS_1A<"ds_sub_src2_u64">;744def DS_RSUB_SRC2_U64 : DS_1A<"ds_rsub_src2_u64">;745def DS_INC_SRC2_U64 : DS_1A<"ds_inc_src2_u64">;746def DS_DEC_SRC2_U64 : DS_1A<"ds_dec_src2_u64">;747def DS_MIN_SRC2_I64 : DS_1A<"ds_min_src2_i64">;748def DS_MAX_SRC2_I64 : DS_1A<"ds_max_src2_i64">;749def DS_MIN_SRC2_U64 : DS_1A<"ds_min_src2_u64">;750def DS_MAX_SRC2_U64 : DS_1A<"ds_max_src2_u64">;751def DS_AND_SRC2_B64 : DS_1A<"ds_and_src2_b64">;752def DS_OR_SRC2_B64 : DS_1A<"ds_or_src2_b64">;753def DS_XOR_SRC2_B64 : DS_1A<"ds_xor_src2_b64">;754def DS_MIN_SRC2_F64 : DS_1A<"ds_min_src2_f64">;755def DS_MAX_SRC2_F64 : DS_1A<"ds_max_src2_f64">;756 757def DS_WRITE_SRC2_B32 : DS_1A<"ds_write_src2_b32">;758def DS_WRITE_SRC2_B64 : DS_1A<"ds_write_src2_b64">;759} // End SubtargetPredicate = HasDsSrc2Insts760 761let Uses = [EXEC], mayLoad = 0, mayStore = 0, isConvergent = 1 in {762def DS_SWIZZLE_B32 : DS_1A_RET <"ds_swizzle_b32", AVLdSt_32, 0, Swizzle>;763}764 765let mayStore = 0 in {766defm DS_READ_I16 : DS_1A_RET_mc<"ds_read_i16">;767defm DS_READ_B32 : DS_1A_RET_mc<"ds_read_b32">;768defm DS_READ_B64 : DS_1A_RET_mc<"ds_read_b64", AVLdSt_64>;769 770defm DS_READ2_B32 : DS_1A_Off8_RET_mc<"ds_read2_b32", AVLdSt_64>;771defm DS_READ2ST64_B32: DS_1A_Off8_RET_mc<"ds_read2st64_b32", AVLdSt_64>;772 773defm DS_READ2_B64 : DS_1A_Off8_RET_mc<"ds_read2_b64", AVLdSt_128>;774defm DS_READ2ST64_B64: DS_1A_Off8_RET_mc<"ds_read2st64_b64", AVLdSt_128>;775 776let has_m0_read = 0 in {777let SubtargetPredicate = HasD16LoadStore, TiedSourceNotRead = 1 in {778def DS_READ_U8_D16 : DS_1A_RET_Tied<"ds_read_u8_d16">;779def DS_READ_U8_D16_HI : DS_1A_RET_Tied<"ds_read_u8_d16_hi">;780def DS_READ_I8_D16 : DS_1A_RET_Tied<"ds_read_i8_d16">;781def DS_READ_I8_D16_HI : DS_1A_RET_Tied<"ds_read_i8_d16_hi">;782def DS_READ_U16_D16 : DS_1A_RET_Tied<"ds_read_u16_d16">;783def DS_READ_U16_D16_HI : DS_1A_RET_Tied<"ds_read_u16_d16_hi">;784}785} // End has_m0_read = 0786 787defm DS_READ_I8 : DS_1A_RET_t16<"ds_read_i8">;788defm DS_READ_U8 : DS_1A_RET_t16<"ds_read_u8">;789defm DS_READ_U16 : DS_1A_RET_t16<"ds_read_u16">;790 791let SubtargetPredicate = HasDSAddTid in {792def DS_READ_ADDTID_B32 : DS_0A_RET<"ds_read_addtid_b32">;793}794 795} // End mayStore = 0796 797def DS_CONSUME : DS_0A_RET<"ds_consume">;798def DS_APPEND : DS_0A_RET<"ds_append">;799 800let SubtargetPredicate = isNotGFX90APlus in801def DS_ORDERED_COUNT : DS_1A_RET_GDS<"ds_ordered_count">;802 803//===----------------------------------------------------------------------===//804// Instruction definitions for CI and newer.805//===----------------------------------------------------------------------===//806 807let SubtargetPredicate = isGFX7Plus in {808 809defm DS_WRAP_RTN_B32 : DS_1A2D_RET_mc<"ds_wrap_rtn_b32", VGPROp_32>;810defm DS_CONDXCHG32_RTN_B64 : DS_1A1D_RET_mc<"ds_condxchg32_rtn_b64", VGPROp_64>;811 812let isConvergent = 1, usesCustomInserter = 1 in {813def DS_GWS_SEMA_RELEASE_ALL : DS_GWS_0D<"ds_gws_sema_release_all">;814}815 816let mayStore = 0 in {817defm DS_READ_B96 : DS_1A_RET_mc<"ds_read_b96", AVLdSt_96>;818defm DS_READ_B128: DS_1A_RET_mc<"ds_read_b128", AVLdSt_128>;819} // End mayStore = 0820 821let mayLoad = 0 in {822defm DS_WRITE_B96 : DS_1A1D_NORET_mc<"ds_write_b96", AVLdSt_96>;823defm DS_WRITE_B128 : DS_1A1D_NORET_mc<"ds_write_b128", AVLdSt_128>;824} // End mayLoad = 0825 826def DS_NOP : DS_VOID<"ds_nop">;827 828} // let SubtargetPredicate = isGFX7Plus829 830//===----------------------------------------------------------------------===//831// Instruction definitions for VI and newer.832//===----------------------------------------------------------------------===//833 834let SubtargetPredicate = isGFX8Plus in {835 836let Uses = [EXEC] in {837defm DS_PERMUTE_B32 : DS_1A1D_PERMUTE_mc<"ds_permute_b32",838 int_amdgcn_ds_permute>;839defm DS_BPERMUTE_B32 : DS_1A1D_PERMUTE_mc<"ds_bpermute_b32",840 int_amdgcn_ds_bpermute>;841}842 843} // let SubtargetPredicate = isGFX8Plus844 845let SubtargetPredicate = HasLDSFPAtomicAddF32, OtherPredicates = [HasDsSrc2Insts] in {846def DS_ADD_SRC2_F32 : DS_1A<"ds_add_src2_f32">;847}848 849 850//===----------------------------------------------------------------------===//851// Instruction definitions for GFX11.852//===----------------------------------------------------------------------===//853 854let SubtargetPredicate = isGFX11Only in {855 856def DS_ADD_GS_REG_RTN : DS_0A1D_RET_GDS<"ds_add_gs_reg_rtn", VGPROp_64, VGPROp_32>;857def DS_SUB_GS_REG_RTN : DS_0A1D_RET_GDS<"ds_sub_gs_reg_rtn", VGPROp_64, VGPROp_32>;858 859} // let SubtargetPredicate = isGFX11Only860 861let SubtargetPredicate = isGFX11Plus in {862 863let OtherPredicates = [HasImageInsts] in864def DS_BVH_STACK_RTN_B32 : DS_BVH_STACK<"ds_bvh_stack_rtn_b32",865 VGPROp_32, VGPROp_128> ;866 867} // let SubtargetPredicate = isGFX11Plus868 869//===----------------------------------------------------------------------===//870// Instruction definitions for GFX12 and newer.871//===----------------------------------------------------------------------===//872 873let SubtargetPredicate = isGFX12Plus in {874 875let OtherPredicates = [HasImageInsts] in {876def DS_BVH_STACK_PUSH8_POP1_RTN_B32 : DS_BVH_STACK<877 "ds_bvh_stack_push8_pop1_rtn_b32", VGPROp_32, VGPROp_256>;878def DS_BVH_STACK_PUSH8_POP2_RTN_B64 : DS_BVH_STACK<879 "ds_bvh_stack_push8_pop2_rtn_b64", VGPROp_64, VGPROp_256>;880} // End OtherPredicates = [HasImageInsts].881 882defm DS_COND_SUB_U32 : DS_1A1D_NORET_mc_gfx9<"ds_cond_sub_u32">;883defm DS_COND_SUB_RTN_U32 : DS_1A1D_RET_mc_gfx9<"ds_cond_sub_rtn_u32", VGPROp_32>;884defm DS_SUB_CLAMP_U32 : DS_1A1D_NORET_mc_gfx9<"ds_sub_clamp_u32">;885defm DS_SUB_CLAMP_RTN_U32 : DS_1A1D_RET_mc_gfx9<"ds_sub_clamp_rtn_u32", VGPROp_32>;886def DS_BPERMUTE_FI_B32 : DS_1A1D_PERMUTE <"ds_bpermute_fi_b32",887 int_amdgcn_ds_bpermute_fi_b32>;888 889multiclass DSAtomicRetNoRetPatIntrinsic_mc<DS_Pseudo inst, DS_Pseudo noRetInst,890 ValueType vt, string frag> {891 def : DSAtomicRetPat<inst, vt,892 !cast<PatFrag>(frag#"_local_addrspace")>;893 894 let OtherPredicates = [HasAtomicCSubNoRtnInsts] in895 def : DSAtomicRetPat<noRetInst, vt,896 !cast<PatFrag>(frag#"_noret_local_addrspace"), /* complexity */ 1>;897}898 899defm : DSAtomicRetNoRetPatIntrinsic_mc<DS_COND_SUB_RTN_U32, DS_COND_SUB_U32, i32, "int_amdgcn_atomic_cond_sub_u32">;900} // let SubtargetPredicate = isGFX12Plus901 902let SubtargetPredicate = isGFX1250Plus in {903 904let WaveSizePredicate = isWave32, mayStore = 0 in {905let OtherPredicates = [HasTransposeLoadF4F6Insts] in {906defm DS_LOAD_TR4_B64 : DS_1A_RET_NoM0<"ds_load_tr4_b64", VGPROp_64>;907defm DS_LOAD_TR6_B96 : DS_1A_RET_NoM0<"ds_load_tr6_b96", VGPROp_96_Align1>;908} // End OtherPredicates = [HasTransposeLoadF4F6Insts]909defm DS_LOAD_TR8_B64 : DS_1A_RET_NoM0<"ds_load_tr8_b64", VGPROp_64>;910defm DS_LOAD_TR16_B128 : DS_1A_RET_NoM0<"ds_load_tr16_b128", VGPROp_128>;911} // End WaveSizePredicate = isWave32, mayStore = 0912 913let OtherPredicates = [HasLdsBarrierArriveAtomic] in {914let ASYNC_CNT = 1, LGKM_CNT = 0, Uses = [EXEC, ASYNCcnt], Defs = [ASYNCcnt] in {915def DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 : DS_1A_Off16_NORET<"ds_atomic_async_barrier_arrive_b64">;916}917 918def : GCNPat <919 (int_amdgcn_ds_atomic_async_barrier_arrive_b64 (DS1Addr1Offset i32:$ptr, i32:$offset)),920 (DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 VGPR_32:$ptr, Offset:$offset, (i1 0))921>;922 923defm DS_ATOMIC_BARRIER_ARRIVE_RTN_B64 : DS_1A1D_RET_mc_gfx9<"ds_atomic_barrier_arrive_rtn_b64", VGPROp_64>;924 925def : GCNPat<926 (i64 (int_amdgcn_ds_atomic_barrier_arrive_rtn_b64 (DS1Addr1Offset i32:$ptr, i32:$offset), i64:$data)),927 (DS_ATOMIC_BARRIER_ARRIVE_RTN_B64 $ptr, $data, Offset:$offset, (i1 0))928>;929} // End OtherPredicates = [HasLdsBarrierArriveAtomic]930 931} // End SubtargetPredicate = isGFX1250Plus932 933let WaveSizePredicate = isWave64, SubtargetPredicate = HasGFX950Insts, mayStore = 0 in {934 defm DS_READ_B64_TR_B4 : DS_1A_RET_NoM0<"ds_read_b64_tr_b4", AVLdSt_64>;935 defm DS_READ_B64_TR_B8 : DS_1A_RET_NoM0<"ds_read_b64_tr_b8", AVLdSt_64>;936 defm DS_READ_B64_TR_B16 : DS_1A_RET_NoM0<"ds_read_b64_tr_b16", AVLdSt_64>;937 defm DS_READ_B96_TR_B6 : DS_1A_RET_NoM0<"ds_read_b96_tr_b6", AVLdSt_96_Align1>;938}939 940//===----------------------------------------------------------------------===//941// DS Patterns942//===----------------------------------------------------------------------===//943 944def : GCNPat <945 (int_amdgcn_ds_swizzle i32:$src, timm:$offset16),946 (DS_SWIZZLE_B32 VGPR_32:$src, (as_i16timm $offset16), (i1 0))947>;948 949class DSReadPat <DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat <950 (vt (frag (DS1Addr1Offset i32:$ptr, i32:$offset))),951 (inst $ptr, Offset:$offset, (i1 gds))952>;953 954class DSReadPat_t16 <DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat <955 (vt (frag (DS1Addr1Offset i32:$ptr, i32:$offset))),956 (EXTRACT_SUBREG (inst $ptr, Offset:$offset, (i1 gds)), lo16)957>;958 959multiclass DSReadPat_mc<DS_Pseudo inst, ValueType vt, string frag> {960 961 let OtherPredicates = [LDSRequiresM0Init] in {962 def : DSReadPat<inst, vt, !cast<PatFrag>(frag#"_m0")>;963 }964 965 let OtherPredicates = [NotLDSRequiresM0Init] in {966 def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;967 }968}969 970multiclass DSReadPat_t16<DS_Pseudo inst, ValueType vt, string frag> {971 972 let OtherPredicates = [LDSRequiresM0Init] in {973 def : DSReadPat<inst, vt, !cast<PatFrag>(frag#"_m0")>;974 }975 976 let OtherPredicates = [NotLDSRequiresM0Init], True16Predicate = NotUseRealTrue16Insts in {977 def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;978 }979 let OtherPredicates = [NotLDSRequiresM0Init, D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts in {980 def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_t16"), vt, !cast<PatFrag>(frag)>;981 }982 let OtherPredicates = [NotLDSRequiresM0Init], True16Predicate = UseTrue16WithSramECC in {983 def : DSReadPat_t16<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;984 }985}986 987class DSReadPat_D16 <DS_Pseudo inst, PatFrag frag, ValueType vt> : GCNPat <988 (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$in),989 (inst $ptr, Offset:$offset, (i1 0), $in)990>;991 992defm : DSReadPat_mc <DS_READ_I8, i32, "sextloadi8_local">;993defm : DSReadPat_mc <DS_READ_U8, i32, "extloadi8_local">;994defm : DSReadPat_mc <DS_READ_U8, i32, "zextloadi8_local">;995defm : DSReadPat_mc <DS_READ_I16, i32, "sextloadi16_local">;996defm : DSReadPat_mc <DS_READ_U16, i32, "extloadi16_local">;997defm : DSReadPat_mc <DS_READ_U16, i32, "zextloadi16_local">;998defm : DSReadPat_t16 <DS_READ_I8, i16, "sextloadi8_local">;999defm : DSReadPat_t16 <DS_READ_U8, i16, "extloadi8_local">;1000defm : DSReadPat_t16 <DS_READ_U8, i16, "zextloadi8_local">;1001defm : DSReadPat_t16 <DS_READ_U16, i16, "load_local">;1002 1003foreach vt = Reg32Types.types in {1004defm : DSReadPat_mc <DS_READ_B32, vt, "load_local">;1005}1006 1007defm : DSReadPat_t16 <DS_READ_U8, i16, "atomic_load_aext_8_local">;1008defm : DSReadPat_mc <DS_READ_U8, i32, "atomic_load_aext_8_local">;1009defm : DSReadPat_t16 <DS_READ_U8, i16, "atomic_load_zext_8_local">;1010defm : DSReadPat_mc <DS_READ_U8, i32, "atomic_load_zext_8_local">;1011defm : DSReadPat_t16 <DS_READ_I8, i16, "atomic_load_sext_8_local">;1012defm : DSReadPat_mc <DS_READ_I8, i32, "atomic_load_sext_8_local">;1013defm : DSReadPat_t16 <DS_READ_U16, i16, "atomic_load_nonext_16_local">;1014defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_aext_16_local">;1015defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_zext_16_local">;1016defm : DSReadPat_mc <DS_READ_I16, i32, "atomic_load_sext_16_local">;1017defm : DSReadPat_mc <DS_READ_B32, i32, "atomic_load_nonext_32_local">;1018defm : DSReadPat_mc <DS_READ_B64, i64, "atomic_load_nonext_64_local">;1019 1020let OtherPredicates = [D16PreservesUnusedBits] in {1021// TODO: Atomic loads1022def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2i16>;1023def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2f16>;1024def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2i16>;1025def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2f16>;1026def : DSReadPat_D16<DS_READ_I8_D16_HI, sextloadi8_d16_hi_local, v2i16>;1027def : DSReadPat_D16<DS_READ_I8_D16_HI, sextloadi8_d16_hi_local, v2f16>;1028 1029def : DSReadPat_D16<DS_READ_U16_D16, load_d16_lo_local, v2i16>;1030def : DSReadPat_D16<DS_READ_U16_D16, load_d16_lo_local, v2f16>;1031def : DSReadPat_D16<DS_READ_U8_D16, az_extloadi8_d16_lo_local, v2i16>;1032def : DSReadPat_D16<DS_READ_U8_D16, az_extloadi8_d16_lo_local, v2f16>;1033def : DSReadPat_D16<DS_READ_I8_D16, sextloadi8_d16_lo_local, v2i16>;1034def : DSReadPat_D16<DS_READ_I8_D16, sextloadi8_d16_lo_local, v2f16>;1035}1036 1037class DSWritePat <DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat <1038 (frag vt:$value, (DS1Addr1Offset i32:$ptr, i32:$offset)),1039 (inst $ptr, getVregSrcForVT<vt>.ret:$value, Offset:$offset, (i1 gds))1040>;1041 1042multiclass DSWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {1043 let OtherPredicates = [LDSRequiresM0Init] in {1044 def : DSWritePat<inst, vt, !cast<PatFrag>(frag#"_m0")>;1045 }1046 1047 let OtherPredicates = [NotLDSRequiresM0Init] in {1048 def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;1049 }1050}1051 1052multiclass DSWritePat_t16 <DS_Pseudo inst, ValueType vt, string frag> {1053 let OtherPredicates = [LDSRequiresM0Init] in {1054 def : DSWritePat<inst, vt, !cast<PatFrag>(frag#"_m0")>;1055 }1056 1057 let OtherPredicates = [NotLDSRequiresM0Init] in {1058 let True16Predicate = NotUseRealTrue16Insts in {1059 def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;1060 }1061 let True16Predicate = UseRealTrue16Insts in {1062 def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_t16"), vt, !cast<PatFrag>(frag)>;1063 }1064 }1065}1066 1067defm : DSWritePat_mc <DS_WRITE_B8, i32, "truncstorei8_local">;1068defm : DSWritePat_mc <DS_WRITE_B16, i32, "truncstorei16_local">;1069defm : DSWritePat_t16 <DS_WRITE_B8, i16, "truncstorei8_local">;1070defm : DSWritePat_t16 <DS_WRITE_B16, i16, "store_local">;1071 1072foreach vt = Reg32Types.types in {1073defm : DSWritePat_mc <DS_WRITE_B32, vt, "store_local">;1074}1075 1076defm : DSWritePat_t16 <DS_WRITE_B8, i16, "atomic_store_8_local">;1077defm : DSWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">;1078defm : DSWritePat_t16 <DS_WRITE_B16, i16, "atomic_store_16_local">;1079defm : DSWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">;1080defm : DSWritePat_mc <DS_WRITE_B32, i32, "atomic_store_32_local">;1081defm : DSWritePat_mc <DS_WRITE_B64, i64, "atomic_store_64_local">;1082 1083let OtherPredicates = [HasD16LoadStore] in {1084def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_hi16_local>;1085def : DSWritePat <DS_WRITE_B8_D16_HI, i32, truncstorei8_hi16_local>;1086}1087 1088class DS64Bit4ByteAlignedReadPat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <1089 (vt:$value (frag (DS64Bit4ByteAligned i32:$ptr, i32:$offset0, i32:$offset1))),1090 (inst $ptr, $offset0, $offset1, (i1 0))1091>;1092 1093// TODO: Should this use AVLdSt_64 for the class?1094class DS64Bit4ByteAlignedWritePat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat<1095 (frag vt:$value, (DS64Bit4ByteAligned i32:$ptr, i32:$offset0, i32:$offset1)),1096 (inst $ptr, (i32 (EXTRACT_SUBREG VReg_64:$value, sub0)),1097 (i32 (EXTRACT_SUBREG VReg_64:$value, sub1)), $offset0, $offset1,1098 (i1 0))1099>;1100 1101class DS128Bit8ByteAlignedReadPat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <1102 (vt:$value (frag (DS128Bit8ByteAligned i32:$ptr, i32:$offset0, i32:$offset1))),1103 (inst $ptr, $offset0, $offset1, (i1 0))1104>;1105 1106class DS128Bit8ByteAlignedWritePat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat<1107 (frag vt:$value, (DS128Bit8ByteAligned i32:$ptr, i32:$offset0, i32:$offset1)),1108 (inst $ptr, (i64 (EXTRACT_SUBREG VReg_128:$value, sub0_sub1)),1109 (i64 (EXTRACT_SUBREG VReg_128:$value, sub2_sub3)), $offset0, $offset1,1110 (i1 0))1111>;1112 1113multiclass DS64Bit4ByteAlignedPat_mc<ValueType vt> {1114 let OtherPredicates = [LDSRequiresM0Init, isGFX7Plus] in {1115 def : DS64Bit4ByteAlignedReadPat<DS_READ2_B32, vt, load_local_m0>;1116 def : DS64Bit4ByteAlignedWritePat<DS_WRITE2_B32, vt, store_local_m0>;1117 }1118 1119 let OtherPredicates = [NotLDSRequiresM0Init] in {1120 def : DS64Bit4ByteAlignedReadPat<DS_READ2_B32_gfx9, vt, load_local>;1121 def : DS64Bit4ByteAlignedWritePat<DS_WRITE2_B32_gfx9, vt, store_local>;1122 }1123}1124 1125multiclass DS128Bit8ByteAlignedPat_mc<ValueType vt> {1126 let OtherPredicates = [LDSRequiresM0Init, isGFX7Plus] in {1127 def : DS128Bit8ByteAlignedReadPat<DS_READ2_B64, vt, load_local_m0>;1128 def : DS128Bit8ByteAlignedWritePat<DS_WRITE2_B64, vt, store_local_m0>;1129 }1130 1131 let OtherPredicates = [NotLDSRequiresM0Init] in {1132 def : DS128Bit8ByteAlignedReadPat<DS_READ2_B64_gfx9, vt, load_local>;1133 def : DS128Bit8ByteAlignedWritePat<DS_WRITE2_B64_gfx9, vt, store_local>;1134 }1135}1136 1137// v2i32 loads are split into i32 loads on SI during lowering, due to a bug1138// related to bounds checking.1139foreach vt = VReg_64.RegTypes in {1140defm : DS64Bit4ByteAlignedPat_mc<vt>;1141}1142 1143foreach vt = VReg_128.RegTypes in {1144defm : DS128Bit8ByteAlignedPat_mc<vt>;1145}1146 1147// Prefer ds_read over ds_read2 and ds_write over ds_write2, all other things1148// being equal, because it has a larger immediate offset range.1149let AddedComplexity = 100 in {1150 1151foreach vt = VReg_64.RegTypes in {1152defm : DSReadPat_mc <DS_READ_B64, vt, "load_align8_local">;1153defm : DSWritePat_mc <DS_WRITE_B64, vt, "store_align8_local">;1154}1155 1156let SubtargetPredicate = isGFX7Plus in {1157 1158foreach vt = VReg_96.RegTypes in {1159defm : DSReadPat_mc <DS_READ_B96, vt, "load_align16_local">;1160defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_align16_local">;1161}1162 1163foreach vt = VReg_128.RegTypes in {1164defm : DSReadPat_mc <DS_READ_B128, vt, "load_align16_local">;1165defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_align16_local">;1166}1167 1168let SubtargetPredicate = HasUnalignedAccessMode in {1169 1170// Select 64 bit loads and stores aligned less than 4 as a single ds_read_b64/1171// ds_write_b64 instruction as this is faster than ds_read2_b32/ds_write2_b321172// which would be used otherwise. In this case a b32 access would still be1173// misaligned, but we will have 2 of them.1174foreach vt = VReg_64.RegTypes in {1175defm : DSReadPat_mc <DS_READ_B64, vt, "load_align_less_than_4_local">;1176defm : DSWritePat_mc <DS_WRITE_B64, vt, "store_align_less_than_4_local">;1177}1178 1179// Selection will split most of the unaligned 3 dword accesses due to performance1180// reasons when beneficial. Keep these two patterns for the rest of the cases.1181foreach vt = VReg_96.RegTypes in {1182defm : DSReadPat_mc <DS_READ_B96, vt, "load_local">;1183defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_local">;1184}1185 1186// Select 128 bit loads and stores aligned less than 4 as a single ds_read_b128/1187// ds_write_b128 instruction as this is faster than ds_read2_b64/ds_write2_b641188// which would be used otherwise. In this case a b64 access would still be1189// misaligned, but we will have 2 of them.1190foreach vt = VReg_128.RegTypes in {1191defm : DSReadPat_mc <DS_READ_B128, vt, "load_align_less_than_4_local">;1192defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_align_less_than_4_local">;1193}1194 1195} // End SubtargetPredicate = HasUnalignedAccessMode1196 1197} // End SubtargetPredicate = isGFX7Plus1198 1199} // End AddedComplexity = 1001200 1201let SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10 in {1202// Caution, the order of src and cmp is the *opposite* of the BUFFER_ATOMIC_CMPSWAP opcode.1203class DSAtomicCmpXChgSwapped<DS_Pseudo inst, ValueType vt, PatFrag frag,1204 int complexity = 0, bit gds=0> : GCNPat<1205 (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$cmp, vt:$swap),1206 (inst $ptr, getVregSrcForVT<vt>.ret:$cmp, getVregSrcForVT<vt>.ret:$swap, Offset:$offset, (i1 gds))> {1207 let AddedComplexity = complexity;1208}1209 1210multiclass DSAtomicCmpXChgSwapped_mc<DS_Pseudo inst, DS_Pseudo noRetInst, ValueType vt,1211 string frag> {1212 let OtherPredicates = [LDSRequiresM0Init] in {1213 def : DSAtomicCmpXChgSwapped<inst, vt, !cast<PatFrag>(frag#"_local_m0_"#vt)>;1214 def : DSAtomicCmpXChgSwapped<noRetInst, vt, !cast<PatFrag>(frag#"_local_m0_noret_"#vt),1215 /* complexity */ 1>;1216 }1217 1218 let OtherPredicates = [NotLDSRequiresM0Init] in {1219 def : DSAtomicCmpXChgSwapped<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,1220 !cast<PatFrag>(frag#"_local_"#vt)>;1221 def : DSAtomicCmpXChgSwapped<!cast<DS_Pseudo>(!cast<string>(noRetInst)#"_gfx9"), vt,1222 !cast<PatFrag>(frag#"_local_noret_"#vt),1223 /* complexity */ 1>;1224 }1225 1226 let OtherPredicates = [HasGDS] in {1227 def : DSAtomicCmpXChgSwapped<inst, vt, !cast<PatFrag>(frag#"_region_m0_"#vt),1228 /* complexity */ 0, /* gds */ 1>;1229 def : DSAtomicCmpXChgSwapped<noRetInst, vt, !cast<PatFrag>(frag#"_region_m0_noret_"#vt),1230 /* complexity */ 1, /* gds */ 1>;1231 }1232}1233} // End SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX101234 1235let SubtargetPredicate = isGFX11Plus in {1236// The order of src and cmp agrees with the BUFFER_ATOMIC_CMPSWAP opcode.1237class DSAtomicCmpXChg<DS_Pseudo inst, ValueType vt, PatFrag frag,1238 int complexity = 0, bit gds=0> : GCNPat<1239 (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$cmp, vt:$swap),1240 (inst $ptr, getVregSrcForVT<vt>.ret:$swap, getVregSrcForVT<vt>.ret:$cmp, Offset:$offset, (i1 gds))> {1241 let AddedComplexity = complexity;1242}1243 1244multiclass DSAtomicCmpXChg_mc<DS_Pseudo inst, DS_Pseudo noRetInst, ValueType vt, string frag> {1245 1246 def : DSAtomicCmpXChg<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,1247 !cast<PatFrag>(frag#"_local_"#vt)>;1248 def : DSAtomicCmpXChg<!cast<DS_Pseudo>(!cast<string>(noRetInst)#"_gfx9"), vt,1249 !cast<PatFrag>(frag#"_local_noret_"#vt), /* complexity */ 1>;1250 1251 let OtherPredicates = [HasGDS] in {1252 def : DSAtomicCmpXChg<inst, vt, !cast<PatFrag>(frag#"_region_m0_"#vt),1253 /* complexity */ 0, /* gds */ 1>;1254 def : DSAtomicCmpXChg<noRetInst, vt, !cast<PatFrag>(frag#"_region_m0_noret_"#vt),1255 /* complexity */ 1, /* gds */ 1>;1256 }1257}1258} // End SubtargetPredicate = isGFX11Plus1259 1260// 32-bit atomics.1261defm : DSAtomicRetPat_mc<DS_WRXCHG_RTN_B32, i32, "atomic_swap">;1262defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_U32, DS_ADD_U32, i32, "atomic_load_add">;1263defm : DSAtomicRetNoRetPat_mc<DS_SUB_RTN_U32, DS_SUB_U32, i32, "atomic_load_sub">;1264defm : DSAtomicRetNoRetPat_mc<DS_INC_RTN_U32, DS_INC_U32, i32, "atomic_load_uinc_wrap">;1265defm : DSAtomicRetNoRetPat_mc<DS_DEC_RTN_U32, DS_DEC_U32, i32, "atomic_load_udec_wrap">;1266defm : DSAtomicRetNoRetPat_mc<DS_AND_RTN_B32, DS_AND_B32, i32, "atomic_load_and">;1267defm : DSAtomicRetNoRetPat_mc<DS_OR_RTN_B32, DS_OR_B32, i32, "atomic_load_or">;1268defm : DSAtomicRetNoRetPat_mc<DS_XOR_RTN_B32, DS_XOR_B32, i32, "atomic_load_xor">;1269defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_I32, DS_MIN_I32, i32, "atomic_load_min">;1270defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_I32, DS_MAX_I32, i32, "atomic_load_max">;1271defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_U32, DS_MIN_U32, i32, "atomic_load_umin">;1272defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_U32, DS_MAX_U32, i32, "atomic_load_umax">;1273defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_F32, DS_MIN_F32, f32, "atomic_load_fmin">;1274defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_F32, DS_MAX_F32, f32, "atomic_load_fmax">;1275 1276 1277let SubtargetPredicate = HasAtomicDsPkAdd16Insts in {1278defm : DSAtomicRetNoRetPat_NoM0_mc<DS_PK_ADD_RTN_F16, DS_PK_ADD_F16, v2f16, "atomic_load_fadd">;1279defm : DSAtomicRetNoRetPat_NoM0_mc<DS_PK_ADD_RTN_BF16, DS_PK_ADD_BF16, v2bf16, "atomic_load_fadd">;1280}1281 1282let SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10 in {1283defm : DSAtomicCmpXChgSwapped_mc<DS_CMPST_RTN_B32, DS_CMPST_B32, i32, "atomic_cmp_swap">;1284}1285 1286let SubtargetPredicate = isGFX11Plus in {1287defm : DSAtomicCmpXChg_mc<DS_CMPSTORE_RTN_B32, DS_CMPSTORE_B32, i32, "atomic_cmp_swap">;1288}1289 1290let SubtargetPredicate = HasLDSFPAtomicAddF32 in {1291defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_F32, DS_ADD_F32, f32, "atomic_load_fadd">;1292}1293 1294// 64-bit atomics.1295defm : DSAtomicRetPat_mc<DS_WRXCHG_RTN_B64, i64, "atomic_swap">;1296defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_U64, DS_ADD_U64, i64, "atomic_load_add">;1297defm : DSAtomicRetNoRetPat_mc<DS_SUB_RTN_U64, DS_SUB_U64, i64, "atomic_load_sub">;1298defm : DSAtomicRetNoRetPat_mc<DS_INC_RTN_U64, DS_INC_U64, i64, "atomic_load_uinc_wrap">;1299defm : DSAtomicRetNoRetPat_mc<DS_DEC_RTN_U64, DS_DEC_U64, i64, "atomic_load_udec_wrap">;1300defm : DSAtomicRetNoRetPat_mc<DS_AND_RTN_B64, DS_AND_B64, i64, "atomic_load_and">;1301defm : DSAtomicRetNoRetPat_mc<DS_OR_RTN_B64, DS_OR_B64, i64, "atomic_load_or">;1302defm : DSAtomicRetNoRetPat_mc<DS_XOR_RTN_B64, DS_XOR_B64, i64, "atomic_load_xor">;1303defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_I64, DS_MIN_I64, i64, "atomic_load_min">;1304defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_I64, DS_MAX_I64, i64, "atomic_load_max">;1305defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_U64, DS_MIN_U64, i64, "atomic_load_umin">;1306defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_U64, DS_MAX_U64, i64, "atomic_load_umax">;1307defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_F64, DS_MIN_F64, f64, "atomic_load_fmin">;1308defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_F64, DS_MAX_F64, f64, "atomic_load_fmax">;1309 1310let SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10 in {1311defm : DSAtomicCmpXChgSwapped_mc<DS_CMPST_RTN_B64, DS_CMPST_B64, i64, "atomic_cmp_swap">;1312} // End SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX101313 1314let SubtargetPredicate = isGFX11Plus in {1315defm : DSAtomicCmpXChg_mc<DS_CMPSTORE_RTN_B64, DS_CMPSTORE_B64, i64, "atomic_cmp_swap">;1316} // End SubtargetPredicate = isGFX11Plus1317 1318let SubtargetPredicate = HasLdsAtomicAddF64 in {1319def : DSAtomicRetPat<DS_ADD_RTN_F64, f64, atomic_load_fadd_local_f64>;1320let AddedComplexity = 1 in1321def : DSAtomicRetPat<DS_ADD_F64, f64, atomic_load_fadd_local_noret_f64>;1322 1323class DSAtomicRetPatIntrinsic<DS_Pseudo inst, ValueType vt, PatFrag frag,1324 bit gds=0> : GCNPat <1325 (vt (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$value)),1326 (inst $ptr, getVregSrcForVT<vt>.ret:$value, Offset:$offset, (i1 gds))> {1327}1328} // End SubtargetPredicate = HasLdsAtomicAddF641329 1330let SubtargetPredicate = HasAtomicDsPkAdd16Insts in {1331defm : DSAtomicRetNoRetPat_NoM0_mc<DS_PK_ADD_RTN_F16, DS_PK_ADD_F16, v2f16, "atomic_load_fadd">;1332} // End SubtargetPredicate = HasAtomicDsPkAdd16Insts1333 1334let OtherPredicates = [HasGDS] in1335def : GCNPat <1336 (SIds_ordered_count i32:$value, i16:$offset),1337 (DS_ORDERED_COUNT $value, (as_i16imm $offset))1338>;1339 1340def : GCNPat <1341 (i64 (int_amdgcn_ds_add_gs_reg_rtn i32:$src, timm:$offset32)),1342 (DS_ADD_GS_REG_RTN VGPR_32:$src, (as_i32timm $offset32))1343>;1344 1345def : GCNPat <1346 (i32 (int_amdgcn_ds_add_gs_reg_rtn i32:$src, timm:$offset32)),1347 (EXTRACT_SUBREG1348 (i64 (COPY_TO_REGCLASS1349 (DS_ADD_GS_REG_RTN VGPR_32:$src, (as_i32timm $offset32)),1350 VReg_64)),1351 sub0)1352>;1353 1354def : GCNPat <1355 (i64 (int_amdgcn_ds_sub_gs_reg_rtn i32:$src, timm:$offset32)),1356 (DS_SUB_GS_REG_RTN VGPR_32:$src, (as_i32timm $offset32))1357>;1358 1359def : GCNPat <1360 (i32 (int_amdgcn_ds_sub_gs_reg_rtn i32:$src, timm:$offset32)),1361 (EXTRACT_SUBREG1362 (i64 (COPY_TO_REGCLASS1363 (DS_SUB_GS_REG_RTN VGPR_32:$src, (as_i32timm $offset32)),1364 VReg_64)),1365 sub0)1366>;1367 1368class DSLoadTrPat <DS_Pseudo inst, ValueType vt, SDPatternOperator node> : GCNPat <1369 (vt (node (DS1Addr1Offset i32:$ptr, i32:$offset))),1370 (inst $ptr, Offset:$offset, (i1 0))1371>;1372 1373let WaveSizePredicate = isWave32, SubtargetPredicate = isGFX1250Plus in {1374let OtherPredicates = [HasTransposeLoadF4F6Insts] in {1375 def : DSLoadTrPat <DS_LOAD_TR4_B64, v2i32, int_amdgcn_ds_load_tr4_b64>;1376 def : DSLoadTrPat <DS_LOAD_TR6_B96, v3i32, int_amdgcn_ds_load_tr6_b96>;1377} // End OtherPredicates = [HasTransposeLoadF4F6Insts]1378 1379 def : DSLoadTrPat <DS_LOAD_TR8_B64, v2i32, int_amdgcn_ds_load_tr8_b64>;1380 foreach vt = [v8i16, v8f16, v8bf16] in1381 def : DSLoadTrPat <DS_LOAD_TR16_B128, vt, int_amdgcn_ds_load_tr16_b128>;1382} // End WaveSizePredicate = isWave32, SubtargetPredicate = isGFX1250Plus1383 1384let SubtargetPredicate = HasGFX950Insts in {1385 def : DSLoadTrPat <DS_READ_B64_TR_B4, v2i32, int_amdgcn_ds_read_tr4_b64>;1386 def : DSLoadTrPat <DS_READ_B64_TR_B8, v2i32, int_amdgcn_ds_read_tr8_b64>;1387 def : DSLoadTrPat <DS_READ_B96_TR_B6, v3i32, int_amdgcn_ds_read_tr6_b96>;1388 def : DSLoadTrPat <DS_READ_B64_TR_B16, v4i16, int_amdgcn_ds_read_tr16_b64>;1389 def : DSLoadTrPat <DS_READ_B64_TR_B16, v4f16, int_amdgcn_ds_read_tr16_b64>;1390 def : DSLoadTrPat <DS_READ_B64_TR_B16, v4bf16, int_amdgcn_ds_read_tr16_b64>;1391}1392 1393//===----------------------------------------------------------------------===//1394// Target-specific instruction encodings.1395//===----------------------------------------------------------------------===//1396 1397//===----------------------------------------------------------------------===//1398// Base ENC_DS for GFX6, GFX7, GFX10, GFX11, GFX12.1399//===----------------------------------------------------------------------===//1400 1401class Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<bits<8> op, DS_Pseudo ps, int ef,1402 string opName = ps.Mnemonic,1403 bit hasGDS = true>1404 : DS_Real<ps, opName>, SIMCInstr <ps.PseudoInstr, ef> {1405 1406 let Inst{7-0} = !if(ps.has_offset0, offset0, 0);1407 let Inst{15-8} = !if(ps.has_offset1, offset1, 0);1408 let Inst{17} = !if(ps.has_gds, gds, ps.gdsValue);1409 let Inst{25-18} = op;1410 let Inst{31-26} = 0x36;1411 let Inst{39-32} = !if(ps.has_addr, addr, !if(ps.has_gws_data0, data0{7-0}, 0));1412 let Inst{47-40} = !if(ps.has_data0, data0{7-0}, 0);1413 let Inst{55-48} = !if(ps.has_data1, data1{7-0}, 0);1414 let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, 0);1415 1416 let gds = !if(hasGDS, ?, 0);1417}1418 1419//===----------------------------------------------------------------------===//1420// GFX12.1421//===----------------------------------------------------------------------===//1422 1423multiclass DS_Real_gfx12<bits<8> op,1424 DS_Pseudo ps = !cast<DS_Pseudo>(NAME),1425 string name = !tolower(NAME)> {1426 1427 let AssemblerPredicate = isGFX12Plus in {1428 let DecoderNamespace = "GFX12" in1429 def _gfx12 :1430 Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op, ps, SIEncodingFamily.GFX12,1431 name, /*hasGDS=*/false>;1432 if !ne(ps.Mnemonic, name) then1433 def : AMDGPUMnemonicAlias<ps.Mnemonic, name>;1434 } // End AssemblerPredicate1435}1436 1437// Helper to avoid repeating the pseudo-name if we only need to set1438// the gfx12 name.1439multiclass DS_Real_gfx12_with_name<bits<8> op, string name> {1440 defm "" : DS_Real_gfx12<op, !cast<DS_Pseudo>(NAME#"_gfx9"), name>;1441}1442 1443defm DS_MIN_F32 : DS_Real_gfx12_with_name<0x012, "ds_min_num_f32">;1444defm DS_MAX_F32 : DS_Real_gfx12_with_name<0x013, "ds_max_num_f32">;1445defm DS_MIN_RTN_F32 : DS_Real_gfx12_with_name<0x032, "ds_min_num_rtn_f32">;1446defm DS_MAX_RTN_F32 : DS_Real_gfx12_with_name<0x033, "ds_max_num_rtn_f32">;1447defm DS_MIN_F64 : DS_Real_gfx12_with_name<0x052, "ds_min_num_f64">;1448defm DS_MAX_F64 : DS_Real_gfx12_with_name<0x053, "ds_max_num_f64">;1449defm DS_MIN_RTN_F64 : DS_Real_gfx12_with_name<0x072, "ds_min_num_rtn_f64">;1450defm DS_MAX_RTN_F64 : DS_Real_gfx12_with_name<0x073, "ds_max_num_rtn_f64">;1451defm DS_COND_SUB_U32 : DS_Real_gfx12<0x098>;1452defm DS_SUB_CLAMP_U32 : DS_Real_gfx12<0x099>;1453defm DS_COND_SUB_RTN_U32 : DS_Real_gfx12<0x0a8>;1454defm DS_SUB_CLAMP_RTN_U32 : DS_Real_gfx12<0x0a9>;1455defm DS_PK_ADD_F16 : DS_Real_gfx12<0x09a>;1456defm DS_PK_ADD_RTN_F16 : DS_Real_gfx12<0x0aa>;1457defm DS_PK_ADD_BF16 : DS_Real_gfx12<0x09b>;1458defm DS_PK_ADD_RTN_BF16 : DS_Real_gfx12<0x0ab>;1459defm DS_BPERMUTE_FI_B32 : DS_Real_gfx12<0x0cd>;1460 1461defm DS_LOAD_TR4_B64 : DS_Real_gfx12<0x0fa>;1462defm DS_LOAD_TR6_B96 : DS_Real_gfx12<0x0fb>;1463defm DS_LOAD_TR16_B128 : DS_Real_gfx12<0x0fc>;1464defm DS_LOAD_TR8_B64 : DS_Real_gfx12<0x0fd>;1465 1466defm DS_BVH_STACK_RTN_B32 : DS_Real_gfx12<0x0e0, DS_BVH_STACK_RTN_B32,1467 "ds_bvh_stack_push4_pop1_rtn_b32">;1468defm DS_BVH_STACK_PUSH8_POP1_RTN_B32 : DS_Real_gfx12<0x0e1>;1469defm DS_BVH_STACK_PUSH8_POP2_RTN_B64 : DS_Real_gfx12<0x0e2>;1470 1471defm DS_ADD_F64 : DS_Real_gfx12<0x054>;1472defm DS_ADD_RTN_F64 : DS_Real_gfx12<0x074>;1473 1474let AssemblerPredicate = HasLdsBarrierArriveAtomic in {1475defm DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 : DS_Real_gfx12<0x056>;1476defm DS_ATOMIC_BARRIER_ARRIVE_RTN_B64 : DS_Real_gfx12<0x075>;1477}1478 1479// New aliases added in GFX12 without renaming the instructions.1480let AssemblerPredicate = isGFX12Plus in {1481 def : AMDGPUMnemonicAlias<"ds_subrev_u32", "ds_rsub_u32">;1482 def : AMDGPUMnemonicAlias<"ds_subrev_rtn_u32", "ds_rsub_rtn_u32">;1483 def : AMDGPUMnemonicAlias<"ds_subrev_u64", "ds_rsub_u64">;1484 def : AMDGPUMnemonicAlias<"ds_subrev_rtn_u64", "ds_rsub_rtn_u64">;1485}1486 1487// Aliases that have existed since these instructions were introduced.1488def : MnemonicAlias<"ds_load_tr_b64", "ds_load_tr8_b64">, Requires<[isGFX1250Plus]>;1489def : MnemonicAlias<"ds_load_tr_b128", "ds_load_tr16_b128">, Requires<[isGFX1250Plus]>;1490 1491// Additional aliases for ds load transpose instructions.1492def : MnemonicAlias<"ds_load_b64_tr_b8", "ds_load_tr8_b64">, Requires<[isGFX125xOnly]>;1493def : MnemonicAlias<"ds_load_b128_tr_b16", "ds_load_tr16_b128">, Requires<[isGFX125xOnly]>;1494def : MnemonicAlias<"ds_load_b64_tr_b4", "ds_load_tr4_b64">, Requires<[isGFX125xOnly]>;1495def : MnemonicAlias<"ds_load_b96_tr_b6", "ds_load_tr6_b96">, Requires<[isGFX125xOnly]>;1496 1497//===----------------------------------------------------------------------===//1498// GFX11.1499//===----------------------------------------------------------------------===//1500 1501multiclass DS_Real_gfx11<bits<8> op, DS_Pseudo ps = !cast<DS_Pseudo>(NAME#"_gfx9"),1502 string name = !tolower(NAME)> {1503 let AssemblerPredicate = isGFX11Only in {1504 let DecoderNamespace = "GFX11" in1505 def _gfx11 :1506 Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op, ps, SIEncodingFamily.GFX11,1507 name>;1508 if !ne(ps.Mnemonic, name) then1509 def : AMDGPUMnemonicAlias<ps.Mnemonic, name>;1510 } // End AssemblerPredicate1511}1512 1513multiclass DS_Real_gfx11_gfx12<bits<8> op,1514 string name = !tolower(NAME),1515 DS_Pseudo ps = !cast<DS_Pseudo>(NAME#"_gfx9")>1516 : DS_Real_gfx11<op, ps, name>,1517 DS_Real_gfx12<op, ps, name>;1518 1519defm DS_WRITE_B32 : DS_Real_gfx11_gfx12<0x00d, "ds_store_b32">;1520defm DS_WRITE2_B32 : DS_Real_gfx11_gfx12<0x00e, "ds_store_2addr_b32">;1521defm DS_WRITE2ST64_B32 : DS_Real_gfx11_gfx12<0x00f, "ds_store_2addr_stride64_b32">;1522defm DS_WRITE_B8 : DS_Real_gfx11_gfx12<0x01e, "ds_store_b8">;1523defm DS_WRITE_B16 : DS_Real_gfx11_gfx12<0x01f, "ds_store_b16">;1524defm DS_WRXCHG_RTN_B32 : DS_Real_gfx11_gfx12<0x02d, "ds_storexchg_rtn_b32">;1525defm DS_WRXCHG2_RTN_B32 : DS_Real_gfx11_gfx12<0x02e, "ds_storexchg_2addr_rtn_b32">;1526defm DS_WRXCHG2ST64_RTN_B32 : DS_Real_gfx11_gfx12<0x02f, "ds_storexchg_2addr_stride64_rtn_b32">;1527defm DS_READ_B32 : DS_Real_gfx11_gfx12<0x036, "ds_load_b32">;1528defm DS_READ2_B32 : DS_Real_gfx11_gfx12<0x037, "ds_load_2addr_b32">;1529defm DS_READ2ST64_B32 : DS_Real_gfx11_gfx12<0x038, "ds_load_2addr_stride64_b32">;1530defm DS_READ_I8 : DS_Real_gfx11_gfx12<0x039, "ds_load_i8">;1531defm DS_READ_U8 : DS_Real_gfx11_gfx12<0x03a, "ds_load_u8">;1532defm DS_READ_I16 : DS_Real_gfx11_gfx12<0x03b, "ds_load_i16">;1533defm DS_READ_U16 : DS_Real_gfx11_gfx12<0x03c, "ds_load_u16">;1534defm DS_WRITE_B64 : DS_Real_gfx11_gfx12<0x04d, "ds_store_b64">;1535defm DS_WRITE2_B64 : DS_Real_gfx11_gfx12<0x04e, "ds_store_2addr_b64">;1536defm DS_WRITE2ST64_B64 : DS_Real_gfx11_gfx12<0x04f, "ds_store_2addr_stride64_b64">;1537defm DS_WRXCHG_RTN_B64 : DS_Real_gfx11_gfx12<0x06d, "ds_storexchg_rtn_b64">;1538defm DS_WRXCHG2_RTN_B64 : DS_Real_gfx11_gfx12<0x06e, "ds_storexchg_2addr_rtn_b64">;1539defm DS_WRXCHG2ST64_RTN_B64 : DS_Real_gfx11_gfx12<0x06f, "ds_storexchg_2addr_stride64_rtn_b64">;1540defm DS_READ_B64 : DS_Real_gfx11_gfx12<0x076, "ds_load_b64">;1541defm DS_READ2_B64 : DS_Real_gfx11_gfx12<0x077, "ds_load_2addr_b64">;1542defm DS_READ2ST64_B64 : DS_Real_gfx11_gfx12<0x078, "ds_load_2addr_stride64_b64">;1543defm DS_WRITE_B8_D16_HI : DS_Real_gfx11_gfx12<0x0a0, "ds_store_b8_d16_hi", DS_WRITE_B8_D16_HI>;1544defm DS_WRITE_B16_D16_HI : DS_Real_gfx11_gfx12<0x0a1, "ds_store_b16_d16_hi", DS_WRITE_B16_D16_HI>;1545defm DS_READ_U8_D16 : DS_Real_gfx11_gfx12<0x0a2, "ds_load_u8_d16", DS_READ_U8_D16>;1546defm DS_READ_U8_D16_HI : DS_Real_gfx11_gfx12<0x0a3, "ds_load_u8_d16_hi", DS_READ_U8_D16_HI>;1547defm DS_READ_I8_D16 : DS_Real_gfx11_gfx12<0x0a4, "ds_load_i8_d16", DS_READ_I8_D16>;1548defm DS_READ_I8_D16_HI : DS_Real_gfx11_gfx12<0x0a5, "ds_load_i8_d16_hi", DS_READ_I8_D16_HI>;1549defm DS_READ_U16_D16 : DS_Real_gfx11_gfx12<0x0a6, "ds_load_u16_d16", DS_READ_U16_D16>;1550defm DS_READ_U16_D16_HI : DS_Real_gfx11_gfx12<0x0a7, "ds_load_u16_d16_hi", DS_READ_U16_D16_HI>;1551defm DS_WRITE_ADDTID_B32 : DS_Real_gfx11_gfx12<0x0b0, "ds_store_addtid_b32", DS_WRITE_ADDTID_B32>;1552defm DS_READ_ADDTID_B32 : DS_Real_gfx11_gfx12<0x0b1, "ds_load_addtid_b32", DS_READ_ADDTID_B32>;1553defm DS_WRITE_B96 : DS_Real_gfx11_gfx12<0x0de, "ds_store_b96">;1554defm DS_WRITE_B128 : DS_Real_gfx11_gfx12<0x0df, "ds_store_b128">;1555defm DS_READ_B96 : DS_Real_gfx11_gfx12<0x0fe, "ds_load_b96">;1556defm DS_READ_B128 : DS_Real_gfx11_gfx12<0x0ff, "ds_load_b128">;1557 1558// DS_CMPST_* are renamed to DS_CMPSTORE_* in GFX11, but also the data operands (src and cmp) are swapped1559// comparing to pre-GFX11.1560// Note: the mnemonic alias is not generated to avoid a potential ambiguity due to the semantics change.1561 1562defm DS_CMPSTORE_B32 : DS_Real_gfx11_gfx12<0x010>;1563defm DS_CMPSTORE_F32 : DS_Real_gfx11<0x011>;1564defm DS_CMPSTORE_RTN_B32 : DS_Real_gfx11_gfx12<0x030>;1565defm DS_CMPSTORE_RTN_F32 : DS_Real_gfx11<0x031>;1566defm DS_CMPSTORE_B64 : DS_Real_gfx11_gfx12<0x050>;1567defm DS_CMPSTORE_F64 : DS_Real_gfx11<0x051>;1568defm DS_CMPSTORE_RTN_B64 : DS_Real_gfx11_gfx12<0x070>;1569defm DS_CMPSTORE_RTN_F64 : DS_Real_gfx11<0x071>;1570 1571defm DS_ADD_RTN_F32 : DS_Real_gfx11_gfx12<0x079>;1572defm DS_ADD_GS_REG_RTN : DS_Real_gfx11<0x07a, DS_ADD_GS_REG_RTN>;1573defm DS_SUB_GS_REG_RTN : DS_Real_gfx11<0x07b, DS_SUB_GS_REG_RTN>;1574defm DS_BVH_STACK_RTN_B32 : DS_Real_gfx11<0x0ad, DS_BVH_STACK_RTN_B32>;1575 1576//===----------------------------------------------------------------------===//1577// GFX10.1578//===----------------------------------------------------------------------===//1579 1580let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in {1581 multiclass DS_Real_gfx10<bits<8> op, DS_Pseudo ps = !cast<DS_Pseudo>(NAME)> {1582 def _gfx10 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op,1583 ps, SIEncodingFamily.GFX10>;1584 }1585} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10"1586 1587defm DS_ADD_RTN_F32 : DS_Real_gfx10<0x055, DS_ADD_RTN_F32_gfx9>;1588defm DS_WRITE_B8_D16_HI : DS_Real_gfx10<0x0a0>;1589defm DS_WRITE_B16_D16_HI : DS_Real_gfx10<0x0a1>;1590defm DS_READ_U8_D16 : DS_Real_gfx10<0x0a2>;1591defm DS_READ_U8_D16_HI : DS_Real_gfx10<0x0a3>;1592defm DS_READ_I8_D16 : DS_Real_gfx10<0x0a4>;1593defm DS_READ_I8_D16_HI : DS_Real_gfx10<0x0a5>;1594defm DS_READ_U16_D16 : DS_Real_gfx10<0x0a6>;1595defm DS_READ_U16_D16_HI : DS_Real_gfx10<0x0a7>;1596defm DS_WRITE_ADDTID_B32 : DS_Real_gfx10<0x0b0>;1597defm DS_READ_ADDTID_B32 : DS_Real_gfx10<0x0b1>;1598 1599//===----------------------------------------------------------------------===//1600// GFX10, GFX11, GFX12.1601//===----------------------------------------------------------------------===//1602 1603multiclass DS_Real_gfx10_gfx11_gfx12<bits<8> op, DS_Pseudo ps = !cast<DS_Pseudo>(NAME#"_gfx9")> :1604 DS_Real_gfx10<op, ps>,1605 DS_Real_gfx11<op, ps>,1606 DS_Real_gfx12<op, ps>;1607 1608multiclass DS_Real_gfx10_gfx11<bits<8> op, DS_Pseudo ps = !cast<DS_Pseudo>(NAME#"_gfx9")> :1609 DS_Real_gfx10<op, ps>, DS_Real_gfx11<op, ps>;1610 1611defm DS_ADD_F32 : DS_Real_gfx10_gfx11_gfx12<0x015>;1612defm DS_ADD_SRC2_F32 : DS_Real_gfx10<0x095>;1613defm DS_PERMUTE_B32 : DS_Real_gfx10_gfx11_gfx12<0x0b2, DS_PERMUTE_B32>;1614defm DS_BPERMUTE_B32 : DS_Real_gfx10_gfx11_gfx12<0x0b3, DS_BPERMUTE_B32>;1615 1616//===----------------------------------------------------------------------===//1617// GFX7, GFX10, GFX11, GFX12.1618//===----------------------------------------------------------------------===//1619 1620let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in {1621 multiclass DS_Real_gfx7<bits<8> op, DS_Pseudo ps> {1622 def _gfx7 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op,1623 ps, SIEncodingFamily.SI>;1624 }1625} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7"1626 1627multiclass DS_Real_gfx7_gfx10_gfx11_gfx12<bits<8> op,1628 DS_Pseudo ps_gfx6 = !cast<DS_Pseudo>(NAME),1629 DS_Pseudo ps_gfx9 = !cast<DS_Pseudo>(NAME#"_gfx9")> :1630 DS_Real_gfx7<op, ps_gfx6>,1631 DS_Real_gfx10_gfx11_gfx12<op, ps_gfx9>;1632 1633multiclass DS_Real_gfx7_gfx10_gfx11<bits<8> op,1634 DS_Pseudo ps_gfx6 = !cast<DS_Pseudo>(NAME),1635 DS_Pseudo ps_gfx9 = !cast<DS_Pseudo>(NAME#"_gfx9")> :1636 DS_Real_gfx7<op, ps_gfx6>, DS_Real_gfx10_gfx11<op, ps_gfx9>;1637 1638multiclass DS_Real_gfx7_gfx10<bits<8> op,1639 DS_Pseudo ps_gfx6 = !cast<DS_Pseudo>(NAME),1640 DS_Pseudo ps_gfx9 = !cast<DS_Pseudo>(NAME#"_gfx9")> :1641 DS_Real_gfx7<op, ps_gfx6>, DS_Real_gfx10<op, ps_gfx9>;1642 1643// FIXME-GFX7: Add tests when upstreaming this part.1644defm DS_GWS_SEMA_RELEASE_ALL : DS_Real_gfx7_gfx10_gfx11<0x018, DS_GWS_SEMA_RELEASE_ALL, DS_GWS_SEMA_RELEASE_ALL>;1645defm DS_WRAP_RTN_B32 : DS_Real_gfx7_gfx10_gfx11<0x034>;1646defm DS_CONDXCHG32_RTN_B64 : DS_Real_gfx7_gfx10_gfx11_gfx12<0x07e>;1647defm DS_WRITE_B96 : DS_Real_gfx7_gfx10<0x0de>;1648defm DS_WRITE_B128 : DS_Real_gfx7_gfx10<0x0df>;1649defm DS_READ_B96 : DS_Real_gfx7_gfx10<0x0fe>;1650defm DS_READ_B128 : DS_Real_gfx7_gfx10<0x0ff>;1651 1652//===----------------------------------------------------------------------===//1653// GFX6, GFX7, GFX10, GFX11.1654//===----------------------------------------------------------------------===//1655 1656let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {1657 multiclass DS_Real_gfx6_gfx7<bits<8> op, DS_Pseudo ps> {1658 def _gfx6_gfx7 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op,1659 ps, SIEncodingFamily.SI>;1660 }1661} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"1662 1663multiclass DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<bits<8> op,1664 DS_Pseudo ps_gfx6 = !cast<DS_Pseudo>(NAME),1665 DS_Pseudo ps_gfx9 = !cast<DS_Pseudo>(NAME#"_gfx9")> :1666 DS_Real_gfx6_gfx7<op, ps_gfx6>,1667 DS_Real_gfx10_gfx11_gfx12<op, ps_gfx9>;1668 1669multiclass DS_Real_gfx6_gfx7_gfx10_gfx11<bits<8> op,1670 DS_Pseudo ps_gfx6 = !cast<DS_Pseudo>(NAME),1671 DS_Pseudo ps_gfx9 = !cast<DS_Pseudo>(NAME#"_gfx9")> :1672 DS_Real_gfx6_gfx7<op, ps_gfx6>, DS_Real_gfx10_gfx11<op, ps_gfx9>;1673 1674multiclass DS_Real_gfx6_gfx7_gfx10<bits<8> op,1675 DS_Pseudo ps_gfx6 = !cast<DS_Pseudo>(NAME),1676 DS_Pseudo ps_gfx9 = !cast<DS_Pseudo>(NAME#"_gfx9")> :1677 DS_Real_gfx6_gfx7<op, ps_gfx6>, DS_Real_gfx10<op, ps_gfx9>;1678 1679defm DS_ADD_U32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x000>;1680defm DS_SUB_U32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x001>;1681defm DS_RSUB_U32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x002>;1682defm DS_INC_U32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x003>;1683defm DS_DEC_U32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x004>;1684defm DS_MIN_I32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x005>;1685defm DS_MAX_I32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x006>;1686defm DS_MIN_U32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x007>;1687defm DS_MAX_U32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x008>;1688defm DS_AND_B32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x009>;1689defm DS_OR_B32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x00a>;1690defm DS_XOR_B32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x00b>;1691defm DS_MSKOR_B32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x00c>;1692 1693defm DS_WRITE_B32 : DS_Real_gfx6_gfx7_gfx10<0x00d>;1694defm DS_WRITE2_B32 : DS_Real_gfx6_gfx7_gfx10<0x00e>;1695defm DS_WRITE2ST64_B32 : DS_Real_gfx6_gfx7_gfx10<0x00f>;1696defm DS_CMPST_B32 : DS_Real_gfx6_gfx7_gfx10<0x010>;1697defm DS_CMPST_F32 : DS_Real_gfx6_gfx7_gfx10<0x011>;1698 1699defm DS_MIN_F32 : DS_Real_gfx6_gfx7_gfx10_gfx11<0x012>;1700defm DS_MAX_F32 : DS_Real_gfx6_gfx7_gfx10_gfx11<0x013>;1701defm DS_NOP : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x014, DS_NOP, DS_NOP>;1702defm DS_GWS_INIT : DS_Real_gfx6_gfx7_gfx10_gfx11<0x019, DS_GWS_INIT, DS_GWS_INIT>;1703defm DS_GWS_SEMA_V : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01a, DS_GWS_SEMA_V, DS_GWS_SEMA_V>;1704defm DS_GWS_SEMA_BR : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01b, DS_GWS_SEMA_BR, DS_GWS_SEMA_BR>;1705defm DS_GWS_SEMA_P : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01c, DS_GWS_SEMA_P, DS_GWS_SEMA_P>;1706defm DS_GWS_BARRIER : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01d, DS_GWS_BARRIER, DS_GWS_BARRIER>;1707 1708defm DS_WRITE_B8 : DS_Real_gfx6_gfx7_gfx10<0x01e>;1709defm DS_WRITE_B16 : DS_Real_gfx6_gfx7_gfx10<0x01f>;1710 1711defm DS_ADD_RTN_U32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x020>;1712defm DS_SUB_RTN_U32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x021>;1713defm DS_RSUB_RTN_U32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x022>;1714defm DS_INC_RTN_U32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x023>;1715defm DS_DEC_RTN_U32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x024>;1716defm DS_MIN_RTN_I32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x025>;1717defm DS_MAX_RTN_I32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x026>;1718defm DS_MIN_RTN_U32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x027>;1719defm DS_MAX_RTN_U32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x028>;1720defm DS_AND_RTN_B32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x029>;1721defm DS_OR_RTN_B32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x02a>;1722defm DS_XOR_RTN_B32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x02b>;1723defm DS_MSKOR_RTN_B32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x02c>;1724 1725defm DS_WRXCHG_RTN_B32 : DS_Real_gfx6_gfx7_gfx10<0x02d>;1726defm DS_WRXCHG2_RTN_B32 : DS_Real_gfx6_gfx7_gfx10<0x02e>;1727defm DS_WRXCHG2ST64_RTN_B32 : DS_Real_gfx6_gfx7_gfx10<0x02f>;1728defm DS_CMPST_RTN_B32 : DS_Real_gfx6_gfx7_gfx10<0x030>;1729defm DS_CMPST_RTN_F32 : DS_Real_gfx6_gfx7_gfx10<0x031>;1730 1731defm DS_MIN_RTN_F32 : DS_Real_gfx6_gfx7_gfx10_gfx11<0x032>;1732defm DS_MAX_RTN_F32 : DS_Real_gfx6_gfx7_gfx10_gfx11<0x033>;1733defm DS_SWIZZLE_B32 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x035, DS_SWIZZLE_B32, DS_SWIZZLE_B32>;1734 1735defm DS_READ_B32 : DS_Real_gfx6_gfx7_gfx10<0x036>;1736defm DS_READ2_B32 : DS_Real_gfx6_gfx7_gfx10<0x037>;1737defm DS_READ2ST64_B32 : DS_Real_gfx6_gfx7_gfx10<0x038>;1738defm DS_READ_I8 : DS_Real_gfx6_gfx7_gfx10<0x039>;1739defm DS_READ_U8 : DS_Real_gfx6_gfx7_gfx10<0x03a>;1740defm DS_READ_I16 : DS_Real_gfx6_gfx7_gfx10<0x03b>;1741defm DS_READ_U16 : DS_Real_gfx6_gfx7_gfx10<0x03c>;1742 1743defm DS_CONSUME : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x03d, DS_CONSUME, DS_CONSUME>;1744defm DS_APPEND : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x03e, DS_APPEND, DS_APPEND>;1745defm DS_ORDERED_COUNT : DS_Real_gfx6_gfx7_gfx10_gfx11<0x03f, DS_ORDERED_COUNT, DS_ORDERED_COUNT>;1746defm DS_ADD_U64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x040>;1747defm DS_SUB_U64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x041>;1748defm DS_RSUB_U64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x042>;1749defm DS_INC_U64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x043>;1750defm DS_DEC_U64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x044>;1751defm DS_MIN_I64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x045>;1752defm DS_MAX_I64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x046>;1753defm DS_MIN_U64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x047>;1754defm DS_MAX_U64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x048>;1755defm DS_AND_B64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x049>;1756defm DS_OR_B64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x04a>;1757defm DS_XOR_B64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x04b>;1758defm DS_MSKOR_B64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x04c>;1759 1760defm DS_WRITE_B64 : DS_Real_gfx6_gfx7_gfx10<0x04d>;1761defm DS_WRITE2_B64 : DS_Real_gfx6_gfx7_gfx10<0x04e>;1762defm DS_WRITE2ST64_B64 : DS_Real_gfx6_gfx7_gfx10<0x04f>;1763defm DS_CMPST_B64 : DS_Real_gfx6_gfx7_gfx10<0x050>;1764defm DS_CMPST_F64 : DS_Real_gfx6_gfx7_gfx10<0x051>;1765 1766defm DS_MIN_F64 : DS_Real_gfx6_gfx7_gfx10_gfx11<0x052>;1767defm DS_MAX_F64 : DS_Real_gfx6_gfx7_gfx10_gfx11<0x053>;1768defm DS_ADD_RTN_U64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x060>;1769defm DS_SUB_RTN_U64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x061>;1770defm DS_RSUB_RTN_U64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x062>;1771defm DS_INC_RTN_U64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x063>;1772defm DS_DEC_RTN_U64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x064>;1773defm DS_MIN_RTN_I64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x065>;1774defm DS_MAX_RTN_I64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x066>;1775defm DS_MIN_RTN_U64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x067>;1776defm DS_MAX_RTN_U64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x068>;1777defm DS_AND_RTN_B64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x069>;1778defm DS_OR_RTN_B64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x06a>;1779defm DS_XOR_RTN_B64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x06b>;1780defm DS_MSKOR_RTN_B64 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x06c>;1781 1782defm DS_WRXCHG_RTN_B64 : DS_Real_gfx6_gfx7_gfx10<0x06d>;1783defm DS_WRXCHG2_RTN_B64 : DS_Real_gfx6_gfx7_gfx10<0x06e>;1784defm DS_WRXCHG2ST64_RTN_B64 : DS_Real_gfx6_gfx7_gfx10<0x06f>;1785defm DS_CMPST_RTN_B64 : DS_Real_gfx6_gfx7_gfx10<0x070>;1786defm DS_CMPST_RTN_F64 : DS_Real_gfx6_gfx7_gfx10<0x071>;1787 1788defm DS_MIN_RTN_F64 : DS_Real_gfx6_gfx7_gfx10_gfx11<0x072>;1789defm DS_MAX_RTN_F64 : DS_Real_gfx6_gfx7_gfx10_gfx11<0x073>;1790 1791defm DS_READ_B64 : DS_Real_gfx6_gfx7_gfx10<0x076>;1792defm DS_READ2_B64 : DS_Real_gfx6_gfx7_gfx10<0x077>;1793defm DS_READ2ST64_B64 : DS_Real_gfx6_gfx7_gfx10<0x078>;1794defm DS_ADD_SRC2_U32 : DS_Real_gfx6_gfx7_gfx10<0x080, DS_ADD_SRC2_U32, DS_ADD_SRC2_U32>;1795defm DS_SUB_SRC2_U32 : DS_Real_gfx6_gfx7_gfx10<0x081, DS_SUB_SRC2_U32, DS_SUB_SRC2_U32>;1796defm DS_RSUB_SRC2_U32 : DS_Real_gfx6_gfx7_gfx10<0x082, DS_RSUB_SRC2_U32, DS_RSUB_SRC2_U32>;1797defm DS_INC_SRC2_U32 : DS_Real_gfx6_gfx7_gfx10<0x083, DS_INC_SRC2_U32, DS_INC_SRC2_U32>;1798defm DS_DEC_SRC2_U32 : DS_Real_gfx6_gfx7_gfx10<0x084, DS_DEC_SRC2_U32, DS_DEC_SRC2_U32>;1799defm DS_MIN_SRC2_I32 : DS_Real_gfx6_gfx7_gfx10<0x085, DS_MIN_SRC2_I32, DS_MIN_SRC2_I32>;1800defm DS_MAX_SRC2_I32 : DS_Real_gfx6_gfx7_gfx10<0x086, DS_MAX_SRC2_I32, DS_MAX_SRC2_I32>;1801defm DS_MIN_SRC2_U32 : DS_Real_gfx6_gfx7_gfx10<0x087, DS_MIN_SRC2_U32, DS_MIN_SRC2_U32>;1802defm DS_MAX_SRC2_U32 : DS_Real_gfx6_gfx7_gfx10<0x088, DS_MAX_SRC2_U32, DS_MAX_SRC2_U32>;1803defm DS_AND_SRC2_B32 : DS_Real_gfx6_gfx7_gfx10<0x089, DS_AND_SRC2_B32, DS_AND_SRC2_B32>;1804defm DS_OR_SRC2_B32 : DS_Real_gfx6_gfx7_gfx10<0x08a, DS_OR_SRC2_B32, DS_OR_SRC2_B32>;1805defm DS_XOR_SRC2_B32 : DS_Real_gfx6_gfx7_gfx10<0x08b, DS_XOR_SRC2_B32, DS_XOR_SRC2_B32>;1806defm DS_WRITE_SRC2_B32 : DS_Real_gfx6_gfx7_gfx10<0x08d, DS_WRITE_SRC2_B32, DS_WRITE_SRC2_B32>;1807defm DS_MIN_SRC2_F32 : DS_Real_gfx6_gfx7_gfx10<0x092, DS_MIN_SRC2_F32, DS_MIN_SRC2_F32>;1808defm DS_MAX_SRC2_F32 : DS_Real_gfx6_gfx7_gfx10<0x093, DS_MAX_SRC2_F32, DS_MAX_SRC2_F32>;1809defm DS_ADD_SRC2_U64 : DS_Real_gfx6_gfx7_gfx10<0x0c0, DS_ADD_SRC2_U64, DS_ADD_SRC2_U64>;1810defm DS_SUB_SRC2_U64 : DS_Real_gfx6_gfx7_gfx10<0x0c1, DS_SUB_SRC2_U64, DS_SUB_SRC2_U64>;1811defm DS_RSUB_SRC2_U64 : DS_Real_gfx6_gfx7_gfx10<0x0c2, DS_RSUB_SRC2_U64, DS_RSUB_SRC2_U64>;1812defm DS_INC_SRC2_U64 : DS_Real_gfx6_gfx7_gfx10<0x0c3, DS_INC_SRC2_U64, DS_INC_SRC2_U64>;1813defm DS_DEC_SRC2_U64 : DS_Real_gfx6_gfx7_gfx10<0x0c4, DS_DEC_SRC2_U64, DS_DEC_SRC2_U64>;1814defm DS_MIN_SRC2_I64 : DS_Real_gfx6_gfx7_gfx10<0x0c5, DS_MIN_SRC2_I64, DS_MIN_SRC2_I64>;1815defm DS_MAX_SRC2_I64 : DS_Real_gfx6_gfx7_gfx10<0x0c6, DS_MAX_SRC2_I64, DS_MAX_SRC2_I64>;1816defm DS_MIN_SRC2_U64 : DS_Real_gfx6_gfx7_gfx10<0x0c7, DS_MIN_SRC2_U64, DS_MIN_SRC2_U64>;1817defm DS_MAX_SRC2_U64 : DS_Real_gfx6_gfx7_gfx10<0x0c8, DS_MAX_SRC2_U64, DS_MAX_SRC2_U64>;1818defm DS_AND_SRC2_B64 : DS_Real_gfx6_gfx7_gfx10<0x0c9, DS_AND_SRC2_B64, DS_AND_SRC2_B64>;1819defm DS_OR_SRC2_B64 : DS_Real_gfx6_gfx7_gfx10<0x0ca, DS_OR_SRC2_B64, DS_OR_SRC2_B64>;1820defm DS_XOR_SRC2_B64 : DS_Real_gfx6_gfx7_gfx10<0x0cb, DS_XOR_SRC2_B64, DS_XOR_SRC2_B64>;1821defm DS_WRITE_SRC2_B64 : DS_Real_gfx6_gfx7_gfx10<0x0cd, DS_WRITE_SRC2_B64, DS_WRITE_SRC2_B64>;1822defm DS_MIN_SRC2_F64 : DS_Real_gfx6_gfx7_gfx10<0x0d2, DS_MIN_SRC2_F64, DS_MIN_SRC2_F64>;1823defm DS_MAX_SRC2_F64 : DS_Real_gfx6_gfx7_gfx10<0x0d3, DS_MAX_SRC2_F64, DS_MAX_SRC2_F64>;1824 1825//===----------------------------------------------------------------------===//1826// GFX8, GFX9 (VI).1827//===----------------------------------------------------------------------===//1828 1829class DS_Real_Base_vi <bits<8> op, DS_Pseudo ps> :1830 DS_Real <ps>,1831 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.VI> {1832 let AssemblerPredicate = isGFX8GFX9;1833 let DecoderNamespace = "GFX8";1834 1835 // encoding1836 let Inst{7-0} = !if(ps.has_offset0, offset0, 0);1837 let Inst{15-8} = !if(ps.has_offset1, offset1, 0);1838 let Inst{16} = !if(ps.has_gds, gds, ps.gdsValue);1839 let Inst{24-17} = op;1840 let Inst{25} = acc;1841 let Inst{31-26} = 0x36; // ds prefix1842 let Inst{39-32} = !if(ps.has_addr, addr, !if(ps.has_gws_data0, data0{7-0}, 0));1843 let Inst{47-40} = !if(ps.has_data0, data0{7-0}, 0);1844 let Inst{55-48} = !if(ps.has_data1, data1{7-0}, 0);1845 let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, 0);1846}1847 1848 1849multiclass DS_Real_vi <bits<8> op, DS_Pseudo base_pseudo, bit need_gfx9_suffix = true> {1850 def "" : DS_Real_Base_vi<op, base_pseudo>;1851 1852 if need_gfx9_suffix then {1853 def _gfx9 : DS_Real_Base_vi<op, !cast<DS_Pseudo>(!cast<string>(base_pseudo)#"_gfx9")> {1854 let DecoderNamespace = "GFX9";1855 }1856 }1857 1858 // Handle cases that are available in all-AGPR or all-VGPR data1859 // operand forms. This should be used for all DS instructions with 21860 // data operands.1861 defvar agpr_suffixed_name = !cast<string>(base_pseudo)#"_agpr";1862 1863 if !exists<DS_Pseudo>(agpr_suffixed_name) then {1864 def _agpr : DS_Real_Base_vi<op, !cast<DS_Pseudo>(agpr_suffixed_name)> {1865 let DecoderNamespace = "GFX9";1866 let AssemblerPredicate = isGFX90APlus;1867 }1868 }1869}1870 1871// Instructions which use m0 or not for both gfx8 and gfx9 (or did not1872// exist on gfx8)1873multiclass DS_Real_m0_vi<bits<8> op, DS_Pseudo ps> : DS_Real_vi<op, ps, false>;1874 1875defm DS_ADD_U32_vi : DS_Real_vi<0x0, DS_ADD_U32>;1876defm DS_SUB_U32_vi : DS_Real_vi<0x1, DS_SUB_U32>;1877defm DS_RSUB_U32_vi : DS_Real_vi<0x2, DS_RSUB_U32>;1878defm DS_INC_U32_vi : DS_Real_vi<0x3, DS_INC_U32>;1879defm DS_DEC_U32_vi : DS_Real_vi<0x4, DS_DEC_U32>;1880defm DS_MIN_I32_vi : DS_Real_vi<0x5, DS_MIN_I32>;1881defm DS_MAX_I32_vi : DS_Real_vi<0x6, DS_MAX_I32>;1882defm DS_MIN_U32_vi : DS_Real_vi<0x7, DS_MIN_U32>;1883defm DS_MAX_U32_vi : DS_Real_vi<0x8, DS_MAX_U32>;1884defm DS_AND_B32_vi : DS_Real_vi<0x9, DS_AND_B32>;1885defm DS_OR_B32_vi : DS_Real_vi<0xa, DS_OR_B32>;1886defm DS_XOR_B32_vi : DS_Real_vi<0xb, DS_XOR_B32>;1887defm DS_MSKOR_B32_vi : DS_Real_vi<0xc, DS_MSKOR_B32>;1888defm DS_WRITE_B32_vi : DS_Real_vi<0xd, DS_WRITE_B32>;1889defm DS_WRITE2_B32_vi : DS_Real_vi<0xe, DS_WRITE2_B32>;1890defm DS_WRITE2ST64_B32_vi : DS_Real_vi<0xf, DS_WRITE2ST64_B32>;1891 1892defm DS_CMPST_B32_vi : DS_Real_vi<0x10, DS_CMPST_B32>;1893defm DS_CMPST_F32_vi : DS_Real_vi<0x11, DS_CMPST_F32>;1894defm DS_MIN_F32_vi : DS_Real_vi<0x12, DS_MIN_F32>;1895defm DS_MAX_F32_vi : DS_Real_vi<0x13, DS_MAX_F32>;1896defm DS_NOP_vi : DS_Real_m0_vi<0x14, DS_NOP>;1897defm DS_ADD_F32_vi : DS_Real_vi<0x15, DS_ADD_F32>;1898defm DS_GWS_INIT_vi : DS_Real_m0_vi<0x99, DS_GWS_INIT>;1899defm DS_GWS_SEMA_V_vi : DS_Real_m0_vi<0x9a, DS_GWS_SEMA_V>;1900defm DS_GWS_SEMA_BR_vi : DS_Real_m0_vi<0x9b, DS_GWS_SEMA_BR>;1901defm DS_GWS_SEMA_P_vi : DS_Real_m0_vi<0x9c, DS_GWS_SEMA_P>;1902defm DS_GWS_BARRIER_vi : DS_Real_m0_vi<0x9d, DS_GWS_BARRIER>;1903defm DS_WRITE_ADDTID_B32_vi: DS_Real_m0_vi<0x1d, DS_WRITE_ADDTID_B32>;1904defm DS_WRITE_B8_vi : DS_Real_vi<0x1e, DS_WRITE_B8>;1905defm DS_WRITE_B16_vi : DS_Real_vi<0x1f, DS_WRITE_B16>;1906defm DS_ADD_RTN_U32_vi : DS_Real_vi<0x20, DS_ADD_RTN_U32>;1907defm DS_SUB_RTN_U32_vi : DS_Real_vi<0x21, DS_SUB_RTN_U32>;1908defm DS_RSUB_RTN_U32_vi : DS_Real_vi<0x22, DS_RSUB_RTN_U32>;1909defm DS_INC_RTN_U32_vi : DS_Real_vi<0x23, DS_INC_RTN_U32>;1910defm DS_DEC_RTN_U32_vi : DS_Real_vi<0x24, DS_DEC_RTN_U32>;1911defm DS_MIN_RTN_I32_vi : DS_Real_vi<0x25, DS_MIN_RTN_I32>;1912defm DS_MAX_RTN_I32_vi : DS_Real_vi<0x26, DS_MAX_RTN_I32>;1913defm DS_MIN_RTN_U32_vi : DS_Real_vi<0x27, DS_MIN_RTN_U32>;1914defm DS_MAX_RTN_U32_vi : DS_Real_vi<0x28, DS_MAX_RTN_U32>;1915defm DS_AND_RTN_B32_vi : DS_Real_vi<0x29, DS_AND_RTN_B32>;1916defm DS_OR_RTN_B32_vi : DS_Real_vi<0x2a, DS_OR_RTN_B32>;1917defm DS_XOR_RTN_B32_vi : DS_Real_vi<0x2b, DS_XOR_RTN_B32>;1918defm DS_MSKOR_RTN_B32_vi : DS_Real_vi<0x2c, DS_MSKOR_RTN_B32>;1919defm DS_WRXCHG_RTN_B32_vi : DS_Real_vi<0x2d, DS_WRXCHG_RTN_B32>;1920defm DS_WRXCHG2_RTN_B32_vi : DS_Real_vi<0x2e, DS_WRXCHG2_RTN_B32>;1921defm DS_WRXCHG2ST64_RTN_B32_vi : DS_Real_vi<0x2f, DS_WRXCHG2ST64_RTN_B32>;1922defm DS_CMPST_RTN_B32_vi : DS_Real_vi<0x30, DS_CMPST_RTN_B32>;1923defm DS_CMPST_RTN_F32_vi : DS_Real_vi<0x31, DS_CMPST_RTN_F32>;1924defm DS_MIN_RTN_F32_vi : DS_Real_vi<0x32, DS_MIN_RTN_F32>;1925defm DS_MAX_RTN_F32_vi : DS_Real_vi<0x33, DS_MAX_RTN_F32>;1926defm DS_WRAP_RTN_B32_vi : DS_Real_vi<0x34, DS_WRAP_RTN_B32>;1927defm DS_ADD_RTN_F32_vi : DS_Real_vi<0x35, DS_ADD_RTN_F32>;1928defm DS_READ_B32_vi : DS_Real_vi<0x36, DS_READ_B32>;1929defm DS_READ2_B32_vi : DS_Real_vi<0x37, DS_READ2_B32>;1930defm DS_READ2ST64_B32_vi : DS_Real_vi<0x38, DS_READ2ST64_B32>;1931defm DS_READ_I8_vi : DS_Real_vi<0x39, DS_READ_I8>;1932defm DS_READ_U8_vi : DS_Real_vi<0x3a, DS_READ_U8>;1933defm DS_READ_I16_vi : DS_Real_vi<0x3b, DS_READ_I16>;1934defm DS_READ_U16_vi : DS_Real_vi<0x3c, DS_READ_U16>;1935defm DS_READ_ADDTID_B32_vi : DS_Real_m0_vi<0xb6, DS_READ_ADDTID_B32>;1936defm DS_CONSUME_vi : DS_Real_m0_vi<0xbd, DS_CONSUME>;1937defm DS_APPEND_vi : DS_Real_m0_vi<0xbe, DS_APPEND>;1938defm DS_ORDERED_COUNT_vi : DS_Real_m0_vi<0xbf, DS_ORDERED_COUNT>;1939defm DS_SWIZZLE_B32_vi : DS_Real_m0_vi<0x3d, DS_SWIZZLE_B32>;1940defm DS_PERMUTE_B32_vi : DS_Real_m0_vi<0x3e, DS_PERMUTE_B32>;1941defm DS_BPERMUTE_B32_vi : DS_Real_m0_vi<0x3f, DS_BPERMUTE_B32>;1942 1943defm DS_ADD_U64_vi : DS_Real_vi<0x40, DS_ADD_U64>;1944defm DS_SUB_U64_vi : DS_Real_vi<0x41, DS_SUB_U64>;1945defm DS_RSUB_U64_vi : DS_Real_vi<0x42, DS_RSUB_U64>;1946defm DS_INC_U64_vi : DS_Real_vi<0x43, DS_INC_U64>;1947defm DS_DEC_U64_vi : DS_Real_vi<0x44, DS_DEC_U64>;1948defm DS_MIN_I64_vi : DS_Real_vi<0x45, DS_MIN_I64>;1949defm DS_MAX_I64_vi : DS_Real_vi<0x46, DS_MAX_I64>;1950defm DS_MIN_U64_vi : DS_Real_vi<0x47, DS_MIN_U64>;1951defm DS_MAX_U64_vi : DS_Real_vi<0x48, DS_MAX_U64>;1952defm DS_AND_B64_vi : DS_Real_vi<0x49, DS_AND_B64>;1953defm DS_OR_B64_vi : DS_Real_vi<0x4a, DS_OR_B64>;1954defm DS_XOR_B64_vi : DS_Real_vi<0x4b, DS_XOR_B64>;1955defm DS_MSKOR_B64_vi : DS_Real_vi<0x4c, DS_MSKOR_B64>;1956defm DS_WRITE_B64_vi : DS_Real_vi<0x4d, DS_WRITE_B64>;1957defm DS_WRITE2_B64_vi : DS_Real_vi<0x4E, DS_WRITE2_B64>;1958defm DS_WRITE2ST64_B64_vi : DS_Real_vi<0x4f, DS_WRITE2ST64_B64>;1959 1960defm DS_CMPST_B64_vi : DS_Real_vi<0x50, DS_CMPST_B64>;1961defm DS_CMPST_F64_vi : DS_Real_vi<0x51, DS_CMPST_F64>;1962defm DS_MIN_F64_vi : DS_Real_vi<0x52, DS_MIN_F64>;1963defm DS_MAX_F64_vi : DS_Real_vi<0x53, DS_MAX_F64>;1964 1965defm DS_WRITE_B8_D16_HI_vi : DS_Real_m0_vi<0x54, DS_WRITE_B8_D16_HI>;1966defm DS_WRITE_B16_D16_HI_vi: DS_Real_m0_vi<0x55, DS_WRITE_B16_D16_HI>;1967 1968defm DS_READ_U8_D16_vi : DS_Real_m0_vi<0x56, DS_READ_U8_D16>;1969defm DS_READ_U8_D16_HI_vi : DS_Real_m0_vi<0x57, DS_READ_U8_D16_HI>;1970defm DS_READ_I8_D16_vi : DS_Real_m0_vi<0x58, DS_READ_I8_D16>;1971defm DS_READ_I8_D16_HI_vi : DS_Real_m0_vi<0x59, DS_READ_I8_D16_HI>;1972defm DS_READ_U16_D16_vi : DS_Real_m0_vi<0x5a, DS_READ_U16_D16>;1973defm DS_READ_U16_D16_HI_vi: DS_Real_m0_vi<0x5b, DS_READ_U16_D16_HI>;1974 1975defm DS_ADD_RTN_U64_vi : DS_Real_vi<0x60, DS_ADD_RTN_U64>;1976defm DS_SUB_RTN_U64_vi : DS_Real_vi<0x61, DS_SUB_RTN_U64>;1977defm DS_RSUB_RTN_U64_vi : DS_Real_vi<0x62, DS_RSUB_RTN_U64>;1978defm DS_INC_RTN_U64_vi : DS_Real_vi<0x63, DS_INC_RTN_U64>;1979defm DS_DEC_RTN_U64_vi : DS_Real_vi<0x64, DS_DEC_RTN_U64>;1980defm DS_MIN_RTN_I64_vi : DS_Real_vi<0x65, DS_MIN_RTN_I64>;1981defm DS_MAX_RTN_I64_vi : DS_Real_vi<0x66, DS_MAX_RTN_I64>;1982defm DS_MIN_RTN_U64_vi : DS_Real_vi<0x67, DS_MIN_RTN_U64>;1983defm DS_MAX_RTN_U64_vi : DS_Real_vi<0x68, DS_MAX_RTN_U64>;1984defm DS_AND_RTN_B64_vi : DS_Real_vi<0x69, DS_AND_RTN_B64>;1985defm DS_OR_RTN_B64_vi : DS_Real_vi<0x6a, DS_OR_RTN_B64>;1986defm DS_XOR_RTN_B64_vi : DS_Real_vi<0x6b, DS_XOR_RTN_B64>;1987defm DS_MSKOR_RTN_B64_vi : DS_Real_vi<0x6c, DS_MSKOR_RTN_B64>;1988defm DS_WRXCHG_RTN_B64_vi : DS_Real_vi<0x6d, DS_WRXCHG_RTN_B64>;1989defm DS_WRXCHG2_RTN_B64_vi : DS_Real_vi<0x6e, DS_WRXCHG2_RTN_B64>;1990defm DS_WRXCHG2ST64_RTN_B64_vi : DS_Real_vi<0x6f, DS_WRXCHG2ST64_RTN_B64>;1991defm DS_CONDXCHG32_RTN_B64_vi : DS_Real_vi<0x7e, DS_CONDXCHG32_RTN_B64>;1992defm DS_GWS_SEMA_RELEASE_ALL_vi: DS_Real_m0_vi<0x98, DS_GWS_SEMA_RELEASE_ALL>;1993defm DS_CMPST_RTN_B64_vi : DS_Real_vi<0x70, DS_CMPST_RTN_B64>;1994defm DS_CMPST_RTN_F64_vi : DS_Real_vi<0x71, DS_CMPST_RTN_F64>;1995defm DS_MIN_RTN_F64_vi : DS_Real_vi<0x72, DS_MIN_RTN_F64>;1996defm DS_MAX_RTN_F64_vi : DS_Real_vi<0x73, DS_MAX_RTN_F64>;1997 1998defm DS_READ_B64_vi : DS_Real_vi<0x76, DS_READ_B64>;1999defm DS_READ2_B64_vi : DS_Real_vi<0x77, DS_READ2_B64>;2000defm DS_READ2ST64_B64_vi : DS_Real_vi<0x78, DS_READ2ST64_B64>;2001 2002defm DS_ADD_SRC2_U32_vi : DS_Real_m0_vi<0x80, DS_ADD_SRC2_U32>;2003defm DS_SUB_SRC2_U32_vi : DS_Real_m0_vi<0x81, DS_SUB_SRC2_U32>;2004defm DS_RSUB_SRC2_U32_vi : DS_Real_m0_vi<0x82, DS_RSUB_SRC2_U32>;2005defm DS_INC_SRC2_U32_vi : DS_Real_m0_vi<0x83, DS_INC_SRC2_U32>;2006defm DS_DEC_SRC2_U32_vi : DS_Real_m0_vi<0x84, DS_DEC_SRC2_U32>;2007defm DS_MIN_SRC2_I32_vi : DS_Real_m0_vi<0x85, DS_MIN_SRC2_I32>;2008defm DS_MAX_SRC2_I32_vi : DS_Real_m0_vi<0x86, DS_MAX_SRC2_I32>;2009defm DS_MIN_SRC2_U32_vi : DS_Real_m0_vi<0x87, DS_MIN_SRC2_U32>;2010defm DS_MAX_SRC2_U32_vi : DS_Real_m0_vi<0x88, DS_MAX_SRC2_U32>;2011defm DS_AND_SRC2_B32_vi : DS_Real_m0_vi<0x89, DS_AND_SRC2_B32>;2012defm DS_OR_SRC2_B32_vi : DS_Real_m0_vi<0x8a, DS_OR_SRC2_B32>;2013defm DS_XOR_SRC2_B32_vi : DS_Real_m0_vi<0x8b, DS_XOR_SRC2_B32>;2014defm DS_WRITE_SRC2_B32_vi : DS_Real_m0_vi<0x8d, DS_WRITE_SRC2_B32>;2015defm DS_MIN_SRC2_F32_vi : DS_Real_m0_vi<0x92, DS_MIN_SRC2_F32>;2016defm DS_MAX_SRC2_F32_vi : DS_Real_m0_vi<0x93, DS_MAX_SRC2_F32>;2017defm DS_ADD_SRC2_F32_vi : DS_Real_m0_vi<0x95, DS_ADD_SRC2_F32>;2018defm DS_ADD_SRC2_U64_vi : DS_Real_m0_vi<0xc0, DS_ADD_SRC2_U64>;2019defm DS_SUB_SRC2_U64_vi : DS_Real_m0_vi<0xc1, DS_SUB_SRC2_U64>;2020defm DS_RSUB_SRC2_U64_vi : DS_Real_m0_vi<0xc2, DS_RSUB_SRC2_U64>;2021defm DS_INC_SRC2_U64_vi : DS_Real_m0_vi<0xc3, DS_INC_SRC2_U64>;2022defm DS_DEC_SRC2_U64_vi : DS_Real_m0_vi<0xc4, DS_DEC_SRC2_U64>;2023defm DS_MIN_SRC2_I64_vi : DS_Real_m0_vi<0xc5, DS_MIN_SRC2_I64>;2024defm DS_MAX_SRC2_I64_vi : DS_Real_m0_vi<0xc6, DS_MAX_SRC2_I64>;2025defm DS_MIN_SRC2_U64_vi : DS_Real_m0_vi<0xc7, DS_MIN_SRC2_U64>;2026defm DS_MAX_SRC2_U64_vi : DS_Real_m0_vi<0xc8, DS_MAX_SRC2_U64>;2027defm DS_AND_SRC2_B64_vi : DS_Real_m0_vi<0xc9, DS_AND_SRC2_B64>;2028defm DS_OR_SRC2_B64_vi : DS_Real_m0_vi<0xca, DS_OR_SRC2_B64>;2029defm DS_XOR_SRC2_B64_vi : DS_Real_m0_vi<0xcb, DS_XOR_SRC2_B64>;2030defm DS_WRITE_SRC2_B64_vi : DS_Real_m0_vi<0xcd, DS_WRITE_SRC2_B64>;2031defm DS_MIN_SRC2_F64_vi : DS_Real_m0_vi<0xd2, DS_MIN_SRC2_F64>;2032defm DS_MAX_SRC2_F64_vi : DS_Real_m0_vi<0xd3, DS_MAX_SRC2_F64>;2033defm DS_WRITE_B96_vi : DS_Real_vi<0xde, DS_WRITE_B96>;2034defm DS_WRITE_B128_vi : DS_Real_vi<0xdf, DS_WRITE_B128>;2035defm DS_READ_B96_vi : DS_Real_vi<0xfe, DS_READ_B96>;2036defm DS_READ_B128_vi : DS_Real_vi<0xff, DS_READ_B128>;2037 2038// GFX90A+.2039defm DS_ADD_F64_vi : DS_Real_m0_vi<0x5c, DS_ADD_F64>;2040defm DS_ADD_RTN_F64_vi: DS_Real_m0_vi<0x7c, DS_ADD_RTN_F64>;2041 2042// GFX942+.2043defm DS_PK_ADD_F16_vi : DS_Real_m0_vi<0x17, DS_PK_ADD_F16>;2044defm DS_PK_ADD_RTN_F16_vi : DS_Real_m0_vi<0xb7, DS_PK_ADD_RTN_F16>;2045defm DS_PK_ADD_BF16_vi : DS_Real_m0_vi<0x18, DS_PK_ADD_BF16>;2046defm DS_PK_ADD_RTN_BF16_vi: DS_Real_m0_vi<0xb8, DS_PK_ADD_RTN_BF16>;2047 2048//===----------------------------------------------------------------------===//2049// GFX950.2050//===----------------------------------------------------------------------===//2051defm DS_READ_B64_TR_B4_vi : DS_Real_m0_vi<0x0e0, DS_READ_B64_TR_B4>;2052defm DS_READ_B96_TR_B6_vi : DS_Real_m0_vi<0x0e1, DS_READ_B96_TR_B6>;2053defm DS_READ_B64_TR_B8_vi : DS_Real_m0_vi<0x0e2, DS_READ_B64_TR_B8>;2054defm DS_READ_B64_TR_B16_vi: DS_Real_m0_vi<0x0e3, DS_READ_B64_TR_B16>;2055