451 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 < %s | FileCheck %s --check-prefix=GFX123 4define amdgpu_ps void @test_swmmac_f32_16x16x32_f16_index_key(<4 x half> %A, <8 x half> %B, <4 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, ptr addrspace(1) %out3) {5; GFX12-LABEL: test_swmmac_f32_16x16x32_f16_index_key:6; GFX12: ; %bb.0: ; %bb7; GFX12-NEXT: global_load_b32 v10, v[10:11], off8; GFX12-NEXT: v_mov_b32_e32 v23, v99; GFX12-NEXT: v_mov_b32_e32 v22, v810; GFX12-NEXT: v_mov_b32_e32 v21, v711; GFX12-NEXT: v_mov_b32_e32 v20, v612; GFX12-NEXT: v_mov_b32_e32 v27, v913; GFX12-NEXT: v_mov_b32_e32 v26, v814; GFX12-NEXT: v_mov_b32_e32 v25, v715; GFX12-NEXT: v_mov_b32_e32 v24, v616; GFX12-NEXT: v_mov_b32_e32 v31, v917; GFX12-NEXT: v_mov_b32_e32 v30, v818; GFX12-NEXT: v_mov_b32_e32 v29, v719; GFX12-NEXT: v_mov_b32_e32 v28, v620; GFX12-NEXT: s_wait_loadcnt 0x021; GFX12-NEXT: v_swmmac_f32_16x16x32_f16 v[20:23], v[0:1], v[2:5], v1022; GFX12-NEXT: v_swmmac_f32_16x16x32_f16 v[24:27], v[0:1], v[2:5], v10 index_key:123; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)24; GFX12-NEXT: v_swmmac_f32_16x16x32_f16 v[28:31], v[0:1], v[2:5], v10 index_key:225; GFX12-NEXT: v_swmmac_f32_16x16x32_f16 v[6:9], v[0:1], v[2:5], v10 index_key:326; GFX12-NEXT: global_store_b128 v[12:13], v[20:23], off27; GFX12-NEXT: global_store_b128 v[14:15], v[24:27], off28; GFX12-NEXT: global_store_b128 v[16:17], v[28:31], off29; GFX12-NEXT: global_store_b128 v[18:19], v[6:9], off30; GFX12-NEXT: s_endpgm31bb:32 %IndexVec = load <4 x i8>, ptr addrspace(1) %IndexVecPtr, align 433 %Index0 = extractelement <4 x i8> %IndexVec, i32 034 %res0 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16.v4f32.v4f16.v8f16.i8(<4 x half> %A, <8 x half> %B, <4 x float> %C, i8 %Index0)35 store <4 x float> %res0, ptr addrspace(1) %out036 %Index1 = extractelement <4 x i8> %IndexVec, i32 137 %res1 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16.v4f32.v4f16.v8f16.i8(<4 x half> %A, <8 x half> %B, <4 x float> %C, i8 %Index1)38 store <4 x float> %res1, ptr addrspace(1) %out139 %Index2 = extractelement <4 x i8> %IndexVec, i32 240 %res2 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16.v4f32.v4f16.v8f16.i8(<4 x half> %A, <8 x half> %B, <4 x float> %C, i8 %Index2)41 store <4 x float> %res2, ptr addrspace(1) %out242 %Index3 = extractelement <4 x i8> %IndexVec, i32 343 %res3 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16.v4f32.v4f16.v8f16.i8(<4 x half> %A, <8 x half> %B, <4 x float> %C, i8 %Index3)44 store <4 x float> %res3, ptr addrspace(1) %out345 ret void46}47 48define amdgpu_ps void @test_swmmac_f32_16x16x32_bf16_index_key(<4 x i16> %A, <8 x i16> %B, <4 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, ptr addrspace(1) %out3) {49; GFX12-LABEL: test_swmmac_f32_16x16x32_bf16_index_key:50; GFX12: ; %bb.0: ; %bb51; GFX12-NEXT: global_load_b32 v10, v[10:11], off52; GFX12-NEXT: v_mov_b32_e32 v23, v953; GFX12-NEXT: v_mov_b32_e32 v22, v854; GFX12-NEXT: v_mov_b32_e32 v21, v755; GFX12-NEXT: v_mov_b32_e32 v20, v656; GFX12-NEXT: v_mov_b32_e32 v27, v957; GFX12-NEXT: v_mov_b32_e32 v26, v858; GFX12-NEXT: v_mov_b32_e32 v25, v759; GFX12-NEXT: v_mov_b32_e32 v24, v660; GFX12-NEXT: v_mov_b32_e32 v31, v961; GFX12-NEXT: v_mov_b32_e32 v30, v862; GFX12-NEXT: v_mov_b32_e32 v29, v763; GFX12-NEXT: v_mov_b32_e32 v28, v664; GFX12-NEXT: s_wait_loadcnt 0x065; GFX12-NEXT: v_swmmac_f32_16x16x32_bf16 v[20:23], v[0:1], v[2:5], v1066; GFX12-NEXT: v_swmmac_f32_16x16x32_bf16 v[24:27], v[0:1], v[2:5], v10 index_key:167; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)68; GFX12-NEXT: v_swmmac_f32_16x16x32_bf16 v[28:31], v[0:1], v[2:5], v10 index_key:269; GFX12-NEXT: v_swmmac_f32_16x16x32_bf16 v[6:9], v[0:1], v[2:5], v10 index_key:370; GFX12-NEXT: global_store_b128 v[12:13], v[20:23], off71; GFX12-NEXT: global_store_b128 v[14:15], v[24:27], off72; GFX12-NEXT: global_store_b128 v[16:17], v[28:31], off73; GFX12-NEXT: global_store_b128 v[18:19], v[6:9], off74; GFX12-NEXT: s_endpgm75bb:76 %IndexVec = load <4 x i8>, ptr addrspace(1) %IndexVecPtr, align 477 %Index0 = extractelement <4 x i8> %IndexVec, i32 078 %res0 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf16.v4f32.v4i16.v8i16.i8(<4 x i16> %A, <8 x i16> %B, <4 x float> %C, i8 %Index0)79 store <4 x float> %res0, ptr addrspace(1) %out080 %Index1 = extractelement <4 x i8> %IndexVec, i32 181 %res1 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf16.v4f32.v4i16.v8i16.i8(<4 x i16> %A, <8 x i16> %B, <4 x float> %C, i8 %Index1)82 store <4 x float> %res1, ptr addrspace(1) %out183 %Index2 = extractelement <4 x i8> %IndexVec, i32 284 %res2 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf16.v4f32.v4i16.v8i16.i8(<4 x i16> %A, <8 x i16> %B, <4 x float> %C, i8 %Index2)85 store <4 x float> %res2, ptr addrspace(1) %out286 %Index3 = extractelement <4 x i8> %IndexVec, i32 387 %res3 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf16.v4f32.v4i16.v8i16.i8(<4 x i16> %A, <8 x i16> %B, <4 x float> %C, i8 %Index3)88 store <4 x float> %res3, ptr addrspace(1) %out389 ret void90}91 92define amdgpu_ps void @test_swmmac_f16_16x16x32_f16_index_key(<4 x half> %A, <8 x half> %B, <4 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, ptr addrspace(1) %out3) {93; GFX12-LABEL: test_swmmac_f16_16x16x32_f16_index_key:94; GFX12: ; %bb.0: ; %bb95; GFX12-NEXT: global_load_b32 v22, v[8:9], off96; GFX12-NEXT: v_mov_b32_e32 v9, v797; GFX12-NEXT: v_mov_b32_e32 v8, v698; GFX12-NEXT: v_mov_b32_e32 v19, v799; GFX12-NEXT: v_mov_b32_e32 v18, v6100; GFX12-NEXT: v_mov_b32_e32 v21, v7101; GFX12-NEXT: v_mov_b32_e32 v20, v6102; GFX12-NEXT: s_wait_loadcnt 0x0103; GFX12-NEXT: v_swmmac_f16_16x16x32_f16 v[8:9], v[0:1], v[2:5], v22104; GFX12-NEXT: v_swmmac_f16_16x16x32_f16 v[18:19], v[0:1], v[2:5], v22 index_key:1105; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)106; GFX12-NEXT: v_swmmac_f16_16x16x32_f16 v[20:21], v[0:1], v[2:5], v22 index_key:2107; GFX12-NEXT: v_swmmac_f16_16x16x32_f16 v[6:7], v[0:1], v[2:5], v22 index_key:3108; GFX12-NEXT: global_store_b64 v[10:11], v[8:9], off109; GFX12-NEXT: global_store_b64 v[12:13], v[18:19], off110; GFX12-NEXT: global_store_b64 v[14:15], v[20:21], off111; GFX12-NEXT: global_store_b64 v[16:17], v[6:7], off112; GFX12-NEXT: s_endpgm113bb:114 %IndexVec = load <4 x i8>, ptr addrspace(1) %IndexVecPtr, align 4115 %Index0 = extractelement <4 x i8> %IndexVec, i32 0116 %res0 = call <4 x half> @llvm.amdgcn.swmmac.f16.16x16x32.f16.v4f16.v4f16.v8f16.i8(<4 x half> %A, <8 x half> %B, <4 x half> %C, i8 %Index0)117 store <4 x half> %res0, ptr addrspace(1) %out0118 %Index1 = extractelement <4 x i8> %IndexVec, i32 1119 %res1 = call <4 x half> @llvm.amdgcn.swmmac.f16.16x16x32.f16.v4f16.v4f16.v8f16.i8(<4 x half> %A, <8 x half> %B, <4 x half> %C, i8 %Index1)120 store <4 x half> %res1, ptr addrspace(1) %out1121 %Index2 = extractelement <4 x i8> %IndexVec, i32 2122 %res2 = call <4 x half> @llvm.amdgcn.swmmac.f16.16x16x32.f16.v4f16.v4f16.v8f16.i8(<4 x half> %A, <8 x half> %B, <4 x half> %C, i8 %Index2)123 store <4 x half> %res2, ptr addrspace(1) %out2124 %Index3 = extractelement <4 x i8> %IndexVec, i32 3125 %res3 = call <4 x half> @llvm.amdgcn.swmmac.f16.16x16x32.f16.v4f16.v4f16.v8f16.i8(<4 x half> %A, <8 x half> %B, <4 x half> %C, i8 %Index3)126 store <4 x half> %res3, ptr addrspace(1) %out3127 ret void128}129 130define amdgpu_ps void @test_swmmac_bf16_16x16x32_bf16_index_key(<4 x i16> %A, <8 x i16> %B, <4 x i16> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, ptr addrspace(1) %out3) {131; GFX12-LABEL: test_swmmac_bf16_16x16x32_bf16_index_key:132; GFX12: ; %bb.0: ; %bb133; GFX12-NEXT: global_load_b32 v22, v[8:9], off134; GFX12-NEXT: v_mov_b32_e32 v9, v7135; GFX12-NEXT: v_mov_b32_e32 v8, v6136; GFX12-NEXT: v_mov_b32_e32 v19, v7137; GFX12-NEXT: v_mov_b32_e32 v18, v6138; GFX12-NEXT: v_mov_b32_e32 v21, v7139; GFX12-NEXT: v_mov_b32_e32 v20, v6140; GFX12-NEXT: s_wait_loadcnt 0x0141; GFX12-NEXT: v_swmmac_bf16_16x16x32_bf16 v[8:9], v[0:1], v[2:5], v22142; GFX12-NEXT: v_swmmac_bf16_16x16x32_bf16 v[18:19], v[0:1], v[2:5], v22 index_key:1143; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)144; GFX12-NEXT: v_swmmac_bf16_16x16x32_bf16 v[20:21], v[0:1], v[2:5], v22 index_key:2145; GFX12-NEXT: v_swmmac_bf16_16x16x32_bf16 v[6:7], v[0:1], v[2:5], v22 index_key:3146; GFX12-NEXT: global_store_b64 v[10:11], v[8:9], off147; GFX12-NEXT: global_store_b64 v[12:13], v[18:19], off148; GFX12-NEXT: global_store_b64 v[14:15], v[20:21], off149; GFX12-NEXT: global_store_b64 v[16:17], v[6:7], off150; GFX12-NEXT: s_endpgm151bb:152 %IndexVec = load <4 x i8>, ptr addrspace(1) %IndexVecPtr, align 4153 %Index0 = extractelement <4 x i8> %IndexVec, i32 0154 %res0 = call <4 x i16> @llvm.amdgcn.swmmac.bf16.16x16x32.bf16.v4i16.v4i16.v8i16.i8(<4 x i16> %A, <8 x i16> %B, <4 x i16> %C, i8 %Index0)155 store <4 x i16> %res0, ptr addrspace(1) %out0156 %Index1 = extractelement <4 x i8> %IndexVec, i32 1157 %res1 = call <4 x i16> @llvm.amdgcn.swmmac.bf16.16x16x32.bf16.v4i16.v4i16.v8i16.i8(<4 x i16> %A, <8 x i16> %B, <4 x i16> %C, i8 %Index1)158 store <4 x i16> %res1, ptr addrspace(1) %out1159 %Index2 = extractelement <4 x i8> %IndexVec, i32 2160 %res2 = call <4 x i16> @llvm.amdgcn.swmmac.bf16.16x16x32.bf16.v4i16.v4i16.v8i16.i8(<4 x i16> %A, <8 x i16> %B, <4 x i16> %C, i8 %Index2)161 store <4 x i16> %res2, ptr addrspace(1) %out2162 %Index3 = extractelement <4 x i8> %IndexVec, i32 3163 %res3 = call <4 x i16> @llvm.amdgcn.swmmac.bf16.16x16x32.bf16.v4i16.v4i16.v8i16.i8(<4 x i16> %A, <8 x i16> %B, <4 x i16> %C, i8 %Index3)164 store <4 x i16> %res3, ptr addrspace(1) %out3165 ret void166}167 168define amdgpu_ps void @test_swmmac_i32_16x16x32_iu8_index_key(i32 %A, <2 x i32> %B, <4 x i32> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, ptr addrspace(1) %out3) {169; GFX12-LABEL: test_swmmac_i32_16x16x32_iu8_index_key:170; GFX12: ; %bb.0: ; %bb171; GFX12-NEXT: global_load_b32 v7, v[7:8], off172; GFX12-NEXT: v_mov_b32_e32 v20, v6173; GFX12-NEXT: v_mov_b32_e32 v19, v5174; GFX12-NEXT: v_mov_b32_e32 v18, v4175; GFX12-NEXT: v_mov_b32_e32 v17, v3176; GFX12-NEXT: v_mov_b32_e32 v24, v6177; GFX12-NEXT: v_mov_b32_e32 v23, v5178; GFX12-NEXT: v_mov_b32_e32 v22, v4179; GFX12-NEXT: v_mov_b32_e32 v21, v3180; GFX12-NEXT: v_mov_b32_e32 v28, v6181; GFX12-NEXT: v_mov_b32_e32 v27, v5182; GFX12-NEXT: v_mov_b32_e32 v26, v4183; GFX12-NEXT: v_mov_b32_e32 v25, v3184; GFX12-NEXT: s_wait_loadcnt 0x0185; GFX12-NEXT: v_swmmac_i32_16x16x32_iu8 v[17:20], v0, v[1:2], v7186; GFX12-NEXT: v_swmmac_i32_16x16x32_iu8 v[21:24], v0, v[1:2], v7 index_key:1187; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)188; GFX12-NEXT: v_swmmac_i32_16x16x32_iu8 v[25:28], v0, v[1:2], v7 index_key:2189; GFX12-NEXT: v_swmmac_i32_16x16x32_iu8 v[3:6], v0, v[1:2], v7 index_key:3190; GFX12-NEXT: global_store_b128 v[9:10], v[17:20], off191; GFX12-NEXT: global_store_b128 v[11:12], v[21:24], off192; GFX12-NEXT: global_store_b128 v[13:14], v[25:28], off193; GFX12-NEXT: global_store_b128 v[15:16], v[3:6], off194; GFX12-NEXT: s_endpgm195bb:196 %IndexVec = load <4 x i8>, ptr addrspace(1) %IndexVecPtr, align 4197 %Index0 = extractelement <4 x i8> %IndexVec, i32 0198 %res0 = call <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x32.iu8.v4i32.i32.v2i32.i8(i1 0, i32 %A, i1 0, <2 x i32> %B, <4 x i32> %C, i8 %Index0, i1 0)199 store <4 x i32> %res0, ptr addrspace(1) %out0200 %Index1 = extractelement <4 x i8> %IndexVec, i32 1201 %res1 = call <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x32.iu8.v4i32.i32.v2i32.i8(i1 0, i32 %A, i1 0, <2 x i32> %B, <4 x i32> %C, i8 %Index1, i1 0)202 store <4 x i32> %res1, ptr addrspace(1) %out1203 %Index2 = extractelement <4 x i8> %IndexVec, i32 2204 %res2 = call <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x32.iu8.v4i32.i32.v2i32.i8(i1 0, i32 %A, i1 0, <2 x i32> %B, <4 x i32> %C, i8 %Index2, i1 0)205 store <4 x i32> %res2, ptr addrspace(1) %out2206 %Index3 = extractelement <4 x i8> %IndexVec, i32 3207 %res3 = call <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x32.iu8.v4i32.i32.v2i32.i8(i1 0, i32 %A, i1 0, <2 x i32> %B, <4 x i32> %C, i8 %Index3, i1 0)208 store <4 x i32> %res3, ptr addrspace(1) %out3209 ret void210}211 212define amdgpu_ps void @test_swmmac_i32_16x16x32_iu4_index_key(i32 %A, i32 %B, <4 x i32> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {213; GFX12-LABEL: test_swmmac_i32_16x16x32_iu4_index_key:214; GFX12: ; %bb.0: ; %bb215; GFX12-NEXT: global_load_b32 v6, v[6:7], off216; GFX12-NEXT: v_mov_b32_e32 v15, v5217; GFX12-NEXT: v_mov_b32_e32 v14, v4218; GFX12-NEXT: v_mov_b32_e32 v13, v3219; GFX12-NEXT: v_mov_b32_e32 v12, v2220; GFX12-NEXT: s_wait_loadcnt 0x0221; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)222; GFX12-NEXT: v_swmmac_i32_16x16x32_iu4 v[12:15], v0, v1, v6223; GFX12-NEXT: v_swmmac_i32_16x16x32_iu4 v[2:5], v0, v1, v6 index_key:1224; GFX12-NEXT: global_store_b128 v[8:9], v[12:15], off225; GFX12-NEXT: global_store_b128 v[10:11], v[2:5], off226; GFX12-NEXT: s_endpgm227bb:228 %IndexVec = load <2 x i16>, ptr addrspace(1) %IndexVecPtr, align 4229 %Index0 = extractelement <2 x i16> %IndexVec, i32 0230 %res0 = call <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x32.iu4.v4i32.i32.i32.i16(i1 0, i32 %A, i1 0, i32 %B, <4 x i32> %C, i16 %Index0, i1 0)231 store <4 x i32> %res0, ptr addrspace(1) %out0232 %Index1 = extractelement <2 x i16> %IndexVec, i32 1233 %res1 = call <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x32.iu4.v4i32.i32.i32.i16(i1 0, i32 %A, i1 0, i32 %B, <4 x i32> %C, i16 %Index1, i1 0)234 store <4 x i32> %res1, ptr addrspace(1) %out1235 ret void236}237 238define amdgpu_ps void @test_swmmac_i32_16x16x64_iu4_index_key(i32 %A, <2 x i32> %B, <4 x i32> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {239; GFX12-LABEL: test_swmmac_i32_16x16x64_iu4_index_key:240; GFX12: ; %bb.0: ; %bb241; GFX12-NEXT: global_load_b32 v7, v[7:8], off242; GFX12-NEXT: v_mov_b32_e32 v16, v6243; GFX12-NEXT: v_mov_b32_e32 v15, v5244; GFX12-NEXT: v_mov_b32_e32 v14, v4245; GFX12-NEXT: v_mov_b32_e32 v13, v3246; GFX12-NEXT: s_wait_loadcnt 0x0247; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)248; GFX12-NEXT: v_swmmac_i32_16x16x64_iu4 v[13:16], v0, v[1:2], v7249; GFX12-NEXT: v_swmmac_i32_16x16x64_iu4 v[3:6], v0, v[1:2], v7 index_key:1250; GFX12-NEXT: global_store_b128 v[9:10], v[13:16], off251; GFX12-NEXT: global_store_b128 v[11:12], v[3:6], off252; GFX12-NEXT: s_endpgm253bb:254 %IndexVec = load <2 x i16>, ptr addrspace(1) %IndexVecPtr, align 4255 %Index0 = extractelement <2 x i16> %IndexVec, i32 0256 %res0 = call <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x64.iu4.v4i32.i32.v2i32.i16(i1 0, i32 %A, i1 0, <2 x i32> %B, <4 x i32> %C, i16 %Index0, i1 0)257 store <4 x i32> %res0, ptr addrspace(1) %out0258 %Index1 = extractelement <2 x i16> %IndexVec, i32 1259 %res1 = call <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x64.iu4.v4i32.i32.v2i32.i16(i1 0, i32 %A, i1 0, <2 x i32> %B, <4 x i32> %C, i16 %Index1, i1 0)260 store <4 x i32> %res1, ptr addrspace(1) %out1261 ret void262}263 264define amdgpu_ps void @test_swmmac_f32_16x16x32_fp8_fp8_index_key(i32 %A, <2 x i32> %B, <4 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, ptr addrspace(1) %out3) {265; GFX12-LABEL: test_swmmac_f32_16x16x32_fp8_fp8_index_key:266; GFX12: ; %bb.0: ; %bb267; GFX12-NEXT: global_load_b32 v7, v[7:8], off268; GFX12-NEXT: v_mov_b32_e32 v20, v6269; GFX12-NEXT: v_mov_b32_e32 v19, v5270; GFX12-NEXT: v_mov_b32_e32 v18, v4271; GFX12-NEXT: v_mov_b32_e32 v17, v3272; GFX12-NEXT: v_mov_b32_e32 v24, v6273; GFX12-NEXT: v_mov_b32_e32 v23, v5274; GFX12-NEXT: v_mov_b32_e32 v22, v4275; GFX12-NEXT: v_mov_b32_e32 v21, v3276; GFX12-NEXT: v_mov_b32_e32 v28, v6277; GFX12-NEXT: v_mov_b32_e32 v27, v5278; GFX12-NEXT: v_mov_b32_e32 v26, v4279; GFX12-NEXT: v_mov_b32_e32 v25, v3280; GFX12-NEXT: s_wait_loadcnt 0x0281; GFX12-NEXT: v_swmmac_f32_16x16x32_fp8_fp8 v[17:20], v0, v[1:2], v7282; GFX12-NEXT: v_swmmac_f32_16x16x32_fp8_fp8 v[21:24], v0, v[1:2], v7 index_key:1283; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)284; GFX12-NEXT: v_swmmac_f32_16x16x32_fp8_fp8 v[25:28], v0, v[1:2], v7 index_key:2285; GFX12-NEXT: v_swmmac_f32_16x16x32_fp8_fp8 v[3:6], v0, v[1:2], v7 index_key:3286; GFX12-NEXT: global_store_b128 v[9:10], v[17:20], off287; GFX12-NEXT: global_store_b128 v[11:12], v[21:24], off288; GFX12-NEXT: global_store_b128 v[13:14], v[25:28], off289; GFX12-NEXT: global_store_b128 v[15:16], v[3:6], off290; GFX12-NEXT: s_endpgm291bb:292 %IndexVec = load <4 x i8>, ptr addrspace(1) %IndexVecPtr, align 4293 %Index0 = extractelement <4 x i8> %IndexVec, i32 0294 %res0 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.fp8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index0)295 store <4 x float> %res0, ptr addrspace(1) %out0296 %Index1 = extractelement <4 x i8> %IndexVec, i32 1297 %res1 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.fp8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index1)298 store <4 x float> %res1, ptr addrspace(1) %out1299 %Index2 = extractelement <4 x i8> %IndexVec, i32 2300 %res2 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.fp8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index2)301 store <4 x float> %res2, ptr addrspace(1) %out2302 %Index3 = extractelement <4 x i8> %IndexVec, i32 3303 %res3 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.fp8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index3)304 store <4 x float> %res3, ptr addrspace(1) %out3305 ret void306}307 308define amdgpu_ps void @test_swmmac_f32_16x16x32_fp8_bf8_index_key(i32 %A, <2 x i32> %B, <4 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, ptr addrspace(1) %out3) {309; GFX12-LABEL: test_swmmac_f32_16x16x32_fp8_bf8_index_key:310; GFX12: ; %bb.0: ; %bb311; GFX12-NEXT: global_load_b32 v7, v[7:8], off312; GFX12-NEXT: v_mov_b32_e32 v20, v6313; GFX12-NEXT: v_mov_b32_e32 v19, v5314; GFX12-NEXT: v_mov_b32_e32 v18, v4315; GFX12-NEXT: v_mov_b32_e32 v17, v3316; GFX12-NEXT: v_mov_b32_e32 v24, v6317; GFX12-NEXT: v_mov_b32_e32 v23, v5318; GFX12-NEXT: v_mov_b32_e32 v22, v4319; GFX12-NEXT: v_mov_b32_e32 v21, v3320; GFX12-NEXT: v_mov_b32_e32 v28, v6321; GFX12-NEXT: v_mov_b32_e32 v27, v5322; GFX12-NEXT: v_mov_b32_e32 v26, v4323; GFX12-NEXT: v_mov_b32_e32 v25, v3324; GFX12-NEXT: s_wait_loadcnt 0x0325; GFX12-NEXT: v_swmmac_f32_16x16x32_fp8_bf8 v[17:20], v0, v[1:2], v7326; GFX12-NEXT: v_swmmac_f32_16x16x32_fp8_bf8 v[21:24], v0, v[1:2], v7 index_key:1327; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)328; GFX12-NEXT: v_swmmac_f32_16x16x32_fp8_bf8 v[25:28], v0, v[1:2], v7 index_key:2329; GFX12-NEXT: v_swmmac_f32_16x16x32_fp8_bf8 v[3:6], v0, v[1:2], v7 index_key:3330; GFX12-NEXT: global_store_b128 v[9:10], v[17:20], off331; GFX12-NEXT: global_store_b128 v[11:12], v[21:24], off332; GFX12-NEXT: global_store_b128 v[13:14], v[25:28], off333; GFX12-NEXT: global_store_b128 v[15:16], v[3:6], off334; GFX12-NEXT: s_endpgm335bb:336 %IndexVec = load <4 x i8>, ptr addrspace(1) %IndexVecPtr, align 4337 %Index0 = extractelement <4 x i8> %IndexVec, i32 0338 %res0 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.bf8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index0)339 store <4 x float> %res0, ptr addrspace(1) %out0340 %Index1 = extractelement <4 x i8> %IndexVec, i32 1341 %res1 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.bf8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index1)342 store <4 x float> %res1, ptr addrspace(1) %out1343 %Index2 = extractelement <4 x i8> %IndexVec, i32 2344 %res2 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.bf8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index2)345 store <4 x float> %res2, ptr addrspace(1) %out2346 %Index3 = extractelement <4 x i8> %IndexVec, i32 3347 %res3 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.bf8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index3)348 store <4 x float> %res3, ptr addrspace(1) %out3349 ret void350}351 352define amdgpu_ps void @test_swmmac_f32_16x16x32_bf8_fp8_index_key(i32 %A, <2 x i32> %B, <4 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, ptr addrspace(1) %out3) {353; GFX12-LABEL: test_swmmac_f32_16x16x32_bf8_fp8_index_key:354; GFX12: ; %bb.0: ; %bb355; GFX12-NEXT: global_load_b32 v7, v[7:8], off356; GFX12-NEXT: v_mov_b32_e32 v20, v6357; GFX12-NEXT: v_mov_b32_e32 v19, v5358; GFX12-NEXT: v_mov_b32_e32 v18, v4359; GFX12-NEXT: v_mov_b32_e32 v17, v3360; GFX12-NEXT: v_mov_b32_e32 v24, v6361; GFX12-NEXT: v_mov_b32_e32 v23, v5362; GFX12-NEXT: v_mov_b32_e32 v22, v4363; GFX12-NEXT: v_mov_b32_e32 v21, v3364; GFX12-NEXT: v_mov_b32_e32 v28, v6365; GFX12-NEXT: v_mov_b32_e32 v27, v5366; GFX12-NEXT: v_mov_b32_e32 v26, v4367; GFX12-NEXT: v_mov_b32_e32 v25, v3368; GFX12-NEXT: s_wait_loadcnt 0x0369; GFX12-NEXT: v_swmmac_f32_16x16x32_bf8_fp8 v[17:20], v0, v[1:2], v7370; GFX12-NEXT: v_swmmac_f32_16x16x32_bf8_fp8 v[21:24], v0, v[1:2], v7 index_key:1371; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)372; GFX12-NEXT: v_swmmac_f32_16x16x32_bf8_fp8 v[25:28], v0, v[1:2], v7 index_key:2373; GFX12-NEXT: v_swmmac_f32_16x16x32_bf8_fp8 v[3:6], v0, v[1:2], v7 index_key:3374; GFX12-NEXT: global_store_b128 v[9:10], v[17:20], off375; GFX12-NEXT: global_store_b128 v[11:12], v[21:24], off376; GFX12-NEXT: global_store_b128 v[13:14], v[25:28], off377; GFX12-NEXT: global_store_b128 v[15:16], v[3:6], off378; GFX12-NEXT: s_endpgm379bb:380 %IndexVec = load <4 x i8>, ptr addrspace(1) %IndexVecPtr, align 4381 %Index0 = extractelement <4 x i8> %IndexVec, i32 0382 %res0 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.fp8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index0)383 store <4 x float> %res0, ptr addrspace(1) %out0384 %Index1 = extractelement <4 x i8> %IndexVec, i32 1385 %res1 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.fp8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index1)386 store <4 x float> %res1, ptr addrspace(1) %out1387 %Index2 = extractelement <4 x i8> %IndexVec, i32 2388 %res2 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.fp8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index2)389 store <4 x float> %res2, ptr addrspace(1) %out2390 %Index3 = extractelement <4 x i8> %IndexVec, i32 3391 %res3 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.fp8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index3)392 store <4 x float> %res3, ptr addrspace(1) %out3393 ret void394}395 396define amdgpu_ps void @test_swmmac_f32_16x16x32_bf8_bf8_index_key(i32 %A, <2 x i32> %B, <4 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, ptr addrspace(1) %out3) {397; GFX12-LABEL: test_swmmac_f32_16x16x32_bf8_bf8_index_key:398; GFX12: ; %bb.0: ; %bb399; GFX12-NEXT: global_load_b32 v7, v[7:8], off400; GFX12-NEXT: v_mov_b32_e32 v20, v6401; GFX12-NEXT: v_mov_b32_e32 v19, v5402; GFX12-NEXT: v_mov_b32_e32 v18, v4403; GFX12-NEXT: v_mov_b32_e32 v17, v3404; GFX12-NEXT: v_mov_b32_e32 v24, v6405; GFX12-NEXT: v_mov_b32_e32 v23, v5406; GFX12-NEXT: v_mov_b32_e32 v22, v4407; GFX12-NEXT: v_mov_b32_e32 v21, v3408; GFX12-NEXT: v_mov_b32_e32 v28, v6409; GFX12-NEXT: v_mov_b32_e32 v27, v5410; GFX12-NEXT: v_mov_b32_e32 v26, v4411; GFX12-NEXT: v_mov_b32_e32 v25, v3412; GFX12-NEXT: s_wait_loadcnt 0x0413; GFX12-NEXT: v_swmmac_f32_16x16x32_bf8_bf8 v[17:20], v0, v[1:2], v7414; GFX12-NEXT: v_swmmac_f32_16x16x32_bf8_bf8 v[21:24], v0, v[1:2], v7 index_key:1415; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)416; GFX12-NEXT: v_swmmac_f32_16x16x32_bf8_bf8 v[25:28], v0, v[1:2], v7 index_key:2417; GFX12-NEXT: v_swmmac_f32_16x16x32_bf8_bf8 v[3:6], v0, v[1:2], v7 index_key:3418; GFX12-NEXT: global_store_b128 v[9:10], v[17:20], off419; GFX12-NEXT: global_store_b128 v[11:12], v[21:24], off420; GFX12-NEXT: global_store_b128 v[13:14], v[25:28], off421; GFX12-NEXT: global_store_b128 v[15:16], v[3:6], off422; GFX12-NEXT: s_endpgm423bb:424 %IndexVec = load <4 x i8>, ptr addrspace(1) %IndexVecPtr, align 4425 %Index0 = extractelement <4 x i8> %IndexVec, i32 0426 %res0 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.bf8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index0)427 store <4 x float> %res0, ptr addrspace(1) %out0428 %Index1 = extractelement <4 x i8> %IndexVec, i32 1429 %res1 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.bf8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index1)430 store <4 x float> %res1, ptr addrspace(1) %out1431 %Index2 = extractelement <4 x i8> %IndexVec, i32 2432 %res2 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.bf8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index2)433 store <4 x float> %res2, ptr addrspace(1) %out2434 %Index3 = extractelement <4 x i8> %IndexVec, i32 3435 %res3 = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.bf8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index3)436 store <4 x float> %res3, ptr addrspace(1) %out3437 ret void438}439 440declare <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16.v4f32.v4f16.v8f16.i8(<4 x half>, <8 x half>, <4 x float>, i8)441declare <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf16.v4f32.v4i16.v8i16.i8(<4 x i16>, <8 x i16>, <4 x float>, i8)442declare <4 x half> @llvm.amdgcn.swmmac.f16.16x16x32.f16.v4f16.v4f16.v8f16.i8(<4 x half>, <8 x half>, <4 x half>, i8)443declare <4 x i16> @llvm.amdgcn.swmmac.bf16.16x16x32.bf16.v4i16.v4i16.v8i16.i8(<4 x i16>, <8 x i16>, <4 x i16>, i8)444declare <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x32.iu8.v4i32.i32.v2i32.i8(i1 immarg, i32, i1 immarg, <2 x i32>, <4 x i32>, i8 %Index, i1 immarg)445declare <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x32.iu4.v4i32.i32.i32.i16(i1 immarg, i32, i1 immarg, i32, <4 x i32>, i16 %Index, i1 immarg)446declare <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x64.iu4.v4i32.i32.v2i32.i16(i1 immarg, i32, i1 immarg, <2 x i32>, <4 x i32>, i16 %Index, i1 immarg)447declare <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.fp8.v4f32.i32.v2i32.i8(i32, <2 x i32>, <4 x float>, i8)448declare <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.bf8.v4f32.i32.v2i32.i8(i32, <2 x i32>, <4 x float>, i8)449declare <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.fp8.v4f32.i32.v2i32.i8(i32, <2 x i32>, <4 x float>, i8)450declare <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.bf8.v4f32.i32.v2i32.i8(i32, <2 x i32>, <4 x float>, i8)451