3340 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 42; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,SDAG %s3; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL %s4 5declare i32 @llvm.amdgcn.workitem.id.x()6 7; --------------------------------------------------------------------8; llvm.amdgcn.smfmac.f32.16x16x64.f169; --------------------------------------------------------------------10 11declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.f16(<8 x half>, <16 x half>, <4 x float>, i32, i32 immarg, i32 immarg)12 13define amdgpu_kernel void @test_smfmac_f32_16x16x64_f16__vgpr(ptr addrspace(1) %arg, <8 x half> %a, <16 x half> %b, i32 %idx) #0 {14; SDAG-LABEL: test_smfmac_f32_16x16x64_f16__vgpr:15; SDAG: ; %bb.0: ; %bb16; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x2417; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x3418; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v019; SDAG-NEXT: v_lshlrev_b32_e32 v0, 4, v020; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x4421; SDAG-NEXT: s_load_dword s16, s[4:5], 0x6422; SDAG-NEXT: s_waitcnt lgkmcnt(0)23; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]24; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[2:3]25; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[0:1]26; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[8:9]27; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[10:11]28; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[12:13]29; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[14:15]30; SDAG-NEXT: v_mov_b32_e32 v1, s1631; SDAG-NEXT: v_mov_b32_e32 v0, 032; SDAG-NEXT: s_waitcnt vmcnt(0)33; SDAG-NEXT: v_smfmac_f32_16x16x64_f16 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:234; SDAG-NEXT: s_nop 735; SDAG-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]36; SDAG-NEXT: s_endpgm37;38; GISEL-LABEL: test_smfmac_f32_16x16x64_f16__vgpr:39; GISEL: ; %bb.0: ; %bb40; GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x2441; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x3442; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v043; GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v044; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x4445; GISEL-NEXT: s_load_dword s16, s[4:5], 0x6446; GISEL-NEXT: s_waitcnt lgkmcnt(0)47; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[6:7]48; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[2:3]49; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]50; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]51; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]52; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]53; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]54; GISEL-NEXT: v_mov_b32_e32 v16, s1655; GISEL-NEXT: s_waitcnt vmcnt(0)56; GISEL-NEXT: s_nop 057; GISEL-NEXT: v_smfmac_f32_16x16x64_f16 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:258; GISEL-NEXT: v_mov_b32_e32 v0, 059; GISEL-NEXT: s_nop 660; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[6:7]61; GISEL-NEXT: s_endpgm62bb:63 %id = call i32 @llvm.amdgcn.workitem.id.x()64 %gep = getelementptr <4 x float>, ptr addrspace(1) %arg, i32 %id65 %in.1 = load <4 x float>, ptr addrspace(1) %gep66 %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.f16(<8 x half> %a, <16 x half> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2)67 store <4 x float> %mai.1, ptr addrspace(1) %arg68 ret void69}70 71define <4 x float> @test_smfmac_f32_16x16x64_f16(<8 x half> %arg0, <16 x half> %arg1, <4 x float> %arg2, i32 %arg3) {72; GCN-LABEL: test_smfmac_f32_16x16x64_f16:73; GCN: ; %bb.0:74; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)75; GCN-NEXT: v_smfmac_f32_16x16x64_f16 v[12:15], v[0:3], v[4:11], v1676; GCN-NEXT: s_nop 777; GCN-NEXT: v_mov_b32_e32 v0, v1278; GCN-NEXT: v_mov_b32_e32 v1, v1379; GCN-NEXT: v_mov_b32_e32 v2, v1480; GCN-NEXT: v_mov_b32_e32 v3, v1581; GCN-NEXT: s_setpc_b64 s[30:31]82 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.f16(<8 x half> %arg0, <16 x half> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)83 ret <4 x float> %result84}85 86define <4 x float> @test_smfmac_f32_16x16x64_f16__flags0(<8 x half> %arg0, <16 x half> %arg1, <4 x float> %arg2, i32 %arg3) {87; GCN-LABEL: test_smfmac_f32_16x16x64_f16__flags0:88; GCN: ; %bb.0:89; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)90; GCN-NEXT: v_smfmac_f32_16x16x64_f16 v[12:15], v[0:3], v[4:11], v16 cbsz:1 abid:391; GCN-NEXT: s_nop 792; GCN-NEXT: v_mov_b32_e32 v0, v1293; GCN-NEXT: v_mov_b32_e32 v1, v1394; GCN-NEXT: v_mov_b32_e32 v2, v1495; GCN-NEXT: v_mov_b32_e32 v3, v1596; GCN-NEXT: s_setpc_b64 s[30:31]97 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.f16(<8 x half> %arg0, <16 x half> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)98 ret <4 x float> %result99}100 101define <4 x float> @test_smfmac_f32_16x16x64_f16__flags1(<8 x half> %arg0, <16 x half> %arg1, <4 x float> %arg2, i32 %arg3) {102; GCN-LABEL: test_smfmac_f32_16x16x64_f16__flags1:103; GCN: ; %bb.0:104; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)105; GCN-NEXT: v_smfmac_f32_16x16x64_f16 v[12:15], v[0:3], v[4:11], v16 cbsz:3 abid:1106; GCN-NEXT: s_nop 7107; GCN-NEXT: v_mov_b32_e32 v0, v12108; GCN-NEXT: v_mov_b32_e32 v1, v13109; GCN-NEXT: v_mov_b32_e32 v2, v14110; GCN-NEXT: v_mov_b32_e32 v3, v15111; GCN-NEXT: s_setpc_b64 s[30:31]112 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.f16(<8 x half> %arg0, <16 x half> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)113 ret <4 x float> %result114}115 116define <4 x float> @test_smfmac_f32_16x16x64_f16__sgpr(<8 x half> inreg %arg0, <16 x half> inreg %arg1, <4 x float> inreg %arg2, i32 inreg %arg3) {117; SDAG-LABEL: test_smfmac_f32_16x16x64_f16__sgpr:118; SDAG: ; %bb.0:119; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)120; SDAG-NEXT: v_mov_b32_e32 v14, s0121; SDAG-NEXT: v_mov_b32_e32 v15, s1122; SDAG-NEXT: v_mov_b32_e32 v16, s2123; SDAG-NEXT: v_mov_b32_e32 v17, s3124; SDAG-NEXT: v_mov_b32_e32 v6, s16125; SDAG-NEXT: v_mov_b32_e32 v7, s17126; SDAG-NEXT: v_mov_b32_e32 v8, s18127; SDAG-NEXT: v_mov_b32_e32 v9, s19128; SDAG-NEXT: v_mov_b32_e32 v10, s20129; SDAG-NEXT: v_mov_b32_e32 v11, s21130; SDAG-NEXT: v_mov_b32_e32 v12, s22131; SDAG-NEXT: v_mov_b32_e32 v13, s23132; SDAG-NEXT: v_mov_b32_e32 v0, s24133; SDAG-NEXT: v_mov_b32_e32 v1, s25134; SDAG-NEXT: v_mov_b32_e32 v2, s26135; SDAG-NEXT: v_mov_b32_e32 v3, s27136; SDAG-NEXT: v_mov_b32_e32 v4, s28137; SDAG-NEXT: s_nop 1138; SDAG-NEXT: v_smfmac_f32_16x16x64_f16 v[0:3], v[14:17], v[6:13], v4139; SDAG-NEXT: s_setpc_b64 s[30:31]140;141; GISEL-LABEL: test_smfmac_f32_16x16x64_f16__sgpr:142; GISEL: ; %bb.0:143; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)144; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[2:3]145; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]146; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17]147; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19]148; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[20:21]149; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[22:23]150; GISEL-NEXT: v_mov_b32_e32 v0, s24151; GISEL-NEXT: v_mov_b32_e32 v1, s25152; GISEL-NEXT: v_mov_b32_e32 v2, s26153; GISEL-NEXT: v_mov_b32_e32 v3, s27154; GISEL-NEXT: v_mov_b32_e32 v16, s28155; GISEL-NEXT: s_nop 1156; GISEL-NEXT: v_smfmac_f32_16x16x64_f16 v[0:3], v[12:15], v[4:11], v16157; GISEL-NEXT: s_setpc_b64 s[30:31]158 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.f16(<8 x half> %arg0, <16 x half> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)159 ret <4 x float> %result160}161 162; --------------------------------------------------------------------163; llvm.amdgcn.smfmac.f32.32x32x32.f16164; --------------------------------------------------------------------165 166declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half>, <16 x half>, <16 x float>, i32, i32 immarg, i32 immarg)167 168define amdgpu_kernel void @test_smfmac_f32_32x32x32_f16__vgpr(ptr addrspace(1) %arg, <8 x half> %a, <16 x half> %b, i32 %idx) #0 {169; SDAG-LABEL: test_smfmac_f32_32x32x32_f16__vgpr:170; SDAG: ; %bb.0: ; %bb171; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24172; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34173; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0174; SDAG-NEXT: v_lshlrev_b32_e32 v16, 6, v0175; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x44176; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64177; SDAG-NEXT: s_waitcnt lgkmcnt(0)178; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:48179; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32180; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16181; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]182; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[2:3]183; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[0:1]184; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[14:15]185; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[12:13]186; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[10:11]187; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[8:9]188; SDAG-NEXT: v_mov_b32_e32 v16, s16189; SDAG-NEXT: s_waitcnt vmcnt(0)190; SDAG-NEXT: s_nop 0191; SDAG-NEXT: v_smfmac_f32_32x32x32_f16 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2192; SDAG-NEXT: v_mov_b32_e32 v16, 0193; SDAG-NEXT: s_nop 10194; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32195; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48196; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]197; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16198; SDAG-NEXT: s_endpgm199;200; GISEL-LABEL: test_smfmac_f32_32x32x32_f16__vgpr:201; GISEL: ; %bb.0: ; %bb202; GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24203; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34204; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0205; GISEL-NEXT: v_lshlrev_b32_e32 v16, 6, v0206; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x44207; GISEL-NEXT: s_load_dword s16, s[4:5], 0x64208; GISEL-NEXT: s_waitcnt lgkmcnt(0)209; GISEL-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]210; GISEL-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16211; GISEL-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32212; GISEL-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:48213; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[2:3]214; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[0:1]215; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[14:15]216; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[12:13]217; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[10:11]218; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]219; GISEL-NEXT: v_mov_b32_e32 v28, s16220; GISEL-NEXT: s_waitcnt vmcnt(0)221; GISEL-NEXT: s_nop 0222; GISEL-NEXT: v_smfmac_f32_32x32x32_f16 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2223; GISEL-NEXT: v_mov_b32_e32 v16, 0224; GISEL-NEXT: s_nop 10225; GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]226; GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16227; GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32228; GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48229; GISEL-NEXT: s_endpgm230bb:231 %id = call i32 @llvm.amdgcn.workitem.id.x()232 %gep = getelementptr <16 x float>, ptr addrspace(1) %arg, i32 %id233 %in.1 = load <16 x float>, ptr addrspace(1) %gep234 %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half> %a, <16 x half> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2)235 store <16 x float> %mai.1, ptr addrspace(1) %arg236 ret void237}238 239define <16 x float> @test_smfmac_f32_32x32x32_f16(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3) {240; SDAG-LABEL: test_smfmac_f32_32x32x32_f16:241; SDAG: ; %bb.0:242; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)243; SDAG-NEXT: v_smfmac_f32_32x32x32_f16 v[12:27], v[0:3], v[4:11], v28244; SDAG-NEXT: s_nop 11245; SDAG-NEXT: v_mov_b32_e32 v0, v12246; SDAG-NEXT: v_mov_b32_e32 v1, v13247; SDAG-NEXT: v_mov_b32_e32 v2, v14248; SDAG-NEXT: v_mov_b32_e32 v3, v15249; SDAG-NEXT: v_mov_b32_e32 v4, v16250; SDAG-NEXT: v_mov_b32_e32 v5, v17251; SDAG-NEXT: v_mov_b32_e32 v6, v18252; SDAG-NEXT: v_mov_b32_e32 v7, v19253; SDAG-NEXT: v_mov_b32_e32 v8, v20254; SDAG-NEXT: v_mov_b32_e32 v9, v21255; SDAG-NEXT: v_mov_b32_e32 v10, v22256; SDAG-NEXT: v_mov_b32_e32 v11, v23257; SDAG-NEXT: v_mov_b32_e32 v12, v24258; SDAG-NEXT: v_mov_b32_e32 v13, v25259; SDAG-NEXT: v_mov_b32_e32 v14, v26260; SDAG-NEXT: v_mov_b32_e32 v15, v27261; SDAG-NEXT: s_setpc_b64 s[30:31]262;263; GISEL-LABEL: test_smfmac_f32_32x32x32_f16:264; GISEL: ; %bb.0:265; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)266; GISEL-NEXT: v_mov_b32_e32 v48, v0267; GISEL-NEXT: v_mov_b32_e32 v49, v1268; GISEL-NEXT: v_mov_b32_e32 v50, v2269; GISEL-NEXT: v_mov_b32_e32 v51, v3270; GISEL-NEXT: v_mov_b32_e32 v30, v4271; GISEL-NEXT: v_mov_b32_e32 v31, v5272; GISEL-NEXT: v_mov_b32_e32 v32, v6273; GISEL-NEXT: v_mov_b32_e32 v33, v7274; GISEL-NEXT: v_mov_b32_e32 v34, v8275; GISEL-NEXT: v_mov_b32_e32 v35, v9276; GISEL-NEXT: v_mov_b32_e32 v36, v10277; GISEL-NEXT: v_mov_b32_e32 v37, v11278; GISEL-NEXT: v_mov_b32_e32 v0, v12279; GISEL-NEXT: v_mov_b32_e32 v1, v13280; GISEL-NEXT: v_mov_b32_e32 v2, v14281; GISEL-NEXT: v_mov_b32_e32 v3, v15282; GISEL-NEXT: v_mov_b32_e32 v4, v16283; GISEL-NEXT: v_mov_b32_e32 v5, v17284; GISEL-NEXT: v_mov_b32_e32 v6, v18285; GISEL-NEXT: v_mov_b32_e32 v7, v19286; GISEL-NEXT: v_mov_b32_e32 v8, v20287; GISEL-NEXT: v_mov_b32_e32 v9, v21288; GISEL-NEXT: v_mov_b32_e32 v10, v22289; GISEL-NEXT: v_mov_b32_e32 v11, v23290; GISEL-NEXT: v_mov_b32_e32 v12, v24291; GISEL-NEXT: v_mov_b32_e32 v13, v25292; GISEL-NEXT: v_mov_b32_e32 v14, v26293; GISEL-NEXT: v_mov_b32_e32 v15, v27294; GISEL-NEXT: s_nop 1295; GISEL-NEXT: v_smfmac_f32_32x32x32_f16 v[0:15], v[48:51], v[30:37], v28296; GISEL-NEXT: s_setpc_b64 s[30:31]297 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)298 ret <16 x float> %result299}300 301define <16 x float> @test_smfmac_f32_32x32x32_f16__flags0(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3) {302; SDAG-LABEL: test_smfmac_f32_32x32x32_f16__flags0:303; SDAG: ; %bb.0:304; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)305; SDAG-NEXT: v_smfmac_f32_32x32x32_f16 v[12:27], v[0:3], v[4:11], v28 cbsz:1 abid:3306; SDAG-NEXT: s_nop 11307; SDAG-NEXT: v_mov_b32_e32 v0, v12308; SDAG-NEXT: v_mov_b32_e32 v1, v13309; SDAG-NEXT: v_mov_b32_e32 v2, v14310; SDAG-NEXT: v_mov_b32_e32 v3, v15311; SDAG-NEXT: v_mov_b32_e32 v4, v16312; SDAG-NEXT: v_mov_b32_e32 v5, v17313; SDAG-NEXT: v_mov_b32_e32 v6, v18314; SDAG-NEXT: v_mov_b32_e32 v7, v19315; SDAG-NEXT: v_mov_b32_e32 v8, v20316; SDAG-NEXT: v_mov_b32_e32 v9, v21317; SDAG-NEXT: v_mov_b32_e32 v10, v22318; SDAG-NEXT: v_mov_b32_e32 v11, v23319; SDAG-NEXT: v_mov_b32_e32 v12, v24320; SDAG-NEXT: v_mov_b32_e32 v13, v25321; SDAG-NEXT: v_mov_b32_e32 v14, v26322; SDAG-NEXT: v_mov_b32_e32 v15, v27323; SDAG-NEXT: s_setpc_b64 s[30:31]324;325; GISEL-LABEL: test_smfmac_f32_32x32x32_f16__flags0:326; GISEL: ; %bb.0:327; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)328; GISEL-NEXT: v_mov_b32_e32 v48, v0329; GISEL-NEXT: v_mov_b32_e32 v49, v1330; GISEL-NEXT: v_mov_b32_e32 v50, v2331; GISEL-NEXT: v_mov_b32_e32 v51, v3332; GISEL-NEXT: v_mov_b32_e32 v30, v4333; GISEL-NEXT: v_mov_b32_e32 v31, v5334; GISEL-NEXT: v_mov_b32_e32 v32, v6335; GISEL-NEXT: v_mov_b32_e32 v33, v7336; GISEL-NEXT: v_mov_b32_e32 v34, v8337; GISEL-NEXT: v_mov_b32_e32 v35, v9338; GISEL-NEXT: v_mov_b32_e32 v36, v10339; GISEL-NEXT: v_mov_b32_e32 v37, v11340; GISEL-NEXT: v_mov_b32_e32 v0, v12341; GISEL-NEXT: v_mov_b32_e32 v1, v13342; GISEL-NEXT: v_mov_b32_e32 v2, v14343; GISEL-NEXT: v_mov_b32_e32 v3, v15344; GISEL-NEXT: v_mov_b32_e32 v4, v16345; GISEL-NEXT: v_mov_b32_e32 v5, v17346; GISEL-NEXT: v_mov_b32_e32 v6, v18347; GISEL-NEXT: v_mov_b32_e32 v7, v19348; GISEL-NEXT: v_mov_b32_e32 v8, v20349; GISEL-NEXT: v_mov_b32_e32 v9, v21350; GISEL-NEXT: v_mov_b32_e32 v10, v22351; GISEL-NEXT: v_mov_b32_e32 v11, v23352; GISEL-NEXT: v_mov_b32_e32 v12, v24353; GISEL-NEXT: v_mov_b32_e32 v13, v25354; GISEL-NEXT: v_mov_b32_e32 v14, v26355; GISEL-NEXT: v_mov_b32_e32 v15, v27356; GISEL-NEXT: s_nop 1357; GISEL-NEXT: v_smfmac_f32_32x32x32_f16 v[0:15], v[48:51], v[30:37], v28 cbsz:1 abid:3358; GISEL-NEXT: s_setpc_b64 s[30:31]359 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)360 ret <16 x float> %result361}362 363define <16 x float> @test_smfmac_f32_32x32x32_f16__flags1(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3) {364; SDAG-LABEL: test_smfmac_f32_32x32x32_f16__flags1:365; SDAG: ; %bb.0:366; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)367; SDAG-NEXT: v_smfmac_f32_32x32x32_f16 v[12:27], v[0:3], v[4:11], v28 cbsz:3 abid:1368; SDAG-NEXT: s_nop 11369; SDAG-NEXT: v_mov_b32_e32 v0, v12370; SDAG-NEXT: v_mov_b32_e32 v1, v13371; SDAG-NEXT: v_mov_b32_e32 v2, v14372; SDAG-NEXT: v_mov_b32_e32 v3, v15373; SDAG-NEXT: v_mov_b32_e32 v4, v16374; SDAG-NEXT: v_mov_b32_e32 v5, v17375; SDAG-NEXT: v_mov_b32_e32 v6, v18376; SDAG-NEXT: v_mov_b32_e32 v7, v19377; SDAG-NEXT: v_mov_b32_e32 v8, v20378; SDAG-NEXT: v_mov_b32_e32 v9, v21379; SDAG-NEXT: v_mov_b32_e32 v10, v22380; SDAG-NEXT: v_mov_b32_e32 v11, v23381; SDAG-NEXT: v_mov_b32_e32 v12, v24382; SDAG-NEXT: v_mov_b32_e32 v13, v25383; SDAG-NEXT: v_mov_b32_e32 v14, v26384; SDAG-NEXT: v_mov_b32_e32 v15, v27385; SDAG-NEXT: s_setpc_b64 s[30:31]386;387; GISEL-LABEL: test_smfmac_f32_32x32x32_f16__flags1:388; GISEL: ; %bb.0:389; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)390; GISEL-NEXT: v_mov_b32_e32 v48, v0391; GISEL-NEXT: v_mov_b32_e32 v49, v1392; GISEL-NEXT: v_mov_b32_e32 v50, v2393; GISEL-NEXT: v_mov_b32_e32 v51, v3394; GISEL-NEXT: v_mov_b32_e32 v30, v4395; GISEL-NEXT: v_mov_b32_e32 v31, v5396; GISEL-NEXT: v_mov_b32_e32 v32, v6397; GISEL-NEXT: v_mov_b32_e32 v33, v7398; GISEL-NEXT: v_mov_b32_e32 v34, v8399; GISEL-NEXT: v_mov_b32_e32 v35, v9400; GISEL-NEXT: v_mov_b32_e32 v36, v10401; GISEL-NEXT: v_mov_b32_e32 v37, v11402; GISEL-NEXT: v_mov_b32_e32 v0, v12403; GISEL-NEXT: v_mov_b32_e32 v1, v13404; GISEL-NEXT: v_mov_b32_e32 v2, v14405; GISEL-NEXT: v_mov_b32_e32 v3, v15406; GISEL-NEXT: v_mov_b32_e32 v4, v16407; GISEL-NEXT: v_mov_b32_e32 v5, v17408; GISEL-NEXT: v_mov_b32_e32 v6, v18409; GISEL-NEXT: v_mov_b32_e32 v7, v19410; GISEL-NEXT: v_mov_b32_e32 v8, v20411; GISEL-NEXT: v_mov_b32_e32 v9, v21412; GISEL-NEXT: v_mov_b32_e32 v10, v22413; GISEL-NEXT: v_mov_b32_e32 v11, v23414; GISEL-NEXT: v_mov_b32_e32 v12, v24415; GISEL-NEXT: v_mov_b32_e32 v13, v25416; GISEL-NEXT: v_mov_b32_e32 v14, v26417; GISEL-NEXT: v_mov_b32_e32 v15, v27418; GISEL-NEXT: s_nop 1419; GISEL-NEXT: v_smfmac_f32_32x32x32_f16 v[0:15], v[48:51], v[30:37], v28 cbsz:3 abid:1420; GISEL-NEXT: s_setpc_b64 s[30:31]421 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)422 ret <16 x float> %result423}424 425define <16 x float> @test_smfmac_f32_32x32x32_f16__sgpr(<8 x half> inreg %arg0, <16 x half> inreg %arg1, <16 x float> inreg %arg2, i32 inreg %arg3) {426; SDAG-LABEL: test_smfmac_f32_32x32x32_f16__sgpr:427; SDAG: ; %bb.0:428; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)429; SDAG-NEXT: v_mov_b32_e32 v26, s0430; SDAG-NEXT: v_mov_b32_e32 v27, s1431; SDAG-NEXT: v_mov_b32_e32 v28, s2432; SDAG-NEXT: v_mov_b32_e32 v29, s3433; SDAG-NEXT: v_mov_b32_e32 v16, v10434; SDAG-NEXT: v_mov_b32_e32 v15, v9435; SDAG-NEXT: v_mov_b32_e32 v14, v8436; SDAG-NEXT: v_mov_b32_e32 v13, v7437; SDAG-NEXT: v_mov_b32_e32 v12, v6438; SDAG-NEXT: v_mov_b32_e32 v11, v5439; SDAG-NEXT: v_mov_b32_e32 v10, v4440; SDAG-NEXT: v_mov_b32_e32 v9, v3441; SDAG-NEXT: v_mov_b32_e32 v8, v2442; SDAG-NEXT: v_mov_b32_e32 v7, v1443; SDAG-NEXT: v_mov_b32_e32 v6, v0444; SDAG-NEXT: v_mov_b32_e32 v0, s24445; SDAG-NEXT: v_mov_b32_e32 v1, s25446; SDAG-NEXT: v_mov_b32_e32 v2, s26447; SDAG-NEXT: v_mov_b32_e32 v3, s27448; SDAG-NEXT: v_mov_b32_e32 v4, s28449; SDAG-NEXT: v_mov_b32_e32 v5, s29450; SDAG-NEXT: v_mov_b32_e32 v18, s16451; SDAG-NEXT: v_mov_b32_e32 v19, s17452; SDAG-NEXT: v_mov_b32_e32 v20, s18453; SDAG-NEXT: v_mov_b32_e32 v21, s19454; SDAG-NEXT: v_mov_b32_e32 v22, s20455; SDAG-NEXT: v_mov_b32_e32 v23, s21456; SDAG-NEXT: v_mov_b32_e32 v24, s22457; SDAG-NEXT: v_mov_b32_e32 v25, s23458; SDAG-NEXT: s_nop 1459; SDAG-NEXT: v_smfmac_f32_32x32x32_f16 v[0:15], v[26:29], v[18:25], v16460; SDAG-NEXT: s_setpc_b64 s[30:31]461;462; GISEL-LABEL: test_smfmac_f32_32x32x32_f16__sgpr:463; GISEL: ; %bb.0:464; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)465; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[2:3]466; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[0:1]467; GISEL-NEXT: v_mov_b32_e32 v11, v0468; GISEL-NEXT: v_mov_b32_e32 v12, v1469; GISEL-NEXT: v_mov_b32_e32 v13, v2470; GISEL-NEXT: v_mov_b32_e32 v14, v3471; GISEL-NEXT: v_mov_b32_e32 v15, v4472; GISEL-NEXT: v_mov_b32_e32 v16, v5473; GISEL-NEXT: v_mov_b32_e32 v17, v6474; GISEL-NEXT: v_mov_b32_e32 v18, v7475; GISEL-NEXT: v_mov_b32_e32 v19, v8476; GISEL-NEXT: v_mov_b32_e32 v20, v9477; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[22:23]478; GISEL-NEXT: v_mov_b32_e32 v21, v10479; GISEL-NEXT: v_mov_b32_e32 v0, s24480; GISEL-NEXT: v_mov_b32_e32 v1, s25481; GISEL-NEXT: v_mov_b32_e32 v2, s26482; GISEL-NEXT: v_mov_b32_e32 v3, s27483; GISEL-NEXT: v_mov_b32_e32 v4, s28484; GISEL-NEXT: v_mov_b32_e32 v5, s29485; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[20:21]486; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[18:19]487; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[16:17]488; GISEL-NEXT: v_mov_b32_e32 v6, v11489; GISEL-NEXT: v_mov_b32_e32 v7, v12490; GISEL-NEXT: v_mov_b32_e32 v8, v13491; GISEL-NEXT: v_mov_b32_e32 v9, v14492; GISEL-NEXT: v_mov_b32_e32 v10, v15493; GISEL-NEXT: v_mov_b32_e32 v11, v16494; GISEL-NEXT: v_mov_b32_e32 v12, v17495; GISEL-NEXT: v_mov_b32_e32 v13, v18496; GISEL-NEXT: v_mov_b32_e32 v14, v19497; GISEL-NEXT: v_mov_b32_e32 v15, v20498; GISEL-NEXT: s_nop 1499; GISEL-NEXT: v_smfmac_f32_32x32x32_f16 v[0:15], v[30:33], v[22:29], v21500; GISEL-NEXT: s_setpc_b64 s[30:31]501 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.f16(<8 x half> %arg0, <16 x half> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)502 ret <16 x float> %result503}504 505; --------------------------------------------------------------------506; llvm.amdgcn.smfmac.f32.16x16x64.bf16507; --------------------------------------------------------------------508 509declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat>, <16 x bfloat>, <4 x float>, i32, i32 immarg, i32 immarg)510 511define amdgpu_kernel void @test_smfmac_f32_16x16x64_bf16__vgpr(ptr addrspace(1) %arg, <8 x bfloat> %a, <16 x bfloat> %b, i32 %idx) #0 {512; GCN-LABEL: test_smfmac_f32_16x16x64_bf16__vgpr:513; GCN: ; %bb.0: ; %bb514; GCN-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24515; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34516; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0517; GCN-NEXT: v_lshlrev_b32_e32 v0, 4, v0518; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x44519; GCN-NEXT: s_load_dword s16, s[4:5], 0x64520; GCN-NEXT: s_waitcnt lgkmcnt(0)521; GCN-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]522; GCN-NEXT: v_mov_b64_e32 v[16:17], s[2:3]523; GCN-NEXT: v_mov_b64_e32 v[14:15], s[0:1]524; GCN-NEXT: v_mov_b64_e32 v[2:3], s[8:9]525; GCN-NEXT: v_mov_b64_e32 v[4:5], s[10:11]526; GCN-NEXT: v_mov_b64_e32 v[6:7], s[12:13]527; GCN-NEXT: v_mov_b64_e32 v[8:9], s[14:15]528; GCN-NEXT: v_mov_b32_e32 v1, s16529; GCN-NEXT: v_mov_b32_e32 v0, 0530; GCN-NEXT: s_waitcnt vmcnt(0)531; GCN-NEXT: v_smfmac_f32_16x16x64_bf16 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2532; GCN-NEXT: s_nop 7533; GCN-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]534; GCN-NEXT: s_endpgm535bb:536 %id = call i32 @llvm.amdgcn.workitem.id.x()537 %gep = getelementptr <4 x float>, ptr addrspace(1) %arg, i32 %id538 %in.1 = load <4 x float>, ptr addrspace(1) %gep539 %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat> %a, <16 x bfloat> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2)540 store <4 x float> %mai.1, ptr addrspace(1) %arg541 ret void542}543 544define <4 x float> @test_smfmac_f32_16x16x64_bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3) {545; GCN-LABEL: test_smfmac_f32_16x16x64_bf16:546; GCN: ; %bb.0:547; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)548; GCN-NEXT: v_smfmac_f32_16x16x64_bf16 v[12:15], v[0:3], v[4:11], v16549; GCN-NEXT: s_nop 7550; GCN-NEXT: v_mov_b32_e32 v0, v12551; GCN-NEXT: v_mov_b32_e32 v1, v13552; GCN-NEXT: v_mov_b32_e32 v2, v14553; GCN-NEXT: v_mov_b32_e32 v3, v15554; GCN-NEXT: s_setpc_b64 s[30:31]555 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)556 ret <4 x float> %result557}558 559define <4 x float> @test_smfmac_f32_16x16x64_bf16__flags0(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3) {560; GCN-LABEL: test_smfmac_f32_16x16x64_bf16__flags0:561; GCN: ; %bb.0:562; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)563; GCN-NEXT: v_smfmac_f32_16x16x64_bf16 v[12:15], v[0:3], v[4:11], v16 cbsz:1 abid:3564; GCN-NEXT: s_nop 7565; GCN-NEXT: v_mov_b32_e32 v0, v12566; GCN-NEXT: v_mov_b32_e32 v1, v13567; GCN-NEXT: v_mov_b32_e32 v2, v14568; GCN-NEXT: v_mov_b32_e32 v3, v15569; GCN-NEXT: s_setpc_b64 s[30:31]570 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)571 ret <4 x float> %result572}573 574define <4 x float> @test_smfmac_f32_16x16x64_bf16__flags1(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3) {575; GCN-LABEL: test_smfmac_f32_16x16x64_bf16__flags1:576; GCN: ; %bb.0:577; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)578; GCN-NEXT: v_smfmac_f32_16x16x64_bf16 v[12:15], v[0:3], v[4:11], v16 cbsz:3 abid:1579; GCN-NEXT: s_nop 7580; GCN-NEXT: v_mov_b32_e32 v0, v12581; GCN-NEXT: v_mov_b32_e32 v1, v13582; GCN-NEXT: v_mov_b32_e32 v2, v14583; GCN-NEXT: v_mov_b32_e32 v3, v15584; GCN-NEXT: s_setpc_b64 s[30:31]585 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)586 ret <4 x float> %result587}588 589define <4 x float> @test_smfmac_f32_16x16x64_bf16__sgpr(<8 x bfloat> inreg %arg0, <16 x bfloat> inreg %arg1, <4 x float> inreg %arg2, i32 inreg %arg3) {590; GCN-LABEL: test_smfmac_f32_16x16x64_bf16__sgpr:591; GCN: ; %bb.0:592; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)593; GCN-NEXT: v_mov_b32_e32 v14, s0594; GCN-NEXT: v_mov_b32_e32 v15, s1595; GCN-NEXT: v_mov_b32_e32 v16, s2596; GCN-NEXT: v_mov_b32_e32 v17, s3597; GCN-NEXT: v_mov_b32_e32 v6, s16598; GCN-NEXT: v_mov_b32_e32 v7, s17599; GCN-NEXT: v_mov_b32_e32 v8, s18600; GCN-NEXT: v_mov_b32_e32 v9, s19601; GCN-NEXT: v_mov_b32_e32 v10, s20602; GCN-NEXT: v_mov_b32_e32 v11, s21603; GCN-NEXT: v_mov_b32_e32 v12, s22604; GCN-NEXT: v_mov_b32_e32 v13, s23605; GCN-NEXT: v_mov_b32_e32 v0, s24606; GCN-NEXT: v_mov_b32_e32 v1, s25607; GCN-NEXT: v_mov_b32_e32 v2, s26608; GCN-NEXT: v_mov_b32_e32 v3, s27609; GCN-NEXT: v_mov_b32_e32 v4, s28610; GCN-NEXT: s_nop 1611; GCN-NEXT: v_smfmac_f32_16x16x64_bf16 v[0:3], v[14:17], v[6:13], v4612; GCN-NEXT: s_setpc_b64 s[30:31]613 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)614 ret <4 x float> %result615}616 617; --------------------------------------------------------------------618; llvm.amdgcn.smfmac.f32.32x32x32.bf16619; --------------------------------------------------------------------620 621declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat>, <16 x bfloat>, <16 x float>, i32, i32 immarg, i32 immarg)622 623define amdgpu_kernel void @test_smfmac_f32_32x32x32_bf16__vgpr(ptr addrspace(1) %arg, <8 x bfloat> %a, <16 x bfloat> %b, i32 %idx) #0 {624; GCN-LABEL: test_smfmac_f32_32x32x32_bf16__vgpr:625; GCN: ; %bb.0: ; %bb626; GCN-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24627; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34628; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0629; GCN-NEXT: v_lshlrev_b32_e32 v16, 6, v0630; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x44631; GCN-NEXT: s_load_dword s16, s[4:5], 0x64632; GCN-NEXT: s_waitcnt lgkmcnt(0)633; GCN-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:48634; GCN-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32635; GCN-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16636; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]637; GCN-NEXT: v_mov_b64_e32 v[28:29], s[2:3]638; GCN-NEXT: v_mov_b64_e32 v[26:27], s[0:1]639; GCN-NEXT: v_mov_b64_e32 v[24:25], s[14:15]640; GCN-NEXT: v_mov_b64_e32 v[22:23], s[12:13]641; GCN-NEXT: v_mov_b64_e32 v[20:21], s[10:11]642; GCN-NEXT: v_mov_b64_e32 v[18:19], s[8:9]643; GCN-NEXT: v_mov_b32_e32 v16, s16644; GCN-NEXT: s_waitcnt vmcnt(0)645; GCN-NEXT: s_nop 0646; GCN-NEXT: v_smfmac_f32_32x32x32_bf16 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2647; GCN-NEXT: v_mov_b32_e32 v16, 0648; GCN-NEXT: s_nop 10649; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32650; GCN-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48651; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]652; GCN-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16653; GCN-NEXT: s_endpgm654bb:655 %id = call i32 @llvm.amdgcn.workitem.id.x()656 %gep = getelementptr <16 x float>, ptr addrspace(1) %arg, i32 %id657 %in.1 = load <16 x float>, ptr addrspace(1) %gep658 %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat> %a, <16 x bfloat> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2)659 store <16 x float> %mai.1, ptr addrspace(1) %arg660 ret void661}662 663define <16 x float> @test_smfmac_f32_32x32x32_bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3) {664; GCN-LABEL: test_smfmac_f32_32x32x32_bf16:665; GCN: ; %bb.0:666; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)667; GCN-NEXT: v_smfmac_f32_32x32x32_bf16 v[12:27], v[0:3], v[4:11], v28668; GCN-NEXT: s_nop 11669; GCN-NEXT: v_mov_b32_e32 v0, v12670; GCN-NEXT: v_mov_b32_e32 v1, v13671; GCN-NEXT: v_mov_b32_e32 v2, v14672; GCN-NEXT: v_mov_b32_e32 v3, v15673; GCN-NEXT: v_mov_b32_e32 v4, v16674; GCN-NEXT: v_mov_b32_e32 v5, v17675; GCN-NEXT: v_mov_b32_e32 v6, v18676; GCN-NEXT: v_mov_b32_e32 v7, v19677; GCN-NEXT: v_mov_b32_e32 v8, v20678; GCN-NEXT: v_mov_b32_e32 v9, v21679; GCN-NEXT: v_mov_b32_e32 v10, v22680; GCN-NEXT: v_mov_b32_e32 v11, v23681; GCN-NEXT: v_mov_b32_e32 v12, v24682; GCN-NEXT: v_mov_b32_e32 v13, v25683; GCN-NEXT: v_mov_b32_e32 v14, v26684; GCN-NEXT: v_mov_b32_e32 v15, v27685; GCN-NEXT: s_setpc_b64 s[30:31]686 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)687 ret <16 x float> %result688}689 690define <16 x float> @test_smfmac_f32_32x32x32_bf16__flags0(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3) {691; GCN-LABEL: test_smfmac_f32_32x32x32_bf16__flags0:692; GCN: ; %bb.0:693; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)694; GCN-NEXT: v_smfmac_f32_32x32x32_bf16 v[12:27], v[0:3], v[4:11], v28 cbsz:1 abid:3695; GCN-NEXT: s_nop 11696; GCN-NEXT: v_mov_b32_e32 v0, v12697; GCN-NEXT: v_mov_b32_e32 v1, v13698; GCN-NEXT: v_mov_b32_e32 v2, v14699; GCN-NEXT: v_mov_b32_e32 v3, v15700; GCN-NEXT: v_mov_b32_e32 v4, v16701; GCN-NEXT: v_mov_b32_e32 v5, v17702; GCN-NEXT: v_mov_b32_e32 v6, v18703; GCN-NEXT: v_mov_b32_e32 v7, v19704; GCN-NEXT: v_mov_b32_e32 v8, v20705; GCN-NEXT: v_mov_b32_e32 v9, v21706; GCN-NEXT: v_mov_b32_e32 v10, v22707; GCN-NEXT: v_mov_b32_e32 v11, v23708; GCN-NEXT: v_mov_b32_e32 v12, v24709; GCN-NEXT: v_mov_b32_e32 v13, v25710; GCN-NEXT: v_mov_b32_e32 v14, v26711; GCN-NEXT: v_mov_b32_e32 v15, v27712; GCN-NEXT: s_setpc_b64 s[30:31]713 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)714 ret <16 x float> %result715}716 717define <16 x float> @test_smfmac_f32_32x32x32_bf16__flags1(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3) {718; GCN-LABEL: test_smfmac_f32_32x32x32_bf16__flags1:719; GCN: ; %bb.0:720; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)721; GCN-NEXT: v_smfmac_f32_32x32x32_bf16 v[12:27], v[0:3], v[4:11], v28 cbsz:3 abid:1722; GCN-NEXT: s_nop 11723; GCN-NEXT: v_mov_b32_e32 v0, v12724; GCN-NEXT: v_mov_b32_e32 v1, v13725; GCN-NEXT: v_mov_b32_e32 v2, v14726; GCN-NEXT: v_mov_b32_e32 v3, v15727; GCN-NEXT: v_mov_b32_e32 v4, v16728; GCN-NEXT: v_mov_b32_e32 v5, v17729; GCN-NEXT: v_mov_b32_e32 v6, v18730; GCN-NEXT: v_mov_b32_e32 v7, v19731; GCN-NEXT: v_mov_b32_e32 v8, v20732; GCN-NEXT: v_mov_b32_e32 v9, v21733; GCN-NEXT: v_mov_b32_e32 v10, v22734; GCN-NEXT: v_mov_b32_e32 v11, v23735; GCN-NEXT: v_mov_b32_e32 v12, v24736; GCN-NEXT: v_mov_b32_e32 v13, v25737; GCN-NEXT: v_mov_b32_e32 v14, v26738; GCN-NEXT: v_mov_b32_e32 v15, v27739; GCN-NEXT: s_setpc_b64 s[30:31]740 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)741 ret <16 x float> %result742}743 744define <16 x float> @test_smfmac_f32_32x32x32_bf16__sgpr(<8 x bfloat> inreg %arg0, <16 x bfloat> inreg %arg1, <16 x float> inreg %arg2, i32 inreg %arg3) {745; GCN-LABEL: test_smfmac_f32_32x32x32_bf16__sgpr:746; GCN: ; %bb.0:747; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)748; GCN-NEXT: v_mov_b32_e32 v26, s0749; GCN-NEXT: v_mov_b32_e32 v27, s1750; GCN-NEXT: v_mov_b32_e32 v28, s2751; GCN-NEXT: v_mov_b32_e32 v29, s3752; GCN-NEXT: v_mov_b32_e32 v16, v10753; GCN-NEXT: v_mov_b32_e32 v15, v9754; GCN-NEXT: v_mov_b32_e32 v14, v8755; GCN-NEXT: v_mov_b32_e32 v13, v7756; GCN-NEXT: v_mov_b32_e32 v12, v6757; GCN-NEXT: v_mov_b32_e32 v11, v5758; GCN-NEXT: v_mov_b32_e32 v10, v4759; GCN-NEXT: v_mov_b32_e32 v9, v3760; GCN-NEXT: v_mov_b32_e32 v8, v2761; GCN-NEXT: v_mov_b32_e32 v7, v1762; GCN-NEXT: v_mov_b32_e32 v6, v0763; GCN-NEXT: v_mov_b32_e32 v0, s24764; GCN-NEXT: v_mov_b32_e32 v1, s25765; GCN-NEXT: v_mov_b32_e32 v2, s26766; GCN-NEXT: v_mov_b32_e32 v3, s27767; GCN-NEXT: v_mov_b32_e32 v4, s28768; GCN-NEXT: v_mov_b32_e32 v5, s29769; GCN-NEXT: v_mov_b32_e32 v18, s16770; GCN-NEXT: v_mov_b32_e32 v19, s17771; GCN-NEXT: v_mov_b32_e32 v20, s18772; GCN-NEXT: v_mov_b32_e32 v21, s19773; GCN-NEXT: v_mov_b32_e32 v22, s20774; GCN-NEXT: v_mov_b32_e32 v23, s21775; GCN-NEXT: v_mov_b32_e32 v24, s22776; GCN-NEXT: v_mov_b32_e32 v25, s23777; GCN-NEXT: s_nop 1778; GCN-NEXT: v_smfmac_f32_32x32x32_bf16 v[0:15], v[26:29], v[18:25], v16779; GCN-NEXT: s_setpc_b64 s[30:31]780 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf16(<8 x bfloat> %arg0, <16 x bfloat> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)781 ret <16 x float> %result782}783 784; --------------------------------------------------------------------785; llvm.amdgcn.smfmac.i32.16x16x128.i8786; --------------------------------------------------------------------787 788declare <4 x i32> @llvm.amdgcn.smfmac.i32.16x16x128.i8(<4 x i32>, <8 x i32>, <4 x i32>, i32, i32, i32)789 790define amdgpu_kernel void @test_smfmac_i32_16x16x128_i8__vgpr(ptr addrspace(1) %arg, <4 x i32> %a, <8 x i32> %b, i32 %idx) #0 {791; SDAG-LABEL: test_smfmac_i32_16x16x128_i8__vgpr:792; SDAG: ; %bb.0: ; %bb793; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24794; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0795; SDAG-NEXT: v_lshlrev_b32_e32 v0, 4, v0796; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34797; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64798; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54799; SDAG-NEXT: s_waitcnt lgkmcnt(0)800; SDAG-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]801; SDAG-NEXT: v_mov_b32_e32 v12, 0802; SDAG-NEXT: v_mov_b32_e32 v8, s8803; SDAG-NEXT: v_mov_b32_e32 v9, s9804; SDAG-NEXT: v_mov_b32_e32 v10, s10805; SDAG-NEXT: v_mov_b32_e32 v11, s11806; SDAG-NEXT: v_mov_b32_e32 v0, s12807; SDAG-NEXT: v_mov_b32_e32 v1, s13808; SDAG-NEXT: v_mov_b32_e32 v2, s14809; SDAG-NEXT: v_mov_b32_e32 v3, s15810; SDAG-NEXT: v_mov_b32_e32 v4, s0811; SDAG-NEXT: v_mov_b32_e32 v5, s1812; SDAG-NEXT: v_mov_b32_e32 v6, s2813; SDAG-NEXT: v_mov_b32_e32 v7, s3814; SDAG-NEXT: v_mov_b32_e32 v13, s16815; SDAG-NEXT: s_waitcnt vmcnt(0)816; SDAG-NEXT: s_nop 0817; SDAG-NEXT: v_smfmac_i32_16x16x128_i8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2818; SDAG-NEXT: s_nop 7819; SDAG-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]820; SDAG-NEXT: s_endpgm821;822; GISEL-LABEL: test_smfmac_i32_16x16x128_i8__vgpr:823; GISEL: ; %bb.0: ; %bb824; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24825; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0826; GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v0827; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34828; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54829; GISEL-NEXT: s_load_dword s2, s[4:5], 0x64830; GISEL-NEXT: s_waitcnt lgkmcnt(0)831; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1]832; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[10:11]833; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[8:9]834; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]835; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]836; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17]837; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19]838; GISEL-NEXT: v_mov_b32_e32 v16, s2839; GISEL-NEXT: s_waitcnt vmcnt(0)840; GISEL-NEXT: s_nop 0841; GISEL-NEXT: v_smfmac_i32_16x16x128_i8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2842; GISEL-NEXT: v_mov_b32_e32 v0, 0843; GISEL-NEXT: s_nop 6844; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1]845; GISEL-NEXT: s_endpgm846bb:847 %id = call i32 @llvm.amdgcn.workitem.id.x()848 %gep = getelementptr <4 x i32>, ptr addrspace(1) %arg, i32 %id849 %in.1 = load <4 x i32>, ptr addrspace(1) %gep850 %mai.1 = tail call <4 x i32> @llvm.amdgcn.smfmac.i32.16x16x128.i8(<4 x i32> %a, <8 x i32> %b, <4 x i32> %in.1, i32 %idx, i32 1, i32 2)851 store <4 x i32> %mai.1, ptr addrspace(1) %arg852 ret void853}854 855define <4 x i32> @test_smfmac_i32_16x16x128_i8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x i32> %arg2, i32 %arg3) {856; GCN-LABEL: test_smfmac_i32_16x16x128_i8:857; GCN: ; %bb.0:858; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)859; GCN-NEXT: v_smfmac_i32_16x16x128_i8 v[12:15], v[0:3], v[4:11], v16860; GCN-NEXT: s_nop 7861; GCN-NEXT: v_mov_b32_e32 v0, v12862; GCN-NEXT: v_mov_b32_e32 v1, v13863; GCN-NEXT: v_mov_b32_e32 v2, v14864; GCN-NEXT: v_mov_b32_e32 v3, v15865; GCN-NEXT: s_setpc_b64 s[30:31]866 %result = call <4 x i32> @llvm.amdgcn.smfmac.i32.16x16x128.i8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x i32> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)867 ret <4 x i32> %result868}869 870define <4 x i32> @test_smfmac_i32_16x16x128_i8__flags0(<4 x i32> %arg0, <8 x i32> %arg1, <4 x i32> %arg2, i32 %arg3) {871; GCN-LABEL: test_smfmac_i32_16x16x128_i8__flags0:872; GCN: ; %bb.0:873; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)874; GCN-NEXT: v_smfmac_i32_16x16x128_i8 v[12:15], v[0:3], v[4:11], v16 cbsz:1 abid:3875; GCN-NEXT: s_nop 7876; GCN-NEXT: v_mov_b32_e32 v0, v12877; GCN-NEXT: v_mov_b32_e32 v1, v13878; GCN-NEXT: v_mov_b32_e32 v2, v14879; GCN-NEXT: v_mov_b32_e32 v3, v15880; GCN-NEXT: s_setpc_b64 s[30:31]881 %result = call <4 x i32> @llvm.amdgcn.smfmac.i32.16x16x128.i8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x i32> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)882 ret <4 x i32> %result883}884 885define <4 x i32> @test_smfmac_i32_16x16x128_i8__flags1(<4 x i32> %arg0, <8 x i32> %arg1, <4 x i32> %arg2, i32 %arg3) {886; GCN-LABEL: test_smfmac_i32_16x16x128_i8__flags1:887; GCN: ; %bb.0:888; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)889; GCN-NEXT: v_smfmac_i32_16x16x128_i8 v[12:15], v[0:3], v[4:11], v16 cbsz:3 abid:1890; GCN-NEXT: s_nop 7891; GCN-NEXT: v_mov_b32_e32 v0, v12892; GCN-NEXT: v_mov_b32_e32 v1, v13893; GCN-NEXT: v_mov_b32_e32 v2, v14894; GCN-NEXT: v_mov_b32_e32 v3, v15895; GCN-NEXT: s_setpc_b64 s[30:31]896 %result = call <4 x i32> @llvm.amdgcn.smfmac.i32.16x16x128.i8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x i32> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)897 ret <4 x i32> %result898}899 900define <4 x i32> @test_smfmac_i32_16x16x128_i8__sgpr(<4 x i32> inreg %arg0, <8 x i32> inreg %arg1, <4 x i32> inreg %arg2, i32 inreg %arg3) {901; SDAG-LABEL: test_smfmac_i32_16x16x128_i8__sgpr:902; SDAG: ; %bb.0:903; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)904; SDAG-NEXT: v_mov_b32_e32 v14, s0905; SDAG-NEXT: v_mov_b32_e32 v15, s1906; SDAG-NEXT: v_mov_b32_e32 v16, s2907; SDAG-NEXT: v_mov_b32_e32 v17, s3908; SDAG-NEXT: v_mov_b32_e32 v6, s16909; SDAG-NEXT: v_mov_b32_e32 v7, s17910; SDAG-NEXT: v_mov_b32_e32 v8, s18911; SDAG-NEXT: v_mov_b32_e32 v9, s19912; SDAG-NEXT: v_mov_b32_e32 v10, s20913; SDAG-NEXT: v_mov_b32_e32 v11, s21914; SDAG-NEXT: v_mov_b32_e32 v12, s22915; SDAG-NEXT: v_mov_b32_e32 v13, s23916; SDAG-NEXT: v_mov_b32_e32 v0, s24917; SDAG-NEXT: v_mov_b32_e32 v1, s25918; SDAG-NEXT: v_mov_b32_e32 v2, s26919; SDAG-NEXT: v_mov_b32_e32 v3, s27920; SDAG-NEXT: v_mov_b32_e32 v4, s28921; SDAG-NEXT: s_nop 1922; SDAG-NEXT: v_smfmac_i32_16x16x128_i8 v[0:3], v[14:17], v[6:13], v4923; SDAG-NEXT: s_setpc_b64 s[30:31]924;925; GISEL-LABEL: test_smfmac_i32_16x16x128_i8__sgpr:926; GISEL: ; %bb.0:927; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)928; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[2:3]929; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]930; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17]931; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19]932; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[20:21]933; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[22:23]934; GISEL-NEXT: v_mov_b32_e32 v0, s24935; GISEL-NEXT: v_mov_b32_e32 v1, s25936; GISEL-NEXT: v_mov_b32_e32 v2, s26937; GISEL-NEXT: v_mov_b32_e32 v3, s27938; GISEL-NEXT: v_mov_b32_e32 v16, s28939; GISEL-NEXT: s_nop 1940; GISEL-NEXT: v_smfmac_i32_16x16x128_i8 v[0:3], v[12:15], v[4:11], v16941; GISEL-NEXT: s_setpc_b64 s[30:31]942 %result = call <4 x i32> @llvm.amdgcn.smfmac.i32.16x16x128.i8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x i32> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)943 ret <4 x i32> %result944}945 946; --------------------------------------------------------------------947; llvm.amdgcn.smfmac.i32.32x32x64.i8948; --------------------------------------------------------------------949 950declare <16 x i32> @llvm.amdgcn.smfmac.i32.32x32x64.i8(<4 x i32>, <8 x i32>, <16 x i32>, i32, i32, i32)951 952define amdgpu_kernel void @test_smfmac_i32_32x32x64_i8__vgpr(ptr addrspace(1) %arg, <4 x i32> %a, <8 x i32> %b, i32 %idx) #0 {953; SDAG-LABEL: test_smfmac_i32_32x32x64_i8__vgpr:954; SDAG: ; %bb.0: ; %bb955; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24956; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0957; SDAG-NEXT: v_lshlrev_b32_e32 v16, 6, v0958; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34959; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64960; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54961; SDAG-NEXT: s_waitcnt lgkmcnt(0)962; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:48963; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32964; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16965; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]966; SDAG-NEXT: v_mov_b32_e32 v24, s8967; SDAG-NEXT: v_mov_b32_e32 v25, s9968; SDAG-NEXT: v_mov_b32_e32 v26, s10969; SDAG-NEXT: v_mov_b32_e32 v27, s11970; SDAG-NEXT: v_mov_b32_e32 v16, s12971; SDAG-NEXT: v_mov_b32_e32 v17, s13972; SDAG-NEXT: v_mov_b32_e32 v18, s14973; SDAG-NEXT: v_mov_b32_e32 v19, s15974; SDAG-NEXT: v_mov_b32_e32 v20, s0975; SDAG-NEXT: v_mov_b32_e32 v21, s1976; SDAG-NEXT: v_mov_b32_e32 v22, s2977; SDAG-NEXT: v_mov_b32_e32 v23, s3978; SDAG-NEXT: v_mov_b32_e32 v28, s16979; SDAG-NEXT: s_waitcnt vmcnt(0)980; SDAG-NEXT: s_nop 0981; SDAG-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2982; SDAG-NEXT: v_mov_b32_e32 v16, 0983; SDAG-NEXT: s_nop 10984; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32985; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48986; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]987; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16988; SDAG-NEXT: s_endpgm989;990; GISEL-LABEL: test_smfmac_i32_32x32x64_i8__vgpr:991; GISEL: ; %bb.0: ; %bb992; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24993; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0994; GISEL-NEXT: v_lshlrev_b32_e32 v16, 6, v0995; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34996; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54997; GISEL-NEXT: s_load_dword s2, s[4:5], 0x64998; GISEL-NEXT: s_waitcnt lgkmcnt(0)999; GISEL-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]1000; GISEL-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:161001; GISEL-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:321002; GISEL-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:481003; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[10:11]1004; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[8:9]1005; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[18:19]1006; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]1007; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[14:15]1008; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[12:13]1009; GISEL-NEXT: v_mov_b32_e32 v28, s21010; GISEL-NEXT: s_waitcnt vmcnt(0)1011; GISEL-NEXT: s_nop 01012; GISEL-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:21013; GISEL-NEXT: v_mov_b32_e32 v16, 01014; GISEL-NEXT: s_nop 101015; GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]1016; GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:161017; GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:321018; GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:481019; GISEL-NEXT: s_endpgm1020bb:1021 %id = call i32 @llvm.amdgcn.workitem.id.x()1022 %gep = getelementptr <16 x i32>, ptr addrspace(1) %arg, i32 %id1023 %in.1 = load <16 x i32>, ptr addrspace(1) %gep1024 %mai.1 = tail call <16 x i32> @llvm.amdgcn.smfmac.i32.32x32x64.i8(<4 x i32> %a, <8 x i32> %b, <16 x i32> %in.1, i32 %idx, i32 1, i32 2)1025 store <16 x i32> %mai.1, ptr addrspace(1) %arg1026 ret void1027}1028 1029define <16 x i32> @test_smfmac_i32_32x32x64_i8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x i32> %arg2, i32 %arg3) {1030; SDAG-LABEL: test_smfmac_i32_32x32x64_i8:1031; SDAG: ; %bb.0:1032; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1033; SDAG-NEXT: v_smfmac_i32_32x32x64_i8 v[12:27], v[0:3], v[4:11], v281034; SDAG-NEXT: s_nop 111035; SDAG-NEXT: v_mov_b32_e32 v0, v121036; SDAG-NEXT: v_mov_b32_e32 v1, v131037; SDAG-NEXT: v_mov_b32_e32 v2, v141038; SDAG-NEXT: v_mov_b32_e32 v3, v151039; SDAG-NEXT: v_mov_b32_e32 v4, v161040; SDAG-NEXT: v_mov_b32_e32 v5, v171041; SDAG-NEXT: v_mov_b32_e32 v6, v181042; SDAG-NEXT: v_mov_b32_e32 v7, v191043; SDAG-NEXT: v_mov_b32_e32 v8, v201044; SDAG-NEXT: v_mov_b32_e32 v9, v211045; SDAG-NEXT: v_mov_b32_e32 v10, v221046; SDAG-NEXT: v_mov_b32_e32 v11, v231047; SDAG-NEXT: v_mov_b32_e32 v12, v241048; SDAG-NEXT: v_mov_b32_e32 v13, v251049; SDAG-NEXT: v_mov_b32_e32 v14, v261050; SDAG-NEXT: v_mov_b32_e32 v15, v271051; SDAG-NEXT: s_setpc_b64 s[30:31]1052;1053; GISEL-LABEL: test_smfmac_i32_32x32x64_i8:1054; GISEL: ; %bb.0:1055; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1056; GISEL-NEXT: v_mov_b32_e32 v48, v01057; GISEL-NEXT: v_mov_b32_e32 v49, v11058; GISEL-NEXT: v_mov_b32_e32 v50, v21059; GISEL-NEXT: v_mov_b32_e32 v51, v31060; GISEL-NEXT: v_mov_b32_e32 v30, v41061; GISEL-NEXT: v_mov_b32_e32 v31, v51062; GISEL-NEXT: v_mov_b32_e32 v32, v61063; GISEL-NEXT: v_mov_b32_e32 v33, v71064; GISEL-NEXT: v_mov_b32_e32 v34, v81065; GISEL-NEXT: v_mov_b32_e32 v35, v91066; GISEL-NEXT: v_mov_b32_e32 v36, v101067; GISEL-NEXT: v_mov_b32_e32 v37, v111068; GISEL-NEXT: v_mov_b32_e32 v0, v121069; GISEL-NEXT: v_mov_b32_e32 v1, v131070; GISEL-NEXT: v_mov_b32_e32 v2, v141071; GISEL-NEXT: v_mov_b32_e32 v3, v151072; GISEL-NEXT: v_mov_b32_e32 v4, v161073; GISEL-NEXT: v_mov_b32_e32 v5, v171074; GISEL-NEXT: v_mov_b32_e32 v6, v181075; GISEL-NEXT: v_mov_b32_e32 v7, v191076; GISEL-NEXT: v_mov_b32_e32 v8, v201077; GISEL-NEXT: v_mov_b32_e32 v9, v211078; GISEL-NEXT: v_mov_b32_e32 v10, v221079; GISEL-NEXT: v_mov_b32_e32 v11, v231080; GISEL-NEXT: v_mov_b32_e32 v12, v241081; GISEL-NEXT: v_mov_b32_e32 v13, v251082; GISEL-NEXT: v_mov_b32_e32 v14, v261083; GISEL-NEXT: v_mov_b32_e32 v15, v271084; GISEL-NEXT: s_nop 11085; GISEL-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[48:51], v[30:37], v281086; GISEL-NEXT: s_setpc_b64 s[30:31]1087 %result = call <16 x i32> @llvm.amdgcn.smfmac.i32.32x32x64.i8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x i32> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)1088 ret <16 x i32> %result1089}1090 1091define <16 x i32> @test_smfmac_i32_32x32x64_i8__flags0(<4 x i32> %arg0, <8 x i32> %arg1, <16 x i32> %arg2, i32 %arg3) {1092; SDAG-LABEL: test_smfmac_i32_32x32x64_i8__flags0:1093; SDAG: ; %bb.0:1094; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1095; SDAG-NEXT: v_smfmac_i32_32x32x64_i8 v[12:27], v[0:3], v[4:11], v28 cbsz:1 abid:31096; SDAG-NEXT: s_nop 111097; SDAG-NEXT: v_mov_b32_e32 v0, v121098; SDAG-NEXT: v_mov_b32_e32 v1, v131099; SDAG-NEXT: v_mov_b32_e32 v2, v141100; SDAG-NEXT: v_mov_b32_e32 v3, v151101; SDAG-NEXT: v_mov_b32_e32 v4, v161102; SDAG-NEXT: v_mov_b32_e32 v5, v171103; SDAG-NEXT: v_mov_b32_e32 v6, v181104; SDAG-NEXT: v_mov_b32_e32 v7, v191105; SDAG-NEXT: v_mov_b32_e32 v8, v201106; SDAG-NEXT: v_mov_b32_e32 v9, v211107; SDAG-NEXT: v_mov_b32_e32 v10, v221108; SDAG-NEXT: v_mov_b32_e32 v11, v231109; SDAG-NEXT: v_mov_b32_e32 v12, v241110; SDAG-NEXT: v_mov_b32_e32 v13, v251111; SDAG-NEXT: v_mov_b32_e32 v14, v261112; SDAG-NEXT: v_mov_b32_e32 v15, v271113; SDAG-NEXT: s_setpc_b64 s[30:31]1114;1115; GISEL-LABEL: test_smfmac_i32_32x32x64_i8__flags0:1116; GISEL: ; %bb.0:1117; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1118; GISEL-NEXT: v_mov_b32_e32 v48, v01119; GISEL-NEXT: v_mov_b32_e32 v49, v11120; GISEL-NEXT: v_mov_b32_e32 v50, v21121; GISEL-NEXT: v_mov_b32_e32 v51, v31122; GISEL-NEXT: v_mov_b32_e32 v30, v41123; GISEL-NEXT: v_mov_b32_e32 v31, v51124; GISEL-NEXT: v_mov_b32_e32 v32, v61125; GISEL-NEXT: v_mov_b32_e32 v33, v71126; GISEL-NEXT: v_mov_b32_e32 v34, v81127; GISEL-NEXT: v_mov_b32_e32 v35, v91128; GISEL-NEXT: v_mov_b32_e32 v36, v101129; GISEL-NEXT: v_mov_b32_e32 v37, v111130; GISEL-NEXT: v_mov_b32_e32 v0, v121131; GISEL-NEXT: v_mov_b32_e32 v1, v131132; GISEL-NEXT: v_mov_b32_e32 v2, v141133; GISEL-NEXT: v_mov_b32_e32 v3, v151134; GISEL-NEXT: v_mov_b32_e32 v4, v161135; GISEL-NEXT: v_mov_b32_e32 v5, v171136; GISEL-NEXT: v_mov_b32_e32 v6, v181137; GISEL-NEXT: v_mov_b32_e32 v7, v191138; GISEL-NEXT: v_mov_b32_e32 v8, v201139; GISEL-NEXT: v_mov_b32_e32 v9, v211140; GISEL-NEXT: v_mov_b32_e32 v10, v221141; GISEL-NEXT: v_mov_b32_e32 v11, v231142; GISEL-NEXT: v_mov_b32_e32 v12, v241143; GISEL-NEXT: v_mov_b32_e32 v13, v251144; GISEL-NEXT: v_mov_b32_e32 v14, v261145; GISEL-NEXT: v_mov_b32_e32 v15, v271146; GISEL-NEXT: s_nop 11147; GISEL-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[48:51], v[30:37], v28 cbsz:1 abid:31148; GISEL-NEXT: s_setpc_b64 s[30:31]1149 %result = call <16 x i32> @llvm.amdgcn.smfmac.i32.32x32x64.i8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x i32> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)1150 ret <16 x i32> %result1151}1152 1153define <16 x i32> @test_smfmac_i32_32x32x64_i8__flags1(<4 x i32> %arg0, <8 x i32> %arg1, <16 x i32> %arg2, i32 %arg3) {1154; SDAG-LABEL: test_smfmac_i32_32x32x64_i8__flags1:1155; SDAG: ; %bb.0:1156; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1157; SDAG-NEXT: v_smfmac_i32_32x32x64_i8 v[12:27], v[0:3], v[4:11], v28 cbsz:3 abid:11158; SDAG-NEXT: s_nop 111159; SDAG-NEXT: v_mov_b32_e32 v0, v121160; SDAG-NEXT: v_mov_b32_e32 v1, v131161; SDAG-NEXT: v_mov_b32_e32 v2, v141162; SDAG-NEXT: v_mov_b32_e32 v3, v151163; SDAG-NEXT: v_mov_b32_e32 v4, v161164; SDAG-NEXT: v_mov_b32_e32 v5, v171165; SDAG-NEXT: v_mov_b32_e32 v6, v181166; SDAG-NEXT: v_mov_b32_e32 v7, v191167; SDAG-NEXT: v_mov_b32_e32 v8, v201168; SDAG-NEXT: v_mov_b32_e32 v9, v211169; SDAG-NEXT: v_mov_b32_e32 v10, v221170; SDAG-NEXT: v_mov_b32_e32 v11, v231171; SDAG-NEXT: v_mov_b32_e32 v12, v241172; SDAG-NEXT: v_mov_b32_e32 v13, v251173; SDAG-NEXT: v_mov_b32_e32 v14, v261174; SDAG-NEXT: v_mov_b32_e32 v15, v271175; SDAG-NEXT: s_setpc_b64 s[30:31]1176;1177; GISEL-LABEL: test_smfmac_i32_32x32x64_i8__flags1:1178; GISEL: ; %bb.0:1179; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1180; GISEL-NEXT: v_mov_b32_e32 v48, v01181; GISEL-NEXT: v_mov_b32_e32 v49, v11182; GISEL-NEXT: v_mov_b32_e32 v50, v21183; GISEL-NEXT: v_mov_b32_e32 v51, v31184; GISEL-NEXT: v_mov_b32_e32 v30, v41185; GISEL-NEXT: v_mov_b32_e32 v31, v51186; GISEL-NEXT: v_mov_b32_e32 v32, v61187; GISEL-NEXT: v_mov_b32_e32 v33, v71188; GISEL-NEXT: v_mov_b32_e32 v34, v81189; GISEL-NEXT: v_mov_b32_e32 v35, v91190; GISEL-NEXT: v_mov_b32_e32 v36, v101191; GISEL-NEXT: v_mov_b32_e32 v37, v111192; GISEL-NEXT: v_mov_b32_e32 v0, v121193; GISEL-NEXT: v_mov_b32_e32 v1, v131194; GISEL-NEXT: v_mov_b32_e32 v2, v141195; GISEL-NEXT: v_mov_b32_e32 v3, v151196; GISEL-NEXT: v_mov_b32_e32 v4, v161197; GISEL-NEXT: v_mov_b32_e32 v5, v171198; GISEL-NEXT: v_mov_b32_e32 v6, v181199; GISEL-NEXT: v_mov_b32_e32 v7, v191200; GISEL-NEXT: v_mov_b32_e32 v8, v201201; GISEL-NEXT: v_mov_b32_e32 v9, v211202; GISEL-NEXT: v_mov_b32_e32 v10, v221203; GISEL-NEXT: v_mov_b32_e32 v11, v231204; GISEL-NEXT: v_mov_b32_e32 v12, v241205; GISEL-NEXT: v_mov_b32_e32 v13, v251206; GISEL-NEXT: v_mov_b32_e32 v14, v261207; GISEL-NEXT: v_mov_b32_e32 v15, v271208; GISEL-NEXT: s_nop 11209; GISEL-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[48:51], v[30:37], v28 cbsz:3 abid:11210; GISEL-NEXT: s_setpc_b64 s[30:31]1211 %result = call <16 x i32> @llvm.amdgcn.smfmac.i32.32x32x64.i8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x i32> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)1212 ret <16 x i32> %result1213}1214 1215define <16 x i32> @test_smfmac_i32_32x32x64_i8__sgpr(<4 x i32> inreg %arg0, <8 x i32> inreg %arg1, <16 x i32> inreg %arg2, i32 inreg %arg3) {1216; SDAG-LABEL: test_smfmac_i32_32x32x64_i8__sgpr:1217; SDAG: ; %bb.0:1218; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1219; SDAG-NEXT: v_mov_b32_e32 v26, s01220; SDAG-NEXT: v_mov_b32_e32 v27, s11221; SDAG-NEXT: v_mov_b32_e32 v28, s21222; SDAG-NEXT: v_mov_b32_e32 v29, s31223; SDAG-NEXT: v_mov_b32_e32 v16, v101224; SDAG-NEXT: v_mov_b32_e32 v15, v91225; SDAG-NEXT: v_mov_b32_e32 v14, v81226; SDAG-NEXT: v_mov_b32_e32 v13, v71227; SDAG-NEXT: v_mov_b32_e32 v12, v61228; SDAG-NEXT: v_mov_b32_e32 v11, v51229; SDAG-NEXT: v_mov_b32_e32 v10, v41230; SDAG-NEXT: v_mov_b32_e32 v9, v31231; SDAG-NEXT: v_mov_b32_e32 v8, v21232; SDAG-NEXT: v_mov_b32_e32 v7, v11233; SDAG-NEXT: v_mov_b32_e32 v6, v01234; SDAG-NEXT: v_mov_b32_e32 v0, s241235; SDAG-NEXT: v_mov_b32_e32 v1, s251236; SDAG-NEXT: v_mov_b32_e32 v2, s261237; SDAG-NEXT: v_mov_b32_e32 v3, s271238; SDAG-NEXT: v_mov_b32_e32 v4, s281239; SDAG-NEXT: v_mov_b32_e32 v5, s291240; SDAG-NEXT: v_mov_b32_e32 v18, s161241; SDAG-NEXT: v_mov_b32_e32 v19, s171242; SDAG-NEXT: v_mov_b32_e32 v20, s181243; SDAG-NEXT: v_mov_b32_e32 v21, s191244; SDAG-NEXT: v_mov_b32_e32 v22, s201245; SDAG-NEXT: v_mov_b32_e32 v23, s211246; SDAG-NEXT: v_mov_b32_e32 v24, s221247; SDAG-NEXT: v_mov_b32_e32 v25, s231248; SDAG-NEXT: s_nop 11249; SDAG-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[26:29], v[18:25], v161250; SDAG-NEXT: s_setpc_b64 s[30:31]1251;1252; GISEL-LABEL: test_smfmac_i32_32x32x64_i8__sgpr:1253; GISEL: ; %bb.0:1254; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1255; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[2:3]1256; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[0:1]1257; GISEL-NEXT: v_mov_b32_e32 v11, v01258; GISEL-NEXT: v_mov_b32_e32 v12, v11259; GISEL-NEXT: v_mov_b32_e32 v13, v21260; GISEL-NEXT: v_mov_b32_e32 v14, v31261; GISEL-NEXT: v_mov_b32_e32 v15, v41262; GISEL-NEXT: v_mov_b32_e32 v16, v51263; GISEL-NEXT: v_mov_b32_e32 v17, v61264; GISEL-NEXT: v_mov_b32_e32 v18, v71265; GISEL-NEXT: v_mov_b32_e32 v19, v81266; GISEL-NEXT: v_mov_b32_e32 v20, v91267; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[22:23]1268; GISEL-NEXT: v_mov_b32_e32 v21, v101269; GISEL-NEXT: v_mov_b32_e32 v0, s241270; GISEL-NEXT: v_mov_b32_e32 v1, s251271; GISEL-NEXT: v_mov_b32_e32 v2, s261272; GISEL-NEXT: v_mov_b32_e32 v3, s271273; GISEL-NEXT: v_mov_b32_e32 v4, s281274; GISEL-NEXT: v_mov_b32_e32 v5, s291275; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[20:21]1276; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[18:19]1277; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[16:17]1278; GISEL-NEXT: v_mov_b32_e32 v6, v111279; GISEL-NEXT: v_mov_b32_e32 v7, v121280; GISEL-NEXT: v_mov_b32_e32 v8, v131281; GISEL-NEXT: v_mov_b32_e32 v9, v141282; GISEL-NEXT: v_mov_b32_e32 v10, v151283; GISEL-NEXT: v_mov_b32_e32 v11, v161284; GISEL-NEXT: v_mov_b32_e32 v12, v171285; GISEL-NEXT: v_mov_b32_e32 v13, v181286; GISEL-NEXT: v_mov_b32_e32 v14, v191287; GISEL-NEXT: v_mov_b32_e32 v15, v201288; GISEL-NEXT: s_nop 11289; GISEL-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[30:33], v[22:29], v211290; GISEL-NEXT: s_setpc_b64 s[30:31]1291 %result = call <16 x i32> @llvm.amdgcn.smfmac.i32.32x32x64.i8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x i32> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)1292 ret <16 x i32> %result1293}1294 1295; --------------------------------------------------------------------1296; llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf81297; --------------------------------------------------------------------1298 1299declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8(<4 x i32>, <8 x i32>, <4 x float>, i32, i32 immarg, i32 immarg)1300 1301define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_bf8__vgpr(ptr addrspace(1) %arg, <4 x i32> %a, <8 x i32> %b, i32 %idx) #0 {1302; SDAG-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__vgpr:1303; SDAG: ; %bb.0: ; %bb1304; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x241305; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v01306; SDAG-NEXT: v_lshlrev_b32_e32 v0, 4, v01307; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x341308; SDAG-NEXT: s_load_dword s16, s[4:5], 0x641309; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x541310; SDAG-NEXT: s_waitcnt lgkmcnt(0)1311; SDAG-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]1312; SDAG-NEXT: v_mov_b32_e32 v12, 01313; SDAG-NEXT: v_mov_b32_e32 v8, s81314; SDAG-NEXT: v_mov_b32_e32 v9, s91315; SDAG-NEXT: v_mov_b32_e32 v10, s101316; SDAG-NEXT: v_mov_b32_e32 v11, s111317; SDAG-NEXT: v_mov_b32_e32 v0, s121318; SDAG-NEXT: v_mov_b32_e32 v1, s131319; SDAG-NEXT: v_mov_b32_e32 v2, s141320; SDAG-NEXT: v_mov_b32_e32 v3, s151321; SDAG-NEXT: v_mov_b32_e32 v4, s01322; SDAG-NEXT: v_mov_b32_e32 v5, s11323; SDAG-NEXT: v_mov_b32_e32 v6, s21324; SDAG-NEXT: v_mov_b32_e32 v7, s31325; SDAG-NEXT: v_mov_b32_e32 v13, s161326; SDAG-NEXT: s_waitcnt vmcnt(0)1327; SDAG-NEXT: s_nop 01328; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:21329; SDAG-NEXT: s_nop 71330; SDAG-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]1331; SDAG-NEXT: s_endpgm1332;1333; GISEL-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__vgpr:1334; GISEL: ; %bb.0: ; %bb1335; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241336; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v01337; GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v01338; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x341339; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x541340; GISEL-NEXT: s_load_dword s2, s[4:5], 0x641341; GISEL-NEXT: s_waitcnt lgkmcnt(0)1342; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1]1343; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[10:11]1344; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[8:9]1345; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]1346; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]1347; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17]1348; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19]1349; GISEL-NEXT: v_mov_b32_e32 v16, s21350; GISEL-NEXT: s_waitcnt vmcnt(0)1351; GISEL-NEXT: s_nop 01352; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:21353; GISEL-NEXT: v_mov_b32_e32 v0, 01354; GISEL-NEXT: s_nop 61355; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1]1356; GISEL-NEXT: s_endpgm1357bb:1358 %id = call i32 @llvm.amdgcn.workitem.id.x()1359 %gep = getelementptr <4 x float>, ptr addrspace(1) %arg, i32 %id1360 %in.1 = load <4 x float>, ptr addrspace(1) %gep1361 %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8(<4 x i32> %a, <8 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2)1362 store <4 x float> %mai.1, ptr addrspace(1) %arg1363 ret void1364}1365 1366define <4 x float> @test_smfmac_f32_16x16x128_bf8_bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3) {1367; GCN-LABEL: test_smfmac_f32_16x16x128_bf8_bf8:1368; GCN: ; %bb.0:1369; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1370; GCN-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[12:15], v[0:3], v[4:11], v161371; GCN-NEXT: s_nop 71372; GCN-NEXT: v_mov_b32_e32 v0, v121373; GCN-NEXT: v_mov_b32_e32 v1, v131374; GCN-NEXT: v_mov_b32_e32 v2, v141375; GCN-NEXT: v_mov_b32_e32 v3, v151376; GCN-NEXT: s_setpc_b64 s[30:31]1377 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)1378 ret <4 x float> %result1379}1380 1381define <4 x float> @test_smfmac_f32_16x16x128_bf8_bf8__flags0(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3) {1382; GCN-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__flags0:1383; GCN: ; %bb.0:1384; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1385; GCN-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[12:15], v[0:3], v[4:11], v16 cbsz:1 abid:31386; GCN-NEXT: s_nop 71387; GCN-NEXT: v_mov_b32_e32 v0, v121388; GCN-NEXT: v_mov_b32_e32 v1, v131389; GCN-NEXT: v_mov_b32_e32 v2, v141390; GCN-NEXT: v_mov_b32_e32 v3, v151391; GCN-NEXT: s_setpc_b64 s[30:31]1392 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)1393 ret <4 x float> %result1394}1395 1396define <4 x float> @test_smfmac_f32_16x16x128_bf8_bf8__flags1(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3) {1397; GCN-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__flags1:1398; GCN: ; %bb.0:1399; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1400; GCN-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[12:15], v[0:3], v[4:11], v16 cbsz:3 abid:11401; GCN-NEXT: s_nop 71402; GCN-NEXT: v_mov_b32_e32 v0, v121403; GCN-NEXT: v_mov_b32_e32 v1, v131404; GCN-NEXT: v_mov_b32_e32 v2, v141405; GCN-NEXT: v_mov_b32_e32 v3, v151406; GCN-NEXT: s_setpc_b64 s[30:31]1407 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)1408 ret <4 x float> %result1409}1410 1411define <4 x float> @test_smfmac_f32_16x16x128_bf8_bf8__sgpr(<4 x i32> inreg %arg0, <8 x i32> inreg %arg1, <4 x float> inreg %arg2, i32 inreg %arg3) {1412; SDAG-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__sgpr:1413; SDAG: ; %bb.0:1414; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1415; SDAG-NEXT: v_mov_b32_e32 v14, s01416; SDAG-NEXT: v_mov_b32_e32 v15, s11417; SDAG-NEXT: v_mov_b32_e32 v16, s21418; SDAG-NEXT: v_mov_b32_e32 v17, s31419; SDAG-NEXT: v_mov_b32_e32 v6, s161420; SDAG-NEXT: v_mov_b32_e32 v7, s171421; SDAG-NEXT: v_mov_b32_e32 v8, s181422; SDAG-NEXT: v_mov_b32_e32 v9, s191423; SDAG-NEXT: v_mov_b32_e32 v10, s201424; SDAG-NEXT: v_mov_b32_e32 v11, s211425; SDAG-NEXT: v_mov_b32_e32 v12, s221426; SDAG-NEXT: v_mov_b32_e32 v13, s231427; SDAG-NEXT: v_mov_b32_e32 v0, s241428; SDAG-NEXT: v_mov_b32_e32 v1, s251429; SDAG-NEXT: v_mov_b32_e32 v2, s261430; SDAG-NEXT: v_mov_b32_e32 v3, s271431; SDAG-NEXT: v_mov_b32_e32 v4, s281432; SDAG-NEXT: s_nop 11433; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[0:3], v[14:17], v[6:13], v41434; SDAG-NEXT: s_setpc_b64 s[30:31]1435;1436; GISEL-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__sgpr:1437; GISEL: ; %bb.0:1438; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1439; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[2:3]1440; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]1441; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17]1442; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19]1443; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[20:21]1444; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[22:23]1445; GISEL-NEXT: v_mov_b32_e32 v0, s241446; GISEL-NEXT: v_mov_b32_e32 v1, s251447; GISEL-NEXT: v_mov_b32_e32 v2, s261448; GISEL-NEXT: v_mov_b32_e32 v3, s271449; GISEL-NEXT: v_mov_b32_e32 v16, s281450; GISEL-NEXT: s_nop 11451; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[0:3], v[12:15], v[4:11], v161452; GISEL-NEXT: s_setpc_b64 s[30:31]1453 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)1454 ret <4 x float> %result1455}1456 1457; --------------------------------------------------------------------1458; llvm.amdgcn.smfmac.f32.16x16x128.bf8.fp81459; --------------------------------------------------------------------1460 1461declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.fp8(<4 x i32>, <8 x i32>, <4 x float>, i32, i32 immarg, i32 immarg)1462 1463define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_fp8__vgpr(ptr addrspace(1) %arg, <4 x i32> %a, <8 x i32> %b, i32 %idx) #0 {1464; SDAG-LABEL: test_smfmac_f32_16x16x128_bf8_fp8__vgpr:1465; SDAG: ; %bb.0: ; %bb1466; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x241467; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v01468; SDAG-NEXT: v_lshlrev_b32_e32 v0, 4, v01469; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x341470; SDAG-NEXT: s_load_dword s16, s[4:5], 0x641471; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x541472; SDAG-NEXT: s_waitcnt lgkmcnt(0)1473; SDAG-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]1474; SDAG-NEXT: v_mov_b32_e32 v12, 01475; SDAG-NEXT: v_mov_b32_e32 v8, s81476; SDAG-NEXT: v_mov_b32_e32 v9, s91477; SDAG-NEXT: v_mov_b32_e32 v10, s101478; SDAG-NEXT: v_mov_b32_e32 v11, s111479; SDAG-NEXT: v_mov_b32_e32 v0, s121480; SDAG-NEXT: v_mov_b32_e32 v1, s131481; SDAG-NEXT: v_mov_b32_e32 v2, s141482; SDAG-NEXT: v_mov_b32_e32 v3, s151483; SDAG-NEXT: v_mov_b32_e32 v4, s01484; SDAG-NEXT: v_mov_b32_e32 v5, s11485; SDAG-NEXT: v_mov_b32_e32 v6, s21486; SDAG-NEXT: v_mov_b32_e32 v7, s31487; SDAG-NEXT: v_mov_b32_e32 v13, s161488; SDAG-NEXT: s_waitcnt vmcnt(0)1489; SDAG-NEXT: s_nop 01490; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:21491; SDAG-NEXT: s_nop 71492; SDAG-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]1493; SDAG-NEXT: s_endpgm1494;1495; GISEL-LABEL: test_smfmac_f32_16x16x128_bf8_fp8__vgpr:1496; GISEL: ; %bb.0: ; %bb1497; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241498; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v01499; GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v01500; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x341501; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x541502; GISEL-NEXT: s_load_dword s2, s[4:5], 0x641503; GISEL-NEXT: s_waitcnt lgkmcnt(0)1504; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1]1505; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[10:11]1506; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[8:9]1507; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]1508; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]1509; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17]1510; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19]1511; GISEL-NEXT: v_mov_b32_e32 v16, s21512; GISEL-NEXT: s_waitcnt vmcnt(0)1513; GISEL-NEXT: s_nop 01514; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:21515; GISEL-NEXT: v_mov_b32_e32 v0, 01516; GISEL-NEXT: s_nop 61517; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1]1518; GISEL-NEXT: s_endpgm1519bb:1520 %id = call i32 @llvm.amdgcn.workitem.id.x()1521 %gep = getelementptr <4 x float>, ptr addrspace(1) %arg, i32 %id1522 %in.1 = load <4 x float>, ptr addrspace(1) %gep1523 %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.fp8(<4 x i32> %a, <8 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2)1524 store <4 x float> %mai.1, ptr addrspace(1) %arg1525 ret void1526}1527 1528define <4 x float> @test_smfmac_f32_16x16x128_bf8_fp8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3) {1529; GCN-LABEL: test_smfmac_f32_16x16x128_bf8_fp8:1530; GCN: ; %bb.0:1531; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1532; GCN-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[12:15], v[0:3], v[4:11], v161533; GCN-NEXT: s_nop 71534; GCN-NEXT: v_mov_b32_e32 v0, v121535; GCN-NEXT: v_mov_b32_e32 v1, v131536; GCN-NEXT: v_mov_b32_e32 v2, v141537; GCN-NEXT: v_mov_b32_e32 v3, v151538; GCN-NEXT: s_setpc_b64 s[30:31]1539 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.fp8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)1540 ret <4 x float> %result1541}1542 1543define <4 x float> @test_smfmac_f32_16x16x128_bf8_fp8__flags0(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3) {1544; GCN-LABEL: test_smfmac_f32_16x16x128_bf8_fp8__flags0:1545; GCN: ; %bb.0:1546; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1547; GCN-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[12:15], v[0:3], v[4:11], v16 cbsz:1 abid:31548; GCN-NEXT: s_nop 71549; GCN-NEXT: v_mov_b32_e32 v0, v121550; GCN-NEXT: v_mov_b32_e32 v1, v131551; GCN-NEXT: v_mov_b32_e32 v2, v141552; GCN-NEXT: v_mov_b32_e32 v3, v151553; GCN-NEXT: s_setpc_b64 s[30:31]1554 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.fp8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)1555 ret <4 x float> %result1556}1557 1558define <4 x float> @test_smfmac_f32_16x16x128_bf8_fp8__flags1(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3) {1559; GCN-LABEL: test_smfmac_f32_16x16x128_bf8_fp8__flags1:1560; GCN: ; %bb.0:1561; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1562; GCN-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[12:15], v[0:3], v[4:11], v16 cbsz:3 abid:11563; GCN-NEXT: s_nop 71564; GCN-NEXT: v_mov_b32_e32 v0, v121565; GCN-NEXT: v_mov_b32_e32 v1, v131566; GCN-NEXT: v_mov_b32_e32 v2, v141567; GCN-NEXT: v_mov_b32_e32 v3, v151568; GCN-NEXT: s_setpc_b64 s[30:31]1569 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.fp8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)1570 ret <4 x float> %result1571}1572 1573define <4 x float> @test_smfmac_f32_16x16x128_bf8_fp8__sgpr(<4 x i32> inreg %arg0, <8 x i32> inreg %arg1, <4 x float> inreg %arg2, i32 inreg %arg3) {1574; SDAG-LABEL: test_smfmac_f32_16x16x128_bf8_fp8__sgpr:1575; SDAG: ; %bb.0:1576; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1577; SDAG-NEXT: v_mov_b32_e32 v14, s01578; SDAG-NEXT: v_mov_b32_e32 v15, s11579; SDAG-NEXT: v_mov_b32_e32 v16, s21580; SDAG-NEXT: v_mov_b32_e32 v17, s31581; SDAG-NEXT: v_mov_b32_e32 v6, s161582; SDAG-NEXT: v_mov_b32_e32 v7, s171583; SDAG-NEXT: v_mov_b32_e32 v8, s181584; SDAG-NEXT: v_mov_b32_e32 v9, s191585; SDAG-NEXT: v_mov_b32_e32 v10, s201586; SDAG-NEXT: v_mov_b32_e32 v11, s211587; SDAG-NEXT: v_mov_b32_e32 v12, s221588; SDAG-NEXT: v_mov_b32_e32 v13, s231589; SDAG-NEXT: v_mov_b32_e32 v0, s241590; SDAG-NEXT: v_mov_b32_e32 v1, s251591; SDAG-NEXT: v_mov_b32_e32 v2, s261592; SDAG-NEXT: v_mov_b32_e32 v3, s271593; SDAG-NEXT: v_mov_b32_e32 v4, s281594; SDAG-NEXT: s_nop 11595; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[0:3], v[14:17], v[6:13], v41596; SDAG-NEXT: s_setpc_b64 s[30:31]1597;1598; GISEL-LABEL: test_smfmac_f32_16x16x128_bf8_fp8__sgpr:1599; GISEL: ; %bb.0:1600; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1601; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[2:3]1602; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]1603; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17]1604; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19]1605; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[20:21]1606; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[22:23]1607; GISEL-NEXT: v_mov_b32_e32 v0, s241608; GISEL-NEXT: v_mov_b32_e32 v1, s251609; GISEL-NEXT: v_mov_b32_e32 v2, s261610; GISEL-NEXT: v_mov_b32_e32 v3, s271611; GISEL-NEXT: v_mov_b32_e32 v16, s281612; GISEL-NEXT: s_nop 11613; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[0:3], v[12:15], v[4:11], v161614; GISEL-NEXT: s_setpc_b64 s[30:31]1615 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.fp8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)1616 ret <4 x float> %result1617}1618 1619; --------------------------------------------------------------------1620; llvm.amdgcn.smfmac.f32.16x16x128.fp8.bf81621; --------------------------------------------------------------------1622 1623declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.fp8.bf8(<4 x i32>, <8 x i32>, <4 x float>, i32, i32 immarg, i32 immarg)1624 1625define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_bf8__vgpr(ptr addrspace(1) %arg, <4 x i32> %a, <8 x i32> %b, i32 %idx) #0 {1626; SDAG-LABEL: test_smfmac_f32_16x16x128_fp8_bf8__vgpr:1627; SDAG: ; %bb.0: ; %bb1628; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x241629; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v01630; SDAG-NEXT: v_lshlrev_b32_e32 v0, 4, v01631; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x341632; SDAG-NEXT: s_load_dword s16, s[4:5], 0x641633; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x541634; SDAG-NEXT: s_waitcnt lgkmcnt(0)1635; SDAG-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]1636; SDAG-NEXT: v_mov_b32_e32 v12, 01637; SDAG-NEXT: v_mov_b32_e32 v8, s81638; SDAG-NEXT: v_mov_b32_e32 v9, s91639; SDAG-NEXT: v_mov_b32_e32 v10, s101640; SDAG-NEXT: v_mov_b32_e32 v11, s111641; SDAG-NEXT: v_mov_b32_e32 v0, s121642; SDAG-NEXT: v_mov_b32_e32 v1, s131643; SDAG-NEXT: v_mov_b32_e32 v2, s141644; SDAG-NEXT: v_mov_b32_e32 v3, s151645; SDAG-NEXT: v_mov_b32_e32 v4, s01646; SDAG-NEXT: v_mov_b32_e32 v5, s11647; SDAG-NEXT: v_mov_b32_e32 v6, s21648; SDAG-NEXT: v_mov_b32_e32 v7, s31649; SDAG-NEXT: v_mov_b32_e32 v13, s161650; SDAG-NEXT: s_waitcnt vmcnt(0)1651; SDAG-NEXT: s_nop 01652; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:21653; SDAG-NEXT: s_nop 71654; SDAG-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]1655; SDAG-NEXT: s_endpgm1656;1657; GISEL-LABEL: test_smfmac_f32_16x16x128_fp8_bf8__vgpr:1658; GISEL: ; %bb.0: ; %bb1659; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241660; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v01661; GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v01662; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x341663; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x541664; GISEL-NEXT: s_load_dword s2, s[4:5], 0x641665; GISEL-NEXT: s_waitcnt lgkmcnt(0)1666; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1]1667; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[10:11]1668; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[8:9]1669; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]1670; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]1671; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17]1672; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19]1673; GISEL-NEXT: v_mov_b32_e32 v16, s21674; GISEL-NEXT: s_waitcnt vmcnt(0)1675; GISEL-NEXT: s_nop 01676; GISEL-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:21677; GISEL-NEXT: v_mov_b32_e32 v0, 01678; GISEL-NEXT: s_nop 61679; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1]1680; GISEL-NEXT: s_endpgm1681bb:1682 %id = call i32 @llvm.amdgcn.workitem.id.x()1683 %gep = getelementptr <4 x float>, ptr addrspace(1) %arg, i32 %id1684 %in.1 = load <4 x float>, ptr addrspace(1) %gep1685 %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.fp8.bf8(<4 x i32> %a, <8 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2)1686 store <4 x float> %mai.1, ptr addrspace(1) %arg1687 ret void1688}1689 1690define <4 x float> @test_smfmac_f32_16x16x128_fp8_bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3) {1691; GCN-LABEL: test_smfmac_f32_16x16x128_fp8_bf8:1692; GCN: ; %bb.0:1693; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1694; GCN-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[12:15], v[0:3], v[4:11], v161695; GCN-NEXT: s_nop 71696; GCN-NEXT: v_mov_b32_e32 v0, v121697; GCN-NEXT: v_mov_b32_e32 v1, v131698; GCN-NEXT: v_mov_b32_e32 v2, v141699; GCN-NEXT: v_mov_b32_e32 v3, v151700; GCN-NEXT: s_setpc_b64 s[30:31]1701 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.fp8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)1702 ret <4 x float> %result1703}1704 1705define <4 x float> @test_smfmac_f32_16x16x128_fp8_bf8__flags0(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3) {1706; GCN-LABEL: test_smfmac_f32_16x16x128_fp8_bf8__flags0:1707; GCN: ; %bb.0:1708; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1709; GCN-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[12:15], v[0:3], v[4:11], v16 cbsz:1 abid:31710; GCN-NEXT: s_nop 71711; GCN-NEXT: v_mov_b32_e32 v0, v121712; GCN-NEXT: v_mov_b32_e32 v1, v131713; GCN-NEXT: v_mov_b32_e32 v2, v141714; GCN-NEXT: v_mov_b32_e32 v3, v151715; GCN-NEXT: s_setpc_b64 s[30:31]1716 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.fp8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)1717 ret <4 x float> %result1718}1719 1720define <4 x float> @test_smfmac_f32_16x16x128_fp8_bf8__flags1(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3) {1721; GCN-LABEL: test_smfmac_f32_16x16x128_fp8_bf8__flags1:1722; GCN: ; %bb.0:1723; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1724; GCN-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[12:15], v[0:3], v[4:11], v16 cbsz:3 abid:11725; GCN-NEXT: s_nop 71726; GCN-NEXT: v_mov_b32_e32 v0, v121727; GCN-NEXT: v_mov_b32_e32 v1, v131728; GCN-NEXT: v_mov_b32_e32 v2, v141729; GCN-NEXT: v_mov_b32_e32 v3, v151730; GCN-NEXT: s_setpc_b64 s[30:31]1731 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.fp8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)1732 ret <4 x float> %result1733}1734 1735define <4 x float> @test_smfmac_f32_16x16x128_fp8_bf8__sgpr(<4 x i32> inreg %arg0, <8 x i32> inreg %arg1, <4 x float> inreg %arg2, i32 inreg %arg3) {1736; SDAG-LABEL: test_smfmac_f32_16x16x128_fp8_bf8__sgpr:1737; SDAG: ; %bb.0:1738; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1739; SDAG-NEXT: v_mov_b32_e32 v14, s01740; SDAG-NEXT: v_mov_b32_e32 v15, s11741; SDAG-NEXT: v_mov_b32_e32 v16, s21742; SDAG-NEXT: v_mov_b32_e32 v17, s31743; SDAG-NEXT: v_mov_b32_e32 v6, s161744; SDAG-NEXT: v_mov_b32_e32 v7, s171745; SDAG-NEXT: v_mov_b32_e32 v8, s181746; SDAG-NEXT: v_mov_b32_e32 v9, s191747; SDAG-NEXT: v_mov_b32_e32 v10, s201748; SDAG-NEXT: v_mov_b32_e32 v11, s211749; SDAG-NEXT: v_mov_b32_e32 v12, s221750; SDAG-NEXT: v_mov_b32_e32 v13, s231751; SDAG-NEXT: v_mov_b32_e32 v0, s241752; SDAG-NEXT: v_mov_b32_e32 v1, s251753; SDAG-NEXT: v_mov_b32_e32 v2, s261754; SDAG-NEXT: v_mov_b32_e32 v3, s271755; SDAG-NEXT: v_mov_b32_e32 v4, s281756; SDAG-NEXT: s_nop 11757; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[0:3], v[14:17], v[6:13], v41758; SDAG-NEXT: s_setpc_b64 s[30:31]1759;1760; GISEL-LABEL: test_smfmac_f32_16x16x128_fp8_bf8__sgpr:1761; GISEL: ; %bb.0:1762; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1763; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[2:3]1764; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]1765; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17]1766; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19]1767; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[20:21]1768; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[22:23]1769; GISEL-NEXT: v_mov_b32_e32 v0, s241770; GISEL-NEXT: v_mov_b32_e32 v1, s251771; GISEL-NEXT: v_mov_b32_e32 v2, s261772; GISEL-NEXT: v_mov_b32_e32 v3, s271773; GISEL-NEXT: v_mov_b32_e32 v16, s281774; GISEL-NEXT: s_nop 11775; GISEL-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[0:3], v[12:15], v[4:11], v161776; GISEL-NEXT: s_setpc_b64 s[30:31]1777 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.fp8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)1778 ret <4 x float> %result1779}1780 1781; --------------------------------------------------------------------1782; llvm.amdgcn.smfmac.f32.16x16x128.fp8.fp81783; --------------------------------------------------------------------1784 1785declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.fp8.fp8(<4 x i32>, <8 x i32>, <4 x float>, i32, i32 immarg, i32 immarg)1786 1787define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_fp8__vgpr(ptr addrspace(1) %arg, <4 x i32> %a, <8 x i32> %b, i32 %idx) #0 {1788; SDAG-LABEL: test_smfmac_f32_16x16x128_fp8_fp8__vgpr:1789; SDAG: ; %bb.0: ; %bb1790; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x241791; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v01792; SDAG-NEXT: v_lshlrev_b32_e32 v0, 4, v01793; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x341794; SDAG-NEXT: s_load_dword s16, s[4:5], 0x641795; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x541796; SDAG-NEXT: s_waitcnt lgkmcnt(0)1797; SDAG-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]1798; SDAG-NEXT: v_mov_b32_e32 v12, 01799; SDAG-NEXT: v_mov_b32_e32 v8, s81800; SDAG-NEXT: v_mov_b32_e32 v9, s91801; SDAG-NEXT: v_mov_b32_e32 v10, s101802; SDAG-NEXT: v_mov_b32_e32 v11, s111803; SDAG-NEXT: v_mov_b32_e32 v0, s121804; SDAG-NEXT: v_mov_b32_e32 v1, s131805; SDAG-NEXT: v_mov_b32_e32 v2, s141806; SDAG-NEXT: v_mov_b32_e32 v3, s151807; SDAG-NEXT: v_mov_b32_e32 v4, s01808; SDAG-NEXT: v_mov_b32_e32 v5, s11809; SDAG-NEXT: v_mov_b32_e32 v6, s21810; SDAG-NEXT: v_mov_b32_e32 v7, s31811; SDAG-NEXT: v_mov_b32_e32 v13, s161812; SDAG-NEXT: s_waitcnt vmcnt(0)1813; SDAG-NEXT: s_nop 01814; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:21815; SDAG-NEXT: s_nop 71816; SDAG-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]1817; SDAG-NEXT: s_endpgm1818;1819; GISEL-LABEL: test_smfmac_f32_16x16x128_fp8_fp8__vgpr:1820; GISEL: ; %bb.0: ; %bb1821; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241822; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v01823; GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v01824; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x341825; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x541826; GISEL-NEXT: s_load_dword s2, s[4:5], 0x641827; GISEL-NEXT: s_waitcnt lgkmcnt(0)1828; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1]1829; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[10:11]1830; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[8:9]1831; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]1832; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]1833; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17]1834; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19]1835; GISEL-NEXT: v_mov_b32_e32 v16, s21836; GISEL-NEXT: s_waitcnt vmcnt(0)1837; GISEL-NEXT: s_nop 01838; GISEL-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:21839; GISEL-NEXT: v_mov_b32_e32 v0, 01840; GISEL-NEXT: s_nop 61841; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1]1842; GISEL-NEXT: s_endpgm1843bb:1844 %id = call i32 @llvm.amdgcn.workitem.id.x()1845 %gep = getelementptr <4 x float>, ptr addrspace(1) %arg, i32 %id1846 %in.1 = load <4 x float>, ptr addrspace(1) %gep1847 %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.fp8.fp8(<4 x i32> %a, <8 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2)1848 store <4 x float> %mai.1, ptr addrspace(1) %arg1849 ret void1850}1851 1852define <4 x float> @test_smfmac_f32_16x16x128_fp8_fp8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3) {1853; GCN-LABEL: test_smfmac_f32_16x16x128_fp8_fp8:1854; GCN: ; %bb.0:1855; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1856; GCN-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[12:15], v[0:3], v[4:11], v161857; GCN-NEXT: s_nop 71858; GCN-NEXT: v_mov_b32_e32 v0, v121859; GCN-NEXT: v_mov_b32_e32 v1, v131860; GCN-NEXT: v_mov_b32_e32 v2, v141861; GCN-NEXT: v_mov_b32_e32 v3, v151862; GCN-NEXT: s_setpc_b64 s[30:31]1863 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.fp8.fp8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)1864 ret <4 x float> %result1865}1866 1867define <4 x float> @test_smfmac_f32_16x16x128_fp8_fp8__flags0(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3) {1868; GCN-LABEL: test_smfmac_f32_16x16x128_fp8_fp8__flags0:1869; GCN: ; %bb.0:1870; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1871; GCN-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[12:15], v[0:3], v[4:11], v16 cbsz:1 abid:31872; GCN-NEXT: s_nop 71873; GCN-NEXT: v_mov_b32_e32 v0, v121874; GCN-NEXT: v_mov_b32_e32 v1, v131875; GCN-NEXT: v_mov_b32_e32 v2, v141876; GCN-NEXT: v_mov_b32_e32 v3, v151877; GCN-NEXT: s_setpc_b64 s[30:31]1878 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.fp8.fp8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)1879 ret <4 x float> %result1880}1881 1882define <4 x float> @test_smfmac_f32_16x16x128_fp8_fp8__flags1(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3) {1883; GCN-LABEL: test_smfmac_f32_16x16x128_fp8_fp8__flags1:1884; GCN: ; %bb.0:1885; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1886; GCN-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[12:15], v[0:3], v[4:11], v16 cbsz:3 abid:11887; GCN-NEXT: s_nop 71888; GCN-NEXT: v_mov_b32_e32 v0, v121889; GCN-NEXT: v_mov_b32_e32 v1, v131890; GCN-NEXT: v_mov_b32_e32 v2, v141891; GCN-NEXT: v_mov_b32_e32 v3, v151892; GCN-NEXT: s_setpc_b64 s[30:31]1893 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.fp8.fp8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)1894 ret <4 x float> %result1895}1896 1897define <4 x float> @test_smfmac_f32_16x16x128_fp8_fp8__sgpr(<4 x i32> inreg %arg0, <8 x i32> inreg %arg1, <4 x float> inreg %arg2, i32 inreg %arg3) {1898; SDAG-LABEL: test_smfmac_f32_16x16x128_fp8_fp8__sgpr:1899; SDAG: ; %bb.0:1900; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1901; SDAG-NEXT: v_mov_b32_e32 v14, s01902; SDAG-NEXT: v_mov_b32_e32 v15, s11903; SDAG-NEXT: v_mov_b32_e32 v16, s21904; SDAG-NEXT: v_mov_b32_e32 v17, s31905; SDAG-NEXT: v_mov_b32_e32 v6, s161906; SDAG-NEXT: v_mov_b32_e32 v7, s171907; SDAG-NEXT: v_mov_b32_e32 v8, s181908; SDAG-NEXT: v_mov_b32_e32 v9, s191909; SDAG-NEXT: v_mov_b32_e32 v10, s201910; SDAG-NEXT: v_mov_b32_e32 v11, s211911; SDAG-NEXT: v_mov_b32_e32 v12, s221912; SDAG-NEXT: v_mov_b32_e32 v13, s231913; SDAG-NEXT: v_mov_b32_e32 v0, s241914; SDAG-NEXT: v_mov_b32_e32 v1, s251915; SDAG-NEXT: v_mov_b32_e32 v2, s261916; SDAG-NEXT: v_mov_b32_e32 v3, s271917; SDAG-NEXT: v_mov_b32_e32 v4, s281918; SDAG-NEXT: s_nop 11919; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[0:3], v[14:17], v[6:13], v41920; SDAG-NEXT: s_setpc_b64 s[30:31]1921;1922; GISEL-LABEL: test_smfmac_f32_16x16x128_fp8_fp8__sgpr:1923; GISEL: ; %bb.0:1924; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1925; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[2:3]1926; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]1927; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17]1928; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19]1929; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[20:21]1930; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[22:23]1931; GISEL-NEXT: v_mov_b32_e32 v0, s241932; GISEL-NEXT: v_mov_b32_e32 v1, s251933; GISEL-NEXT: v_mov_b32_e32 v2, s261934; GISEL-NEXT: v_mov_b32_e32 v3, s271935; GISEL-NEXT: v_mov_b32_e32 v16, s281936; GISEL-NEXT: s_nop 11937; GISEL-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[0:3], v[12:15], v[4:11], v161938; GISEL-NEXT: s_setpc_b64 s[30:31]1939 %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.fp8.fp8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)1940 ret <4 x float> %result1941}1942 1943; --------------------------------------------------------------------1944; llvm.amdgcn.smfmac.f32.32x32x64.bf8.bf81945; --------------------------------------------------------------------1946 1947declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.bf8.bf8(<4 x i32>, <8 x i32>, <16 x float>, i32, i32 immarg, i32 immarg)1948 1949define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_bf8__vgpr(ptr addrspace(1) %arg, <4 x i32> %a, <8 x i32> %b, i32 %idx) #0 {1950; SDAG-LABEL: test_smfmac_f32_32x32x64_bf8_bf8__vgpr:1951; SDAG: ; %bb.0: ; %bb1952; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x241953; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v01954; SDAG-NEXT: v_lshlrev_b32_e32 v16, 6, v01955; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x341956; SDAG-NEXT: s_load_dword s16, s[4:5], 0x641957; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x541958; SDAG-NEXT: s_waitcnt lgkmcnt(0)1959; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:481960; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:321961; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:161962; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]1963; SDAG-NEXT: v_mov_b32_e32 v24, s81964; SDAG-NEXT: v_mov_b32_e32 v25, s91965; SDAG-NEXT: v_mov_b32_e32 v26, s101966; SDAG-NEXT: v_mov_b32_e32 v27, s111967; SDAG-NEXT: v_mov_b32_e32 v16, s121968; SDAG-NEXT: v_mov_b32_e32 v17, s131969; SDAG-NEXT: v_mov_b32_e32 v18, s141970; SDAG-NEXT: v_mov_b32_e32 v19, s151971; SDAG-NEXT: v_mov_b32_e32 v20, s01972; SDAG-NEXT: v_mov_b32_e32 v21, s11973; SDAG-NEXT: v_mov_b32_e32 v22, s21974; SDAG-NEXT: v_mov_b32_e32 v23, s31975; SDAG-NEXT: v_mov_b32_e32 v28, s161976; SDAG-NEXT: s_waitcnt vmcnt(0)1977; SDAG-NEXT: s_nop 01978; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:21979; SDAG-NEXT: v_mov_b32_e32 v16, 01980; SDAG-NEXT: s_nop 101981; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:321982; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:481983; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]1984; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:161985; SDAG-NEXT: s_endpgm1986;1987; GISEL-LABEL: test_smfmac_f32_32x32x64_bf8_bf8__vgpr:1988; GISEL: ; %bb.0: ; %bb1989; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241990; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v01991; GISEL-NEXT: v_lshlrev_b32_e32 v16, 6, v01992; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x341993; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x541994; GISEL-NEXT: s_load_dword s2, s[4:5], 0x641995; GISEL-NEXT: s_waitcnt lgkmcnt(0)1996; GISEL-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]1997; GISEL-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:161998; GISEL-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:321999; GISEL-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:482000; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[10:11]2001; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[8:9]2002; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[18:19]2003; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]2004; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[14:15]2005; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[12:13]2006; GISEL-NEXT: v_mov_b32_e32 v28, s22007; GISEL-NEXT: s_waitcnt vmcnt(0)2008; GISEL-NEXT: s_nop 02009; GISEL-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:22010; GISEL-NEXT: v_mov_b32_e32 v16, 02011; GISEL-NEXT: s_nop 102012; GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]2013; GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:162014; GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:322015; GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:482016; GISEL-NEXT: s_endpgm2017bb:2018 %id = call i32 @llvm.amdgcn.workitem.id.x()2019 %gep = getelementptr <16 x float>, ptr addrspace(1) %arg, i32 %id2020 %in.1 = load <16 x float>, ptr addrspace(1) %gep2021 %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.bf8.bf8(<4 x i32> %a, <8 x i32> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2)2022 store <16 x float> %mai.1, ptr addrspace(1) %arg2023 ret void2024}2025 2026define <16 x float> @test_smfmac_f32_32x32x64_bf8_bf8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3) {2027; SDAG-LABEL: test_smfmac_f32_32x32x64_bf8_bf8:2028; SDAG: ; %bb.0:2029; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2030; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[12:27], v[0:3], v[4:11], v282031; SDAG-NEXT: s_nop 112032; SDAG-NEXT: v_mov_b32_e32 v0, v122033; SDAG-NEXT: v_mov_b32_e32 v1, v132034; SDAG-NEXT: v_mov_b32_e32 v2, v142035; SDAG-NEXT: v_mov_b32_e32 v3, v152036; SDAG-NEXT: v_mov_b32_e32 v4, v162037; SDAG-NEXT: v_mov_b32_e32 v5, v172038; SDAG-NEXT: v_mov_b32_e32 v6, v182039; SDAG-NEXT: v_mov_b32_e32 v7, v192040; SDAG-NEXT: v_mov_b32_e32 v8, v202041; SDAG-NEXT: v_mov_b32_e32 v9, v212042; SDAG-NEXT: v_mov_b32_e32 v10, v222043; SDAG-NEXT: v_mov_b32_e32 v11, v232044; SDAG-NEXT: v_mov_b32_e32 v12, v242045; SDAG-NEXT: v_mov_b32_e32 v13, v252046; SDAG-NEXT: v_mov_b32_e32 v14, v262047; SDAG-NEXT: v_mov_b32_e32 v15, v272048; SDAG-NEXT: s_setpc_b64 s[30:31]2049;2050; GISEL-LABEL: test_smfmac_f32_32x32x64_bf8_bf8:2051; GISEL: ; %bb.0:2052; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2053; GISEL-NEXT: v_mov_b32_e32 v48, v02054; GISEL-NEXT: v_mov_b32_e32 v49, v12055; GISEL-NEXT: v_mov_b32_e32 v50, v22056; GISEL-NEXT: v_mov_b32_e32 v51, v32057; GISEL-NEXT: v_mov_b32_e32 v30, v42058; GISEL-NEXT: v_mov_b32_e32 v31, v52059; GISEL-NEXT: v_mov_b32_e32 v32, v62060; GISEL-NEXT: v_mov_b32_e32 v33, v72061; GISEL-NEXT: v_mov_b32_e32 v34, v82062; GISEL-NEXT: v_mov_b32_e32 v35, v92063; GISEL-NEXT: v_mov_b32_e32 v36, v102064; GISEL-NEXT: v_mov_b32_e32 v37, v112065; GISEL-NEXT: v_mov_b32_e32 v0, v122066; GISEL-NEXT: v_mov_b32_e32 v1, v132067; GISEL-NEXT: v_mov_b32_e32 v2, v142068; GISEL-NEXT: v_mov_b32_e32 v3, v152069; GISEL-NEXT: v_mov_b32_e32 v4, v162070; GISEL-NEXT: v_mov_b32_e32 v5, v172071; GISEL-NEXT: v_mov_b32_e32 v6, v182072; GISEL-NEXT: v_mov_b32_e32 v7, v192073; GISEL-NEXT: v_mov_b32_e32 v8, v202074; GISEL-NEXT: v_mov_b32_e32 v9, v212075; GISEL-NEXT: v_mov_b32_e32 v10, v222076; GISEL-NEXT: v_mov_b32_e32 v11, v232077; GISEL-NEXT: v_mov_b32_e32 v12, v242078; GISEL-NEXT: v_mov_b32_e32 v13, v252079; GISEL-NEXT: v_mov_b32_e32 v14, v262080; GISEL-NEXT: v_mov_b32_e32 v15, v272081; GISEL-NEXT: s_nop 12082; GISEL-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[48:51], v[30:37], v282083; GISEL-NEXT: s_setpc_b64 s[30:31]2084 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.bf8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)2085 ret <16 x float> %result2086}2087 2088define <16 x float> @test_smfmac_f32_32x32x64_bf8_bf8__flags0(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3) {2089; SDAG-LABEL: test_smfmac_f32_32x32x64_bf8_bf8__flags0:2090; SDAG: ; %bb.0:2091; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2092; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[12:27], v[0:3], v[4:11], v28 cbsz:1 abid:32093; SDAG-NEXT: s_nop 112094; SDAG-NEXT: v_mov_b32_e32 v0, v122095; SDAG-NEXT: v_mov_b32_e32 v1, v132096; SDAG-NEXT: v_mov_b32_e32 v2, v142097; SDAG-NEXT: v_mov_b32_e32 v3, v152098; SDAG-NEXT: v_mov_b32_e32 v4, v162099; SDAG-NEXT: v_mov_b32_e32 v5, v172100; SDAG-NEXT: v_mov_b32_e32 v6, v182101; SDAG-NEXT: v_mov_b32_e32 v7, v192102; SDAG-NEXT: v_mov_b32_e32 v8, v202103; SDAG-NEXT: v_mov_b32_e32 v9, v212104; SDAG-NEXT: v_mov_b32_e32 v10, v222105; SDAG-NEXT: v_mov_b32_e32 v11, v232106; SDAG-NEXT: v_mov_b32_e32 v12, v242107; SDAG-NEXT: v_mov_b32_e32 v13, v252108; SDAG-NEXT: v_mov_b32_e32 v14, v262109; SDAG-NEXT: v_mov_b32_e32 v15, v272110; SDAG-NEXT: s_setpc_b64 s[30:31]2111;2112; GISEL-LABEL: test_smfmac_f32_32x32x64_bf8_bf8__flags0:2113; GISEL: ; %bb.0:2114; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2115; GISEL-NEXT: v_mov_b32_e32 v48, v02116; GISEL-NEXT: v_mov_b32_e32 v49, v12117; GISEL-NEXT: v_mov_b32_e32 v50, v22118; GISEL-NEXT: v_mov_b32_e32 v51, v32119; GISEL-NEXT: v_mov_b32_e32 v30, v42120; GISEL-NEXT: v_mov_b32_e32 v31, v52121; GISEL-NEXT: v_mov_b32_e32 v32, v62122; GISEL-NEXT: v_mov_b32_e32 v33, v72123; GISEL-NEXT: v_mov_b32_e32 v34, v82124; GISEL-NEXT: v_mov_b32_e32 v35, v92125; GISEL-NEXT: v_mov_b32_e32 v36, v102126; GISEL-NEXT: v_mov_b32_e32 v37, v112127; GISEL-NEXT: v_mov_b32_e32 v0, v122128; GISEL-NEXT: v_mov_b32_e32 v1, v132129; GISEL-NEXT: v_mov_b32_e32 v2, v142130; GISEL-NEXT: v_mov_b32_e32 v3, v152131; GISEL-NEXT: v_mov_b32_e32 v4, v162132; GISEL-NEXT: v_mov_b32_e32 v5, v172133; GISEL-NEXT: v_mov_b32_e32 v6, v182134; GISEL-NEXT: v_mov_b32_e32 v7, v192135; GISEL-NEXT: v_mov_b32_e32 v8, v202136; GISEL-NEXT: v_mov_b32_e32 v9, v212137; GISEL-NEXT: v_mov_b32_e32 v10, v222138; GISEL-NEXT: v_mov_b32_e32 v11, v232139; GISEL-NEXT: v_mov_b32_e32 v12, v242140; GISEL-NEXT: v_mov_b32_e32 v13, v252141; GISEL-NEXT: v_mov_b32_e32 v14, v262142; GISEL-NEXT: v_mov_b32_e32 v15, v272143; GISEL-NEXT: s_nop 12144; GISEL-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[48:51], v[30:37], v28 cbsz:1 abid:32145; GISEL-NEXT: s_setpc_b64 s[30:31]2146 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.bf8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)2147 ret <16 x float> %result2148}2149 2150define <16 x float> @test_smfmac_f32_32x32x64_bf8_bf8__flags1(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3) {2151; SDAG-LABEL: test_smfmac_f32_32x32x64_bf8_bf8__flags1:2152; SDAG: ; %bb.0:2153; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2154; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[12:27], v[0:3], v[4:11], v28 cbsz:3 abid:12155; SDAG-NEXT: s_nop 112156; SDAG-NEXT: v_mov_b32_e32 v0, v122157; SDAG-NEXT: v_mov_b32_e32 v1, v132158; SDAG-NEXT: v_mov_b32_e32 v2, v142159; SDAG-NEXT: v_mov_b32_e32 v3, v152160; SDAG-NEXT: v_mov_b32_e32 v4, v162161; SDAG-NEXT: v_mov_b32_e32 v5, v172162; SDAG-NEXT: v_mov_b32_e32 v6, v182163; SDAG-NEXT: v_mov_b32_e32 v7, v192164; SDAG-NEXT: v_mov_b32_e32 v8, v202165; SDAG-NEXT: v_mov_b32_e32 v9, v212166; SDAG-NEXT: v_mov_b32_e32 v10, v222167; SDAG-NEXT: v_mov_b32_e32 v11, v232168; SDAG-NEXT: v_mov_b32_e32 v12, v242169; SDAG-NEXT: v_mov_b32_e32 v13, v252170; SDAG-NEXT: v_mov_b32_e32 v14, v262171; SDAG-NEXT: v_mov_b32_e32 v15, v272172; SDAG-NEXT: s_setpc_b64 s[30:31]2173;2174; GISEL-LABEL: test_smfmac_f32_32x32x64_bf8_bf8__flags1:2175; GISEL: ; %bb.0:2176; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2177; GISEL-NEXT: v_mov_b32_e32 v48, v02178; GISEL-NEXT: v_mov_b32_e32 v49, v12179; GISEL-NEXT: v_mov_b32_e32 v50, v22180; GISEL-NEXT: v_mov_b32_e32 v51, v32181; GISEL-NEXT: v_mov_b32_e32 v30, v42182; GISEL-NEXT: v_mov_b32_e32 v31, v52183; GISEL-NEXT: v_mov_b32_e32 v32, v62184; GISEL-NEXT: v_mov_b32_e32 v33, v72185; GISEL-NEXT: v_mov_b32_e32 v34, v82186; GISEL-NEXT: v_mov_b32_e32 v35, v92187; GISEL-NEXT: v_mov_b32_e32 v36, v102188; GISEL-NEXT: v_mov_b32_e32 v37, v112189; GISEL-NEXT: v_mov_b32_e32 v0, v122190; GISEL-NEXT: v_mov_b32_e32 v1, v132191; GISEL-NEXT: v_mov_b32_e32 v2, v142192; GISEL-NEXT: v_mov_b32_e32 v3, v152193; GISEL-NEXT: v_mov_b32_e32 v4, v162194; GISEL-NEXT: v_mov_b32_e32 v5, v172195; GISEL-NEXT: v_mov_b32_e32 v6, v182196; GISEL-NEXT: v_mov_b32_e32 v7, v192197; GISEL-NEXT: v_mov_b32_e32 v8, v202198; GISEL-NEXT: v_mov_b32_e32 v9, v212199; GISEL-NEXT: v_mov_b32_e32 v10, v222200; GISEL-NEXT: v_mov_b32_e32 v11, v232201; GISEL-NEXT: v_mov_b32_e32 v12, v242202; GISEL-NEXT: v_mov_b32_e32 v13, v252203; GISEL-NEXT: v_mov_b32_e32 v14, v262204; GISEL-NEXT: v_mov_b32_e32 v15, v272205; GISEL-NEXT: s_nop 12206; GISEL-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[48:51], v[30:37], v28 cbsz:3 abid:12207; GISEL-NEXT: s_setpc_b64 s[30:31]2208 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.bf8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)2209 ret <16 x float> %result2210}2211 2212define <16 x float> @test_smfmac_f32_32x32x64_bf8_bf8__sgpr(<4 x i32> inreg %arg0, <8 x i32> inreg %arg1, <16 x float> inreg %arg2, i32 inreg %arg3) {2213; SDAG-LABEL: test_smfmac_f32_32x32x64_bf8_bf8__sgpr:2214; SDAG: ; %bb.0:2215; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2216; SDAG-NEXT: v_mov_b32_e32 v26, s02217; SDAG-NEXT: v_mov_b32_e32 v27, s12218; SDAG-NEXT: v_mov_b32_e32 v28, s22219; SDAG-NEXT: v_mov_b32_e32 v29, s32220; SDAG-NEXT: v_mov_b32_e32 v16, v102221; SDAG-NEXT: v_mov_b32_e32 v15, v92222; SDAG-NEXT: v_mov_b32_e32 v14, v82223; SDAG-NEXT: v_mov_b32_e32 v13, v72224; SDAG-NEXT: v_mov_b32_e32 v12, v62225; SDAG-NEXT: v_mov_b32_e32 v11, v52226; SDAG-NEXT: v_mov_b32_e32 v10, v42227; SDAG-NEXT: v_mov_b32_e32 v9, v32228; SDAG-NEXT: v_mov_b32_e32 v8, v22229; SDAG-NEXT: v_mov_b32_e32 v7, v12230; SDAG-NEXT: v_mov_b32_e32 v6, v02231; SDAG-NEXT: v_mov_b32_e32 v0, s242232; SDAG-NEXT: v_mov_b32_e32 v1, s252233; SDAG-NEXT: v_mov_b32_e32 v2, s262234; SDAG-NEXT: v_mov_b32_e32 v3, s272235; SDAG-NEXT: v_mov_b32_e32 v4, s282236; SDAG-NEXT: v_mov_b32_e32 v5, s292237; SDAG-NEXT: v_mov_b32_e32 v18, s162238; SDAG-NEXT: v_mov_b32_e32 v19, s172239; SDAG-NEXT: v_mov_b32_e32 v20, s182240; SDAG-NEXT: v_mov_b32_e32 v21, s192241; SDAG-NEXT: v_mov_b32_e32 v22, s202242; SDAG-NEXT: v_mov_b32_e32 v23, s212243; SDAG-NEXT: v_mov_b32_e32 v24, s222244; SDAG-NEXT: v_mov_b32_e32 v25, s232245; SDAG-NEXT: s_nop 12246; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[26:29], v[18:25], v162247; SDAG-NEXT: s_setpc_b64 s[30:31]2248;2249; GISEL-LABEL: test_smfmac_f32_32x32x64_bf8_bf8__sgpr:2250; GISEL: ; %bb.0:2251; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2252; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[2:3]2253; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[0:1]2254; GISEL-NEXT: v_mov_b32_e32 v11, v02255; GISEL-NEXT: v_mov_b32_e32 v12, v12256; GISEL-NEXT: v_mov_b32_e32 v13, v22257; GISEL-NEXT: v_mov_b32_e32 v14, v32258; GISEL-NEXT: v_mov_b32_e32 v15, v42259; GISEL-NEXT: v_mov_b32_e32 v16, v52260; GISEL-NEXT: v_mov_b32_e32 v17, v62261; GISEL-NEXT: v_mov_b32_e32 v18, v72262; GISEL-NEXT: v_mov_b32_e32 v19, v82263; GISEL-NEXT: v_mov_b32_e32 v20, v92264; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[22:23]2265; GISEL-NEXT: v_mov_b32_e32 v21, v102266; GISEL-NEXT: v_mov_b32_e32 v0, s242267; GISEL-NEXT: v_mov_b32_e32 v1, s252268; GISEL-NEXT: v_mov_b32_e32 v2, s262269; GISEL-NEXT: v_mov_b32_e32 v3, s272270; GISEL-NEXT: v_mov_b32_e32 v4, s282271; GISEL-NEXT: v_mov_b32_e32 v5, s292272; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[20:21]2273; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[18:19]2274; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[16:17]2275; GISEL-NEXT: v_mov_b32_e32 v6, v112276; GISEL-NEXT: v_mov_b32_e32 v7, v122277; GISEL-NEXT: v_mov_b32_e32 v8, v132278; GISEL-NEXT: v_mov_b32_e32 v9, v142279; GISEL-NEXT: v_mov_b32_e32 v10, v152280; GISEL-NEXT: v_mov_b32_e32 v11, v162281; GISEL-NEXT: v_mov_b32_e32 v12, v172282; GISEL-NEXT: v_mov_b32_e32 v13, v182283; GISEL-NEXT: v_mov_b32_e32 v14, v192284; GISEL-NEXT: v_mov_b32_e32 v15, v202285; GISEL-NEXT: s_nop 12286; GISEL-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[30:33], v[22:29], v212287; GISEL-NEXT: s_setpc_b64 s[30:31]2288 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.bf8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)2289 ret <16 x float> %result2290}2291 2292; --------------------------------------------------------------------2293; llvm.amdgcn.smfmac.f32.32x32x64.bf8.fp82294; --------------------------------------------------------------------2295 2296declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.bf8.fp8(<4 x i32>, <8 x i32>, <16 x float>, i32, i32 immarg, i32 immarg)2297 2298define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_fp8__vgpr(ptr addrspace(1) %arg, <4 x i32> %a, <8 x i32> %b, i32 %idx) #0 {2299; SDAG-LABEL: test_smfmac_f32_32x32x64_bf8_fp8__vgpr:2300; SDAG: ; %bb.0: ; %bb2301; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x242302; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v02303; SDAG-NEXT: v_lshlrev_b32_e32 v16, 6, v02304; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x342305; SDAG-NEXT: s_load_dword s16, s[4:5], 0x642306; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x542307; SDAG-NEXT: s_waitcnt lgkmcnt(0)2308; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:482309; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:322310; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:162311; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]2312; SDAG-NEXT: v_mov_b32_e32 v24, s82313; SDAG-NEXT: v_mov_b32_e32 v25, s92314; SDAG-NEXT: v_mov_b32_e32 v26, s102315; SDAG-NEXT: v_mov_b32_e32 v27, s112316; SDAG-NEXT: v_mov_b32_e32 v16, s122317; SDAG-NEXT: v_mov_b32_e32 v17, s132318; SDAG-NEXT: v_mov_b32_e32 v18, s142319; SDAG-NEXT: v_mov_b32_e32 v19, s152320; SDAG-NEXT: v_mov_b32_e32 v20, s02321; SDAG-NEXT: v_mov_b32_e32 v21, s12322; SDAG-NEXT: v_mov_b32_e32 v22, s22323; SDAG-NEXT: v_mov_b32_e32 v23, s32324; SDAG-NEXT: v_mov_b32_e32 v28, s162325; SDAG-NEXT: s_waitcnt vmcnt(0)2326; SDAG-NEXT: s_nop 02327; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:22328; SDAG-NEXT: v_mov_b32_e32 v16, 02329; SDAG-NEXT: s_nop 102330; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:322331; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:482332; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]2333; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:162334; SDAG-NEXT: s_endpgm2335;2336; GISEL-LABEL: test_smfmac_f32_32x32x64_bf8_fp8__vgpr:2337; GISEL: ; %bb.0: ; %bb2338; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x242339; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v02340; GISEL-NEXT: v_lshlrev_b32_e32 v16, 6, v02341; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x342342; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x542343; GISEL-NEXT: s_load_dword s2, s[4:5], 0x642344; GISEL-NEXT: s_waitcnt lgkmcnt(0)2345; GISEL-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]2346; GISEL-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:162347; GISEL-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:322348; GISEL-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:482349; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[10:11]2350; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[8:9]2351; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[18:19]2352; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]2353; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[14:15]2354; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[12:13]2355; GISEL-NEXT: v_mov_b32_e32 v28, s22356; GISEL-NEXT: s_waitcnt vmcnt(0)2357; GISEL-NEXT: s_nop 02358; GISEL-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:22359; GISEL-NEXT: v_mov_b32_e32 v16, 02360; GISEL-NEXT: s_nop 102361; GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]2362; GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:162363; GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:322364; GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:482365; GISEL-NEXT: s_endpgm2366bb:2367 %id = call i32 @llvm.amdgcn.workitem.id.x()2368 %gep = getelementptr <16 x float>, ptr addrspace(1) %arg, i32 %id2369 %in.1 = load <16 x float>, ptr addrspace(1) %gep2370 %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.bf8.fp8(<4 x i32> %a, <8 x i32> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2)2371 store <16 x float> %mai.1, ptr addrspace(1) %arg2372 ret void2373}2374 2375define <16 x float> @test_smfmac_f32_32x32x64_bf8_fp8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3) {2376; SDAG-LABEL: test_smfmac_f32_32x32x64_bf8_fp8:2377; SDAG: ; %bb.0:2378; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2379; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[12:27], v[0:3], v[4:11], v282380; SDAG-NEXT: s_nop 112381; SDAG-NEXT: v_mov_b32_e32 v0, v122382; SDAG-NEXT: v_mov_b32_e32 v1, v132383; SDAG-NEXT: v_mov_b32_e32 v2, v142384; SDAG-NEXT: v_mov_b32_e32 v3, v152385; SDAG-NEXT: v_mov_b32_e32 v4, v162386; SDAG-NEXT: v_mov_b32_e32 v5, v172387; SDAG-NEXT: v_mov_b32_e32 v6, v182388; SDAG-NEXT: v_mov_b32_e32 v7, v192389; SDAG-NEXT: v_mov_b32_e32 v8, v202390; SDAG-NEXT: v_mov_b32_e32 v9, v212391; SDAG-NEXT: v_mov_b32_e32 v10, v222392; SDAG-NEXT: v_mov_b32_e32 v11, v232393; SDAG-NEXT: v_mov_b32_e32 v12, v242394; SDAG-NEXT: v_mov_b32_e32 v13, v252395; SDAG-NEXT: v_mov_b32_e32 v14, v262396; SDAG-NEXT: v_mov_b32_e32 v15, v272397; SDAG-NEXT: s_setpc_b64 s[30:31]2398;2399; GISEL-LABEL: test_smfmac_f32_32x32x64_bf8_fp8:2400; GISEL: ; %bb.0:2401; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2402; GISEL-NEXT: v_mov_b32_e32 v48, v02403; GISEL-NEXT: v_mov_b32_e32 v49, v12404; GISEL-NEXT: v_mov_b32_e32 v50, v22405; GISEL-NEXT: v_mov_b32_e32 v51, v32406; GISEL-NEXT: v_mov_b32_e32 v30, v42407; GISEL-NEXT: v_mov_b32_e32 v31, v52408; GISEL-NEXT: v_mov_b32_e32 v32, v62409; GISEL-NEXT: v_mov_b32_e32 v33, v72410; GISEL-NEXT: v_mov_b32_e32 v34, v82411; GISEL-NEXT: v_mov_b32_e32 v35, v92412; GISEL-NEXT: v_mov_b32_e32 v36, v102413; GISEL-NEXT: v_mov_b32_e32 v37, v112414; GISEL-NEXT: v_mov_b32_e32 v0, v122415; GISEL-NEXT: v_mov_b32_e32 v1, v132416; GISEL-NEXT: v_mov_b32_e32 v2, v142417; GISEL-NEXT: v_mov_b32_e32 v3, v152418; GISEL-NEXT: v_mov_b32_e32 v4, v162419; GISEL-NEXT: v_mov_b32_e32 v5, v172420; GISEL-NEXT: v_mov_b32_e32 v6, v182421; GISEL-NEXT: v_mov_b32_e32 v7, v192422; GISEL-NEXT: v_mov_b32_e32 v8, v202423; GISEL-NEXT: v_mov_b32_e32 v9, v212424; GISEL-NEXT: v_mov_b32_e32 v10, v222425; GISEL-NEXT: v_mov_b32_e32 v11, v232426; GISEL-NEXT: v_mov_b32_e32 v12, v242427; GISEL-NEXT: v_mov_b32_e32 v13, v252428; GISEL-NEXT: v_mov_b32_e32 v14, v262429; GISEL-NEXT: v_mov_b32_e32 v15, v272430; GISEL-NEXT: s_nop 12431; GISEL-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[48:51], v[30:37], v282432; GISEL-NEXT: s_setpc_b64 s[30:31]2433 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.bf8.fp8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)2434 ret <16 x float> %result2435}2436 2437define <16 x float> @test_smfmac_f32_32x32x64_bf8_fp8__flags0(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3) {2438; SDAG-LABEL: test_smfmac_f32_32x32x64_bf8_fp8__flags0:2439; SDAG: ; %bb.0:2440; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2441; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[12:27], v[0:3], v[4:11], v28 cbsz:1 abid:32442; SDAG-NEXT: s_nop 112443; SDAG-NEXT: v_mov_b32_e32 v0, v122444; SDAG-NEXT: v_mov_b32_e32 v1, v132445; SDAG-NEXT: v_mov_b32_e32 v2, v142446; SDAG-NEXT: v_mov_b32_e32 v3, v152447; SDAG-NEXT: v_mov_b32_e32 v4, v162448; SDAG-NEXT: v_mov_b32_e32 v5, v172449; SDAG-NEXT: v_mov_b32_e32 v6, v182450; SDAG-NEXT: v_mov_b32_e32 v7, v192451; SDAG-NEXT: v_mov_b32_e32 v8, v202452; SDAG-NEXT: v_mov_b32_e32 v9, v212453; SDAG-NEXT: v_mov_b32_e32 v10, v222454; SDAG-NEXT: v_mov_b32_e32 v11, v232455; SDAG-NEXT: v_mov_b32_e32 v12, v242456; SDAG-NEXT: v_mov_b32_e32 v13, v252457; SDAG-NEXT: v_mov_b32_e32 v14, v262458; SDAG-NEXT: v_mov_b32_e32 v15, v272459; SDAG-NEXT: s_setpc_b64 s[30:31]2460;2461; GISEL-LABEL: test_smfmac_f32_32x32x64_bf8_fp8__flags0:2462; GISEL: ; %bb.0:2463; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2464; GISEL-NEXT: v_mov_b32_e32 v48, v02465; GISEL-NEXT: v_mov_b32_e32 v49, v12466; GISEL-NEXT: v_mov_b32_e32 v50, v22467; GISEL-NEXT: v_mov_b32_e32 v51, v32468; GISEL-NEXT: v_mov_b32_e32 v30, v42469; GISEL-NEXT: v_mov_b32_e32 v31, v52470; GISEL-NEXT: v_mov_b32_e32 v32, v62471; GISEL-NEXT: v_mov_b32_e32 v33, v72472; GISEL-NEXT: v_mov_b32_e32 v34, v82473; GISEL-NEXT: v_mov_b32_e32 v35, v92474; GISEL-NEXT: v_mov_b32_e32 v36, v102475; GISEL-NEXT: v_mov_b32_e32 v37, v112476; GISEL-NEXT: v_mov_b32_e32 v0, v122477; GISEL-NEXT: v_mov_b32_e32 v1, v132478; GISEL-NEXT: v_mov_b32_e32 v2, v142479; GISEL-NEXT: v_mov_b32_e32 v3, v152480; GISEL-NEXT: v_mov_b32_e32 v4, v162481; GISEL-NEXT: v_mov_b32_e32 v5, v172482; GISEL-NEXT: v_mov_b32_e32 v6, v182483; GISEL-NEXT: v_mov_b32_e32 v7, v192484; GISEL-NEXT: v_mov_b32_e32 v8, v202485; GISEL-NEXT: v_mov_b32_e32 v9, v212486; GISEL-NEXT: v_mov_b32_e32 v10, v222487; GISEL-NEXT: v_mov_b32_e32 v11, v232488; GISEL-NEXT: v_mov_b32_e32 v12, v242489; GISEL-NEXT: v_mov_b32_e32 v13, v252490; GISEL-NEXT: v_mov_b32_e32 v14, v262491; GISEL-NEXT: v_mov_b32_e32 v15, v272492; GISEL-NEXT: s_nop 12493; GISEL-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[48:51], v[30:37], v28 cbsz:1 abid:32494; GISEL-NEXT: s_setpc_b64 s[30:31]2495 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.bf8.fp8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)2496 ret <16 x float> %result2497}2498 2499define <16 x float> @test_smfmac_f32_32x32x64_bf8_fp8__flags1(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3) {2500; SDAG-LABEL: test_smfmac_f32_32x32x64_bf8_fp8__flags1:2501; SDAG: ; %bb.0:2502; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2503; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[12:27], v[0:3], v[4:11], v28 cbsz:3 abid:12504; SDAG-NEXT: s_nop 112505; SDAG-NEXT: v_mov_b32_e32 v0, v122506; SDAG-NEXT: v_mov_b32_e32 v1, v132507; SDAG-NEXT: v_mov_b32_e32 v2, v142508; SDAG-NEXT: v_mov_b32_e32 v3, v152509; SDAG-NEXT: v_mov_b32_e32 v4, v162510; SDAG-NEXT: v_mov_b32_e32 v5, v172511; SDAG-NEXT: v_mov_b32_e32 v6, v182512; SDAG-NEXT: v_mov_b32_e32 v7, v192513; SDAG-NEXT: v_mov_b32_e32 v8, v202514; SDAG-NEXT: v_mov_b32_e32 v9, v212515; SDAG-NEXT: v_mov_b32_e32 v10, v222516; SDAG-NEXT: v_mov_b32_e32 v11, v232517; SDAG-NEXT: v_mov_b32_e32 v12, v242518; SDAG-NEXT: v_mov_b32_e32 v13, v252519; SDAG-NEXT: v_mov_b32_e32 v14, v262520; SDAG-NEXT: v_mov_b32_e32 v15, v272521; SDAG-NEXT: s_setpc_b64 s[30:31]2522;2523; GISEL-LABEL: test_smfmac_f32_32x32x64_bf8_fp8__flags1:2524; GISEL: ; %bb.0:2525; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2526; GISEL-NEXT: v_mov_b32_e32 v48, v02527; GISEL-NEXT: v_mov_b32_e32 v49, v12528; GISEL-NEXT: v_mov_b32_e32 v50, v22529; GISEL-NEXT: v_mov_b32_e32 v51, v32530; GISEL-NEXT: v_mov_b32_e32 v30, v42531; GISEL-NEXT: v_mov_b32_e32 v31, v52532; GISEL-NEXT: v_mov_b32_e32 v32, v62533; GISEL-NEXT: v_mov_b32_e32 v33, v72534; GISEL-NEXT: v_mov_b32_e32 v34, v82535; GISEL-NEXT: v_mov_b32_e32 v35, v92536; GISEL-NEXT: v_mov_b32_e32 v36, v102537; GISEL-NEXT: v_mov_b32_e32 v37, v112538; GISEL-NEXT: v_mov_b32_e32 v0, v122539; GISEL-NEXT: v_mov_b32_e32 v1, v132540; GISEL-NEXT: v_mov_b32_e32 v2, v142541; GISEL-NEXT: v_mov_b32_e32 v3, v152542; GISEL-NEXT: v_mov_b32_e32 v4, v162543; GISEL-NEXT: v_mov_b32_e32 v5, v172544; GISEL-NEXT: v_mov_b32_e32 v6, v182545; GISEL-NEXT: v_mov_b32_e32 v7, v192546; GISEL-NEXT: v_mov_b32_e32 v8, v202547; GISEL-NEXT: v_mov_b32_e32 v9, v212548; GISEL-NEXT: v_mov_b32_e32 v10, v222549; GISEL-NEXT: v_mov_b32_e32 v11, v232550; GISEL-NEXT: v_mov_b32_e32 v12, v242551; GISEL-NEXT: v_mov_b32_e32 v13, v252552; GISEL-NEXT: v_mov_b32_e32 v14, v262553; GISEL-NEXT: v_mov_b32_e32 v15, v272554; GISEL-NEXT: s_nop 12555; GISEL-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[48:51], v[30:37], v28 cbsz:3 abid:12556; GISEL-NEXT: s_setpc_b64 s[30:31]2557 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.bf8.fp8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)2558 ret <16 x float> %result2559}2560 2561define <16 x float> @test_smfmac_f32_32x32x64_bf8_fp8__sgpr(<4 x i32> inreg %arg0, <8 x i32> inreg %arg1, <16 x float> inreg %arg2, i32 inreg %arg3) {2562; SDAG-LABEL: test_smfmac_f32_32x32x64_bf8_fp8__sgpr:2563; SDAG: ; %bb.0:2564; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2565; SDAG-NEXT: v_mov_b32_e32 v26, s02566; SDAG-NEXT: v_mov_b32_e32 v27, s12567; SDAG-NEXT: v_mov_b32_e32 v28, s22568; SDAG-NEXT: v_mov_b32_e32 v29, s32569; SDAG-NEXT: v_mov_b32_e32 v16, v102570; SDAG-NEXT: v_mov_b32_e32 v15, v92571; SDAG-NEXT: v_mov_b32_e32 v14, v82572; SDAG-NEXT: v_mov_b32_e32 v13, v72573; SDAG-NEXT: v_mov_b32_e32 v12, v62574; SDAG-NEXT: v_mov_b32_e32 v11, v52575; SDAG-NEXT: v_mov_b32_e32 v10, v42576; SDAG-NEXT: v_mov_b32_e32 v9, v32577; SDAG-NEXT: v_mov_b32_e32 v8, v22578; SDAG-NEXT: v_mov_b32_e32 v7, v12579; SDAG-NEXT: v_mov_b32_e32 v6, v02580; SDAG-NEXT: v_mov_b32_e32 v0, s242581; SDAG-NEXT: v_mov_b32_e32 v1, s252582; SDAG-NEXT: v_mov_b32_e32 v2, s262583; SDAG-NEXT: v_mov_b32_e32 v3, s272584; SDAG-NEXT: v_mov_b32_e32 v4, s282585; SDAG-NEXT: v_mov_b32_e32 v5, s292586; SDAG-NEXT: v_mov_b32_e32 v18, s162587; SDAG-NEXT: v_mov_b32_e32 v19, s172588; SDAG-NEXT: v_mov_b32_e32 v20, s182589; SDAG-NEXT: v_mov_b32_e32 v21, s192590; SDAG-NEXT: v_mov_b32_e32 v22, s202591; SDAG-NEXT: v_mov_b32_e32 v23, s212592; SDAG-NEXT: v_mov_b32_e32 v24, s222593; SDAG-NEXT: v_mov_b32_e32 v25, s232594; SDAG-NEXT: s_nop 12595; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[26:29], v[18:25], v162596; SDAG-NEXT: s_setpc_b64 s[30:31]2597;2598; GISEL-LABEL: test_smfmac_f32_32x32x64_bf8_fp8__sgpr:2599; GISEL: ; %bb.0:2600; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2601; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[2:3]2602; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[0:1]2603; GISEL-NEXT: v_mov_b32_e32 v11, v02604; GISEL-NEXT: v_mov_b32_e32 v12, v12605; GISEL-NEXT: v_mov_b32_e32 v13, v22606; GISEL-NEXT: v_mov_b32_e32 v14, v32607; GISEL-NEXT: v_mov_b32_e32 v15, v42608; GISEL-NEXT: v_mov_b32_e32 v16, v52609; GISEL-NEXT: v_mov_b32_e32 v17, v62610; GISEL-NEXT: v_mov_b32_e32 v18, v72611; GISEL-NEXT: v_mov_b32_e32 v19, v82612; GISEL-NEXT: v_mov_b32_e32 v20, v92613; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[22:23]2614; GISEL-NEXT: v_mov_b32_e32 v21, v102615; GISEL-NEXT: v_mov_b32_e32 v0, s242616; GISEL-NEXT: v_mov_b32_e32 v1, s252617; GISEL-NEXT: v_mov_b32_e32 v2, s262618; GISEL-NEXT: v_mov_b32_e32 v3, s272619; GISEL-NEXT: v_mov_b32_e32 v4, s282620; GISEL-NEXT: v_mov_b32_e32 v5, s292621; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[20:21]2622; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[18:19]2623; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[16:17]2624; GISEL-NEXT: v_mov_b32_e32 v6, v112625; GISEL-NEXT: v_mov_b32_e32 v7, v122626; GISEL-NEXT: v_mov_b32_e32 v8, v132627; GISEL-NEXT: v_mov_b32_e32 v9, v142628; GISEL-NEXT: v_mov_b32_e32 v10, v152629; GISEL-NEXT: v_mov_b32_e32 v11, v162630; GISEL-NEXT: v_mov_b32_e32 v12, v172631; GISEL-NEXT: v_mov_b32_e32 v13, v182632; GISEL-NEXT: v_mov_b32_e32 v14, v192633; GISEL-NEXT: v_mov_b32_e32 v15, v202634; GISEL-NEXT: s_nop 12635; GISEL-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[30:33], v[22:29], v212636; GISEL-NEXT: s_setpc_b64 s[30:31]2637 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.bf8.fp8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)2638 ret <16 x float> %result2639}2640 2641; --------------------------------------------------------------------2642; llvm.amdgcn.smfmac.f32.32x32x64.fp8.bf82643; --------------------------------------------------------------------2644 2645declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.fp8.bf8(<4 x i32>, <8 x i32>, <16 x float>, i32, i32 immarg, i32 immarg)2646 2647define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_bf8__vgpr(ptr addrspace(1) %arg, <4 x i32> %a, <8 x i32> %b, i32 %idx) #0 {2648; SDAG-LABEL: test_smfmac_f32_32x32x64_fp8_bf8__vgpr:2649; SDAG: ; %bb.0: ; %bb2650; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x242651; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v02652; SDAG-NEXT: v_lshlrev_b32_e32 v16, 6, v02653; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x342654; SDAG-NEXT: s_load_dword s16, s[4:5], 0x642655; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x542656; SDAG-NEXT: s_waitcnt lgkmcnt(0)2657; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:482658; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:322659; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:162660; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]2661; SDAG-NEXT: v_mov_b32_e32 v24, s82662; SDAG-NEXT: v_mov_b32_e32 v25, s92663; SDAG-NEXT: v_mov_b32_e32 v26, s102664; SDAG-NEXT: v_mov_b32_e32 v27, s112665; SDAG-NEXT: v_mov_b32_e32 v16, s122666; SDAG-NEXT: v_mov_b32_e32 v17, s132667; SDAG-NEXT: v_mov_b32_e32 v18, s142668; SDAG-NEXT: v_mov_b32_e32 v19, s152669; SDAG-NEXT: v_mov_b32_e32 v20, s02670; SDAG-NEXT: v_mov_b32_e32 v21, s12671; SDAG-NEXT: v_mov_b32_e32 v22, s22672; SDAG-NEXT: v_mov_b32_e32 v23, s32673; SDAG-NEXT: v_mov_b32_e32 v28, s162674; SDAG-NEXT: s_waitcnt vmcnt(0)2675; SDAG-NEXT: s_nop 02676; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:22677; SDAG-NEXT: v_mov_b32_e32 v16, 02678; SDAG-NEXT: s_nop 102679; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:322680; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:482681; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]2682; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:162683; SDAG-NEXT: s_endpgm2684;2685; GISEL-LABEL: test_smfmac_f32_32x32x64_fp8_bf8__vgpr:2686; GISEL: ; %bb.0: ; %bb2687; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x242688; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v02689; GISEL-NEXT: v_lshlrev_b32_e32 v16, 6, v02690; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x342691; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x542692; GISEL-NEXT: s_load_dword s2, s[4:5], 0x642693; GISEL-NEXT: s_waitcnt lgkmcnt(0)2694; GISEL-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]2695; GISEL-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:162696; GISEL-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:322697; GISEL-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:482698; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[10:11]2699; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[8:9]2700; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[18:19]2701; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]2702; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[14:15]2703; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[12:13]2704; GISEL-NEXT: v_mov_b32_e32 v28, s22705; GISEL-NEXT: s_waitcnt vmcnt(0)2706; GISEL-NEXT: s_nop 02707; GISEL-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:22708; GISEL-NEXT: v_mov_b32_e32 v16, 02709; GISEL-NEXT: s_nop 102710; GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]2711; GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:162712; GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:322713; GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:482714; GISEL-NEXT: s_endpgm2715bb:2716 %id = call i32 @llvm.amdgcn.workitem.id.x()2717 %gep = getelementptr <16 x float>, ptr addrspace(1) %arg, i32 %id2718 %in.1 = load <16 x float>, ptr addrspace(1) %gep2719 %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.fp8.bf8(<4 x i32> %a, <8 x i32> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2)2720 store <16 x float> %mai.1, ptr addrspace(1) %arg2721 ret void2722}2723 2724define <16 x float> @test_smfmac_f32_32x32x64_fp8_bf8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3) {2725; SDAG-LABEL: test_smfmac_f32_32x32x64_fp8_bf8:2726; SDAG: ; %bb.0:2727; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2728; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[12:27], v[0:3], v[4:11], v282729; SDAG-NEXT: s_nop 112730; SDAG-NEXT: v_mov_b32_e32 v0, v122731; SDAG-NEXT: v_mov_b32_e32 v1, v132732; SDAG-NEXT: v_mov_b32_e32 v2, v142733; SDAG-NEXT: v_mov_b32_e32 v3, v152734; SDAG-NEXT: v_mov_b32_e32 v4, v162735; SDAG-NEXT: v_mov_b32_e32 v5, v172736; SDAG-NEXT: v_mov_b32_e32 v6, v182737; SDAG-NEXT: v_mov_b32_e32 v7, v192738; SDAG-NEXT: v_mov_b32_e32 v8, v202739; SDAG-NEXT: v_mov_b32_e32 v9, v212740; SDAG-NEXT: v_mov_b32_e32 v10, v222741; SDAG-NEXT: v_mov_b32_e32 v11, v232742; SDAG-NEXT: v_mov_b32_e32 v12, v242743; SDAG-NEXT: v_mov_b32_e32 v13, v252744; SDAG-NEXT: v_mov_b32_e32 v14, v262745; SDAG-NEXT: v_mov_b32_e32 v15, v272746; SDAG-NEXT: s_setpc_b64 s[30:31]2747;2748; GISEL-LABEL: test_smfmac_f32_32x32x64_fp8_bf8:2749; GISEL: ; %bb.0:2750; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2751; GISEL-NEXT: v_mov_b32_e32 v48, v02752; GISEL-NEXT: v_mov_b32_e32 v49, v12753; GISEL-NEXT: v_mov_b32_e32 v50, v22754; GISEL-NEXT: v_mov_b32_e32 v51, v32755; GISEL-NEXT: v_mov_b32_e32 v30, v42756; GISEL-NEXT: v_mov_b32_e32 v31, v52757; GISEL-NEXT: v_mov_b32_e32 v32, v62758; GISEL-NEXT: v_mov_b32_e32 v33, v72759; GISEL-NEXT: v_mov_b32_e32 v34, v82760; GISEL-NEXT: v_mov_b32_e32 v35, v92761; GISEL-NEXT: v_mov_b32_e32 v36, v102762; GISEL-NEXT: v_mov_b32_e32 v37, v112763; GISEL-NEXT: v_mov_b32_e32 v0, v122764; GISEL-NEXT: v_mov_b32_e32 v1, v132765; GISEL-NEXT: v_mov_b32_e32 v2, v142766; GISEL-NEXT: v_mov_b32_e32 v3, v152767; GISEL-NEXT: v_mov_b32_e32 v4, v162768; GISEL-NEXT: v_mov_b32_e32 v5, v172769; GISEL-NEXT: v_mov_b32_e32 v6, v182770; GISEL-NEXT: v_mov_b32_e32 v7, v192771; GISEL-NEXT: v_mov_b32_e32 v8, v202772; GISEL-NEXT: v_mov_b32_e32 v9, v212773; GISEL-NEXT: v_mov_b32_e32 v10, v222774; GISEL-NEXT: v_mov_b32_e32 v11, v232775; GISEL-NEXT: v_mov_b32_e32 v12, v242776; GISEL-NEXT: v_mov_b32_e32 v13, v252777; GISEL-NEXT: v_mov_b32_e32 v14, v262778; GISEL-NEXT: v_mov_b32_e32 v15, v272779; GISEL-NEXT: s_nop 12780; GISEL-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[48:51], v[30:37], v282781; GISEL-NEXT: s_setpc_b64 s[30:31]2782 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.fp8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)2783 ret <16 x float> %result2784}2785 2786define <16 x float> @test_smfmac_f32_32x32x64_fp8_bf8__flags0(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3) {2787; SDAG-LABEL: test_smfmac_f32_32x32x64_fp8_bf8__flags0:2788; SDAG: ; %bb.0:2789; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2790; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[12:27], v[0:3], v[4:11], v28 cbsz:1 abid:32791; SDAG-NEXT: s_nop 112792; SDAG-NEXT: v_mov_b32_e32 v0, v122793; SDAG-NEXT: v_mov_b32_e32 v1, v132794; SDAG-NEXT: v_mov_b32_e32 v2, v142795; SDAG-NEXT: v_mov_b32_e32 v3, v152796; SDAG-NEXT: v_mov_b32_e32 v4, v162797; SDAG-NEXT: v_mov_b32_e32 v5, v172798; SDAG-NEXT: v_mov_b32_e32 v6, v182799; SDAG-NEXT: v_mov_b32_e32 v7, v192800; SDAG-NEXT: v_mov_b32_e32 v8, v202801; SDAG-NEXT: v_mov_b32_e32 v9, v212802; SDAG-NEXT: v_mov_b32_e32 v10, v222803; SDAG-NEXT: v_mov_b32_e32 v11, v232804; SDAG-NEXT: v_mov_b32_e32 v12, v242805; SDAG-NEXT: v_mov_b32_e32 v13, v252806; SDAG-NEXT: v_mov_b32_e32 v14, v262807; SDAG-NEXT: v_mov_b32_e32 v15, v272808; SDAG-NEXT: s_setpc_b64 s[30:31]2809;2810; GISEL-LABEL: test_smfmac_f32_32x32x64_fp8_bf8__flags0:2811; GISEL: ; %bb.0:2812; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2813; GISEL-NEXT: v_mov_b32_e32 v48, v02814; GISEL-NEXT: v_mov_b32_e32 v49, v12815; GISEL-NEXT: v_mov_b32_e32 v50, v22816; GISEL-NEXT: v_mov_b32_e32 v51, v32817; GISEL-NEXT: v_mov_b32_e32 v30, v42818; GISEL-NEXT: v_mov_b32_e32 v31, v52819; GISEL-NEXT: v_mov_b32_e32 v32, v62820; GISEL-NEXT: v_mov_b32_e32 v33, v72821; GISEL-NEXT: v_mov_b32_e32 v34, v82822; GISEL-NEXT: v_mov_b32_e32 v35, v92823; GISEL-NEXT: v_mov_b32_e32 v36, v102824; GISEL-NEXT: v_mov_b32_e32 v37, v112825; GISEL-NEXT: v_mov_b32_e32 v0, v122826; GISEL-NEXT: v_mov_b32_e32 v1, v132827; GISEL-NEXT: v_mov_b32_e32 v2, v142828; GISEL-NEXT: v_mov_b32_e32 v3, v152829; GISEL-NEXT: v_mov_b32_e32 v4, v162830; GISEL-NEXT: v_mov_b32_e32 v5, v172831; GISEL-NEXT: v_mov_b32_e32 v6, v182832; GISEL-NEXT: v_mov_b32_e32 v7, v192833; GISEL-NEXT: v_mov_b32_e32 v8, v202834; GISEL-NEXT: v_mov_b32_e32 v9, v212835; GISEL-NEXT: v_mov_b32_e32 v10, v222836; GISEL-NEXT: v_mov_b32_e32 v11, v232837; GISEL-NEXT: v_mov_b32_e32 v12, v242838; GISEL-NEXT: v_mov_b32_e32 v13, v252839; GISEL-NEXT: v_mov_b32_e32 v14, v262840; GISEL-NEXT: v_mov_b32_e32 v15, v272841; GISEL-NEXT: s_nop 12842; GISEL-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[48:51], v[30:37], v28 cbsz:1 abid:32843; GISEL-NEXT: s_setpc_b64 s[30:31]2844 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.fp8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)2845 ret <16 x float> %result2846}2847 2848define <16 x float> @test_smfmac_f32_32x32x64_fp8_bf8__flags1(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3) {2849; SDAG-LABEL: test_smfmac_f32_32x32x64_fp8_bf8__flags1:2850; SDAG: ; %bb.0:2851; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2852; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[12:27], v[0:3], v[4:11], v28 cbsz:3 abid:12853; SDAG-NEXT: s_nop 112854; SDAG-NEXT: v_mov_b32_e32 v0, v122855; SDAG-NEXT: v_mov_b32_e32 v1, v132856; SDAG-NEXT: v_mov_b32_e32 v2, v142857; SDAG-NEXT: v_mov_b32_e32 v3, v152858; SDAG-NEXT: v_mov_b32_e32 v4, v162859; SDAG-NEXT: v_mov_b32_e32 v5, v172860; SDAG-NEXT: v_mov_b32_e32 v6, v182861; SDAG-NEXT: v_mov_b32_e32 v7, v192862; SDAG-NEXT: v_mov_b32_e32 v8, v202863; SDAG-NEXT: v_mov_b32_e32 v9, v212864; SDAG-NEXT: v_mov_b32_e32 v10, v222865; SDAG-NEXT: v_mov_b32_e32 v11, v232866; SDAG-NEXT: v_mov_b32_e32 v12, v242867; SDAG-NEXT: v_mov_b32_e32 v13, v252868; SDAG-NEXT: v_mov_b32_e32 v14, v262869; SDAG-NEXT: v_mov_b32_e32 v15, v272870; SDAG-NEXT: s_setpc_b64 s[30:31]2871;2872; GISEL-LABEL: test_smfmac_f32_32x32x64_fp8_bf8__flags1:2873; GISEL: ; %bb.0:2874; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2875; GISEL-NEXT: v_mov_b32_e32 v48, v02876; GISEL-NEXT: v_mov_b32_e32 v49, v12877; GISEL-NEXT: v_mov_b32_e32 v50, v22878; GISEL-NEXT: v_mov_b32_e32 v51, v32879; GISEL-NEXT: v_mov_b32_e32 v30, v42880; GISEL-NEXT: v_mov_b32_e32 v31, v52881; GISEL-NEXT: v_mov_b32_e32 v32, v62882; GISEL-NEXT: v_mov_b32_e32 v33, v72883; GISEL-NEXT: v_mov_b32_e32 v34, v82884; GISEL-NEXT: v_mov_b32_e32 v35, v92885; GISEL-NEXT: v_mov_b32_e32 v36, v102886; GISEL-NEXT: v_mov_b32_e32 v37, v112887; GISEL-NEXT: v_mov_b32_e32 v0, v122888; GISEL-NEXT: v_mov_b32_e32 v1, v132889; GISEL-NEXT: v_mov_b32_e32 v2, v142890; GISEL-NEXT: v_mov_b32_e32 v3, v152891; GISEL-NEXT: v_mov_b32_e32 v4, v162892; GISEL-NEXT: v_mov_b32_e32 v5, v172893; GISEL-NEXT: v_mov_b32_e32 v6, v182894; GISEL-NEXT: v_mov_b32_e32 v7, v192895; GISEL-NEXT: v_mov_b32_e32 v8, v202896; GISEL-NEXT: v_mov_b32_e32 v9, v212897; GISEL-NEXT: v_mov_b32_e32 v10, v222898; GISEL-NEXT: v_mov_b32_e32 v11, v232899; GISEL-NEXT: v_mov_b32_e32 v12, v242900; GISEL-NEXT: v_mov_b32_e32 v13, v252901; GISEL-NEXT: v_mov_b32_e32 v14, v262902; GISEL-NEXT: v_mov_b32_e32 v15, v272903; GISEL-NEXT: s_nop 12904; GISEL-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[48:51], v[30:37], v28 cbsz:3 abid:12905; GISEL-NEXT: s_setpc_b64 s[30:31]2906 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.fp8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)2907 ret <16 x float> %result2908}2909 2910define <16 x float> @test_smfmac_f32_32x32x64_fp8_bf8__sgpr(<4 x i32> inreg %arg0, <8 x i32> inreg %arg1, <16 x float> inreg %arg2, i32 inreg %arg3) {2911; SDAG-LABEL: test_smfmac_f32_32x32x64_fp8_bf8__sgpr:2912; SDAG: ; %bb.0:2913; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2914; SDAG-NEXT: v_mov_b32_e32 v26, s02915; SDAG-NEXT: v_mov_b32_e32 v27, s12916; SDAG-NEXT: v_mov_b32_e32 v28, s22917; SDAG-NEXT: v_mov_b32_e32 v29, s32918; SDAG-NEXT: v_mov_b32_e32 v16, v102919; SDAG-NEXT: v_mov_b32_e32 v15, v92920; SDAG-NEXT: v_mov_b32_e32 v14, v82921; SDAG-NEXT: v_mov_b32_e32 v13, v72922; SDAG-NEXT: v_mov_b32_e32 v12, v62923; SDAG-NEXT: v_mov_b32_e32 v11, v52924; SDAG-NEXT: v_mov_b32_e32 v10, v42925; SDAG-NEXT: v_mov_b32_e32 v9, v32926; SDAG-NEXT: v_mov_b32_e32 v8, v22927; SDAG-NEXT: v_mov_b32_e32 v7, v12928; SDAG-NEXT: v_mov_b32_e32 v6, v02929; SDAG-NEXT: v_mov_b32_e32 v0, s242930; SDAG-NEXT: v_mov_b32_e32 v1, s252931; SDAG-NEXT: v_mov_b32_e32 v2, s262932; SDAG-NEXT: v_mov_b32_e32 v3, s272933; SDAG-NEXT: v_mov_b32_e32 v4, s282934; SDAG-NEXT: v_mov_b32_e32 v5, s292935; SDAG-NEXT: v_mov_b32_e32 v18, s162936; SDAG-NEXT: v_mov_b32_e32 v19, s172937; SDAG-NEXT: v_mov_b32_e32 v20, s182938; SDAG-NEXT: v_mov_b32_e32 v21, s192939; SDAG-NEXT: v_mov_b32_e32 v22, s202940; SDAG-NEXT: v_mov_b32_e32 v23, s212941; SDAG-NEXT: v_mov_b32_e32 v24, s222942; SDAG-NEXT: v_mov_b32_e32 v25, s232943; SDAG-NEXT: s_nop 12944; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[26:29], v[18:25], v162945; SDAG-NEXT: s_setpc_b64 s[30:31]2946;2947; GISEL-LABEL: test_smfmac_f32_32x32x64_fp8_bf8__sgpr:2948; GISEL: ; %bb.0:2949; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2950; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[2:3]2951; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[0:1]2952; GISEL-NEXT: v_mov_b32_e32 v11, v02953; GISEL-NEXT: v_mov_b32_e32 v12, v12954; GISEL-NEXT: v_mov_b32_e32 v13, v22955; GISEL-NEXT: v_mov_b32_e32 v14, v32956; GISEL-NEXT: v_mov_b32_e32 v15, v42957; GISEL-NEXT: v_mov_b32_e32 v16, v52958; GISEL-NEXT: v_mov_b32_e32 v17, v62959; GISEL-NEXT: v_mov_b32_e32 v18, v72960; GISEL-NEXT: v_mov_b32_e32 v19, v82961; GISEL-NEXT: v_mov_b32_e32 v20, v92962; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[22:23]2963; GISEL-NEXT: v_mov_b32_e32 v21, v102964; GISEL-NEXT: v_mov_b32_e32 v0, s242965; GISEL-NEXT: v_mov_b32_e32 v1, s252966; GISEL-NEXT: v_mov_b32_e32 v2, s262967; GISEL-NEXT: v_mov_b32_e32 v3, s272968; GISEL-NEXT: v_mov_b32_e32 v4, s282969; GISEL-NEXT: v_mov_b32_e32 v5, s292970; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[20:21]2971; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[18:19]2972; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[16:17]2973; GISEL-NEXT: v_mov_b32_e32 v6, v112974; GISEL-NEXT: v_mov_b32_e32 v7, v122975; GISEL-NEXT: v_mov_b32_e32 v8, v132976; GISEL-NEXT: v_mov_b32_e32 v9, v142977; GISEL-NEXT: v_mov_b32_e32 v10, v152978; GISEL-NEXT: v_mov_b32_e32 v11, v162979; GISEL-NEXT: v_mov_b32_e32 v12, v172980; GISEL-NEXT: v_mov_b32_e32 v13, v182981; GISEL-NEXT: v_mov_b32_e32 v14, v192982; GISEL-NEXT: v_mov_b32_e32 v15, v202983; GISEL-NEXT: s_nop 12984; GISEL-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[30:33], v[22:29], v212985; GISEL-NEXT: s_setpc_b64 s[30:31]2986 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.fp8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)2987 ret <16 x float> %result2988}2989 2990; --------------------------------------------------------------------2991; llvm.amdgcn.smfmac.f32.32x32x64.fp8.fp82992; --------------------------------------------------------------------2993 2994declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.fp8.fp8(<4 x i32>, <8 x i32>, <16 x float>, i32, i32 immarg, i32 immarg)2995 2996define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_fp8__vgpr(ptr addrspace(1) %arg, <4 x i32> %a, <8 x i32> %b, i32 %idx) #0 {2997; SDAG-LABEL: test_smfmac_f32_32x32x64_fp8_fp8__vgpr:2998; SDAG: ; %bb.0: ; %bb2999; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x243000; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v03001; SDAG-NEXT: v_lshlrev_b32_e32 v16, 6, v03002; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x343003; SDAG-NEXT: s_load_dword s16, s[4:5], 0x643004; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x543005; SDAG-NEXT: s_waitcnt lgkmcnt(0)3006; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:483007; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:323008; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:163009; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]3010; SDAG-NEXT: v_mov_b32_e32 v24, s83011; SDAG-NEXT: v_mov_b32_e32 v25, s93012; SDAG-NEXT: v_mov_b32_e32 v26, s103013; SDAG-NEXT: v_mov_b32_e32 v27, s113014; SDAG-NEXT: v_mov_b32_e32 v16, s123015; SDAG-NEXT: v_mov_b32_e32 v17, s133016; SDAG-NEXT: v_mov_b32_e32 v18, s143017; SDAG-NEXT: v_mov_b32_e32 v19, s153018; SDAG-NEXT: v_mov_b32_e32 v20, s03019; SDAG-NEXT: v_mov_b32_e32 v21, s13020; SDAG-NEXT: v_mov_b32_e32 v22, s23021; SDAG-NEXT: v_mov_b32_e32 v23, s33022; SDAG-NEXT: v_mov_b32_e32 v28, s163023; SDAG-NEXT: s_waitcnt vmcnt(0)3024; SDAG-NEXT: s_nop 03025; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:23026; SDAG-NEXT: v_mov_b32_e32 v16, 03027; SDAG-NEXT: s_nop 103028; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:323029; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:483030; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]3031; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:163032; SDAG-NEXT: s_endpgm3033;3034; GISEL-LABEL: test_smfmac_f32_32x32x64_fp8_fp8__vgpr:3035; GISEL: ; %bb.0: ; %bb3036; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x243037; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v03038; GISEL-NEXT: v_lshlrev_b32_e32 v16, 6, v03039; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x343040; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x543041; GISEL-NEXT: s_load_dword s2, s[4:5], 0x643042; GISEL-NEXT: s_waitcnt lgkmcnt(0)3043; GISEL-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]3044; GISEL-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:163045; GISEL-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:323046; GISEL-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:483047; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[10:11]3048; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[8:9]3049; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[18:19]3050; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]3051; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[14:15]3052; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[12:13]3053; GISEL-NEXT: v_mov_b32_e32 v28, s23054; GISEL-NEXT: s_waitcnt vmcnt(0)3055; GISEL-NEXT: s_nop 03056; GISEL-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:23057; GISEL-NEXT: v_mov_b32_e32 v16, 03058; GISEL-NEXT: s_nop 103059; GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]3060; GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:163061; GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:323062; GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:483063; GISEL-NEXT: s_endpgm3064bb:3065 %id = call i32 @llvm.amdgcn.workitem.id.x()3066 %gep = getelementptr <16 x float>, ptr addrspace(1) %arg, i32 %id3067 %in.1 = load <16 x float>, ptr addrspace(1) %gep3068 %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.fp8.fp8(<4 x i32> %a, <8 x i32> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2)3069 store <16 x float> %mai.1, ptr addrspace(1) %arg3070 ret void3071}3072 3073define <16 x float> @test_smfmac_f32_32x32x64_fp8_fp8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3) {3074; SDAG-LABEL: test_smfmac_f32_32x32x64_fp8_fp8:3075; SDAG: ; %bb.0:3076; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3077; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[12:27], v[0:3], v[4:11], v283078; SDAG-NEXT: s_nop 113079; SDAG-NEXT: v_mov_b32_e32 v0, v123080; SDAG-NEXT: v_mov_b32_e32 v1, v133081; SDAG-NEXT: v_mov_b32_e32 v2, v143082; SDAG-NEXT: v_mov_b32_e32 v3, v153083; SDAG-NEXT: v_mov_b32_e32 v4, v163084; SDAG-NEXT: v_mov_b32_e32 v5, v173085; SDAG-NEXT: v_mov_b32_e32 v6, v183086; SDAG-NEXT: v_mov_b32_e32 v7, v193087; SDAG-NEXT: v_mov_b32_e32 v8, v203088; SDAG-NEXT: v_mov_b32_e32 v9, v213089; SDAG-NEXT: v_mov_b32_e32 v10, v223090; SDAG-NEXT: v_mov_b32_e32 v11, v233091; SDAG-NEXT: v_mov_b32_e32 v12, v243092; SDAG-NEXT: v_mov_b32_e32 v13, v253093; SDAG-NEXT: v_mov_b32_e32 v14, v263094; SDAG-NEXT: v_mov_b32_e32 v15, v273095; SDAG-NEXT: s_setpc_b64 s[30:31]3096;3097; GISEL-LABEL: test_smfmac_f32_32x32x64_fp8_fp8:3098; GISEL: ; %bb.0:3099; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3100; GISEL-NEXT: v_mov_b32_e32 v48, v03101; GISEL-NEXT: v_mov_b32_e32 v49, v13102; GISEL-NEXT: v_mov_b32_e32 v50, v23103; GISEL-NEXT: v_mov_b32_e32 v51, v33104; GISEL-NEXT: v_mov_b32_e32 v30, v43105; GISEL-NEXT: v_mov_b32_e32 v31, v53106; GISEL-NEXT: v_mov_b32_e32 v32, v63107; GISEL-NEXT: v_mov_b32_e32 v33, v73108; GISEL-NEXT: v_mov_b32_e32 v34, v83109; GISEL-NEXT: v_mov_b32_e32 v35, v93110; GISEL-NEXT: v_mov_b32_e32 v36, v103111; GISEL-NEXT: v_mov_b32_e32 v37, v113112; GISEL-NEXT: v_mov_b32_e32 v0, v123113; GISEL-NEXT: v_mov_b32_e32 v1, v133114; GISEL-NEXT: v_mov_b32_e32 v2, v143115; GISEL-NEXT: v_mov_b32_e32 v3, v153116; GISEL-NEXT: v_mov_b32_e32 v4, v163117; GISEL-NEXT: v_mov_b32_e32 v5, v173118; GISEL-NEXT: v_mov_b32_e32 v6, v183119; GISEL-NEXT: v_mov_b32_e32 v7, v193120; GISEL-NEXT: v_mov_b32_e32 v8, v203121; GISEL-NEXT: v_mov_b32_e32 v9, v213122; GISEL-NEXT: v_mov_b32_e32 v10, v223123; GISEL-NEXT: v_mov_b32_e32 v11, v233124; GISEL-NEXT: v_mov_b32_e32 v12, v243125; GISEL-NEXT: v_mov_b32_e32 v13, v253126; GISEL-NEXT: v_mov_b32_e32 v14, v263127; GISEL-NEXT: v_mov_b32_e32 v15, v273128; GISEL-NEXT: s_nop 13129; GISEL-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[48:51], v[30:37], v283130; GISEL-NEXT: s_setpc_b64 s[30:31]3131 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.fp8.fp8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)3132 ret <16 x float> %result3133}3134 3135define <16 x float> @test_smfmac_f32_32x32x64_fp8_fp8__flags0(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3) {3136; SDAG-LABEL: test_smfmac_f32_32x32x64_fp8_fp8__flags0:3137; SDAG: ; %bb.0:3138; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3139; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[12:27], v[0:3], v[4:11], v28 cbsz:1 abid:33140; SDAG-NEXT: s_nop 113141; SDAG-NEXT: v_mov_b32_e32 v0, v123142; SDAG-NEXT: v_mov_b32_e32 v1, v133143; SDAG-NEXT: v_mov_b32_e32 v2, v143144; SDAG-NEXT: v_mov_b32_e32 v3, v153145; SDAG-NEXT: v_mov_b32_e32 v4, v163146; SDAG-NEXT: v_mov_b32_e32 v5, v173147; SDAG-NEXT: v_mov_b32_e32 v6, v183148; SDAG-NEXT: v_mov_b32_e32 v7, v193149; SDAG-NEXT: v_mov_b32_e32 v8, v203150; SDAG-NEXT: v_mov_b32_e32 v9, v213151; SDAG-NEXT: v_mov_b32_e32 v10, v223152; SDAG-NEXT: v_mov_b32_e32 v11, v233153; SDAG-NEXT: v_mov_b32_e32 v12, v243154; SDAG-NEXT: v_mov_b32_e32 v13, v253155; SDAG-NEXT: v_mov_b32_e32 v14, v263156; SDAG-NEXT: v_mov_b32_e32 v15, v273157; SDAG-NEXT: s_setpc_b64 s[30:31]3158;3159; GISEL-LABEL: test_smfmac_f32_32x32x64_fp8_fp8__flags0:3160; GISEL: ; %bb.0:3161; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3162; GISEL-NEXT: v_mov_b32_e32 v48, v03163; GISEL-NEXT: v_mov_b32_e32 v49, v13164; GISEL-NEXT: v_mov_b32_e32 v50, v23165; GISEL-NEXT: v_mov_b32_e32 v51, v33166; GISEL-NEXT: v_mov_b32_e32 v30, v43167; GISEL-NEXT: v_mov_b32_e32 v31, v53168; GISEL-NEXT: v_mov_b32_e32 v32, v63169; GISEL-NEXT: v_mov_b32_e32 v33, v73170; GISEL-NEXT: v_mov_b32_e32 v34, v83171; GISEL-NEXT: v_mov_b32_e32 v35, v93172; GISEL-NEXT: v_mov_b32_e32 v36, v103173; GISEL-NEXT: v_mov_b32_e32 v37, v113174; GISEL-NEXT: v_mov_b32_e32 v0, v123175; GISEL-NEXT: v_mov_b32_e32 v1, v133176; GISEL-NEXT: v_mov_b32_e32 v2, v143177; GISEL-NEXT: v_mov_b32_e32 v3, v153178; GISEL-NEXT: v_mov_b32_e32 v4, v163179; GISEL-NEXT: v_mov_b32_e32 v5, v173180; GISEL-NEXT: v_mov_b32_e32 v6, v183181; GISEL-NEXT: v_mov_b32_e32 v7, v193182; GISEL-NEXT: v_mov_b32_e32 v8, v203183; GISEL-NEXT: v_mov_b32_e32 v9, v213184; GISEL-NEXT: v_mov_b32_e32 v10, v223185; GISEL-NEXT: v_mov_b32_e32 v11, v233186; GISEL-NEXT: v_mov_b32_e32 v12, v243187; GISEL-NEXT: v_mov_b32_e32 v13, v253188; GISEL-NEXT: v_mov_b32_e32 v14, v263189; GISEL-NEXT: v_mov_b32_e32 v15, v273190; GISEL-NEXT: s_nop 13191; GISEL-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[48:51], v[30:37], v28 cbsz:1 abid:33192; GISEL-NEXT: s_setpc_b64 s[30:31]3193 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.fp8.fp8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)3194 ret <16 x float> %result3195}3196 3197define <16 x float> @test_smfmac_f32_32x32x64_fp8_fp8__flags1(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3) {3198; SDAG-LABEL: test_smfmac_f32_32x32x64_fp8_fp8__flags1:3199; SDAG: ; %bb.0:3200; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3201; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[12:27], v[0:3], v[4:11], v28 cbsz:3 abid:13202; SDAG-NEXT: s_nop 113203; SDAG-NEXT: v_mov_b32_e32 v0, v123204; SDAG-NEXT: v_mov_b32_e32 v1, v133205; SDAG-NEXT: v_mov_b32_e32 v2, v143206; SDAG-NEXT: v_mov_b32_e32 v3, v153207; SDAG-NEXT: v_mov_b32_e32 v4, v163208; SDAG-NEXT: v_mov_b32_e32 v5, v173209; SDAG-NEXT: v_mov_b32_e32 v6, v183210; SDAG-NEXT: v_mov_b32_e32 v7, v193211; SDAG-NEXT: v_mov_b32_e32 v8, v203212; SDAG-NEXT: v_mov_b32_e32 v9, v213213; SDAG-NEXT: v_mov_b32_e32 v10, v223214; SDAG-NEXT: v_mov_b32_e32 v11, v233215; SDAG-NEXT: v_mov_b32_e32 v12, v243216; SDAG-NEXT: v_mov_b32_e32 v13, v253217; SDAG-NEXT: v_mov_b32_e32 v14, v263218; SDAG-NEXT: v_mov_b32_e32 v15, v273219; SDAG-NEXT: s_setpc_b64 s[30:31]3220;3221; GISEL-LABEL: test_smfmac_f32_32x32x64_fp8_fp8__flags1:3222; GISEL: ; %bb.0:3223; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3224; GISEL-NEXT: v_mov_b32_e32 v48, v03225; GISEL-NEXT: v_mov_b32_e32 v49, v13226; GISEL-NEXT: v_mov_b32_e32 v50, v23227; GISEL-NEXT: v_mov_b32_e32 v51, v33228; GISEL-NEXT: v_mov_b32_e32 v30, v43229; GISEL-NEXT: v_mov_b32_e32 v31, v53230; GISEL-NEXT: v_mov_b32_e32 v32, v63231; GISEL-NEXT: v_mov_b32_e32 v33, v73232; GISEL-NEXT: v_mov_b32_e32 v34, v83233; GISEL-NEXT: v_mov_b32_e32 v35, v93234; GISEL-NEXT: v_mov_b32_e32 v36, v103235; GISEL-NEXT: v_mov_b32_e32 v37, v113236; GISEL-NEXT: v_mov_b32_e32 v0, v123237; GISEL-NEXT: v_mov_b32_e32 v1, v133238; GISEL-NEXT: v_mov_b32_e32 v2, v143239; GISEL-NEXT: v_mov_b32_e32 v3, v153240; GISEL-NEXT: v_mov_b32_e32 v4, v163241; GISEL-NEXT: v_mov_b32_e32 v5, v173242; GISEL-NEXT: v_mov_b32_e32 v6, v183243; GISEL-NEXT: v_mov_b32_e32 v7, v193244; GISEL-NEXT: v_mov_b32_e32 v8, v203245; GISEL-NEXT: v_mov_b32_e32 v9, v213246; GISEL-NEXT: v_mov_b32_e32 v10, v223247; GISEL-NEXT: v_mov_b32_e32 v11, v233248; GISEL-NEXT: v_mov_b32_e32 v12, v243249; GISEL-NEXT: v_mov_b32_e32 v13, v253250; GISEL-NEXT: v_mov_b32_e32 v14, v263251; GISEL-NEXT: v_mov_b32_e32 v15, v273252; GISEL-NEXT: s_nop 13253; GISEL-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[48:51], v[30:37], v28 cbsz:3 abid:13254; GISEL-NEXT: s_setpc_b64 s[30:31]3255 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.fp8.fp8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)3256 ret <16 x float> %result3257}3258 3259define <16 x float> @test_smfmac_f32_32x32x64_fp8_fp8__sgpr(<4 x i32> inreg %arg0, <8 x i32> inreg %arg1, <16 x float> inreg %arg2, i32 inreg %arg3) {3260; SDAG-LABEL: test_smfmac_f32_32x32x64_fp8_fp8__sgpr:3261; SDAG: ; %bb.0:3262; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3263; SDAG-NEXT: v_mov_b32_e32 v26, s03264; SDAG-NEXT: v_mov_b32_e32 v27, s13265; SDAG-NEXT: v_mov_b32_e32 v28, s23266; SDAG-NEXT: v_mov_b32_e32 v29, s33267; SDAG-NEXT: v_mov_b32_e32 v16, v103268; SDAG-NEXT: v_mov_b32_e32 v15, v93269; SDAG-NEXT: v_mov_b32_e32 v14, v83270; SDAG-NEXT: v_mov_b32_e32 v13, v73271; SDAG-NEXT: v_mov_b32_e32 v12, v63272; SDAG-NEXT: v_mov_b32_e32 v11, v53273; SDAG-NEXT: v_mov_b32_e32 v10, v43274; SDAG-NEXT: v_mov_b32_e32 v9, v33275; SDAG-NEXT: v_mov_b32_e32 v8, v23276; SDAG-NEXT: v_mov_b32_e32 v7, v13277; SDAG-NEXT: v_mov_b32_e32 v6, v03278; SDAG-NEXT: v_mov_b32_e32 v0, s243279; SDAG-NEXT: v_mov_b32_e32 v1, s253280; SDAG-NEXT: v_mov_b32_e32 v2, s263281; SDAG-NEXT: v_mov_b32_e32 v3, s273282; SDAG-NEXT: v_mov_b32_e32 v4, s283283; SDAG-NEXT: v_mov_b32_e32 v5, s293284; SDAG-NEXT: v_mov_b32_e32 v18, s163285; SDAG-NEXT: v_mov_b32_e32 v19, s173286; SDAG-NEXT: v_mov_b32_e32 v20, s183287; SDAG-NEXT: v_mov_b32_e32 v21, s193288; SDAG-NEXT: v_mov_b32_e32 v22, s203289; SDAG-NEXT: v_mov_b32_e32 v23, s213290; SDAG-NEXT: v_mov_b32_e32 v24, s223291; SDAG-NEXT: v_mov_b32_e32 v25, s233292; SDAG-NEXT: s_nop 13293; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[26:29], v[18:25], v163294; SDAG-NEXT: s_setpc_b64 s[30:31]3295;3296; GISEL-LABEL: test_smfmac_f32_32x32x64_fp8_fp8__sgpr:3297; GISEL: ; %bb.0:3298; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3299; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[2:3]3300; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[0:1]3301; GISEL-NEXT: v_mov_b32_e32 v11, v03302; GISEL-NEXT: v_mov_b32_e32 v12, v13303; GISEL-NEXT: v_mov_b32_e32 v13, v23304; GISEL-NEXT: v_mov_b32_e32 v14, v33305; GISEL-NEXT: v_mov_b32_e32 v15, v43306; GISEL-NEXT: v_mov_b32_e32 v16, v53307; GISEL-NEXT: v_mov_b32_e32 v17, v63308; GISEL-NEXT: v_mov_b32_e32 v18, v73309; GISEL-NEXT: v_mov_b32_e32 v19, v83310; GISEL-NEXT: v_mov_b32_e32 v20, v93311; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[22:23]3312; GISEL-NEXT: v_mov_b32_e32 v21, v103313; GISEL-NEXT: v_mov_b32_e32 v0, s243314; GISEL-NEXT: v_mov_b32_e32 v1, s253315; GISEL-NEXT: v_mov_b32_e32 v2, s263316; GISEL-NEXT: v_mov_b32_e32 v3, s273317; GISEL-NEXT: v_mov_b32_e32 v4, s283318; GISEL-NEXT: v_mov_b32_e32 v5, s293319; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[20:21]3320; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[18:19]3321; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[16:17]3322; GISEL-NEXT: v_mov_b32_e32 v6, v113323; GISEL-NEXT: v_mov_b32_e32 v7, v123324; GISEL-NEXT: v_mov_b32_e32 v8, v133325; GISEL-NEXT: v_mov_b32_e32 v9, v143326; GISEL-NEXT: v_mov_b32_e32 v10, v153327; GISEL-NEXT: v_mov_b32_e32 v11, v163328; GISEL-NEXT: v_mov_b32_e32 v12, v173329; GISEL-NEXT: v_mov_b32_e32 v13, v183330; GISEL-NEXT: v_mov_b32_e32 v14, v193331; GISEL-NEXT: v_mov_b32_e32 v15, v203332; GISEL-NEXT: s_nop 13333; GISEL-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[30:33], v[22:29], v213334; GISEL-NEXT: s_setpc_b64 s[30:31]3335 %result = call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.fp8.fp8(<4 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)3336 ret <16 x float> %result3337}3338 3339attributes #0 = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-agpr-alloc"="0,0" }3340