5620 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 42; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,SDAG %s3; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 --amdgpu-mfma-vgpr-form=0 < %s | FileCheck -enable-var-scope --check-prefixes=HEURRC %s5; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 --amdgpu-mfma-vgpr-form=1 < %s | FileCheck -enable-var-scope --check-prefixes=VGPRRC %s6 7declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half>, <8 x half>, <4 x float>, i32 immarg, i32 immarg, i32 immarg)8declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half>, <8 x half>, <16 x float>, i32 immarg, i32 immarg, i32 immarg)9 10; --------------------------------------------------------------------11; llvm.amdgcn.mfma.f32.16x16x32.f1612; --------------------------------------------------------------------13 14define <4 x float> @test_mfma_f32_16x16x32_f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2) {15; GCN-LABEL: test_mfma_f32_16x16x32_f16:16; GCN: ; %bb.0:17; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18; GCN-NEXT: v_accvgpr_write_b32 a0, v819; GCN-NEXT: v_accvgpr_write_b32 a1, v920; GCN-NEXT: v_accvgpr_write_b32 a2, v1021; GCN-NEXT: v_accvgpr_write_b32 a3, v1122; GCN-NEXT: s_nop 123; GCN-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3]24; GCN-NEXT: s_nop 725; GCN-NEXT: v_accvgpr_read_b32 v0, a026; GCN-NEXT: v_accvgpr_read_b32 v1, a127; GCN-NEXT: v_accvgpr_read_b32 v2, a228; GCN-NEXT: v_accvgpr_read_b32 v3, a329; GCN-NEXT: s_setpc_b64 s[30:31]30;31; HEURRC-LABEL: test_mfma_f32_16x16x32_f16:32; HEURRC: ; %bb.0:33; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)34; HEURRC-NEXT: v_accvgpr_write_b32 a0, v835; HEURRC-NEXT: v_accvgpr_write_b32 a1, v936; HEURRC-NEXT: v_accvgpr_write_b32 a2, v1037; HEURRC-NEXT: v_accvgpr_write_b32 a3, v1138; HEURRC-NEXT: s_nop 139; HEURRC-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3]40; HEURRC-NEXT: s_nop 741; HEURRC-NEXT: v_accvgpr_read_b32 v0, a042; HEURRC-NEXT: v_accvgpr_read_b32 v1, a143; HEURRC-NEXT: v_accvgpr_read_b32 v2, a244; HEURRC-NEXT: v_accvgpr_read_b32 v3, a345; HEURRC-NEXT: s_setpc_b64 s[30:31]46;47; VGPRRC-LABEL: test_mfma_f32_16x16x32_f16:48; VGPRRC: ; %bb.0:49; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)50; VGPRRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11]51; VGPRRC-NEXT: s_setpc_b64 s[30:31]52; AGPR-LABEL: test_mfma_f32_16x16x32_f16:53; AGPR: ; %bb.0:54; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)55; AGPR-NEXT: v_accvgpr_write_b32 a0, v856; AGPR-NEXT: v_accvgpr_write_b32 a1, v957; AGPR-NEXT: v_accvgpr_write_b32 a2, v1058; AGPR-NEXT: v_accvgpr_write_b32 a3, v1159; AGPR-NEXT: s_nop 160; AGPR-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3]61; AGPR-NEXT: s_nop 762; AGPR-NEXT: v_accvgpr_read_b32 v0, a063; AGPR-NEXT: v_accvgpr_read_b32 v1, a164; AGPR-NEXT: v_accvgpr_read_b32 v2, a265; AGPR-NEXT: v_accvgpr_read_b32 v3, a366; AGPR-NEXT: s_setpc_b64 s[30:31]67; VGPR-LABEL: test_mfma_f32_16x16x32_f16:68; VGPR: ; %bb.0:69; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)70; VGPR-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11]71; VGPR-NEXT: s_setpc_b64 s[30:31]72 %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0)73 ret <4 x float> %result74}75 76define <4 x float> @test_mfma_f32_16x16x32_f16__flags(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2) {77; GCN-LABEL: test_mfma_f32_16x16x32_f16__flags:78; GCN: ; %bb.0:79; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)80; GCN-NEXT: v_accvgpr_write_b32 a0, v881; GCN-NEXT: v_accvgpr_write_b32 a1, v982; GCN-NEXT: v_accvgpr_write_b32 a2, v1083; GCN-NEXT: v_accvgpr_write_b32 a3, v1184; GCN-NEXT: s_nop 185; GCN-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:186; GCN-NEXT: s_nop 787; GCN-NEXT: v_accvgpr_read_b32 v0, a088; GCN-NEXT: v_accvgpr_read_b32 v1, a189; GCN-NEXT: v_accvgpr_read_b32 v2, a290; GCN-NEXT: v_accvgpr_read_b32 v3, a391; GCN-NEXT: s_setpc_b64 s[30:31]92;93; HEURRC-LABEL: test_mfma_f32_16x16x32_f16__flags:94; HEURRC: ; %bb.0:95; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)96; HEURRC-NEXT: v_accvgpr_write_b32 a0, v897; HEURRC-NEXT: v_accvgpr_write_b32 a1, v998; HEURRC-NEXT: v_accvgpr_write_b32 a2, v1099; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11100; HEURRC-NEXT: s_nop 1101; HEURRC-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1102; HEURRC-NEXT: s_nop 7103; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0104; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1105; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2106; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3107; HEURRC-NEXT: s_setpc_b64 s[30:31]108;109; VGPRRC-LABEL: test_mfma_f32_16x16x32_f16__flags:110; VGPRRC: ; %bb.0:111; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)112; VGPRRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:1113; VGPRRC-NEXT: s_setpc_b64 s[30:31]114; AGPR-LABEL: test_mfma_f32_16x16x32_f16__flags:115; AGPR: ; %bb.0:116; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)117; AGPR-NEXT: v_accvgpr_write_b32 a0, v8118; AGPR-NEXT: v_accvgpr_write_b32 a1, v9119; AGPR-NEXT: v_accvgpr_write_b32 a2, v10120; AGPR-NEXT: v_accvgpr_write_b32 a3, v11121; AGPR-NEXT: s_nop 1122; AGPR-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1123; AGPR-NEXT: s_nop 7124; AGPR-NEXT: v_accvgpr_read_b32 v0, a0125; AGPR-NEXT: v_accvgpr_read_b32 v1, a1126; AGPR-NEXT: v_accvgpr_read_b32 v2, a2127; AGPR-NEXT: v_accvgpr_read_b32 v3, a3128; AGPR-NEXT: s_setpc_b64 s[30:31]129; VGPR-LABEL: test_mfma_f32_16x16x32_f16__flags:130; VGPR: ; %bb.0:131; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)132; VGPR-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:1133; VGPR-NEXT: s_setpc_b64 s[30:31]134 %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 1, i32 1, i32 1)135 ret <4 x float> %result136}137 138define amdgpu_kernel void @test_mfma_f32_16x16x32_f16_no_agpr__vgprcd(ptr addrspace(1) %out, <8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2) #0 {139; SDAG-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:140; SDAG: ; %bb.0:141; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34142; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54143; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24144; SDAG-NEXT: v_mov_b32_e32 v4, 0145; SDAG-NEXT: s_waitcnt lgkmcnt(0)146; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[8:9]147; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[10:11]148; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[12:13]149; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]150; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[14:15]151; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]152; SDAG-NEXT: s_nop 1153; SDAG-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[6:9], v[10:13], v[0:3]154; SDAG-NEXT: s_nop 7155; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]156; SDAG-NEXT: s_endpgm157;158; GISEL-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:159; GISEL: ; %bb.0:160; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34161; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54162; GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24163; GISEL-NEXT: s_waitcnt lgkmcnt(0)164; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]165; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]166; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]167; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]168; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]169; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]170; GISEL-NEXT: s_nop 1171; GISEL-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11]172; GISEL-NEXT: v_mov_b32_e32 v4, 0173; GISEL-NEXT: s_nop 6174; GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]175; GISEL-NEXT: s_endpgm176;177; HEURRC-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:178; HEURRC: ; %bb.0:179; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34180; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54181; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24182; HEURRC-NEXT: v_mov_b32_e32 v4, 0183; HEURRC-NEXT: s_waitcnt lgkmcnt(0)184; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[8:9]185; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[10:11]186; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[12:13]187; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[0:1]188; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[14:15]189; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[2:3]190; HEURRC-NEXT: s_nop 1191; HEURRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[6:9], v[10:13], v[0:3]192; HEURRC-NEXT: s_nop 7193; HEURRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]194; HEURRC-NEXT: s_endpgm195;196; VGPRRC-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:197; VGPRRC: ; %bb.0:198; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34199; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54200; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24201; VGPRRC-NEXT: v_mov_b32_e32 v4, 0202; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)203; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[8:9]204; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[10:11]205; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[12:13]206; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[0:1]207; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[14:15]208; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[2:3]209; VGPRRC-NEXT: s_nop 1210; VGPRRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[6:9], v[10:13], v[0:3]211; VGPRRC-NEXT: s_nop 7212; VGPRRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]213; VGPRRC-NEXT: s_endpgm214; AGPR-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:215; AGPR: ; %bb.0:216; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34217; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54218; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24219; AGPR-NEXT: v_mov_b32_e32 v8, 0220; AGPR-NEXT: s_waitcnt lgkmcnt(0)221; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]222; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]223; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]224; AGPR-NEXT: v_accvgpr_write_b32 a0, s0225; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]226; AGPR-NEXT: v_accvgpr_write_b32 a1, s1227; AGPR-NEXT: v_accvgpr_write_b32 a2, s2228; AGPR-NEXT: v_accvgpr_write_b32 a3, s3229; AGPR-NEXT: s_nop 1230; AGPR-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3]231; AGPR-NEXT: s_nop 7232; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7]233; AGPR-NEXT: s_endpgm234; VGPR-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:235; VGPR: ; %bb.0:236; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34237; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54238; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24239; VGPR-NEXT: v_mov_b32_e32 v12, 0240; VGPR-NEXT: s_waitcnt lgkmcnt(0)241; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]242; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]243; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]244; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3]245; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]246; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[0:1]247; VGPR-NEXT: s_nop 1248; VGPR-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11]249; VGPR-NEXT: s_nop 7250; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]251; VGPR-NEXT: s_endpgm252 %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0)253 store <4 x float> %result, ptr addrspace(1) %out254 ret void255}256 257define amdgpu_kernel void @test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags(ptr addrspace(1) %out, <8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2) #0 {258; SDAG-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:259; SDAG: ; %bb.0:260; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34261; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54262; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24263; SDAG-NEXT: v_mov_b32_e32 v4, 0264; SDAG-NEXT: s_waitcnt lgkmcnt(0)265; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[8:9]266; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[10:11]267; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[12:13]268; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]269; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[14:15]270; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]271; SDAG-NEXT: s_nop 1272; SDAG-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1273; SDAG-NEXT: s_nop 7274; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]275; SDAG-NEXT: s_endpgm276;277; GISEL-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:278; GISEL: ; %bb.0:279; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34280; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54281; GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24282; GISEL-NEXT: s_waitcnt lgkmcnt(0)283; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]284; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]285; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]286; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]287; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]288; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]289; GISEL-NEXT: s_nop 1290; GISEL-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1291; GISEL-NEXT: v_mov_b32_e32 v4, 0292; GISEL-NEXT: s_nop 6293; GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]294; GISEL-NEXT: s_endpgm295;296; HEURRC-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:297; HEURRC: ; %bb.0:298; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34299; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54300; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24301; HEURRC-NEXT: v_mov_b32_e32 v4, 0302; HEURRC-NEXT: s_waitcnt lgkmcnt(0)303; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[8:9]304; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[10:11]305; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[12:13]306; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[0:1]307; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[14:15]308; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[2:3]309; HEURRC-NEXT: s_nop 1310; HEURRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1311; HEURRC-NEXT: s_nop 7312; HEURRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]313; HEURRC-NEXT: s_endpgm314;315; VGPRRC-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:316; VGPRRC: ; %bb.0:317; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34318; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54319; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24320; VGPRRC-NEXT: v_mov_b32_e32 v4, 0321; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)322; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[8:9]323; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[10:11]324; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[12:13]325; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[0:1]326; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[14:15]327; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[2:3]328; VGPRRC-NEXT: s_nop 1329; VGPRRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1330; VGPRRC-NEXT: s_nop 7331; VGPRRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]332; VGPRRC-NEXT: s_endpgm333; AGPR-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:334; AGPR: ; %bb.0:335; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34336; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54337; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24338; AGPR-NEXT: v_mov_b32_e32 v8, 0339; AGPR-NEXT: s_waitcnt lgkmcnt(0)340; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]341; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]342; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]343; AGPR-NEXT: v_accvgpr_write_b32 a0, s0344; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]345; AGPR-NEXT: v_accvgpr_write_b32 a1, s1346; AGPR-NEXT: v_accvgpr_write_b32 a2, s2347; AGPR-NEXT: v_accvgpr_write_b32 a3, s3348; AGPR-NEXT: s_nop 1349; AGPR-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1350; AGPR-NEXT: s_nop 7351; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7]352; AGPR-NEXT: s_endpgm353; VGPR-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:354; VGPR: ; %bb.0:355; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34356; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54357; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24358; VGPR-NEXT: v_mov_b32_e32 v12, 0359; VGPR-NEXT: s_waitcnt lgkmcnt(0)360; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]361; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]362; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]363; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3]364; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]365; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[0:1]366; VGPR-NEXT: s_nop 1367; VGPR-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1368; VGPR-NEXT: s_nop 7369; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]370; VGPR-NEXT: s_endpgm371 %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 3, i32 2, i32 1)372 store <4 x float> %result, ptr addrspace(1) %out373 ret void374}375 376; --------------------------------------------------------------------377; llvm.amdgcn.mfma.f32.32x32x16.f16378; --------------------------------------------------------------------379 380define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2) #1 {381; SDAG-LABEL: test_mfma_f32_32x32x16_f16:382; SDAG: ; %bb.0:383; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24384; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64385; SDAG-NEXT: v_mov_b64_e32 v[0:1], 48386; SDAG-NEXT: v_mov_b64_e32 v[2:3], 32387; SDAG-NEXT: v_mov_b64_e32 v[4:5], 16388; SDAG-NEXT: s_waitcnt lgkmcnt(0)389; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[24:25]390; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[26:27]391; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[28:29]392; SDAG-NEXT: v_accvgpr_write_b32 a0, s8393; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[30:31]394; SDAG-NEXT: v_accvgpr_write_b32 a1, s9395; SDAG-NEXT: v_accvgpr_write_b32 a2, s10396; SDAG-NEXT: v_accvgpr_write_b32 a3, s11397; SDAG-NEXT: v_accvgpr_write_b32 a4, s12398; SDAG-NEXT: v_accvgpr_write_b32 a5, s13399; SDAG-NEXT: v_accvgpr_write_b32 a6, s14400; SDAG-NEXT: v_accvgpr_write_b32 a7, s15401; SDAG-NEXT: v_accvgpr_write_b32 a8, s16402; SDAG-NEXT: v_accvgpr_write_b32 a9, s17403; SDAG-NEXT: v_accvgpr_write_b32 a10, s18404; SDAG-NEXT: v_accvgpr_write_b32 a11, s19405; SDAG-NEXT: v_accvgpr_write_b32 a12, s20406; SDAG-NEXT: v_accvgpr_write_b32 a13, s21407; SDAG-NEXT: v_accvgpr_write_b32 a14, s22408; SDAG-NEXT: v_accvgpr_write_b32 a15, s23409; SDAG-NEXT: v_mov_b32_e32 v16, s16410; SDAG-NEXT: v_mov_b32_e32 v17, s17411; SDAG-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[8:11], v[12:15], a[0:15]412; SDAG-NEXT: v_mov_b32_e32 v18, s18413; SDAG-NEXT: v_mov_b32_e32 v19, s19414; SDAG-NEXT: v_mov_b32_e32 v8, s20415; SDAG-NEXT: v_mov_b32_e32 v9, s21416; SDAG-NEXT: v_mov_b32_e32 v10, s22417; SDAG-NEXT: v_mov_b32_e32 v11, s23418; SDAG-NEXT: v_mov_b64_e32 v[6:7], 0419; SDAG-NEXT: s_nop 4420; SDAG-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1421; SDAG-NEXT: s_waitcnt vmcnt(0)422; SDAG-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1423; SDAG-NEXT: s_waitcnt vmcnt(0)424; SDAG-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1425; SDAG-NEXT: s_waitcnt vmcnt(0)426; SDAG-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1427; SDAG-NEXT: s_waitcnt vmcnt(0)428; SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1429; SDAG-NEXT: s_waitcnt vmcnt(0)430; SDAG-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1431; SDAG-NEXT: s_waitcnt vmcnt(0)432; SDAG-NEXT: v_mov_b32_e32 v0, s8433; SDAG-NEXT: v_mov_b32_e32 v1, s9434; SDAG-NEXT: v_mov_b32_e32 v2, s10435; SDAG-NEXT: v_mov_b32_e32 v3, s11436; SDAG-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1437; SDAG-NEXT: s_waitcnt vmcnt(0)438; SDAG-NEXT: s_nop 0439; SDAG-NEXT: v_mov_b32_e32 v0, s12440; SDAG-NEXT: v_mov_b32_e32 v1, s13441; SDAG-NEXT: v_mov_b32_e32 v2, s14442; SDAG-NEXT: v_mov_b32_e32 v3, s15443; SDAG-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1444; SDAG-NEXT: s_waitcnt vmcnt(0)445; SDAG-NEXT: s_endpgm446;447; GISEL-LABEL: test_mfma_f32_32x32x16_f16:448; GISEL: ; %bb.0:449; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24450; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64451; GISEL-NEXT: v_mov_b64_e32 v[12:13], 0452; GISEL-NEXT: v_mov_b64_e32 v[14:15], 16453; GISEL-NEXT: v_mov_b64_e32 v[16:17], 32454; GISEL-NEXT: s_waitcnt lgkmcnt(0)455; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[24:25]456; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[26:27]457; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[28:29]458; GISEL-NEXT: v_accvgpr_write_b32 a0, s8459; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[30:31]460; GISEL-NEXT: v_accvgpr_write_b32 a1, s9461; GISEL-NEXT: v_accvgpr_write_b32 a2, s10462; GISEL-NEXT: v_accvgpr_write_b32 a3, s11463; GISEL-NEXT: v_accvgpr_write_b32 a4, s12464; GISEL-NEXT: v_accvgpr_write_b32 a5, s13465; GISEL-NEXT: v_accvgpr_write_b32 a6, s14466; GISEL-NEXT: v_accvgpr_write_b32 a7, s15467; GISEL-NEXT: v_accvgpr_write_b32 a8, s16468; GISEL-NEXT: v_accvgpr_write_b32 a9, s17469; GISEL-NEXT: v_accvgpr_write_b32 a10, s18470; GISEL-NEXT: v_accvgpr_write_b32 a11, s19471; GISEL-NEXT: v_accvgpr_write_b32 a12, s20472; GISEL-NEXT: v_accvgpr_write_b32 a13, s21473; GISEL-NEXT: v_accvgpr_write_b32 a14, s22474; GISEL-NEXT: v_accvgpr_write_b32 a15, s23475; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]476; GISEL-NEXT: v_mov_b64_e32 v[18:19], 48477; GISEL-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15]478; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]479; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]480; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]481; GISEL-NEXT: s_nop 8482; GISEL-NEXT: global_store_dwordx4 v[12:13], a[16:19], off sc0 sc1483; GISEL-NEXT: s_waitcnt vmcnt(0)484; GISEL-NEXT: global_store_dwordx4 v[14:15], a[20:23], off sc0 sc1485; GISEL-NEXT: s_waitcnt vmcnt(0)486; GISEL-NEXT: global_store_dwordx4 v[16:17], a[24:27], off sc0 sc1487; GISEL-NEXT: s_waitcnt vmcnt(0)488; GISEL-NEXT: global_store_dwordx4 v[18:19], a[28:31], off sc0 sc1489; GISEL-NEXT: s_waitcnt vmcnt(0)490; GISEL-NEXT: global_store_dwordx4 v[12:13], v[8:11], off sc0 sc1491; GISEL-NEXT: s_waitcnt vmcnt(0)492; GISEL-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1493; GISEL-NEXT: s_waitcnt vmcnt(0)494; GISEL-NEXT: s_nop 0495; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[16:17]496; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19]497; GISEL-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1498; GISEL-NEXT: s_waitcnt vmcnt(0)499; GISEL-NEXT: s_nop 0500; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[20:21]501; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[22:23]502; GISEL-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1503; GISEL-NEXT: s_waitcnt vmcnt(0)504; GISEL-NEXT: s_endpgm505;506; HEURRC-LABEL: test_mfma_f32_32x32x16_f16:507; HEURRC: ; %bb.0:508; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24509; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64510; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48511; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32512; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16513; HEURRC-NEXT: s_waitcnt lgkmcnt(0)514; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[24:25]515; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[26:27]516; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[28:29]517; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8518; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[30:31]519; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9520; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10521; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11522; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12523; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13524; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14525; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15526; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16527; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17528; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18529; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19530; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20531; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21532; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22533; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23534; HEURRC-NEXT: v_mov_b32_e32 v16, s16535; HEURRC-NEXT: v_mov_b32_e32 v17, s17536; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[8:11], v[12:15], a[0:15]537; HEURRC-NEXT: v_mov_b32_e32 v18, s18538; HEURRC-NEXT: v_mov_b32_e32 v19, s19539; HEURRC-NEXT: v_mov_b32_e32 v8, s20540; HEURRC-NEXT: v_mov_b32_e32 v9, s21541; HEURRC-NEXT: v_mov_b32_e32 v10, s22542; HEURRC-NEXT: v_mov_b32_e32 v11, s23543; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0544; HEURRC-NEXT: s_nop 4545; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1546; HEURRC-NEXT: s_waitcnt vmcnt(0)547; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1548; HEURRC-NEXT: s_waitcnt vmcnt(0)549; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1550; HEURRC-NEXT: s_waitcnt vmcnt(0)551; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1552; HEURRC-NEXT: s_waitcnt vmcnt(0)553; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1554; HEURRC-NEXT: s_waitcnt vmcnt(0)555; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1556; HEURRC-NEXT: s_waitcnt vmcnt(0)557; HEURRC-NEXT: v_mov_b32_e32 v0, s8558; HEURRC-NEXT: v_mov_b32_e32 v1, s9559; HEURRC-NEXT: v_mov_b32_e32 v2, s10560; HEURRC-NEXT: v_mov_b32_e32 v3, s11561; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1562; HEURRC-NEXT: s_waitcnt vmcnt(0)563; HEURRC-NEXT: s_nop 0564; HEURRC-NEXT: v_mov_b32_e32 v0, s12565; HEURRC-NEXT: v_mov_b32_e32 v1, s13566; HEURRC-NEXT: v_mov_b32_e32 v2, s14567; HEURRC-NEXT: v_mov_b32_e32 v3, s15568; HEURRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1569; HEURRC-NEXT: s_waitcnt vmcnt(0)570; HEURRC-NEXT: s_endpgm571;572; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16:573; VGPRRC: ; %bb.0:574; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24575; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64576; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48577; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32578; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], 16579; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)580; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[26:27]581; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[24:25]582; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], s[30:31]583; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]584; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[28:29]585; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]586; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]587; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]588; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]589; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]590; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]591; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]592; VGPRRC-NEXT: v_mov_b32_e32 v48, s16593; VGPRRC-NEXT: v_mov_b32_e32 v49, s17594; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[40:43], v[44:47], v[0:15]595; VGPRRC-NEXT: v_mov_b32_e32 v50, s18596; VGPRRC-NEXT: v_mov_b32_e32 v51, s19597; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], 0598; VGPRRC-NEXT: s_nop 8599; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1600; VGPRRC-NEXT: s_waitcnt vmcnt(0)601; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1602; VGPRRC-NEXT: s_waitcnt vmcnt(0)603; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1604; VGPRRC-NEXT: s_waitcnt vmcnt(0)605; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1606; VGPRRC-NEXT: s_waitcnt vmcnt(0)607; VGPRRC-NEXT: v_mov_b32_e32 v0, s20608; VGPRRC-NEXT: v_mov_b32_e32 v1, s21609; VGPRRC-NEXT: v_mov_b32_e32 v2, s22610; VGPRRC-NEXT: v_mov_b32_e32 v3, s23611; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1612; VGPRRC-NEXT: s_waitcnt vmcnt(0)613; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1614; VGPRRC-NEXT: s_waitcnt vmcnt(0)615; VGPRRC-NEXT: s_nop 0616; VGPRRC-NEXT: v_mov_b32_e32 v0, s8617; VGPRRC-NEXT: v_mov_b32_e32 v1, s9618; VGPRRC-NEXT: v_mov_b32_e32 v2, s10619; VGPRRC-NEXT: v_mov_b32_e32 v3, s11620; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1621; VGPRRC-NEXT: s_waitcnt vmcnt(0)622; VGPRRC-NEXT: s_nop 0623; VGPRRC-NEXT: v_mov_b32_e32 v0, s12624; VGPRRC-NEXT: v_mov_b32_e32 v1, s13625; VGPRRC-NEXT: v_mov_b32_e32 v2, s14626; VGPRRC-NEXT: v_mov_b32_e32 v3, s15627; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1628; VGPRRC-NEXT: s_waitcnt vmcnt(0)629; VGPRRC-NEXT: s_endpgm630; AGPR-LABEL: test_mfma_f32_32x32x16_f16:631; AGPR: ; %bb.0:632; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24633; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64634; AGPR-NEXT: v_mov_b64_e32 v[12:13], 48635; AGPR-NEXT: v_mov_b64_e32 v[14:15], 32636; AGPR-NEXT: v_mov_b64_e32 v[16:17], 16637; AGPR-NEXT: s_waitcnt lgkmcnt(0)638; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25]639; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27]640; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29]641; AGPR-NEXT: v_accvgpr_write_b32 a0, s8642; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31]643; AGPR-NEXT: v_accvgpr_write_b32 a1, s9644; AGPR-NEXT: v_accvgpr_write_b32 a2, s10645; AGPR-NEXT: v_accvgpr_write_b32 a3, s11646; AGPR-NEXT: v_accvgpr_write_b32 a4, s12647; AGPR-NEXT: v_accvgpr_write_b32 a5, s13648; AGPR-NEXT: v_accvgpr_write_b32 a6, s14649; AGPR-NEXT: v_accvgpr_write_b32 a7, s15650; AGPR-NEXT: v_accvgpr_write_b32 a8, s16651; AGPR-NEXT: v_accvgpr_write_b32 a9, s17652; AGPR-NEXT: v_accvgpr_write_b32 a10, s18653; AGPR-NEXT: v_accvgpr_write_b32 a11, s19654; AGPR-NEXT: v_accvgpr_write_b32 a12, s20655; AGPR-NEXT: v_accvgpr_write_b32 a13, s21656; AGPR-NEXT: v_accvgpr_write_b32 a14, s22657; AGPR-NEXT: v_accvgpr_write_b32 a15, s23658; AGPR-NEXT: v_mov_b64_e32 v[18:19], 0659; AGPR-NEXT: v_mov_b32_e32 v8, s16660; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15]661; AGPR-NEXT: v_mov_b32_e32 v0, s20662; AGPR-NEXT: v_mov_b32_e32 v1, s21663; AGPR-NEXT: v_mov_b32_e32 v2, s22664; AGPR-NEXT: v_mov_b32_e32 v3, s23665; AGPR-NEXT: v_mov_b32_e32 v9, s17666; AGPR-NEXT: v_mov_b32_e32 v10, s18667; AGPR-NEXT: v_mov_b32_e32 v11, s19668; AGPR-NEXT: s_nop 4669; AGPR-NEXT: global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1670; AGPR-NEXT: s_waitcnt vmcnt(0)671; AGPR-NEXT: global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1672; AGPR-NEXT: s_waitcnt vmcnt(0)673; AGPR-NEXT: global_store_dwordx4 v[16:17], a[20:23], off sc0 sc1674; AGPR-NEXT: s_waitcnt vmcnt(0)675; AGPR-NEXT: global_store_dwordx4 v[18:19], a[16:19], off sc0 sc1676; AGPR-NEXT: s_waitcnt vmcnt(0)677; AGPR-NEXT: global_store_dwordx4 v[14:15], v[8:11], off sc0 sc1678; AGPR-NEXT: s_waitcnt vmcnt(0)679; AGPR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1680; AGPR-NEXT: s_waitcnt vmcnt(0)681; AGPR-NEXT: s_nop 0682; AGPR-NEXT: v_mov_b32_e32 v0, s8683; AGPR-NEXT: v_mov_b32_e32 v1, s9684; AGPR-NEXT: v_mov_b32_e32 v2, s10685; AGPR-NEXT: v_mov_b32_e32 v3, s11686; AGPR-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1687; AGPR-NEXT: s_waitcnt vmcnt(0)688; AGPR-NEXT: s_nop 0689; AGPR-NEXT: v_mov_b32_e32 v0, s12690; AGPR-NEXT: v_mov_b32_e32 v1, s13691; AGPR-NEXT: v_mov_b32_e32 v2, s14692; AGPR-NEXT: v_mov_b32_e32 v3, s15693; AGPR-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1694; AGPR-NEXT: s_waitcnt vmcnt(0)695; AGPR-NEXT: s_endpgm696; VGPR-LABEL: test_mfma_f32_32x32x16_f16:697; VGPR: ; %bb.0:698; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24699; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64700; VGPR-NEXT: v_mov_b64_e32 v[44:45], 48701; VGPR-NEXT: v_mov_b64_e32 v[46:47], 32702; VGPR-NEXT: v_mov_b64_e32 v[48:49], 16703; VGPR-NEXT: s_waitcnt lgkmcnt(0)704; VGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27]705; VGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25]706; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31]707; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]708; VGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29]709; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]710; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]711; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]712; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]713; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]714; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]715; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]716; VGPR-NEXT: v_mov_b64_e32 v[50:51], 0717; VGPR-NEXT: v_mov_b32_e32 v40, s16718; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15]719; VGPR-NEXT: v_mov_b32_e32 v41, s17720; VGPR-NEXT: v_mov_b32_e32 v42, s18721; VGPR-NEXT: v_mov_b32_e32 v43, s19722; VGPR-NEXT: s_nop 7723; VGPR-NEXT: s_nop 0724; VGPR-NEXT: global_store_dwordx4 v[44:45], v[28:31], off sc0 sc1725; VGPR-NEXT: s_waitcnt vmcnt(0)726; VGPR-NEXT: global_store_dwordx4 v[46:47], v[24:27], off sc0 sc1727; VGPR-NEXT: s_waitcnt vmcnt(0)728; VGPR-NEXT: global_store_dwordx4 v[48:49], v[20:23], off sc0 sc1729; VGPR-NEXT: s_waitcnt vmcnt(0)730; VGPR-NEXT: global_store_dwordx4 v[50:51], v[16:19], off sc0 sc1731; VGPR-NEXT: s_waitcnt vmcnt(0)732; VGPR-NEXT: global_store_dwordx4 v[46:47], v[40:43], off sc0 sc1733; VGPR-NEXT: s_waitcnt vmcnt(0)734; VGPR-NEXT: v_mov_b32_e32 v0, s20735; VGPR-NEXT: v_mov_b32_e32 v1, s21736; VGPR-NEXT: v_mov_b32_e32 v2, s22737; VGPR-NEXT: v_mov_b32_e32 v3, s23738; VGPR-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1739; VGPR-NEXT: s_waitcnt vmcnt(0)740; VGPR-NEXT: s_nop 0741; VGPR-NEXT: v_mov_b32_e32 v0, s8742; VGPR-NEXT: v_mov_b32_e32 v1, s9743; VGPR-NEXT: v_mov_b32_e32 v2, s10744; VGPR-NEXT: v_mov_b32_e32 v3, s11745; VGPR-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1746; VGPR-NEXT: s_waitcnt vmcnt(0)747; VGPR-NEXT: s_nop 0748; VGPR-NEXT: v_mov_b32_e32 v0, s12749; VGPR-NEXT: v_mov_b32_e32 v1, s13750; VGPR-NEXT: v_mov_b32_e32 v2, s14751; VGPR-NEXT: v_mov_b32_e32 v3, s15752; VGPR-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1753; VGPR-NEXT: s_waitcnt vmcnt(0)754; VGPR-NEXT: s_endpgm755 %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0)756 store volatile <16 x float> %result, ptr addrspace(1) null757 store volatile <16 x float> %arg2, ptr addrspace(1) null758 ret void759}760 761define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2) #1 {762; SDAG-LABEL: test_mfma_f32_32x32x16_f16__flags:763; SDAG: ; %bb.0:764; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24765; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64766; SDAG-NEXT: v_mov_b64_e32 v[0:1], 48767; SDAG-NEXT: v_mov_b64_e32 v[2:3], 32768; SDAG-NEXT: v_mov_b64_e32 v[4:5], 16769; SDAG-NEXT: s_waitcnt lgkmcnt(0)770; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[24:25]771; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[26:27]772; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[28:29]773; SDAG-NEXT: v_accvgpr_write_b32 a0, s8774; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[30:31]775; SDAG-NEXT: v_accvgpr_write_b32 a1, s9776; SDAG-NEXT: v_accvgpr_write_b32 a2, s10777; SDAG-NEXT: v_accvgpr_write_b32 a3, s11778; SDAG-NEXT: v_accvgpr_write_b32 a4, s12779; SDAG-NEXT: v_accvgpr_write_b32 a5, s13780; SDAG-NEXT: v_accvgpr_write_b32 a6, s14781; SDAG-NEXT: v_accvgpr_write_b32 a7, s15782; SDAG-NEXT: v_accvgpr_write_b32 a8, s16783; SDAG-NEXT: v_accvgpr_write_b32 a9, s17784; SDAG-NEXT: v_accvgpr_write_b32 a10, s18785; SDAG-NEXT: v_accvgpr_write_b32 a11, s19786; SDAG-NEXT: v_accvgpr_write_b32 a12, s20787; SDAG-NEXT: v_accvgpr_write_b32 a13, s21788; SDAG-NEXT: v_accvgpr_write_b32 a14, s22789; SDAG-NEXT: v_accvgpr_write_b32 a15, s23790; SDAG-NEXT: v_mov_b32_e32 v16, s16791; SDAG-NEXT: v_mov_b32_e32 v17, s17792; SDAG-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[8:11], v[12:15], a[0:15] cbsz:2 abid:3 blgp:1793; SDAG-NEXT: v_mov_b32_e32 v18, s18794; SDAG-NEXT: v_mov_b32_e32 v19, s19795; SDAG-NEXT: v_mov_b32_e32 v8, s20796; SDAG-NEXT: v_mov_b32_e32 v9, s21797; SDAG-NEXT: v_mov_b32_e32 v10, s22798; SDAG-NEXT: v_mov_b32_e32 v11, s23799; SDAG-NEXT: v_mov_b64_e32 v[6:7], 0800; SDAG-NEXT: s_nop 4801; SDAG-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1802; SDAG-NEXT: s_waitcnt vmcnt(0)803; SDAG-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1804; SDAG-NEXT: s_waitcnt vmcnt(0)805; SDAG-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1806; SDAG-NEXT: s_waitcnt vmcnt(0)807; SDAG-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1808; SDAG-NEXT: s_waitcnt vmcnt(0)809; SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1810; SDAG-NEXT: s_waitcnt vmcnt(0)811; SDAG-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1812; SDAG-NEXT: s_waitcnt vmcnt(0)813; SDAG-NEXT: v_mov_b32_e32 v0, s8814; SDAG-NEXT: v_mov_b32_e32 v1, s9815; SDAG-NEXT: v_mov_b32_e32 v2, s10816; SDAG-NEXT: v_mov_b32_e32 v3, s11817; SDAG-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1818; SDAG-NEXT: s_waitcnt vmcnt(0)819; SDAG-NEXT: s_nop 0820; SDAG-NEXT: v_mov_b32_e32 v0, s12821; SDAG-NEXT: v_mov_b32_e32 v1, s13822; SDAG-NEXT: v_mov_b32_e32 v2, s14823; SDAG-NEXT: v_mov_b32_e32 v3, s15824; SDAG-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1825; SDAG-NEXT: s_waitcnt vmcnt(0)826; SDAG-NEXT: s_endpgm827;828; GISEL-LABEL: test_mfma_f32_32x32x16_f16__flags:829; GISEL: ; %bb.0:830; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24831; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64832; GISEL-NEXT: v_mov_b64_e32 v[12:13], 0833; GISEL-NEXT: v_mov_b64_e32 v[14:15], 16834; GISEL-NEXT: v_mov_b64_e32 v[16:17], 32835; GISEL-NEXT: s_waitcnt lgkmcnt(0)836; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[24:25]837; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[26:27]838; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[28:29]839; GISEL-NEXT: v_accvgpr_write_b32 a0, s8840; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[30:31]841; GISEL-NEXT: v_accvgpr_write_b32 a1, s9842; GISEL-NEXT: v_accvgpr_write_b32 a2, s10843; GISEL-NEXT: v_accvgpr_write_b32 a3, s11844; GISEL-NEXT: v_accvgpr_write_b32 a4, s12845; GISEL-NEXT: v_accvgpr_write_b32 a5, s13846; GISEL-NEXT: v_accvgpr_write_b32 a6, s14847; GISEL-NEXT: v_accvgpr_write_b32 a7, s15848; GISEL-NEXT: v_accvgpr_write_b32 a8, s16849; GISEL-NEXT: v_accvgpr_write_b32 a9, s17850; GISEL-NEXT: v_accvgpr_write_b32 a10, s18851; GISEL-NEXT: v_accvgpr_write_b32 a11, s19852; GISEL-NEXT: v_accvgpr_write_b32 a12, s20853; GISEL-NEXT: v_accvgpr_write_b32 a13, s21854; GISEL-NEXT: v_accvgpr_write_b32 a14, s22855; GISEL-NEXT: v_accvgpr_write_b32 a15, s23856; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]857; GISEL-NEXT: v_mov_b64_e32 v[18:19], 48858; GISEL-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1859; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]860; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]861; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]862; GISEL-NEXT: s_nop 8863; GISEL-NEXT: global_store_dwordx4 v[12:13], a[16:19], off sc0 sc1864; GISEL-NEXT: s_waitcnt vmcnt(0)865; GISEL-NEXT: global_store_dwordx4 v[14:15], a[20:23], off sc0 sc1866; GISEL-NEXT: s_waitcnt vmcnt(0)867; GISEL-NEXT: global_store_dwordx4 v[16:17], a[24:27], off sc0 sc1868; GISEL-NEXT: s_waitcnt vmcnt(0)869; GISEL-NEXT: global_store_dwordx4 v[18:19], a[28:31], off sc0 sc1870; GISEL-NEXT: s_waitcnt vmcnt(0)871; GISEL-NEXT: global_store_dwordx4 v[12:13], v[8:11], off sc0 sc1872; GISEL-NEXT: s_waitcnt vmcnt(0)873; GISEL-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1874; GISEL-NEXT: s_waitcnt vmcnt(0)875; GISEL-NEXT: s_nop 0876; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[16:17]877; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19]878; GISEL-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1879; GISEL-NEXT: s_waitcnt vmcnt(0)880; GISEL-NEXT: s_nop 0881; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[20:21]882; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[22:23]883; GISEL-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1884; GISEL-NEXT: s_waitcnt vmcnt(0)885; GISEL-NEXT: s_endpgm886;887; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__flags:888; HEURRC: ; %bb.0:889; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24890; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64891; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48892; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32893; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16894; HEURRC-NEXT: s_waitcnt lgkmcnt(0)895; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[24:25]896; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[26:27]897; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[28:29]898; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8899; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[30:31]900; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9901; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10902; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11903; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12904; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13905; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14906; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15907; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16908; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17909; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18910; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19911; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20912; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21913; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22914; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23915; HEURRC-NEXT: v_mov_b32_e32 v16, s16916; HEURRC-NEXT: v_mov_b32_e32 v17, s17917; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[8:11], v[12:15], a[0:15] cbsz:2 abid:3 blgp:1918; HEURRC-NEXT: v_mov_b32_e32 v18, s18919; HEURRC-NEXT: v_mov_b32_e32 v19, s19920; HEURRC-NEXT: v_mov_b32_e32 v8, s20921; HEURRC-NEXT: v_mov_b32_e32 v9, s21922; HEURRC-NEXT: v_mov_b32_e32 v10, s22923; HEURRC-NEXT: v_mov_b32_e32 v11, s23924; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0925; HEURRC-NEXT: s_nop 4926; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1927; HEURRC-NEXT: s_waitcnt vmcnt(0)928; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1929; HEURRC-NEXT: s_waitcnt vmcnt(0)930; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1931; HEURRC-NEXT: s_waitcnt vmcnt(0)932; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1933; HEURRC-NEXT: s_waitcnt vmcnt(0)934; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1935; HEURRC-NEXT: s_waitcnt vmcnt(0)936; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1937; HEURRC-NEXT: s_waitcnt vmcnt(0)938; HEURRC-NEXT: v_mov_b32_e32 v0, s8939; HEURRC-NEXT: v_mov_b32_e32 v1, s9940; HEURRC-NEXT: v_mov_b32_e32 v2, s10941; HEURRC-NEXT: v_mov_b32_e32 v3, s11942; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1943; HEURRC-NEXT: s_waitcnt vmcnt(0)944; HEURRC-NEXT: s_nop 0945; HEURRC-NEXT: v_mov_b32_e32 v0, s12946; HEURRC-NEXT: v_mov_b32_e32 v1, s13947; HEURRC-NEXT: v_mov_b32_e32 v2, s14948; HEURRC-NEXT: v_mov_b32_e32 v3, s15949; HEURRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1950; HEURRC-NEXT: s_waitcnt vmcnt(0)951; HEURRC-NEXT: s_endpgm952;953; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__flags:954; VGPRRC: ; %bb.0:955; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24956; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64957; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48958; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32959; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], 16960; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)961; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[26:27]962; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[24:25]963; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], s[30:31]964; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]965; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[28:29]966; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]967; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]968; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]969; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]970; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]971; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]972; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]973; VGPRRC-NEXT: v_mov_b32_e32 v48, s16974; VGPRRC-NEXT: v_mov_b32_e32 v49, s17975; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1976; VGPRRC-NEXT: v_mov_b32_e32 v50, s18977; VGPRRC-NEXT: v_mov_b32_e32 v51, s19978; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], 0979; VGPRRC-NEXT: s_nop 8980; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1981; VGPRRC-NEXT: s_waitcnt vmcnt(0)982; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1983; VGPRRC-NEXT: s_waitcnt vmcnt(0)984; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1985; VGPRRC-NEXT: s_waitcnt vmcnt(0)986; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1987; VGPRRC-NEXT: s_waitcnt vmcnt(0)988; VGPRRC-NEXT: v_mov_b32_e32 v0, s20989; VGPRRC-NEXT: v_mov_b32_e32 v1, s21990; VGPRRC-NEXT: v_mov_b32_e32 v2, s22991; VGPRRC-NEXT: v_mov_b32_e32 v3, s23992; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[48:51], off sc0 sc1993; VGPRRC-NEXT: s_waitcnt vmcnt(0)994; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1995; VGPRRC-NEXT: s_waitcnt vmcnt(0)996; VGPRRC-NEXT: s_nop 0997; VGPRRC-NEXT: v_mov_b32_e32 v0, s8998; VGPRRC-NEXT: v_mov_b32_e32 v1, s9999; VGPRRC-NEXT: v_mov_b32_e32 v2, s101000; VGPRRC-NEXT: v_mov_b32_e32 v3, s111001; VGPRRC-NEXT: global_store_dwordx4 v[38:39], v[0:3], off sc0 sc11002; VGPRRC-NEXT: s_waitcnt vmcnt(0)1003; VGPRRC-NEXT: s_nop 01004; VGPRRC-NEXT: v_mov_b32_e32 v0, s121005; VGPRRC-NEXT: v_mov_b32_e32 v1, s131006; VGPRRC-NEXT: v_mov_b32_e32 v2, s141007; VGPRRC-NEXT: v_mov_b32_e32 v3, s151008; VGPRRC-NEXT: global_store_dwordx4 v[36:37], v[0:3], off sc0 sc11009; VGPRRC-NEXT: s_waitcnt vmcnt(0)1010; VGPRRC-NEXT: s_endpgm1011; AGPR-LABEL: test_mfma_f32_32x32x16_f16__flags:1012; AGPR: ; %bb.0:1013; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x241014; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x641015; AGPR-NEXT: v_mov_b64_e32 v[12:13], 481016; AGPR-NEXT: v_mov_b64_e32 v[14:15], 321017; AGPR-NEXT: v_mov_b64_e32 v[16:17], 161018; AGPR-NEXT: s_waitcnt lgkmcnt(0)1019; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25]1020; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27]1021; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29]1022; AGPR-NEXT: v_accvgpr_write_b32 a0, s81023; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31]1024; AGPR-NEXT: v_accvgpr_write_b32 a1, s91025; AGPR-NEXT: v_accvgpr_write_b32 a2, s101026; AGPR-NEXT: v_accvgpr_write_b32 a3, s111027; AGPR-NEXT: v_accvgpr_write_b32 a4, s121028; AGPR-NEXT: v_accvgpr_write_b32 a5, s131029; AGPR-NEXT: v_accvgpr_write_b32 a6, s141030; AGPR-NEXT: v_accvgpr_write_b32 a7, s151031; AGPR-NEXT: v_accvgpr_write_b32 a8, s161032; AGPR-NEXT: v_accvgpr_write_b32 a9, s171033; AGPR-NEXT: v_accvgpr_write_b32 a10, s181034; AGPR-NEXT: v_accvgpr_write_b32 a11, s191035; AGPR-NEXT: v_accvgpr_write_b32 a12, s201036; AGPR-NEXT: v_accvgpr_write_b32 a13, s211037; AGPR-NEXT: v_accvgpr_write_b32 a14, s221038; AGPR-NEXT: v_accvgpr_write_b32 a15, s231039; AGPR-NEXT: v_mov_b64_e32 v[18:19], 01040; AGPR-NEXT: v_mov_b32_e32 v8, s161041; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:11042; AGPR-NEXT: v_mov_b32_e32 v0, s201043; AGPR-NEXT: v_mov_b32_e32 v1, s211044; AGPR-NEXT: v_mov_b32_e32 v2, s221045; AGPR-NEXT: v_mov_b32_e32 v3, s231046; AGPR-NEXT: v_mov_b32_e32 v9, s171047; AGPR-NEXT: v_mov_b32_e32 v10, s181048; AGPR-NEXT: v_mov_b32_e32 v11, s191049; AGPR-NEXT: s_nop 41050; AGPR-NEXT: global_store_dwordx4 v[12:13], a[28:31], off sc0 sc11051; AGPR-NEXT: s_waitcnt vmcnt(0)1052; AGPR-NEXT: global_store_dwordx4 v[14:15], a[24:27], off sc0 sc11053; AGPR-NEXT: s_waitcnt vmcnt(0)1054; AGPR-NEXT: global_store_dwordx4 v[16:17], a[20:23], off sc0 sc11055; AGPR-NEXT: s_waitcnt vmcnt(0)1056; AGPR-NEXT: global_store_dwordx4 v[18:19], a[16:19], off sc0 sc11057; AGPR-NEXT: s_waitcnt vmcnt(0)1058; AGPR-NEXT: global_store_dwordx4 v[14:15], v[8:11], off sc0 sc11059; AGPR-NEXT: s_waitcnt vmcnt(0)1060; AGPR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc11061; AGPR-NEXT: s_waitcnt vmcnt(0)1062; AGPR-NEXT: s_nop 01063; AGPR-NEXT: v_mov_b32_e32 v0, s81064; AGPR-NEXT: v_mov_b32_e32 v1, s91065; AGPR-NEXT: v_mov_b32_e32 v2, s101066; AGPR-NEXT: v_mov_b32_e32 v3, s111067; AGPR-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc11068; AGPR-NEXT: s_waitcnt vmcnt(0)1069; AGPR-NEXT: s_nop 01070; AGPR-NEXT: v_mov_b32_e32 v0, s121071; AGPR-NEXT: v_mov_b32_e32 v1, s131072; AGPR-NEXT: v_mov_b32_e32 v2, s141073; AGPR-NEXT: v_mov_b32_e32 v3, s151074; AGPR-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc11075; AGPR-NEXT: s_waitcnt vmcnt(0)1076; AGPR-NEXT: s_endpgm1077; VGPR-LABEL: test_mfma_f32_32x32x16_f16__flags:1078; VGPR: ; %bb.0:1079; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x241080; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x641081; VGPR-NEXT: v_mov_b64_e32 v[44:45], 481082; VGPR-NEXT: v_mov_b64_e32 v[46:47], 321083; VGPR-NEXT: v_mov_b64_e32 v[48:49], 161084; VGPR-NEXT: s_waitcnt lgkmcnt(0)1085; VGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27]1086; VGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25]1087; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31]1088; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]1089; VGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29]1090; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]1091; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]1092; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]1093; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]1094; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]1095; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]1096; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]1097; VGPR-NEXT: v_mov_b64_e32 v[50:51], 01098; VGPR-NEXT: v_mov_b32_e32 v40, s161099; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:11100; VGPR-NEXT: v_mov_b32_e32 v41, s171101; VGPR-NEXT: v_mov_b32_e32 v42, s181102; VGPR-NEXT: v_mov_b32_e32 v43, s191103; VGPR-NEXT: s_nop 71104; VGPR-NEXT: s_nop 01105; VGPR-NEXT: global_store_dwordx4 v[44:45], v[28:31], off sc0 sc11106; VGPR-NEXT: s_waitcnt vmcnt(0)1107; VGPR-NEXT: global_store_dwordx4 v[46:47], v[24:27], off sc0 sc11108; VGPR-NEXT: s_waitcnt vmcnt(0)1109; VGPR-NEXT: global_store_dwordx4 v[48:49], v[20:23], off sc0 sc11110; VGPR-NEXT: s_waitcnt vmcnt(0)1111; VGPR-NEXT: global_store_dwordx4 v[50:51], v[16:19], off sc0 sc11112; VGPR-NEXT: s_waitcnt vmcnt(0)1113; VGPR-NEXT: global_store_dwordx4 v[46:47], v[40:43], off sc0 sc11114; VGPR-NEXT: s_waitcnt vmcnt(0)1115; VGPR-NEXT: v_mov_b32_e32 v0, s201116; VGPR-NEXT: v_mov_b32_e32 v1, s211117; VGPR-NEXT: v_mov_b32_e32 v2, s221118; VGPR-NEXT: v_mov_b32_e32 v3, s231119; VGPR-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc11120; VGPR-NEXT: s_waitcnt vmcnt(0)1121; VGPR-NEXT: s_nop 01122; VGPR-NEXT: v_mov_b32_e32 v0, s81123; VGPR-NEXT: v_mov_b32_e32 v1, s91124; VGPR-NEXT: v_mov_b32_e32 v2, s101125; VGPR-NEXT: v_mov_b32_e32 v3, s111126; VGPR-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc11127; VGPR-NEXT: s_waitcnt vmcnt(0)1128; VGPR-NEXT: s_nop 01129; VGPR-NEXT: v_mov_b32_e32 v0, s121130; VGPR-NEXT: v_mov_b32_e32 v1, s131131; VGPR-NEXT: v_mov_b32_e32 v2, s141132; VGPR-NEXT: v_mov_b32_e32 v3, s151133; VGPR-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc11134; VGPR-NEXT: s_waitcnt vmcnt(0)1135; VGPR-NEXT: s_endpgm1136 %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 2, i32 3, i32 1)1137 store volatile <16 x float> %result, ptr addrspace(1) null1138 store volatile <16 x float> %arg2, ptr addrspace(1) null1139 ret void1140}1141 1142define <16 x float> @test_mfma_f32_32x32x16_f16__mac(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2) {1143; GCN-LABEL: test_mfma_f32_32x32x16_f16__mac:1144; GCN: ; %bb.0:1145; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1146; GCN-NEXT: v_accvgpr_write_b32 a0, v81147; GCN-NEXT: v_accvgpr_write_b32 a1, v91148; GCN-NEXT: v_accvgpr_write_b32 a2, v101149; GCN-NEXT: v_accvgpr_write_b32 a3, v111150; GCN-NEXT: v_accvgpr_write_b32 a4, v121151; GCN-NEXT: v_accvgpr_write_b32 a5, v131152; GCN-NEXT: v_accvgpr_write_b32 a6, v141153; GCN-NEXT: v_accvgpr_write_b32 a7, v151154; GCN-NEXT: v_accvgpr_write_b32 a8, v161155; GCN-NEXT: v_accvgpr_write_b32 a9, v171156; GCN-NEXT: v_accvgpr_write_b32 a10, v181157; GCN-NEXT: v_accvgpr_write_b32 a11, v191158; GCN-NEXT: v_accvgpr_write_b32 a12, v201159; GCN-NEXT: v_accvgpr_write_b32 a13, v211160; GCN-NEXT: v_accvgpr_write_b32 a14, v221161; GCN-NEXT: v_accvgpr_write_b32 a15, v231162; GCN-NEXT: s_nop 11163; GCN-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15]1164; GCN-NEXT: s_nop 111165; GCN-NEXT: v_accvgpr_read_b32 v0, a01166; GCN-NEXT: v_accvgpr_read_b32 v1, a11167; GCN-NEXT: v_accvgpr_read_b32 v2, a21168; GCN-NEXT: v_accvgpr_read_b32 v3, a31169; GCN-NEXT: v_accvgpr_read_b32 v4, a41170; GCN-NEXT: v_accvgpr_read_b32 v5, a51171; GCN-NEXT: v_accvgpr_read_b32 v6, a61172; GCN-NEXT: v_accvgpr_read_b32 v7, a71173; GCN-NEXT: v_accvgpr_read_b32 v8, a81174; GCN-NEXT: v_accvgpr_read_b32 v9, a91175; GCN-NEXT: v_accvgpr_read_b32 v10, a101176; GCN-NEXT: v_accvgpr_read_b32 v11, a111177; GCN-NEXT: v_accvgpr_read_b32 v12, a121178; GCN-NEXT: v_accvgpr_read_b32 v13, a131179; GCN-NEXT: v_accvgpr_read_b32 v14, a141180; GCN-NEXT: v_accvgpr_read_b32 v15, a151181; GCN-NEXT: s_setpc_b64 s[30:31]1182;1183; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__mac:1184; HEURRC: ; %bb.0:1185; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1186; HEURRC-NEXT: v_accvgpr_write_b32 a0, v81187; HEURRC-NEXT: v_accvgpr_write_b32 a1, v91188; HEURRC-NEXT: v_accvgpr_write_b32 a2, v101189; HEURRC-NEXT: v_accvgpr_write_b32 a3, v111190; HEURRC-NEXT: v_accvgpr_write_b32 a4, v121191; HEURRC-NEXT: v_accvgpr_write_b32 a5, v131192; HEURRC-NEXT: v_accvgpr_write_b32 a6, v141193; HEURRC-NEXT: v_accvgpr_write_b32 a7, v151194; HEURRC-NEXT: v_accvgpr_write_b32 a8, v161195; HEURRC-NEXT: v_accvgpr_write_b32 a9, v171196; HEURRC-NEXT: v_accvgpr_write_b32 a10, v181197; HEURRC-NEXT: v_accvgpr_write_b32 a11, v191198; HEURRC-NEXT: v_accvgpr_write_b32 a12, v201199; HEURRC-NEXT: v_accvgpr_write_b32 a13, v211200; HEURRC-NEXT: v_accvgpr_write_b32 a14, v221201; HEURRC-NEXT: v_accvgpr_write_b32 a15, v231202; HEURRC-NEXT: s_nop 11203; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15]1204; HEURRC-NEXT: s_nop 111205; HEURRC-NEXT: v_accvgpr_read_b32 v0, a01206; HEURRC-NEXT: v_accvgpr_read_b32 v1, a11207; HEURRC-NEXT: v_accvgpr_read_b32 v2, a21208; HEURRC-NEXT: v_accvgpr_read_b32 v3, a31209; HEURRC-NEXT: v_accvgpr_read_b32 v4, a41210; HEURRC-NEXT: v_accvgpr_read_b32 v5, a51211; HEURRC-NEXT: v_accvgpr_read_b32 v6, a61212; HEURRC-NEXT: v_accvgpr_read_b32 v7, a71213; HEURRC-NEXT: v_accvgpr_read_b32 v8, a81214; HEURRC-NEXT: v_accvgpr_read_b32 v9, a91215; HEURRC-NEXT: v_accvgpr_read_b32 v10, a101216; HEURRC-NEXT: v_accvgpr_read_b32 v11, a111217; HEURRC-NEXT: v_accvgpr_read_b32 v12, a121218; HEURRC-NEXT: v_accvgpr_read_b32 v13, a131219; HEURRC-NEXT: v_accvgpr_read_b32 v14, a141220; HEURRC-NEXT: v_accvgpr_read_b32 v15, a151221; HEURRC-NEXT: s_setpc_b64 s[30:31]1222;1223; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__mac:1224; VGPRRC: ; %bb.0:1225; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1226; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[8:23], v[0:3], v[4:7], v[8:23]1227; VGPRRC-NEXT: s_nop 111228; VGPRRC-NEXT: v_mov_b32_e32 v0, v81229; VGPRRC-NEXT: v_mov_b32_e32 v1, v91230; VGPRRC-NEXT: v_mov_b32_e32 v2, v101231; VGPRRC-NEXT: v_mov_b32_e32 v3, v111232; VGPRRC-NEXT: v_mov_b32_e32 v4, v121233; VGPRRC-NEXT: v_mov_b32_e32 v5, v131234; VGPRRC-NEXT: v_mov_b32_e32 v6, v141235; VGPRRC-NEXT: v_mov_b32_e32 v7, v151236; VGPRRC-NEXT: v_mov_b32_e32 v8, v161237; VGPRRC-NEXT: v_mov_b32_e32 v9, v171238; VGPRRC-NEXT: v_mov_b32_e32 v10, v181239; VGPRRC-NEXT: v_mov_b32_e32 v11, v191240; VGPRRC-NEXT: v_mov_b32_e32 v12, v201241; VGPRRC-NEXT: v_mov_b32_e32 v13, v211242; VGPRRC-NEXT: v_mov_b32_e32 v14, v221243; VGPRRC-NEXT: v_mov_b32_e32 v15, v231244; VGPRRC-NEXT: s_setpc_b64 s[30:31]1245; AGPR-LABEL: test_mfma_f32_32x32x16_f16__mac:1246; AGPR: ; %bb.0:1247; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1248; AGPR-NEXT: v_accvgpr_write_b32 a0, v81249; AGPR-NEXT: v_accvgpr_write_b32 a1, v91250; AGPR-NEXT: v_accvgpr_write_b32 a2, v101251; AGPR-NEXT: v_accvgpr_write_b32 a3, v111252; AGPR-NEXT: v_accvgpr_write_b32 a4, v121253; AGPR-NEXT: v_accvgpr_write_b32 a5, v131254; AGPR-NEXT: v_accvgpr_write_b32 a6, v141255; AGPR-NEXT: v_accvgpr_write_b32 a7, v151256; AGPR-NEXT: v_accvgpr_write_b32 a8, v161257; AGPR-NEXT: v_accvgpr_write_b32 a9, v171258; AGPR-NEXT: v_accvgpr_write_b32 a10, v181259; AGPR-NEXT: v_accvgpr_write_b32 a11, v191260; AGPR-NEXT: v_accvgpr_write_b32 a12, v201261; AGPR-NEXT: v_accvgpr_write_b32 a13, v211262; AGPR-NEXT: v_accvgpr_write_b32 a14, v221263; AGPR-NEXT: v_accvgpr_write_b32 a15, v231264; AGPR-NEXT: s_nop 11265; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15]1266; AGPR-NEXT: s_nop 71267; AGPR-NEXT: s_nop 31268; AGPR-NEXT: v_accvgpr_read_b32 v0, a01269; AGPR-NEXT: v_accvgpr_read_b32 v1, a11270; AGPR-NEXT: v_accvgpr_read_b32 v2, a21271; AGPR-NEXT: v_accvgpr_read_b32 v3, a31272; AGPR-NEXT: v_accvgpr_read_b32 v4, a41273; AGPR-NEXT: v_accvgpr_read_b32 v5, a51274; AGPR-NEXT: v_accvgpr_read_b32 v6, a61275; AGPR-NEXT: v_accvgpr_read_b32 v7, a71276; AGPR-NEXT: v_accvgpr_read_b32 v8, a81277; AGPR-NEXT: v_accvgpr_read_b32 v9, a91278; AGPR-NEXT: v_accvgpr_read_b32 v10, a101279; AGPR-NEXT: v_accvgpr_read_b32 v11, a111280; AGPR-NEXT: v_accvgpr_read_b32 v12, a121281; AGPR-NEXT: v_accvgpr_read_b32 v13, a131282; AGPR-NEXT: v_accvgpr_read_b32 v14, a141283; AGPR-NEXT: v_accvgpr_read_b32 v15, a151284; AGPR-NEXT: s_setpc_b64 s[30:31]1285; VGPR-LABEL: test_mfma_f32_32x32x16_f16__mac:1286; VGPR: ; %bb.0:1287; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1288; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[8:23], v[0:3], v[4:7], v[8:23]1289; VGPR-NEXT: s_nop 71290; VGPR-NEXT: s_nop 31291; VGPR-NEXT: v_mov_b32_e32 v0, v81292; VGPR-NEXT: v_mov_b32_e32 v1, v91293; VGPR-NEXT: v_mov_b32_e32 v2, v101294; VGPR-NEXT: v_mov_b32_e32 v3, v111295; VGPR-NEXT: v_mov_b32_e32 v4, v121296; VGPR-NEXT: v_mov_b32_e32 v5, v131297; VGPR-NEXT: v_mov_b32_e32 v6, v141298; VGPR-NEXT: v_mov_b32_e32 v7, v151299; VGPR-NEXT: v_mov_b32_e32 v8, v161300; VGPR-NEXT: v_mov_b32_e32 v9, v171301; VGPR-NEXT: v_mov_b32_e32 v10, v181302; VGPR-NEXT: v_mov_b32_e32 v11, v191303; VGPR-NEXT: v_mov_b32_e32 v12, v201304; VGPR-NEXT: v_mov_b32_e32 v13, v211305; VGPR-NEXT: v_mov_b32_e32 v14, v221306; VGPR-NEXT: v_mov_b32_e32 v15, v231307; VGPR-NEXT: s_setpc_b64 s[30:31]1308 %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0)1309 ret <16 x float> %result1310}1311 1312define <16 x float> @test_mfma_f32_32x32x16_f16__mac__flags(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2) {1313; GCN-LABEL: test_mfma_f32_32x32x16_f16__mac__flags:1314; GCN: ; %bb.0:1315; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1316; GCN-NEXT: v_accvgpr_write_b32 a0, v81317; GCN-NEXT: v_accvgpr_write_b32 a1, v91318; GCN-NEXT: v_accvgpr_write_b32 a2, v101319; GCN-NEXT: v_accvgpr_write_b32 a3, v111320; GCN-NEXT: v_accvgpr_write_b32 a4, v121321; GCN-NEXT: v_accvgpr_write_b32 a5, v131322; GCN-NEXT: v_accvgpr_write_b32 a6, v141323; GCN-NEXT: v_accvgpr_write_b32 a7, v151324; GCN-NEXT: v_accvgpr_write_b32 a8, v161325; GCN-NEXT: v_accvgpr_write_b32 a9, v171326; GCN-NEXT: v_accvgpr_write_b32 a10, v181327; GCN-NEXT: v_accvgpr_write_b32 a11, v191328; GCN-NEXT: v_accvgpr_write_b32 a12, v201329; GCN-NEXT: v_accvgpr_write_b32 a13, v211330; GCN-NEXT: v_accvgpr_write_b32 a14, v221331; GCN-NEXT: v_accvgpr_write_b32 a15, v231332; GCN-NEXT: s_nop 11333; GCN-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:11334; GCN-NEXT: s_nop 111335; GCN-NEXT: v_accvgpr_read_b32 v0, a01336; GCN-NEXT: v_accvgpr_read_b32 v1, a11337; GCN-NEXT: v_accvgpr_read_b32 v2, a21338; GCN-NEXT: v_accvgpr_read_b32 v3, a31339; GCN-NEXT: v_accvgpr_read_b32 v4, a41340; GCN-NEXT: v_accvgpr_read_b32 v5, a51341; GCN-NEXT: v_accvgpr_read_b32 v6, a61342; GCN-NEXT: v_accvgpr_read_b32 v7, a71343; GCN-NEXT: v_accvgpr_read_b32 v8, a81344; GCN-NEXT: v_accvgpr_read_b32 v9, a91345; GCN-NEXT: v_accvgpr_read_b32 v10, a101346; GCN-NEXT: v_accvgpr_read_b32 v11, a111347; GCN-NEXT: v_accvgpr_read_b32 v12, a121348; GCN-NEXT: v_accvgpr_read_b32 v13, a131349; GCN-NEXT: v_accvgpr_read_b32 v14, a141350; GCN-NEXT: v_accvgpr_read_b32 v15, a151351; GCN-NEXT: s_setpc_b64 s[30:31]1352;1353; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__mac__flags:1354; HEURRC: ; %bb.0:1355; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1356; HEURRC-NEXT: v_accvgpr_write_b32 a0, v81357; HEURRC-NEXT: v_accvgpr_write_b32 a1, v91358; HEURRC-NEXT: v_accvgpr_write_b32 a2, v101359; HEURRC-NEXT: v_accvgpr_write_b32 a3, v111360; HEURRC-NEXT: v_accvgpr_write_b32 a4, v121361; HEURRC-NEXT: v_accvgpr_write_b32 a5, v131362; HEURRC-NEXT: v_accvgpr_write_b32 a6, v141363; HEURRC-NEXT: v_accvgpr_write_b32 a7, v151364; HEURRC-NEXT: v_accvgpr_write_b32 a8, v161365; HEURRC-NEXT: v_accvgpr_write_b32 a9, v171366; HEURRC-NEXT: v_accvgpr_write_b32 a10, v181367; HEURRC-NEXT: v_accvgpr_write_b32 a11, v191368; HEURRC-NEXT: v_accvgpr_write_b32 a12, v201369; HEURRC-NEXT: v_accvgpr_write_b32 a13, v211370; HEURRC-NEXT: v_accvgpr_write_b32 a14, v221371; HEURRC-NEXT: v_accvgpr_write_b32 a15, v231372; HEURRC-NEXT: s_nop 11373; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:11374; HEURRC-NEXT: s_nop 111375; HEURRC-NEXT: v_accvgpr_read_b32 v0, a01376; HEURRC-NEXT: v_accvgpr_read_b32 v1, a11377; HEURRC-NEXT: v_accvgpr_read_b32 v2, a21378; HEURRC-NEXT: v_accvgpr_read_b32 v3, a31379; HEURRC-NEXT: v_accvgpr_read_b32 v4, a41380; HEURRC-NEXT: v_accvgpr_read_b32 v5, a51381; HEURRC-NEXT: v_accvgpr_read_b32 v6, a61382; HEURRC-NEXT: v_accvgpr_read_b32 v7, a71383; HEURRC-NEXT: v_accvgpr_read_b32 v8, a81384; HEURRC-NEXT: v_accvgpr_read_b32 v9, a91385; HEURRC-NEXT: v_accvgpr_read_b32 v10, a101386; HEURRC-NEXT: v_accvgpr_read_b32 v11, a111387; HEURRC-NEXT: v_accvgpr_read_b32 v12, a121388; HEURRC-NEXT: v_accvgpr_read_b32 v13, a131389; HEURRC-NEXT: v_accvgpr_read_b32 v14, a141390; HEURRC-NEXT: v_accvgpr_read_b32 v15, a151391; HEURRC-NEXT: s_setpc_b64 s[30:31]1392;1393; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__mac__flags:1394; VGPRRC: ; %bb.0:1395; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1396; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[8:23], v[0:3], v[4:7], v[8:23] cbsz:1 abid:1 blgp:11397; VGPRRC-NEXT: s_nop 111398; VGPRRC-NEXT: v_mov_b32_e32 v0, v81399; VGPRRC-NEXT: v_mov_b32_e32 v1, v91400; VGPRRC-NEXT: v_mov_b32_e32 v2, v101401; VGPRRC-NEXT: v_mov_b32_e32 v3, v111402; VGPRRC-NEXT: v_mov_b32_e32 v4, v121403; VGPRRC-NEXT: v_mov_b32_e32 v5, v131404; VGPRRC-NEXT: v_mov_b32_e32 v6, v141405; VGPRRC-NEXT: v_mov_b32_e32 v7, v151406; VGPRRC-NEXT: v_mov_b32_e32 v8, v161407; VGPRRC-NEXT: v_mov_b32_e32 v9, v171408; VGPRRC-NEXT: v_mov_b32_e32 v10, v181409; VGPRRC-NEXT: v_mov_b32_e32 v11, v191410; VGPRRC-NEXT: v_mov_b32_e32 v12, v201411; VGPRRC-NEXT: v_mov_b32_e32 v13, v211412; VGPRRC-NEXT: v_mov_b32_e32 v14, v221413; VGPRRC-NEXT: v_mov_b32_e32 v15, v231414; VGPRRC-NEXT: s_setpc_b64 s[30:31]1415; AGPR-LABEL: test_mfma_f32_32x32x16_f16__mac__flags:1416; AGPR: ; %bb.0:1417; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1418; AGPR-NEXT: v_accvgpr_write_b32 a0, v81419; AGPR-NEXT: v_accvgpr_write_b32 a1, v91420; AGPR-NEXT: v_accvgpr_write_b32 a2, v101421; AGPR-NEXT: v_accvgpr_write_b32 a3, v111422; AGPR-NEXT: v_accvgpr_write_b32 a4, v121423; AGPR-NEXT: v_accvgpr_write_b32 a5, v131424; AGPR-NEXT: v_accvgpr_write_b32 a6, v141425; AGPR-NEXT: v_accvgpr_write_b32 a7, v151426; AGPR-NEXT: v_accvgpr_write_b32 a8, v161427; AGPR-NEXT: v_accvgpr_write_b32 a9, v171428; AGPR-NEXT: v_accvgpr_write_b32 a10, v181429; AGPR-NEXT: v_accvgpr_write_b32 a11, v191430; AGPR-NEXT: v_accvgpr_write_b32 a12, v201431; AGPR-NEXT: v_accvgpr_write_b32 a13, v211432; AGPR-NEXT: v_accvgpr_write_b32 a14, v221433; AGPR-NEXT: v_accvgpr_write_b32 a15, v231434; AGPR-NEXT: s_nop 11435; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:11436; AGPR-NEXT: s_nop 71437; AGPR-NEXT: s_nop 31438; AGPR-NEXT: v_accvgpr_read_b32 v0, a01439; AGPR-NEXT: v_accvgpr_read_b32 v1, a11440; AGPR-NEXT: v_accvgpr_read_b32 v2, a21441; AGPR-NEXT: v_accvgpr_read_b32 v3, a31442; AGPR-NEXT: v_accvgpr_read_b32 v4, a41443; AGPR-NEXT: v_accvgpr_read_b32 v5, a51444; AGPR-NEXT: v_accvgpr_read_b32 v6, a61445; AGPR-NEXT: v_accvgpr_read_b32 v7, a71446; AGPR-NEXT: v_accvgpr_read_b32 v8, a81447; AGPR-NEXT: v_accvgpr_read_b32 v9, a91448; AGPR-NEXT: v_accvgpr_read_b32 v10, a101449; AGPR-NEXT: v_accvgpr_read_b32 v11, a111450; AGPR-NEXT: v_accvgpr_read_b32 v12, a121451; AGPR-NEXT: v_accvgpr_read_b32 v13, a131452; AGPR-NEXT: v_accvgpr_read_b32 v14, a141453; AGPR-NEXT: v_accvgpr_read_b32 v15, a151454; AGPR-NEXT: s_setpc_b64 s[30:31]1455; VGPR-LABEL: test_mfma_f32_32x32x16_f16__mac__flags:1456; VGPR: ; %bb.0:1457; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1458; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[8:23], v[0:3], v[4:7], v[8:23] cbsz:1 abid:1 blgp:11459; VGPR-NEXT: s_nop 71460; VGPR-NEXT: s_nop 31461; VGPR-NEXT: v_mov_b32_e32 v0, v81462; VGPR-NEXT: v_mov_b32_e32 v1, v91463; VGPR-NEXT: v_mov_b32_e32 v2, v101464; VGPR-NEXT: v_mov_b32_e32 v3, v111465; VGPR-NEXT: v_mov_b32_e32 v4, v121466; VGPR-NEXT: v_mov_b32_e32 v5, v131467; VGPR-NEXT: v_mov_b32_e32 v6, v141468; VGPR-NEXT: v_mov_b32_e32 v7, v151469; VGPR-NEXT: v_mov_b32_e32 v8, v161470; VGPR-NEXT: v_mov_b32_e32 v9, v171471; VGPR-NEXT: v_mov_b32_e32 v10, v181472; VGPR-NEXT: v_mov_b32_e32 v11, v191473; VGPR-NEXT: v_mov_b32_e32 v12, v201474; VGPR-NEXT: v_mov_b32_e32 v13, v211475; VGPR-NEXT: v_mov_b32_e32 v14, v221476; VGPR-NEXT: v_mov_b32_e32 v15, v231477; VGPR-NEXT: s_setpc_b64 s[30:31]1478 %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 1, i32 1, i32 1)1479 ret <16 x float> %result1480}1481 1482define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, ptr addrspace(1) %out) #0 {1483; SDAG-LABEL: test_mfma_f32_32x32x16_f16__vgprcd:1484; SDAG: ; %bb.0:1485; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x241486; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x641487; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa41488; SDAG-NEXT: v_mov_b32_e32 v36, 01489; SDAG-NEXT: s_waitcnt lgkmcnt(0)1490; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[26:27]1491; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[24:25]1492; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[30:31]1493; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]1494; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[28:29]1495; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]1496; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]1497; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]1498; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]1499; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]1500; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]1501; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]1502; SDAG-NEXT: v_mov_b32_e32 v32, s201503; SDAG-NEXT: v_mov_b32_e32 v33, s211504; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31]1505; SDAG-NEXT: v_mov_b32_e32 v34, s221506; SDAG-NEXT: v_mov_b32_e32 v35, s231507; SDAG-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc11508; SDAG-NEXT: s_waitcnt vmcnt(0)1509; SDAG-NEXT: s_nop 21510; SDAG-NEXT: v_mov_b32_e32 v16, s161511; SDAG-NEXT: v_mov_b32_e32 v17, s171512; SDAG-NEXT: v_mov_b32_e32 v18, s181513; SDAG-NEXT: v_mov_b32_e32 v19, s191514; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc11515; SDAG-NEXT: s_waitcnt vmcnt(0)1516; SDAG-NEXT: s_nop 01517; SDAG-NEXT: v_mov_b32_e32 v16, s121518; SDAG-NEXT: v_mov_b32_e32 v17, s131519; SDAG-NEXT: v_mov_b32_e32 v18, s141520; SDAG-NEXT: v_mov_b32_e32 v19, s151521; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc11522; SDAG-NEXT: s_waitcnt vmcnt(0)1523; SDAG-NEXT: s_nop 01524; SDAG-NEXT: v_mov_b32_e32 v16, s81525; SDAG-NEXT: v_mov_b32_e32 v17, s91526; SDAG-NEXT: v_mov_b32_e32 v18, s101527; SDAG-NEXT: v_mov_b32_e32 v19, s111528; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc11529; SDAG-NEXT: s_waitcnt vmcnt(0)1530; SDAG-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc11531; SDAG-NEXT: s_waitcnt vmcnt(0)1532; SDAG-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc11533; SDAG-NEXT: s_waitcnt vmcnt(0)1534; SDAG-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc11535; SDAG-NEXT: s_waitcnt vmcnt(0)1536; SDAG-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc11537; SDAG-NEXT: s_waitcnt vmcnt(0)1538; SDAG-NEXT: s_endpgm1539;1540; GISEL-LABEL: test_mfma_f32_32x32x16_f16__vgprcd:1541; GISEL: ; %bb.0:1542; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x241543; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x641544; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa41545; GISEL-NEXT: v_mov_b32_e32 v56, 01546; GISEL-NEXT: s_waitcnt lgkmcnt(0)1547; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]1548; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]1549; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]1550; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]1551; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]1552; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]1553; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]1554; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]1555; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]1556; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]1557; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]1558; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]1559; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]1560; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]1561; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15]1562; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[14:15]1563; GISEL-NEXT: v_mov_b64_e32 v[50:51], s[18:19]1564; GISEL-NEXT: v_mov_b64_e32 v[54:55], s[22:23]1565; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[12:13]1566; GISEL-NEXT: v_mov_b64_e32 v[48:49], s[16:17]1567; GISEL-NEXT: v_mov_b64_e32 v[52:53], s[20:21]1568; GISEL-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc11569; GISEL-NEXT: s_waitcnt vmcnt(0)1570; GISEL-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc11571; GISEL-NEXT: s_waitcnt vmcnt(0)1572; GISEL-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc11573; GISEL-NEXT: s_waitcnt vmcnt(0)1574; GISEL-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc11575; GISEL-NEXT: s_waitcnt vmcnt(0)1576; GISEL-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc11577; GISEL-NEXT: s_waitcnt vmcnt(0)1578; GISEL-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc11579; GISEL-NEXT: s_waitcnt vmcnt(0)1580; GISEL-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc11581; GISEL-NEXT: s_waitcnt vmcnt(0)1582; GISEL-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc11583; GISEL-NEXT: s_waitcnt vmcnt(0)1584; GISEL-NEXT: s_endpgm1585;1586; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd:1587; HEURRC: ; %bb.0:1588; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x241589; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x641590; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa41591; HEURRC-NEXT: v_mov_b32_e32 v36, 01592; HEURRC-NEXT: s_waitcnt lgkmcnt(0)1593; HEURRC-NEXT: v_mov_b64_e32 v[40:41], s[26:27]1594; HEURRC-NEXT: v_mov_b64_e32 v[38:39], s[24:25]1595; HEURRC-NEXT: v_mov_b64_e32 v[44:45], s[30:31]1596; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]1597; HEURRC-NEXT: v_mov_b64_e32 v[42:43], s[28:29]1598; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]1599; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]1600; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]1601; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]1602; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]1603; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]1604; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]1605; HEURRC-NEXT: v_mov_b32_e32 v32, s201606; HEURRC-NEXT: v_mov_b32_e32 v33, s211607; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31]1608; HEURRC-NEXT: v_mov_b32_e32 v34, s221609; HEURRC-NEXT: v_mov_b32_e32 v35, s231610; HEURRC-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc11611; HEURRC-NEXT: s_waitcnt vmcnt(0)1612; HEURRC-NEXT: s_nop 21613; HEURRC-NEXT: v_mov_b32_e32 v16, s161614; HEURRC-NEXT: v_mov_b32_e32 v17, s171615; HEURRC-NEXT: v_mov_b32_e32 v18, s181616; HEURRC-NEXT: v_mov_b32_e32 v19, s191617; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc11618; HEURRC-NEXT: s_waitcnt vmcnt(0)1619; HEURRC-NEXT: s_nop 01620; HEURRC-NEXT: v_mov_b32_e32 v16, s121621; HEURRC-NEXT: v_mov_b32_e32 v17, s131622; HEURRC-NEXT: v_mov_b32_e32 v18, s141623; HEURRC-NEXT: v_mov_b32_e32 v19, s151624; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc11625; HEURRC-NEXT: s_waitcnt vmcnt(0)1626; HEURRC-NEXT: s_nop 01627; HEURRC-NEXT: v_mov_b32_e32 v16, s81628; HEURRC-NEXT: v_mov_b32_e32 v17, s91629; HEURRC-NEXT: v_mov_b32_e32 v18, s101630; HEURRC-NEXT: v_mov_b32_e32 v19, s111631; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc11632; HEURRC-NEXT: s_waitcnt vmcnt(0)1633; HEURRC-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc11634; HEURRC-NEXT: s_waitcnt vmcnt(0)1635; HEURRC-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc11636; HEURRC-NEXT: s_waitcnt vmcnt(0)1637; HEURRC-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc11638; HEURRC-NEXT: s_waitcnt vmcnt(0)1639; HEURRC-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc11640; HEURRC-NEXT: s_waitcnt vmcnt(0)1641; HEURRC-NEXT: s_endpgm1642;1643; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd:1644; VGPRRC: ; %bb.0:1645; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x241646; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x641647; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa41648; VGPRRC-NEXT: v_mov_b32_e32 v36, 01649; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)1650; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[26:27]1651; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[24:25]1652; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[30:31]1653; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]1654; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[28:29]1655; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]1656; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]1657; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]1658; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]1659; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]1660; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]1661; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]1662; VGPRRC-NEXT: v_mov_b32_e32 v32, s201663; VGPRRC-NEXT: v_mov_b32_e32 v33, s211664; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31]1665; VGPRRC-NEXT: v_mov_b32_e32 v34, s221666; VGPRRC-NEXT: v_mov_b32_e32 v35, s231667; VGPRRC-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc11668; VGPRRC-NEXT: s_waitcnt vmcnt(0)1669; VGPRRC-NEXT: s_nop 21670; VGPRRC-NEXT: v_mov_b32_e32 v16, s161671; VGPRRC-NEXT: v_mov_b32_e32 v17, s171672; VGPRRC-NEXT: v_mov_b32_e32 v18, s181673; VGPRRC-NEXT: v_mov_b32_e32 v19, s191674; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc11675; VGPRRC-NEXT: s_waitcnt vmcnt(0)1676; VGPRRC-NEXT: s_nop 01677; VGPRRC-NEXT: v_mov_b32_e32 v16, s121678; VGPRRC-NEXT: v_mov_b32_e32 v17, s131679; VGPRRC-NEXT: v_mov_b32_e32 v18, s141680; VGPRRC-NEXT: v_mov_b32_e32 v19, s151681; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc11682; VGPRRC-NEXT: s_waitcnt vmcnt(0)1683; VGPRRC-NEXT: s_nop 01684; VGPRRC-NEXT: v_mov_b32_e32 v16, s81685; VGPRRC-NEXT: v_mov_b32_e32 v17, s91686; VGPRRC-NEXT: v_mov_b32_e32 v18, s101687; VGPRRC-NEXT: v_mov_b32_e32 v19, s111688; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc11689; VGPRRC-NEXT: s_waitcnt vmcnt(0)1690; VGPRRC-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc11691; VGPRRC-NEXT: s_waitcnt vmcnt(0)1692; VGPRRC-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc11693; VGPRRC-NEXT: s_waitcnt vmcnt(0)1694; VGPRRC-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc11695; VGPRRC-NEXT: s_waitcnt vmcnt(0)1696; VGPRRC-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc11697; VGPRRC-NEXT: s_waitcnt vmcnt(0)1698; VGPRRC-NEXT: s_endpgm1699; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd:1700; AGPR: ; %bb.0:1701; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x241702; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x641703; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa41704; AGPR-NEXT: v_mov_b32_e32 v12, 01705; AGPR-NEXT: s_waitcnt lgkmcnt(0)1706; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25]1707; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27]1708; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29]1709; AGPR-NEXT: v_accvgpr_write_b32 a31, s231710; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31]1711; AGPR-NEXT: v_accvgpr_write_b32 a30, s221712; AGPR-NEXT: v_accvgpr_write_b32 a29, s211713; AGPR-NEXT: v_accvgpr_write_b32 a28, s201714; AGPR-NEXT: v_accvgpr_write_b32 a27, s191715; AGPR-NEXT: v_accvgpr_write_b32 a26, s181716; AGPR-NEXT: v_accvgpr_write_b32 a25, s171717; AGPR-NEXT: v_accvgpr_write_b32 a24, s161718; AGPR-NEXT: v_accvgpr_write_b32 a23, s151719; AGPR-NEXT: v_accvgpr_write_b32 a22, s141720; AGPR-NEXT: v_accvgpr_write_b32 a21, s131721; AGPR-NEXT: v_accvgpr_write_b32 a20, s121722; AGPR-NEXT: v_accvgpr_write_b32 a19, s111723; AGPR-NEXT: v_accvgpr_write_b32 a18, s101724; AGPR-NEXT: v_accvgpr_write_b32 a17, s91725; AGPR-NEXT: v_accvgpr_write_b32 a16, s81726; AGPR-NEXT: v_mov_b32_e32 v8, s201727; AGPR-NEXT: v_mov_b32_e32 v9, s211728; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[16:31]1729; AGPR-NEXT: v_mov_b32_e32 v10, s221730; AGPR-NEXT: v_mov_b32_e32 v11, s231731; AGPR-NEXT: v_mov_b32_e32 v0, s161732; AGPR-NEXT: v_mov_b32_e32 v1, s171733; AGPR-NEXT: v_mov_b32_e32 v2, s181734; AGPR-NEXT: v_mov_b32_e32 v3, s191735; AGPR-NEXT: global_store_dwordx4 v12, v[8:11], s[0:1] offset:48 sc0 sc11736; AGPR-NEXT: s_waitcnt vmcnt(0)1737; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:32 sc0 sc11738; AGPR-NEXT: s_waitcnt vmcnt(0)1739; AGPR-NEXT: s_nop 01740; AGPR-NEXT: v_mov_b32_e32 v0, s121741; AGPR-NEXT: v_mov_b32_e32 v1, s131742; AGPR-NEXT: v_mov_b32_e32 v2, s141743; AGPR-NEXT: v_mov_b32_e32 v3, s151744; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:16 sc0 sc11745; AGPR-NEXT: s_waitcnt vmcnt(0)1746; AGPR-NEXT: s_nop 01747; AGPR-NEXT: v_mov_b32_e32 v0, s81748; AGPR-NEXT: v_mov_b32_e32 v1, s91749; AGPR-NEXT: v_mov_b32_e32 v2, s101750; AGPR-NEXT: v_mov_b32_e32 v3, s111751; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] sc0 sc11752; AGPR-NEXT: s_waitcnt vmcnt(0)1753; AGPR-NEXT: global_store_dwordx4 v12, a[8:11], s[0:1] offset:32 sc0 sc11754; AGPR-NEXT: s_waitcnt vmcnt(0)1755; AGPR-NEXT: global_store_dwordx4 v12, a[12:15], s[0:1] offset:48 sc0 sc11756; AGPR-NEXT: s_waitcnt vmcnt(0)1757; AGPR-NEXT: global_store_dwordx4 v12, a[0:3], s[0:1] sc0 sc11758; AGPR-NEXT: s_waitcnt vmcnt(0)1759; AGPR-NEXT: global_store_dwordx4 v12, a[4:7], s[0:1] offset:16 sc0 sc11760; AGPR-NEXT: s_waitcnt vmcnt(0)1761; AGPR-NEXT: s_endpgm1762; VGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd:1763; VGPR: ; %bb.0:1764; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x241765; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x641766; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa41767; VGPR-NEXT: v_mov_b32_e32 v44, 01768; VGPR-NEXT: s_waitcnt lgkmcnt(0)1769; VGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27]1770; VGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25]1771; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31]1772; VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]1773; VGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29]1774; VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]1775; VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]1776; VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]1777; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]1778; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]1779; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]1780; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]1781; VGPR-NEXT: v_mov_b32_e32 v40, s201782; VGPR-NEXT: v_mov_b32_e32 v41, s211783; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31]1784; VGPR-NEXT: v_mov_b32_e32 v42, s221785; VGPR-NEXT: v_mov_b32_e32 v43, s231786; VGPR-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] offset:48 sc0 sc11787; VGPR-NEXT: s_waitcnt vmcnt(0)1788; VGPR-NEXT: s_nop 21789; VGPR-NEXT: v_mov_b32_e32 v16, s161790; VGPR-NEXT: v_mov_b32_e32 v17, s171791; VGPR-NEXT: v_mov_b32_e32 v18, s181792; VGPR-NEXT: v_mov_b32_e32 v19, s191793; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:32 sc0 sc11794; VGPR-NEXT: s_waitcnt vmcnt(0)1795; VGPR-NEXT: s_nop 01796; VGPR-NEXT: v_mov_b32_e32 v16, s121797; VGPR-NEXT: v_mov_b32_e32 v17, s131798; VGPR-NEXT: v_mov_b32_e32 v18, s141799; VGPR-NEXT: v_mov_b32_e32 v19, s151800; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc11801; VGPR-NEXT: s_waitcnt vmcnt(0)1802; VGPR-NEXT: s_nop 01803; VGPR-NEXT: v_mov_b32_e32 v16, s81804; VGPR-NEXT: v_mov_b32_e32 v17, s91805; VGPR-NEXT: v_mov_b32_e32 v18, s101806; VGPR-NEXT: v_mov_b32_e32 v19, s111807; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] sc0 sc11808; VGPR-NEXT: s_waitcnt vmcnt(0)1809; VGPR-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc11810; VGPR-NEXT: s_waitcnt vmcnt(0)1811; VGPR-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc11812; VGPR-NEXT: s_waitcnt vmcnt(0)1813; VGPR-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc11814; VGPR-NEXT: s_waitcnt vmcnt(0)1815; VGPR-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc11816; VGPR-NEXT: s_waitcnt vmcnt(0)1817; VGPR-NEXT: s_endpgm1818 %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0)1819 store volatile <16 x float> %arg2, ptr addrspace(1) %out1820 store volatile <16 x float> %result, ptr addrspace(1) %out1821 ret void1822}1823 1824define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, ptr addrspace(1) %out) #0 {1825; SDAG-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags:1826; SDAG: ; %bb.0:1827; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x241828; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x641829; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa41830; SDAG-NEXT: v_mov_b32_e32 v36, 01831; SDAG-NEXT: s_waitcnt lgkmcnt(0)1832; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[26:27]1833; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[24:25]1834; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[30:31]1835; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]1836; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[28:29]1837; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]1838; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]1839; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]1840; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]1841; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]1842; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]1843; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]1844; SDAG-NEXT: v_mov_b32_e32 v32, s201845; SDAG-NEXT: v_mov_b32_e32 v33, s211846; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31] cbsz:1 abid:2 blgp:31847; SDAG-NEXT: v_mov_b32_e32 v34, s221848; SDAG-NEXT: v_mov_b32_e32 v35, s231849; SDAG-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc11850; SDAG-NEXT: s_waitcnt vmcnt(0)1851; SDAG-NEXT: s_nop 21852; SDAG-NEXT: v_mov_b32_e32 v16, s161853; SDAG-NEXT: v_mov_b32_e32 v17, s171854; SDAG-NEXT: v_mov_b32_e32 v18, s181855; SDAG-NEXT: v_mov_b32_e32 v19, s191856; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc11857; SDAG-NEXT: s_waitcnt vmcnt(0)1858; SDAG-NEXT: s_nop 01859; SDAG-NEXT: v_mov_b32_e32 v16, s121860; SDAG-NEXT: v_mov_b32_e32 v17, s131861; SDAG-NEXT: v_mov_b32_e32 v18, s141862; SDAG-NEXT: v_mov_b32_e32 v19, s151863; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc11864; SDAG-NEXT: s_waitcnt vmcnt(0)1865; SDAG-NEXT: s_nop 01866; SDAG-NEXT: v_mov_b32_e32 v16, s81867; SDAG-NEXT: v_mov_b32_e32 v17, s91868; SDAG-NEXT: v_mov_b32_e32 v18, s101869; SDAG-NEXT: v_mov_b32_e32 v19, s111870; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc11871; SDAG-NEXT: s_waitcnt vmcnt(0)1872; SDAG-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc11873; SDAG-NEXT: s_waitcnt vmcnt(0)1874; SDAG-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc11875; SDAG-NEXT: s_waitcnt vmcnt(0)1876; SDAG-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc11877; SDAG-NEXT: s_waitcnt vmcnt(0)1878; SDAG-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc11879; SDAG-NEXT: s_waitcnt vmcnt(0)1880; SDAG-NEXT: s_endpgm1881;1882; GISEL-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags:1883; GISEL: ; %bb.0:1884; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x241885; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x641886; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa41887; GISEL-NEXT: v_mov_b32_e32 v56, 01888; GISEL-NEXT: s_waitcnt lgkmcnt(0)1889; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]1890; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]1891; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]1892; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]1893; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]1894; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]1895; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]1896; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]1897; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]1898; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]1899; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]1900; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]1901; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]1902; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]1903; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:1 abid:2 blgp:31904; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[14:15]1905; GISEL-NEXT: v_mov_b64_e32 v[50:51], s[18:19]1906; GISEL-NEXT: v_mov_b64_e32 v[54:55], s[22:23]1907; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[12:13]1908; GISEL-NEXT: v_mov_b64_e32 v[48:49], s[16:17]1909; GISEL-NEXT: v_mov_b64_e32 v[52:53], s[20:21]1910; GISEL-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc11911; GISEL-NEXT: s_waitcnt vmcnt(0)1912; GISEL-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc11913; GISEL-NEXT: s_waitcnt vmcnt(0)1914; GISEL-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc11915; GISEL-NEXT: s_waitcnt vmcnt(0)1916; GISEL-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc11917; GISEL-NEXT: s_waitcnt vmcnt(0)1918; GISEL-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc11919; GISEL-NEXT: s_waitcnt vmcnt(0)1920; GISEL-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc11921; GISEL-NEXT: s_waitcnt vmcnt(0)1922; GISEL-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc11923; GISEL-NEXT: s_waitcnt vmcnt(0)1924; GISEL-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc11925; GISEL-NEXT: s_waitcnt vmcnt(0)1926; GISEL-NEXT: s_endpgm1927;1928; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags:1929; HEURRC: ; %bb.0:1930; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x241931; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x641932; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa41933; HEURRC-NEXT: v_mov_b32_e32 v36, 01934; HEURRC-NEXT: s_waitcnt lgkmcnt(0)1935; HEURRC-NEXT: v_mov_b64_e32 v[40:41], s[26:27]1936; HEURRC-NEXT: v_mov_b64_e32 v[38:39], s[24:25]1937; HEURRC-NEXT: v_mov_b64_e32 v[44:45], s[30:31]1938; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]1939; HEURRC-NEXT: v_mov_b64_e32 v[42:43], s[28:29]1940; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]1941; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]1942; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]1943; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]1944; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]1945; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]1946; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]1947; HEURRC-NEXT: v_mov_b32_e32 v32, s201948; HEURRC-NEXT: v_mov_b32_e32 v33, s211949; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31] cbsz:1 abid:2 blgp:31950; HEURRC-NEXT: v_mov_b32_e32 v34, s221951; HEURRC-NEXT: v_mov_b32_e32 v35, s231952; HEURRC-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc11953; HEURRC-NEXT: s_waitcnt vmcnt(0)1954; HEURRC-NEXT: s_nop 21955; HEURRC-NEXT: v_mov_b32_e32 v16, s161956; HEURRC-NEXT: v_mov_b32_e32 v17, s171957; HEURRC-NEXT: v_mov_b32_e32 v18, s181958; HEURRC-NEXT: v_mov_b32_e32 v19, s191959; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc11960; HEURRC-NEXT: s_waitcnt vmcnt(0)1961; HEURRC-NEXT: s_nop 01962; HEURRC-NEXT: v_mov_b32_e32 v16, s121963; HEURRC-NEXT: v_mov_b32_e32 v17, s131964; HEURRC-NEXT: v_mov_b32_e32 v18, s141965; HEURRC-NEXT: v_mov_b32_e32 v19, s151966; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc11967; HEURRC-NEXT: s_waitcnt vmcnt(0)1968; HEURRC-NEXT: s_nop 01969; HEURRC-NEXT: v_mov_b32_e32 v16, s81970; HEURRC-NEXT: v_mov_b32_e32 v17, s91971; HEURRC-NEXT: v_mov_b32_e32 v18, s101972; HEURRC-NEXT: v_mov_b32_e32 v19, s111973; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc11974; HEURRC-NEXT: s_waitcnt vmcnt(0)1975; HEURRC-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc11976; HEURRC-NEXT: s_waitcnt vmcnt(0)1977; HEURRC-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc11978; HEURRC-NEXT: s_waitcnt vmcnt(0)1979; HEURRC-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc11980; HEURRC-NEXT: s_waitcnt vmcnt(0)1981; HEURRC-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc11982; HEURRC-NEXT: s_waitcnt vmcnt(0)1983; HEURRC-NEXT: s_endpgm1984;1985; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags:1986; VGPRRC: ; %bb.0:1987; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x241988; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x641989; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa41990; VGPRRC-NEXT: v_mov_b32_e32 v36, 01991; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)1992; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[26:27]1993; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[24:25]1994; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[30:31]1995; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]1996; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[28:29]1997; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]1998; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]1999; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]2000; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]2001; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]2002; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]2003; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]2004; VGPRRC-NEXT: v_mov_b32_e32 v32, s202005; VGPRRC-NEXT: v_mov_b32_e32 v33, s212006; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31] cbsz:1 abid:2 blgp:32007; VGPRRC-NEXT: v_mov_b32_e32 v34, s222008; VGPRRC-NEXT: v_mov_b32_e32 v35, s232009; VGPRRC-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc12010; VGPRRC-NEXT: s_waitcnt vmcnt(0)2011; VGPRRC-NEXT: s_nop 22012; VGPRRC-NEXT: v_mov_b32_e32 v16, s162013; VGPRRC-NEXT: v_mov_b32_e32 v17, s172014; VGPRRC-NEXT: v_mov_b32_e32 v18, s182015; VGPRRC-NEXT: v_mov_b32_e32 v19, s192016; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc12017; VGPRRC-NEXT: s_waitcnt vmcnt(0)2018; VGPRRC-NEXT: s_nop 02019; VGPRRC-NEXT: v_mov_b32_e32 v16, s122020; VGPRRC-NEXT: v_mov_b32_e32 v17, s132021; VGPRRC-NEXT: v_mov_b32_e32 v18, s142022; VGPRRC-NEXT: v_mov_b32_e32 v19, s152023; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc12024; VGPRRC-NEXT: s_waitcnt vmcnt(0)2025; VGPRRC-NEXT: s_nop 02026; VGPRRC-NEXT: v_mov_b32_e32 v16, s82027; VGPRRC-NEXT: v_mov_b32_e32 v17, s92028; VGPRRC-NEXT: v_mov_b32_e32 v18, s102029; VGPRRC-NEXT: v_mov_b32_e32 v19, s112030; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc12031; VGPRRC-NEXT: s_waitcnt vmcnt(0)2032; VGPRRC-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc12033; VGPRRC-NEXT: s_waitcnt vmcnt(0)2034; VGPRRC-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc12035; VGPRRC-NEXT: s_waitcnt vmcnt(0)2036; VGPRRC-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc12037; VGPRRC-NEXT: s_waitcnt vmcnt(0)2038; VGPRRC-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc12039; VGPRRC-NEXT: s_waitcnt vmcnt(0)2040; VGPRRC-NEXT: s_endpgm2041; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags:2042; AGPR: ; %bb.0:2043; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x242044; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x642045; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa42046; AGPR-NEXT: v_mov_b32_e32 v12, 02047; AGPR-NEXT: s_waitcnt lgkmcnt(0)2048; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25]2049; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27]2050; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29]2051; AGPR-NEXT: v_accvgpr_write_b32 a31, s232052; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31]2053; AGPR-NEXT: v_accvgpr_write_b32 a30, s222054; AGPR-NEXT: v_accvgpr_write_b32 a29, s212055; AGPR-NEXT: v_accvgpr_write_b32 a28, s202056; AGPR-NEXT: v_accvgpr_write_b32 a27, s192057; AGPR-NEXT: v_accvgpr_write_b32 a26, s182058; AGPR-NEXT: v_accvgpr_write_b32 a25, s172059; AGPR-NEXT: v_accvgpr_write_b32 a24, s162060; AGPR-NEXT: v_accvgpr_write_b32 a23, s152061; AGPR-NEXT: v_accvgpr_write_b32 a22, s142062; AGPR-NEXT: v_accvgpr_write_b32 a21, s132063; AGPR-NEXT: v_accvgpr_write_b32 a20, s122064; AGPR-NEXT: v_accvgpr_write_b32 a19, s112065; AGPR-NEXT: v_accvgpr_write_b32 a18, s102066; AGPR-NEXT: v_accvgpr_write_b32 a17, s92067; AGPR-NEXT: v_accvgpr_write_b32 a16, s82068; AGPR-NEXT: v_mov_b32_e32 v8, s202069; AGPR-NEXT: v_mov_b32_e32 v9, s212070; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[16:31] cbsz:1 abid:2 blgp:32071; AGPR-NEXT: v_mov_b32_e32 v10, s222072; AGPR-NEXT: v_mov_b32_e32 v11, s232073; AGPR-NEXT: v_mov_b32_e32 v0, s162074; AGPR-NEXT: v_mov_b32_e32 v1, s172075; AGPR-NEXT: v_mov_b32_e32 v2, s182076; AGPR-NEXT: v_mov_b32_e32 v3, s192077; AGPR-NEXT: global_store_dwordx4 v12, v[8:11], s[0:1] offset:48 sc0 sc12078; AGPR-NEXT: s_waitcnt vmcnt(0)2079; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:32 sc0 sc12080; AGPR-NEXT: s_waitcnt vmcnt(0)2081; AGPR-NEXT: s_nop 02082; AGPR-NEXT: v_mov_b32_e32 v0, s122083; AGPR-NEXT: v_mov_b32_e32 v1, s132084; AGPR-NEXT: v_mov_b32_e32 v2, s142085; AGPR-NEXT: v_mov_b32_e32 v3, s152086; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:16 sc0 sc12087; AGPR-NEXT: s_waitcnt vmcnt(0)2088; AGPR-NEXT: s_nop 02089; AGPR-NEXT: v_mov_b32_e32 v0, s82090; AGPR-NEXT: v_mov_b32_e32 v1, s92091; AGPR-NEXT: v_mov_b32_e32 v2, s102092; AGPR-NEXT: v_mov_b32_e32 v3, s112093; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] sc0 sc12094; AGPR-NEXT: s_waitcnt vmcnt(0)2095; AGPR-NEXT: global_store_dwordx4 v12, a[8:11], s[0:1] offset:32 sc0 sc12096; AGPR-NEXT: s_waitcnt vmcnt(0)2097; AGPR-NEXT: global_store_dwordx4 v12, a[12:15], s[0:1] offset:48 sc0 sc12098; AGPR-NEXT: s_waitcnt vmcnt(0)2099; AGPR-NEXT: global_store_dwordx4 v12, a[0:3], s[0:1] sc0 sc12100; AGPR-NEXT: s_waitcnt vmcnt(0)2101; AGPR-NEXT: global_store_dwordx4 v12, a[4:7], s[0:1] offset:16 sc0 sc12102; AGPR-NEXT: s_waitcnt vmcnt(0)2103; AGPR-NEXT: s_endpgm2104; VGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags:2105; VGPR: ; %bb.0:2106; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x242107; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x642108; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa42109; VGPR-NEXT: v_mov_b32_e32 v44, 02110; VGPR-NEXT: s_waitcnt lgkmcnt(0)2111; VGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27]2112; VGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25]2113; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31]2114; VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]2115; VGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29]2116; VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]2117; VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]2118; VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]2119; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]2120; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]2121; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]2122; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]2123; VGPR-NEXT: v_mov_b32_e32 v40, s202124; VGPR-NEXT: v_mov_b32_e32 v41, s212125; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:32126; VGPR-NEXT: v_mov_b32_e32 v42, s222127; VGPR-NEXT: v_mov_b32_e32 v43, s232128; VGPR-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] offset:48 sc0 sc12129; VGPR-NEXT: s_waitcnt vmcnt(0)2130; VGPR-NEXT: s_nop 22131; VGPR-NEXT: v_mov_b32_e32 v16, s162132; VGPR-NEXT: v_mov_b32_e32 v17, s172133; VGPR-NEXT: v_mov_b32_e32 v18, s182134; VGPR-NEXT: v_mov_b32_e32 v19, s192135; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:32 sc0 sc12136; VGPR-NEXT: s_waitcnt vmcnt(0)2137; VGPR-NEXT: s_nop 02138; VGPR-NEXT: v_mov_b32_e32 v16, s122139; VGPR-NEXT: v_mov_b32_e32 v17, s132140; VGPR-NEXT: v_mov_b32_e32 v18, s142141; VGPR-NEXT: v_mov_b32_e32 v19, s152142; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc12143; VGPR-NEXT: s_waitcnt vmcnt(0)2144; VGPR-NEXT: s_nop 02145; VGPR-NEXT: v_mov_b32_e32 v16, s82146; VGPR-NEXT: v_mov_b32_e32 v17, s92147; VGPR-NEXT: v_mov_b32_e32 v18, s102148; VGPR-NEXT: v_mov_b32_e32 v19, s112149; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] sc0 sc12150; VGPR-NEXT: s_waitcnt vmcnt(0)2151; VGPR-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc12152; VGPR-NEXT: s_waitcnt vmcnt(0)2153; VGPR-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc12154; VGPR-NEXT: s_waitcnt vmcnt(0)2155; VGPR-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc12156; VGPR-NEXT: s_waitcnt vmcnt(0)2157; VGPR-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc12158; VGPR-NEXT: s_waitcnt vmcnt(0)2159; VGPR-NEXT: s_endpgm2160 %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 1, i32 2, i32 3)2161 store volatile <16 x float> %arg2, ptr addrspace(1) %out2162 store volatile <16 x float> %result, ptr addrspace(1) %out2163 ret void2164}2165 2166define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd_mac(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, ptr addrspace(1) %out) #0 {2167; SDAG-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac:2168; SDAG: ; %bb.0:2169; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x242170; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x642171; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa42172; SDAG-NEXT: s_waitcnt lgkmcnt(0)2173; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[24:25]2174; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[26:27]2175; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[28:29]2176; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]2177; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[30:31]2178; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]2179; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]2180; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]2181; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]2182; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]2183; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]2184; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]2185; SDAG-NEXT: s_nop 12186; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15]2187; SDAG-NEXT: v_mov_b32_e32 v16, 02188; SDAG-NEXT: s_nop 102189; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:482190; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:322191; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:162192; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]2193; SDAG-NEXT: s_endpgm2194;2195; GISEL-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac:2196; GISEL: ; %bb.0:2197; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x242198; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x642199; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa42200; GISEL-NEXT: s_waitcnt lgkmcnt(0)2201; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[24:25]2202; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[26:27]2203; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[28:29]2204; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]2205; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[30:31]2206; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]2207; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]2208; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]2209; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]2210; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]2211; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]2212; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]2213; GISEL-NEXT: s_nop 12214; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15]2215; GISEL-NEXT: v_mov_b32_e32 v16, 02216; GISEL-NEXT: s_nop 102217; GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]2218; GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:162219; GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:322220; GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:482221; GISEL-NEXT: s_endpgm2222;2223; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac:2224; HEURRC: ; %bb.0:2225; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x242226; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x642227; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa42228; HEURRC-NEXT: s_waitcnt lgkmcnt(0)2229; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[24:25]2230; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[26:27]2231; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[28:29]2232; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]2233; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[30:31]2234; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]2235; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]2236; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]2237; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]2238; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]2239; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]2240; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]2241; HEURRC-NEXT: s_nop 12242; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15]2243; HEURRC-NEXT: v_mov_b32_e32 v16, 02244; HEURRC-NEXT: s_nop 102245; HEURRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:482246; HEURRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:322247; HEURRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:162248; HEURRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]2249; HEURRC-NEXT: s_endpgm2250;2251; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac:2252; VGPRRC: ; %bb.0:2253; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x242254; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x642255; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa42256; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)2257; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[24:25]2258; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[26:27]2259; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[28:29]2260; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]2261; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[30:31]2262; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]2263; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]2264; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]2265; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]2266; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]2267; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]2268; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]2269; VGPRRC-NEXT: s_nop 12270; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15]2271; VGPRRC-NEXT: v_mov_b32_e32 v16, 02272; VGPRRC-NEXT: s_nop 102273; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:482274; VGPRRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:322275; VGPRRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:162276; VGPRRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]2277; VGPRRC-NEXT: s_endpgm2278; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac:2279; AGPR: ; %bb.0:2280; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x242281; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x642282; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa42283; AGPR-NEXT: s_waitcnt lgkmcnt(0)2284; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25]2285; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27]2286; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29]2287; AGPR-NEXT: v_accvgpr_write_b32 a0, s82288; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31]2289; AGPR-NEXT: v_accvgpr_write_b32 a1, s92290; AGPR-NEXT: v_accvgpr_write_b32 a2, s102291; AGPR-NEXT: v_accvgpr_write_b32 a3, s112292; AGPR-NEXT: v_accvgpr_write_b32 a4, s122293; AGPR-NEXT: v_accvgpr_write_b32 a5, s132294; AGPR-NEXT: v_accvgpr_write_b32 a6, s142295; AGPR-NEXT: v_accvgpr_write_b32 a7, s152296; AGPR-NEXT: v_accvgpr_write_b32 a8, s162297; AGPR-NEXT: v_accvgpr_write_b32 a9, s172298; AGPR-NEXT: v_accvgpr_write_b32 a10, s182299; AGPR-NEXT: v_accvgpr_write_b32 a11, s192300; AGPR-NEXT: v_accvgpr_write_b32 a12, s202301; AGPR-NEXT: v_accvgpr_write_b32 a13, s212302; AGPR-NEXT: v_accvgpr_write_b32 a14, s222303; AGPR-NEXT: v_accvgpr_write_b32 a15, s232304; AGPR-NEXT: s_nop 12305; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15]2306; AGPR-NEXT: v_mov_b32_e32 v0, 02307; AGPR-NEXT: s_nop 72308; AGPR-NEXT: s_nop 22309; AGPR-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:482310; AGPR-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:322311; AGPR-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:162312; AGPR-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1]2313; AGPR-NEXT: s_endpgm2314; VGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac:2315; VGPR: ; %bb.0:2316; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x242317; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x642318; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa42319; VGPR-NEXT: s_waitcnt lgkmcnt(0)2320; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[24:25]2321; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[26:27]2322; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[28:29]2323; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]2324; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[30:31]2325; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]2326; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]2327; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]2328; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]2329; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]2330; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]2331; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]2332; VGPR-NEXT: s_nop 12333; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15]2334; VGPR-NEXT: v_mov_b32_e32 v16, 02335; VGPR-NEXT: s_nop 72336; VGPR-NEXT: s_nop 22337; VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:482338; VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:322339; VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:162340; VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]2341; VGPR-NEXT: s_endpgm2342 %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0)2343 store <16 x float> %result, ptr addrspace(1) %out2344 ret void2345}2346 2347define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd_mac_flags(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, ptr addrspace(1) %out) #0 {2348; SDAG-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac_flags:2349; SDAG: ; %bb.0:2350; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x242351; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x642352; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa42353; SDAG-NEXT: s_waitcnt lgkmcnt(0)2354; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[24:25]2355; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[26:27]2356; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[28:29]2357; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]2358; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[30:31]2359; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]2360; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]2361; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]2362; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]2363; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]2364; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]2365; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]2366; SDAG-NEXT: s_nop 12367; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:12368; SDAG-NEXT: v_mov_b32_e32 v16, 02369; SDAG-NEXT: s_nop 102370; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:482371; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:322372; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:162373; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]2374; SDAG-NEXT: s_endpgm2375;2376; GISEL-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac_flags:2377; GISEL: ; %bb.0:2378; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x242379; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x642380; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa42381; GISEL-NEXT: s_waitcnt lgkmcnt(0)2382; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[24:25]2383; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[26:27]2384; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[28:29]2385; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]2386; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[30:31]2387; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]2388; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]2389; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]2390; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]2391; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]2392; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]2393; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]2394; GISEL-NEXT: s_nop 12395; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:12396; GISEL-NEXT: v_mov_b32_e32 v16, 02397; GISEL-NEXT: s_nop 102398; GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]2399; GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:162400; GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:322401; GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:482402; GISEL-NEXT: s_endpgm2403;2404; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac_flags:2405; HEURRC: ; %bb.0:2406; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x242407; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x642408; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa42409; HEURRC-NEXT: s_waitcnt lgkmcnt(0)2410; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[24:25]2411; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[26:27]2412; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[28:29]2413; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]2414; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[30:31]2415; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]2416; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]2417; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]2418; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]2419; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]2420; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]2421; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]2422; HEURRC-NEXT: s_nop 12423; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:12424; HEURRC-NEXT: v_mov_b32_e32 v16, 02425; HEURRC-NEXT: s_nop 102426; HEURRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:482427; HEURRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:322428; HEURRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:162429; HEURRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]2430; HEURRC-NEXT: s_endpgm2431;2432; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac_flags:2433; VGPRRC: ; %bb.0:2434; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x242435; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x642436; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa42437; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)2438; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[24:25]2439; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[26:27]2440; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[28:29]2441; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]2442; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[30:31]2443; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]2444; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]2445; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]2446; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]2447; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]2448; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]2449; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]2450; VGPRRC-NEXT: s_nop 12451; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:12452; VGPRRC-NEXT: v_mov_b32_e32 v16, 02453; VGPRRC-NEXT: s_nop 102454; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:482455; VGPRRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:322456; VGPRRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:162457; VGPRRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]2458; VGPRRC-NEXT: s_endpgm2459; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac_flags:2460; AGPR: ; %bb.0:2461; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x242462; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x642463; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa42464; AGPR-NEXT: s_waitcnt lgkmcnt(0)2465; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25]2466; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27]2467; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29]2468; AGPR-NEXT: v_accvgpr_write_b32 a0, s82469; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31]2470; AGPR-NEXT: v_accvgpr_write_b32 a1, s92471; AGPR-NEXT: v_accvgpr_write_b32 a2, s102472; AGPR-NEXT: v_accvgpr_write_b32 a3, s112473; AGPR-NEXT: v_accvgpr_write_b32 a4, s122474; AGPR-NEXT: v_accvgpr_write_b32 a5, s132475; AGPR-NEXT: v_accvgpr_write_b32 a6, s142476; AGPR-NEXT: v_accvgpr_write_b32 a7, s152477; AGPR-NEXT: v_accvgpr_write_b32 a8, s162478; AGPR-NEXT: v_accvgpr_write_b32 a9, s172479; AGPR-NEXT: v_accvgpr_write_b32 a10, s182480; AGPR-NEXT: v_accvgpr_write_b32 a11, s192481; AGPR-NEXT: v_accvgpr_write_b32 a12, s202482; AGPR-NEXT: v_accvgpr_write_b32 a13, s212483; AGPR-NEXT: v_accvgpr_write_b32 a14, s222484; AGPR-NEXT: v_accvgpr_write_b32 a15, s232485; AGPR-NEXT: s_nop 12486; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:3 abid:2 blgp:12487; AGPR-NEXT: v_mov_b32_e32 v0, 02488; AGPR-NEXT: s_nop 72489; AGPR-NEXT: s_nop 22490; AGPR-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:482491; AGPR-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:322492; AGPR-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:162493; AGPR-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1]2494; AGPR-NEXT: s_endpgm2495; VGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac_flags:2496; VGPR: ; %bb.0:2497; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x242498; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x642499; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa42500; VGPR-NEXT: s_waitcnt lgkmcnt(0)2501; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[24:25]2502; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[26:27]2503; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[28:29]2504; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]2505; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[30:31]2506; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]2507; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]2508; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]2509; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]2510; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]2511; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]2512; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]2513; VGPR-NEXT: s_nop 12514; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:12515; VGPR-NEXT: v_mov_b32_e32 v16, 02516; VGPR-NEXT: s_nop 72517; VGPR-NEXT: s_nop 22518; VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:482519; VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:322520; VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:162521; VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]2522; VGPR-NEXT: s_endpgm2523 %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 3, i32 2, i32 1)2524 store <16 x float> %result, ptr addrspace(1) %out2525 ret void2526}2527 2528; --------------------------------------------------------------------2529; llvm.amdgcn.mfma.i32.16x16x64.i82530; --------------------------------------------------------------------2531 2532declare <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32>, <4 x i32>, <4 x i32>, i32 immarg, i32 immarg, i32 immarg)2533 2534define <4 x i32> @test_mfma_i32_16x16x64_i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2) {2535; GCN-LABEL: test_mfma_i32_16x16x64_i8:2536; GCN: ; %bb.0:2537; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2538; GCN-NEXT: v_accvgpr_write_b32 a0, v82539; GCN-NEXT: v_accvgpr_write_b32 a1, v92540; GCN-NEXT: v_accvgpr_write_b32 a2, v102541; GCN-NEXT: v_accvgpr_write_b32 a3, v112542; GCN-NEXT: s_nop 12543; GCN-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3]2544; GCN-NEXT: s_nop 72545; GCN-NEXT: v_accvgpr_read_b32 v0, a02546; GCN-NEXT: v_accvgpr_read_b32 v1, a12547; GCN-NEXT: v_accvgpr_read_b32 v2, a22548; GCN-NEXT: v_accvgpr_read_b32 v3, a32549; GCN-NEXT: s_setpc_b64 s[30:31]2550;2551; HEURRC-LABEL: test_mfma_i32_16x16x64_i8:2552; HEURRC: ; %bb.0:2553; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2554; HEURRC-NEXT: v_accvgpr_write_b32 a0, v82555; HEURRC-NEXT: v_accvgpr_write_b32 a1, v92556; HEURRC-NEXT: v_accvgpr_write_b32 a2, v102557; HEURRC-NEXT: v_accvgpr_write_b32 a3, v112558; HEURRC-NEXT: s_nop 12559; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3]2560; HEURRC-NEXT: s_nop 72561; HEURRC-NEXT: v_accvgpr_read_b32 v0, a02562; HEURRC-NEXT: v_accvgpr_read_b32 v1, a12563; HEURRC-NEXT: v_accvgpr_read_b32 v2, a22564; HEURRC-NEXT: v_accvgpr_read_b32 v3, a32565; HEURRC-NEXT: s_setpc_b64 s[30:31]2566;2567; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8:2568; VGPRRC: ; %bb.0:2569; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2570; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]2571; VGPRRC-NEXT: s_setpc_b64 s[30:31]2572; AGPR-LABEL: test_mfma_i32_16x16x64_i8:2573; AGPR: ; %bb.0:2574; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2575; AGPR-NEXT: v_accvgpr_write_b32 a0, v82576; AGPR-NEXT: v_accvgpr_write_b32 a1, v92577; AGPR-NEXT: v_accvgpr_write_b32 a2, v102578; AGPR-NEXT: v_accvgpr_write_b32 a3, v112579; AGPR-NEXT: s_nop 12580; AGPR-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3]2581; AGPR-NEXT: s_nop 72582; AGPR-NEXT: v_accvgpr_read_b32 v0, a02583; AGPR-NEXT: v_accvgpr_read_b32 v1, a12584; AGPR-NEXT: v_accvgpr_read_b32 v2, a22585; AGPR-NEXT: v_accvgpr_read_b32 v3, a32586; AGPR-NEXT: s_setpc_b64 s[30:31]2587; VGPR-LABEL: test_mfma_i32_16x16x64_i8:2588; VGPR: ; %bb.0:2589; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2590; VGPR-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]2591; VGPR-NEXT: s_setpc_b64 s[30:31]2592 %result = call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2, i32 0, i32 0, i32 0)2593 ret <4 x i32> %result2594}2595 2596define <4 x i32> @test_mfma_i32_16x16x64_i8__flags(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2) {2597; GCN-LABEL: test_mfma_i32_16x16x64_i8__flags:2598; GCN: ; %bb.0:2599; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2600; GCN-NEXT: v_accvgpr_write_b32 a0, v82601; GCN-NEXT: v_accvgpr_write_b32 a1, v92602; GCN-NEXT: v_accvgpr_write_b32 a2, v102603; GCN-NEXT: v_accvgpr_write_b32 a3, v112604; GCN-NEXT: s_nop 12605; GCN-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:12606; GCN-NEXT: s_nop 72607; GCN-NEXT: v_accvgpr_read_b32 v0, a02608; GCN-NEXT: v_accvgpr_read_b32 v1, a12609; GCN-NEXT: v_accvgpr_read_b32 v2, a22610; GCN-NEXT: v_accvgpr_read_b32 v3, a32611; GCN-NEXT: s_setpc_b64 s[30:31]2612;2613; HEURRC-LABEL: test_mfma_i32_16x16x64_i8__flags:2614; HEURRC: ; %bb.0:2615; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2616; HEURRC-NEXT: v_accvgpr_write_b32 a0, v82617; HEURRC-NEXT: v_accvgpr_write_b32 a1, v92618; HEURRC-NEXT: v_accvgpr_write_b32 a2, v102619; HEURRC-NEXT: v_accvgpr_write_b32 a3, v112620; HEURRC-NEXT: s_nop 12621; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:12622; HEURRC-NEXT: s_nop 72623; HEURRC-NEXT: v_accvgpr_read_b32 v0, a02624; HEURRC-NEXT: v_accvgpr_read_b32 v1, a12625; HEURRC-NEXT: v_accvgpr_read_b32 v2, a22626; HEURRC-NEXT: v_accvgpr_read_b32 v3, a32627; HEURRC-NEXT: s_setpc_b64 s[30:31]2628;2629; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8__flags:2630; VGPRRC: ; %bb.0:2631; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2632; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:12633; VGPRRC-NEXT: s_setpc_b64 s[30:31]2634; AGPR-LABEL: test_mfma_i32_16x16x64_i8__flags:2635; AGPR: ; %bb.0:2636; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2637; AGPR-NEXT: v_accvgpr_write_b32 a0, v82638; AGPR-NEXT: v_accvgpr_write_b32 a1, v92639; AGPR-NEXT: v_accvgpr_write_b32 a2, v102640; AGPR-NEXT: v_accvgpr_write_b32 a3, v112641; AGPR-NEXT: s_nop 12642; AGPR-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:12643; AGPR-NEXT: s_nop 72644; AGPR-NEXT: v_accvgpr_read_b32 v0, a02645; AGPR-NEXT: v_accvgpr_read_b32 v1, a12646; AGPR-NEXT: v_accvgpr_read_b32 v2, a22647; AGPR-NEXT: v_accvgpr_read_b32 v3, a32648; AGPR-NEXT: s_setpc_b64 s[30:31]2649; VGPR-LABEL: test_mfma_i32_16x16x64_i8__flags:2650; VGPR: ; %bb.0:2651; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2652; VGPR-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:12653; VGPR-NEXT: s_setpc_b64 s[30:31]2654 %result = call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2, i32 1, i32 1, i32 1)2655 ret <4 x i32> %result2656}2657 2658define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspace(1) %out, <4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2) #0 {2659; SDAG-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:2660; SDAG: ; %bb.0:2661; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x342662; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x242663; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x542664; SDAG-NEXT: v_mov_b32_e32 v12, 02665; SDAG-NEXT: s_waitcnt lgkmcnt(0)2666; SDAG-NEXT: v_mov_b32_e32 v0, s82667; SDAG-NEXT: v_mov_b32_e32 v1, s92668; SDAG-NEXT: v_mov_b32_e32 v2, s102669; SDAG-NEXT: v_mov_b32_e32 v3, s112670; SDAG-NEXT: v_mov_b32_e32 v4, s122671; SDAG-NEXT: v_mov_b32_e32 v5, s132672; SDAG-NEXT: v_mov_b32_e32 v6, s142673; SDAG-NEXT: v_mov_b32_e32 v7, s152674; SDAG-NEXT: v_mov_b32_e32 v8, s02675; SDAG-NEXT: v_mov_b32_e32 v9, s12676; SDAG-NEXT: v_mov_b32_e32 v10, s22677; SDAG-NEXT: v_mov_b32_e32 v11, s32678; SDAG-NEXT: s_nop 12679; SDAG-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]2680; SDAG-NEXT: s_nop 72681; SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]2682; SDAG-NEXT: s_endpgm2683;2684; GISEL-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:2685; GISEL: ; %bb.0:2686; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x342687; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x542688; GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x242689; GISEL-NEXT: s_waitcnt lgkmcnt(0)2690; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]2691; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]2692; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]2693; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]2694; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]2695; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]2696; GISEL-NEXT: s_nop 12697; GISEL-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]2698; GISEL-NEXT: v_mov_b32_e32 v4, 02699; GISEL-NEXT: s_nop 62700; GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]2701; GISEL-NEXT: s_endpgm2702;2703; HEURRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:2704; HEURRC: ; %bb.0:2705; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x342706; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x242707; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x542708; HEURRC-NEXT: v_mov_b32_e32 v12, 02709; HEURRC-NEXT: s_waitcnt lgkmcnt(0)2710; HEURRC-NEXT: v_mov_b32_e32 v0, s82711; HEURRC-NEXT: v_mov_b32_e32 v1, s92712; HEURRC-NEXT: v_mov_b32_e32 v2, s102713; HEURRC-NEXT: v_mov_b32_e32 v3, s112714; HEURRC-NEXT: v_mov_b32_e32 v4, s122715; HEURRC-NEXT: v_mov_b32_e32 v5, s132716; HEURRC-NEXT: v_mov_b32_e32 v6, s142717; HEURRC-NEXT: v_mov_b32_e32 v7, s152718; HEURRC-NEXT: v_mov_b32_e32 v8, s02719; HEURRC-NEXT: v_mov_b32_e32 v9, s12720; HEURRC-NEXT: v_mov_b32_e32 v10, s22721; HEURRC-NEXT: v_mov_b32_e32 v11, s32722; HEURRC-NEXT: s_nop 12723; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]2724; HEURRC-NEXT: s_nop 72725; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]2726; HEURRC-NEXT: s_endpgm2727;2728; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:2729; VGPRRC: ; %bb.0:2730; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x342731; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x242732; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x542733; VGPRRC-NEXT: v_mov_b32_e32 v12, 02734; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)2735; VGPRRC-NEXT: v_mov_b32_e32 v0, s82736; VGPRRC-NEXT: v_mov_b32_e32 v1, s92737; VGPRRC-NEXT: v_mov_b32_e32 v2, s102738; VGPRRC-NEXT: v_mov_b32_e32 v3, s112739; VGPRRC-NEXT: v_mov_b32_e32 v4, s122740; VGPRRC-NEXT: v_mov_b32_e32 v5, s132741; VGPRRC-NEXT: v_mov_b32_e32 v6, s142742; VGPRRC-NEXT: v_mov_b32_e32 v7, s152743; VGPRRC-NEXT: v_mov_b32_e32 v8, s02744; VGPRRC-NEXT: v_mov_b32_e32 v9, s12745; VGPRRC-NEXT: v_mov_b32_e32 v10, s22746; VGPRRC-NEXT: v_mov_b32_e32 v11, s32747; VGPRRC-NEXT: s_nop 12748; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]2749; VGPRRC-NEXT: s_nop 72750; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]2751; VGPRRC-NEXT: s_endpgm2752; AGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:2753; AGPR: ; %bb.0:2754; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x342755; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x242756; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x542757; AGPR-NEXT: v_mov_b32_e32 v8, 02758; AGPR-NEXT: s_waitcnt lgkmcnt(0)2759; AGPR-NEXT: v_mov_b32_e32 v0, s82760; AGPR-NEXT: v_mov_b32_e32 v1, s92761; AGPR-NEXT: v_mov_b32_e32 v2, s102762; AGPR-NEXT: v_mov_b32_e32 v3, s112763; AGPR-NEXT: v_mov_b32_e32 v4, s122764; AGPR-NEXT: v_mov_b32_e32 v5, s132765; AGPR-NEXT: v_mov_b32_e32 v6, s142766; AGPR-NEXT: v_mov_b32_e32 v7, s152767; AGPR-NEXT: v_accvgpr_write_b32 a0, s02768; AGPR-NEXT: v_accvgpr_write_b32 a1, s12769; AGPR-NEXT: v_accvgpr_write_b32 a2, s22770; AGPR-NEXT: v_accvgpr_write_b32 a3, s32771; AGPR-NEXT: s_nop 12772; AGPR-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3]2773; AGPR-NEXT: s_nop 72774; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7]2775; AGPR-NEXT: s_endpgm2776; VGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:2777; VGPR: ; %bb.0:2778; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x342779; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x242780; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x542781; VGPR-NEXT: v_mov_b32_e32 v12, 02782; VGPR-NEXT: s_waitcnt lgkmcnt(0)2783; VGPR-NEXT: v_mov_b32_e32 v0, s82784; VGPR-NEXT: v_mov_b32_e32 v1, s92785; VGPR-NEXT: v_mov_b32_e32 v2, s102786; VGPR-NEXT: v_mov_b32_e32 v3, s112787; VGPR-NEXT: v_mov_b32_e32 v4, s122788; VGPR-NEXT: v_mov_b32_e32 v5, s132789; VGPR-NEXT: v_mov_b32_e32 v6, s142790; VGPR-NEXT: v_mov_b32_e32 v7, s152791; VGPR-NEXT: v_mov_b32_e32 v8, s02792; VGPR-NEXT: v_mov_b32_e32 v9, s12793; VGPR-NEXT: v_mov_b32_e32 v10, s22794; VGPR-NEXT: v_mov_b32_e32 v11, s32795; VGPR-NEXT: s_nop 12796; VGPR-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]2797; VGPR-NEXT: s_nop 72798; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]2799; VGPR-NEXT: s_endpgm2800 %result = call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2, i32 0, i32 0, i32 0)2801 store <4 x i32> %result, ptr addrspace(1) %out2802 ret void2803}2804 2805define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr addrspace(1) %out, <4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2) #0 {2806; SDAG-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:2807; SDAG: ; %bb.0:2808; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x342809; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x242810; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x542811; SDAG-NEXT: v_mov_b32_e32 v12, 02812; SDAG-NEXT: s_waitcnt lgkmcnt(0)2813; SDAG-NEXT: v_mov_b32_e32 v0, s82814; SDAG-NEXT: v_mov_b32_e32 v1, s92815; SDAG-NEXT: v_mov_b32_e32 v2, s102816; SDAG-NEXT: v_mov_b32_e32 v3, s112817; SDAG-NEXT: v_mov_b32_e32 v4, s122818; SDAG-NEXT: v_mov_b32_e32 v5, s132819; SDAG-NEXT: v_mov_b32_e32 v6, s142820; SDAG-NEXT: v_mov_b32_e32 v7, s152821; SDAG-NEXT: v_mov_b32_e32 v8, s02822; SDAG-NEXT: v_mov_b32_e32 v9, s12823; SDAG-NEXT: v_mov_b32_e32 v10, s22824; SDAG-NEXT: v_mov_b32_e32 v11, s32825; SDAG-NEXT: s_nop 12826; SDAG-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:12827; SDAG-NEXT: s_nop 72828; SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]2829; SDAG-NEXT: s_endpgm2830;2831; GISEL-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:2832; GISEL: ; %bb.0:2833; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x342834; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x542835; GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x242836; GISEL-NEXT: s_waitcnt lgkmcnt(0)2837; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]2838; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]2839; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]2840; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]2841; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]2842; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]2843; GISEL-NEXT: s_nop 12844; GISEL-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:12845; GISEL-NEXT: v_mov_b32_e32 v4, 02846; GISEL-NEXT: s_nop 62847; GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]2848; GISEL-NEXT: s_endpgm2849;2850; HEURRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:2851; HEURRC: ; %bb.0:2852; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x342853; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x242854; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x542855; HEURRC-NEXT: v_mov_b32_e32 v12, 02856; HEURRC-NEXT: s_waitcnt lgkmcnt(0)2857; HEURRC-NEXT: v_mov_b32_e32 v0, s82858; HEURRC-NEXT: v_mov_b32_e32 v1, s92859; HEURRC-NEXT: v_mov_b32_e32 v2, s102860; HEURRC-NEXT: v_mov_b32_e32 v3, s112861; HEURRC-NEXT: v_mov_b32_e32 v4, s122862; HEURRC-NEXT: v_mov_b32_e32 v5, s132863; HEURRC-NEXT: v_mov_b32_e32 v6, s142864; HEURRC-NEXT: v_mov_b32_e32 v7, s152865; HEURRC-NEXT: v_mov_b32_e32 v8, s02866; HEURRC-NEXT: v_mov_b32_e32 v9, s12867; HEURRC-NEXT: v_mov_b32_e32 v10, s22868; HEURRC-NEXT: v_mov_b32_e32 v11, s32869; HEURRC-NEXT: s_nop 12870; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:12871; HEURRC-NEXT: s_nop 72872; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]2873; HEURRC-NEXT: s_endpgm2874;2875; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:2876; VGPRRC: ; %bb.0:2877; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x342878; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x242879; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x542880; VGPRRC-NEXT: v_mov_b32_e32 v12, 02881; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)2882; VGPRRC-NEXT: v_mov_b32_e32 v0, s82883; VGPRRC-NEXT: v_mov_b32_e32 v1, s92884; VGPRRC-NEXT: v_mov_b32_e32 v2, s102885; VGPRRC-NEXT: v_mov_b32_e32 v3, s112886; VGPRRC-NEXT: v_mov_b32_e32 v4, s122887; VGPRRC-NEXT: v_mov_b32_e32 v5, s132888; VGPRRC-NEXT: v_mov_b32_e32 v6, s142889; VGPRRC-NEXT: v_mov_b32_e32 v7, s152890; VGPRRC-NEXT: v_mov_b32_e32 v8, s02891; VGPRRC-NEXT: v_mov_b32_e32 v9, s12892; VGPRRC-NEXT: v_mov_b32_e32 v10, s22893; VGPRRC-NEXT: v_mov_b32_e32 v11, s32894; VGPRRC-NEXT: s_nop 12895; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:12896; VGPRRC-NEXT: s_nop 72897; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]2898; VGPRRC-NEXT: s_endpgm2899; AGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:2900; AGPR: ; %bb.0:2901; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x342902; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x242903; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x542904; AGPR-NEXT: v_mov_b32_e32 v8, 02905; AGPR-NEXT: s_waitcnt lgkmcnt(0)2906; AGPR-NEXT: v_mov_b32_e32 v0, s82907; AGPR-NEXT: v_mov_b32_e32 v1, s92908; AGPR-NEXT: v_mov_b32_e32 v2, s102909; AGPR-NEXT: v_mov_b32_e32 v3, s112910; AGPR-NEXT: v_mov_b32_e32 v4, s122911; AGPR-NEXT: v_mov_b32_e32 v5, s132912; AGPR-NEXT: v_mov_b32_e32 v6, s142913; AGPR-NEXT: v_mov_b32_e32 v7, s152914; AGPR-NEXT: v_accvgpr_write_b32 a0, s02915; AGPR-NEXT: v_accvgpr_write_b32 a1, s12916; AGPR-NEXT: v_accvgpr_write_b32 a2, s22917; AGPR-NEXT: v_accvgpr_write_b32 a3, s32918; AGPR-NEXT: s_nop 12919; AGPR-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:12920; AGPR-NEXT: s_nop 72921; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7]2922; AGPR-NEXT: s_endpgm2923; VGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:2924; VGPR: ; %bb.0:2925; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x342926; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x242927; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x542928; VGPR-NEXT: v_mov_b32_e32 v12, 02929; VGPR-NEXT: s_waitcnt lgkmcnt(0)2930; VGPR-NEXT: v_mov_b32_e32 v0, s82931; VGPR-NEXT: v_mov_b32_e32 v1, s92932; VGPR-NEXT: v_mov_b32_e32 v2, s102933; VGPR-NEXT: v_mov_b32_e32 v3, s112934; VGPR-NEXT: v_mov_b32_e32 v4, s122935; VGPR-NEXT: v_mov_b32_e32 v5, s132936; VGPR-NEXT: v_mov_b32_e32 v6, s142937; VGPR-NEXT: v_mov_b32_e32 v7, s152938; VGPR-NEXT: v_mov_b32_e32 v8, s02939; VGPR-NEXT: v_mov_b32_e32 v9, s12940; VGPR-NEXT: v_mov_b32_e32 v10, s22941; VGPR-NEXT: v_mov_b32_e32 v11, s32942; VGPR-NEXT: s_nop 12943; VGPR-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:12944; VGPR-NEXT: s_nop 72945; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]2946; VGPR-NEXT: s_endpgm2947 %result = call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2, i32 3, i32 2, i32 1)2948 store <4 x i32> %result, ptr addrspace(1) %out2949 ret void2950}2951 2952; --------------------------------------------------------------------2953; llvm.amdgcn.mfma.i32.32x32x32.i82954; --------------------------------------------------------------------2955 2956declare <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32>, <4 x i32>, <16 x i32>, i32 immarg, i32 immarg, i32 immarg)2957 2958define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2) #1 {2959; SDAG-LABEL: test_mfma_i32_32x32x32_i8:2960; SDAG: ; %bb.0:2961; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x242962; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x642963; SDAG-NEXT: v_mov_b64_e32 v[0:1], 482964; SDAG-NEXT: v_mov_b64_e32 v[2:3], 322965; SDAG-NEXT: s_waitcnt lgkmcnt(0)2966; SDAG-NEXT: v_mov_b32_e32 v4, s242967; SDAG-NEXT: v_mov_b32_e32 v5, s252968; SDAG-NEXT: v_mov_b32_e32 v6, s262969; SDAG-NEXT: v_mov_b32_e32 v7, s272970; SDAG-NEXT: v_accvgpr_write_b32 a31, s232971; SDAG-NEXT: v_mov_b32_e32 v8, s282972; SDAG-NEXT: v_mov_b32_e32 v9, s292973; SDAG-NEXT: v_mov_b32_e32 v10, s302974; SDAG-NEXT: v_mov_b32_e32 v11, s312975; SDAG-NEXT: v_accvgpr_write_b32 a30, s222976; SDAG-NEXT: v_accvgpr_write_b32 a29, s212977; SDAG-NEXT: v_accvgpr_write_b32 a28, s202978; SDAG-NEXT: v_accvgpr_write_b32 a27, s192979; SDAG-NEXT: v_accvgpr_write_b32 a26, s182980; SDAG-NEXT: v_accvgpr_write_b32 a25, s172981; SDAG-NEXT: v_accvgpr_write_b32 a24, s162982; SDAG-NEXT: v_accvgpr_write_b32 a23, s152983; SDAG-NEXT: v_accvgpr_write_b32 a22, s142984; SDAG-NEXT: v_accvgpr_write_b32 a21, s132985; SDAG-NEXT: v_accvgpr_write_b32 a20, s122986; SDAG-NEXT: v_accvgpr_write_b32 a19, s112987; SDAG-NEXT: v_accvgpr_write_b32 a18, s102988; SDAG-NEXT: v_accvgpr_write_b32 a17, s92989; SDAG-NEXT: v_accvgpr_write_b32 a16, s82990; SDAG-NEXT: s_nop 12991; SDAG-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31]2992; SDAG-NEXT: v_mov_b64_e32 v[4:5], 162993; SDAG-NEXT: v_mov_b64_e32 v[6:7], 02994; SDAG-NEXT: v_mov_b32_e32 v8, s162995; SDAG-NEXT: v_mov_b32_e32 v9, s172996; SDAG-NEXT: v_mov_b32_e32 v10, s182997; SDAG-NEXT: v_mov_b32_e32 v11, s192998; SDAG-NEXT: s_nop 52999; SDAG-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc13000; SDAG-NEXT: s_waitcnt vmcnt(0)3001; SDAG-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc13002; SDAG-NEXT: s_waitcnt vmcnt(0)3003; SDAG-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc13004; SDAG-NEXT: s_waitcnt vmcnt(0)3005; SDAG-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc13006; SDAG-NEXT: s_waitcnt vmcnt(0)3007; SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc13008; SDAG-NEXT: s_waitcnt vmcnt(0)3009; SDAG-NEXT: v_mov_b32_e32 v2, s103010; SDAG-NEXT: v_mov_b32_e32 v3, s113011; SDAG-NEXT: v_mov_b32_e32 v8, s203012; SDAG-NEXT: v_mov_b32_e32 v9, s213013; SDAG-NEXT: v_mov_b32_e32 v10, s223014; SDAG-NEXT: v_mov_b32_e32 v11, s233015; SDAG-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc13016; SDAG-NEXT: s_waitcnt vmcnt(0)3017; SDAG-NEXT: v_mov_b32_e32 v0, s83018; SDAG-NEXT: v_mov_b32_e32 v1, s93019; SDAG-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc13020; SDAG-NEXT: s_waitcnt vmcnt(0)3021; SDAG-NEXT: s_nop 03022; SDAG-NEXT: v_mov_b32_e32 v0, s123023; SDAG-NEXT: v_mov_b32_e32 v1, s133024; SDAG-NEXT: v_mov_b32_e32 v2, s143025; SDAG-NEXT: v_mov_b32_e32 v3, s153026; SDAG-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc13027; SDAG-NEXT: s_waitcnt vmcnt(0)3028; SDAG-NEXT: s_endpgm3029;3030; GISEL-LABEL: test_mfma_i32_32x32x32_i8:3031; GISEL: ; %bb.0:3032; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x243033; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x643034; GISEL-NEXT: v_mov_b64_e32 v[12:13], 03035; GISEL-NEXT: v_mov_b64_e32 v[14:15], 163036; GISEL-NEXT: v_mov_b64_e32 v[16:17], 323037; GISEL-NEXT: s_waitcnt lgkmcnt(0)3038; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[24:25]3039; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[26:27]3040; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[28:29]3041; GISEL-NEXT: v_accvgpr_write_b32 a0, s83042; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[30:31]3043; GISEL-NEXT: v_accvgpr_write_b32 a1, s93044; GISEL-NEXT: v_accvgpr_write_b32 a2, s103045; GISEL-NEXT: v_accvgpr_write_b32 a3, s113046; GISEL-NEXT: v_accvgpr_write_b32 a4, s123047; GISEL-NEXT: v_accvgpr_write_b32 a5, s133048; GISEL-NEXT: v_accvgpr_write_b32 a6, s143049; GISEL-NEXT: v_accvgpr_write_b32 a7, s153050; GISEL-NEXT: v_accvgpr_write_b32 a8, s163051; GISEL-NEXT: v_accvgpr_write_b32 a9, s173052; GISEL-NEXT: v_accvgpr_write_b32 a10, s183053; GISEL-NEXT: v_accvgpr_write_b32 a11, s193054; GISEL-NEXT: v_accvgpr_write_b32 a12, s203055; GISEL-NEXT: v_accvgpr_write_b32 a13, s213056; GISEL-NEXT: v_accvgpr_write_b32 a14, s223057; GISEL-NEXT: v_accvgpr_write_b32 a15, s233058; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]3059; GISEL-NEXT: v_mov_b64_e32 v[18:19], 483060; GISEL-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15]3061; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]3062; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]3063; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]3064; GISEL-NEXT: s_nop 83065; GISEL-NEXT: global_store_dwordx4 v[12:13], a[16:19], off sc0 sc13066; GISEL-NEXT: s_waitcnt vmcnt(0)3067; GISEL-NEXT: global_store_dwordx4 v[14:15], a[20:23], off sc0 sc13068; GISEL-NEXT: s_waitcnt vmcnt(0)3069; GISEL-NEXT: global_store_dwordx4 v[16:17], a[24:27], off sc0 sc13070; GISEL-NEXT: s_waitcnt vmcnt(0)3071; GISEL-NEXT: global_store_dwordx4 v[18:19], a[28:31], off sc0 sc13072; GISEL-NEXT: s_waitcnt vmcnt(0)3073; GISEL-NEXT: global_store_dwordx4 v[12:13], v[8:11], off sc0 sc13074; GISEL-NEXT: s_waitcnt vmcnt(0)3075; GISEL-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc13076; GISEL-NEXT: s_waitcnt vmcnt(0)3077; GISEL-NEXT: s_nop 03078; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[16:17]3079; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19]3080; GISEL-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc13081; GISEL-NEXT: s_waitcnt vmcnt(0)3082; GISEL-NEXT: s_nop 03083; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[20:21]3084; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[22:23]3085; GISEL-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc13086; GISEL-NEXT: s_waitcnt vmcnt(0)3087; GISEL-NEXT: s_endpgm3088;3089; HEURRC-LABEL: test_mfma_i32_32x32x32_i8:3090; HEURRC: ; %bb.0:3091; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x243092; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x643093; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 483094; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 323095; HEURRC-NEXT: s_waitcnt lgkmcnt(0)3096; HEURRC-NEXT: v_mov_b32_e32 v4, s243097; HEURRC-NEXT: v_mov_b32_e32 v5, s253098; HEURRC-NEXT: v_mov_b32_e32 v6, s263099; HEURRC-NEXT: v_mov_b32_e32 v7, s273100; HEURRC-NEXT: v_accvgpr_write_b32 a31, s233101; HEURRC-NEXT: v_mov_b32_e32 v8, s283102; HEURRC-NEXT: v_mov_b32_e32 v9, s293103; HEURRC-NEXT: v_mov_b32_e32 v10, s303104; HEURRC-NEXT: v_mov_b32_e32 v11, s313105; HEURRC-NEXT: v_accvgpr_write_b32 a30, s223106; HEURRC-NEXT: v_accvgpr_write_b32 a29, s213107; HEURRC-NEXT: v_accvgpr_write_b32 a28, s203108; HEURRC-NEXT: v_accvgpr_write_b32 a27, s193109; HEURRC-NEXT: v_accvgpr_write_b32 a26, s183110; HEURRC-NEXT: v_accvgpr_write_b32 a25, s173111; HEURRC-NEXT: v_accvgpr_write_b32 a24, s163112; HEURRC-NEXT: v_accvgpr_write_b32 a23, s153113; HEURRC-NEXT: v_accvgpr_write_b32 a22, s143114; HEURRC-NEXT: v_accvgpr_write_b32 a21, s133115; HEURRC-NEXT: v_accvgpr_write_b32 a20, s123116; HEURRC-NEXT: v_accvgpr_write_b32 a19, s113117; HEURRC-NEXT: v_accvgpr_write_b32 a18, s103118; HEURRC-NEXT: v_accvgpr_write_b32 a17, s93119; HEURRC-NEXT: v_accvgpr_write_b32 a16, s83120; HEURRC-NEXT: s_nop 13121; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31]3122; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 163123; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 03124; HEURRC-NEXT: v_mov_b32_e32 v8, s163125; HEURRC-NEXT: v_mov_b32_e32 v9, s173126; HEURRC-NEXT: v_mov_b32_e32 v10, s183127; HEURRC-NEXT: v_mov_b32_e32 v11, s193128; HEURRC-NEXT: s_nop 53129; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc13130; HEURRC-NEXT: s_waitcnt vmcnt(0)3131; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc13132; HEURRC-NEXT: s_waitcnt vmcnt(0)3133; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc13134; HEURRC-NEXT: s_waitcnt vmcnt(0)3135; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc13136; HEURRC-NEXT: s_waitcnt vmcnt(0)3137; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc13138; HEURRC-NEXT: s_waitcnt vmcnt(0)3139; HEURRC-NEXT: v_mov_b32_e32 v2, s103140; HEURRC-NEXT: v_mov_b32_e32 v3, s113141; HEURRC-NEXT: v_mov_b32_e32 v8, s203142; HEURRC-NEXT: v_mov_b32_e32 v9, s213143; HEURRC-NEXT: v_mov_b32_e32 v10, s223144; HEURRC-NEXT: v_mov_b32_e32 v11, s233145; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc13146; HEURRC-NEXT: s_waitcnt vmcnt(0)3147; HEURRC-NEXT: v_mov_b32_e32 v0, s83148; HEURRC-NEXT: v_mov_b32_e32 v1, s93149; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc13150; HEURRC-NEXT: s_waitcnt vmcnt(0)3151; HEURRC-NEXT: s_nop 03152; HEURRC-NEXT: v_mov_b32_e32 v0, s123153; HEURRC-NEXT: v_mov_b32_e32 v1, s133154; HEURRC-NEXT: v_mov_b32_e32 v2, s143155; HEURRC-NEXT: v_mov_b32_e32 v3, s153156; HEURRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc13157; HEURRC-NEXT: s_waitcnt vmcnt(0)3158; HEURRC-NEXT: s_endpgm3159;3160; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8:3161; VGPRRC: ; %bb.0:3162; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x243163; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x643164; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 483165; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 323166; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)3167; VGPRRC-NEXT: v_mov_b32_e32 v36, s243168; VGPRRC-NEXT: v_mov_b32_e32 v37, s253169; VGPRRC-NEXT: v_mov_b32_e32 v38, s263170; VGPRRC-NEXT: v_mov_b32_e32 v39, s273171; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]3172; VGPRRC-NEXT: v_mov_b32_e32 v40, s283173; VGPRRC-NEXT: v_mov_b32_e32 v41, s293174; VGPRRC-NEXT: v_mov_b32_e32 v42, s303175; VGPRRC-NEXT: v_mov_b32_e32 v43, s313176; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]3177; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]3178; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]3179; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]3180; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]3181; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]3182; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]3183; VGPRRC-NEXT: s_nop 13184; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31]3185; VGPRRC-NEXT: s_nop 113186; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc13187; VGPRRC-NEXT: s_waitcnt vmcnt(0)3188; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc13189; VGPRRC-NEXT: s_waitcnt vmcnt(0)3190; VGPRRC-NEXT: s_nop 03191; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], 163192; VGPRRC-NEXT: global_store_dwordx4 v[8:9], v[4:7], off sc0 sc13193; VGPRRC-NEXT: s_waitcnt vmcnt(0)3194; VGPRRC-NEXT: s_nop 03195; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], 03196; VGPRRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc13197; VGPRRC-NEXT: s_waitcnt vmcnt(0)3198; VGPRRC-NEXT: s_nop 03199; VGPRRC-NEXT: v_mov_b32_e32 v0, s163200; VGPRRC-NEXT: v_mov_b32_e32 v1, s173201; VGPRRC-NEXT: v_mov_b32_e32 v2, s183202; VGPRRC-NEXT: v_mov_b32_e32 v3, s193203; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc13204; VGPRRC-NEXT: s_waitcnt vmcnt(0)3205; VGPRRC-NEXT: s_nop 03206; VGPRRC-NEXT: v_mov_b32_e32 v0, s203207; VGPRRC-NEXT: v_mov_b32_e32 v1, s213208; VGPRRC-NEXT: v_mov_b32_e32 v2, s223209; VGPRRC-NEXT: v_mov_b32_e32 v3, s233210; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc13211; VGPRRC-NEXT: s_waitcnt vmcnt(0)3212; VGPRRC-NEXT: s_nop 03213; VGPRRC-NEXT: v_mov_b32_e32 v0, s83214; VGPRRC-NEXT: v_mov_b32_e32 v1, s93215; VGPRRC-NEXT: v_mov_b32_e32 v2, s103216; VGPRRC-NEXT: v_mov_b32_e32 v3, s113217; VGPRRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc13218; VGPRRC-NEXT: s_waitcnt vmcnt(0)3219; VGPRRC-NEXT: s_nop 03220; VGPRRC-NEXT: v_mov_b32_e32 v0, s123221; VGPRRC-NEXT: v_mov_b32_e32 v1, s133222; VGPRRC-NEXT: v_mov_b32_e32 v2, s143223; VGPRRC-NEXT: v_mov_b32_e32 v3, s153224; VGPRRC-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc13225; VGPRRC-NEXT: s_waitcnt vmcnt(0)3226; VGPRRC-NEXT: s_endpgm3227; AGPR-LABEL: test_mfma_i32_32x32x32_i8:3228; AGPR: ; %bb.0:3229; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x243230; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x643231; AGPR-NEXT: v_mov_b64_e32 v[8:9], 483232; AGPR-NEXT: v_mov_b64_e32 v[10:11], 323233; AGPR-NEXT: v_mov_b64_e32 v[12:13], 163234; AGPR-NEXT: s_waitcnt lgkmcnt(0)3235; AGPR-NEXT: v_mov_b32_e32 v0, s243236; AGPR-NEXT: v_mov_b32_e32 v1, s253237; AGPR-NEXT: v_mov_b32_e32 v2, s263238; AGPR-NEXT: v_mov_b32_e32 v3, s273239; AGPR-NEXT: v_accvgpr_write_b32 a0, s83240; AGPR-NEXT: v_mov_b32_e32 v4, s283241; AGPR-NEXT: v_mov_b32_e32 v5, s293242; AGPR-NEXT: v_mov_b32_e32 v6, s303243; AGPR-NEXT: v_mov_b32_e32 v7, s313244; AGPR-NEXT: v_accvgpr_write_b32 a1, s93245; AGPR-NEXT: v_accvgpr_write_b32 a2, s103246; AGPR-NEXT: v_accvgpr_write_b32 a3, s113247; AGPR-NEXT: v_accvgpr_write_b32 a4, s123248; AGPR-NEXT: v_accvgpr_write_b32 a5, s133249; AGPR-NEXT: v_accvgpr_write_b32 a6, s143250; AGPR-NEXT: v_accvgpr_write_b32 a7, s153251; AGPR-NEXT: v_accvgpr_write_b32 a8, s163252; AGPR-NEXT: v_accvgpr_write_b32 a9, s173253; AGPR-NEXT: v_accvgpr_write_b32 a10, s183254; AGPR-NEXT: v_accvgpr_write_b32 a11, s193255; AGPR-NEXT: v_accvgpr_write_b32 a12, s203256; AGPR-NEXT: v_accvgpr_write_b32 a13, s213257; AGPR-NEXT: v_accvgpr_write_b32 a14, s223258; AGPR-NEXT: v_accvgpr_write_b32 a15, s233259; AGPR-NEXT: v_mov_b64_e32 v[14:15], 03260; AGPR-NEXT: s_nop 03261; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15]3262; AGPR-NEXT: v_mov_b32_e32 v0, s163263; AGPR-NEXT: v_mov_b32_e32 v1, s173264; AGPR-NEXT: v_mov_b32_e32 v2, s183265; AGPR-NEXT: v_mov_b32_e32 v3, s193266; AGPR-NEXT: s_nop 73267; AGPR-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc13268; AGPR-NEXT: s_waitcnt vmcnt(0)3269; AGPR-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc13270; AGPR-NEXT: s_waitcnt vmcnt(0)3271; AGPR-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc13272; AGPR-NEXT: s_waitcnt vmcnt(0)3273; AGPR-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc13274; AGPR-NEXT: s_waitcnt vmcnt(0)3275; AGPR-NEXT: global_store_dwordx4 v[10:11], v[0:3], off sc0 sc13276; AGPR-NEXT: s_waitcnt vmcnt(0)3277; AGPR-NEXT: s_nop 03278; AGPR-NEXT: v_mov_b32_e32 v0, s203279; AGPR-NEXT: v_mov_b32_e32 v1, s213280; AGPR-NEXT: v_mov_b32_e32 v2, s223281; AGPR-NEXT: v_mov_b32_e32 v3, s233282; AGPR-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc13283; AGPR-NEXT: s_waitcnt vmcnt(0)3284; AGPR-NEXT: s_nop 03285; AGPR-NEXT: v_mov_b32_e32 v0, s83286; AGPR-NEXT: v_mov_b32_e32 v1, s93287; AGPR-NEXT: v_mov_b32_e32 v2, s103288; AGPR-NEXT: v_mov_b32_e32 v3, s113289; AGPR-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc13290; AGPR-NEXT: s_waitcnt vmcnt(0)3291; AGPR-NEXT: s_nop 03292; AGPR-NEXT: v_mov_b32_e32 v0, s123293; AGPR-NEXT: v_mov_b32_e32 v1, s133294; AGPR-NEXT: v_mov_b32_e32 v2, s143295; AGPR-NEXT: v_mov_b32_e32 v3, s153296; AGPR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc13297; AGPR-NEXT: s_waitcnt vmcnt(0)3298; AGPR-NEXT: s_endpgm3299; VGPR-LABEL: test_mfma_i32_32x32x32_i8:3300; VGPR: ; %bb.0:3301; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x243302; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x643303; VGPR-NEXT: v_mov_b64_e32 v[40:41], 483304; VGPR-NEXT: v_mov_b64_e32 v[42:43], 323305; VGPR-NEXT: v_mov_b64_e32 v[44:45], 163306; VGPR-NEXT: s_waitcnt lgkmcnt(0)3307; VGPR-NEXT: v_mov_b32_e32 v32, s243308; VGPR-NEXT: v_mov_b32_e32 v33, s253309; VGPR-NEXT: v_mov_b32_e32 v34, s263310; VGPR-NEXT: v_mov_b32_e32 v35, s273311; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]3312; VGPR-NEXT: v_mov_b32_e32 v36, s283313; VGPR-NEXT: v_mov_b32_e32 v37, s293314; VGPR-NEXT: v_mov_b32_e32 v38, s303315; VGPR-NEXT: v_mov_b32_e32 v39, s313316; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]3317; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]3318; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]3319; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]3320; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]3321; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]3322; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]3323; VGPR-NEXT: v_mov_b64_e32 v[46:47], 03324; VGPR-NEXT: s_nop 03325; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15]3326; VGPR-NEXT: s_nop 73327; VGPR-NEXT: s_nop 33328; VGPR-NEXT: global_store_dwordx4 v[40:41], v[28:31], off sc0 sc13329; VGPR-NEXT: s_waitcnt vmcnt(0)3330; VGPR-NEXT: global_store_dwordx4 v[42:43], v[24:27], off sc0 sc13331; VGPR-NEXT: s_waitcnt vmcnt(0)3332; VGPR-NEXT: global_store_dwordx4 v[44:45], v[20:23], off sc0 sc13333; VGPR-NEXT: s_waitcnt vmcnt(0)3334; VGPR-NEXT: global_store_dwordx4 v[46:47], v[16:19], off sc0 sc13335; VGPR-NEXT: s_waitcnt vmcnt(0)3336; VGPR-NEXT: v_mov_b32_e32 v0, s163337; VGPR-NEXT: v_mov_b32_e32 v1, s173338; VGPR-NEXT: v_mov_b32_e32 v2, s183339; VGPR-NEXT: v_mov_b32_e32 v3, s193340; VGPR-NEXT: global_store_dwordx4 v[42:43], v[0:3], off sc0 sc13341; VGPR-NEXT: s_waitcnt vmcnt(0)3342; VGPR-NEXT: s_nop 03343; VGPR-NEXT: v_mov_b32_e32 v0, s203344; VGPR-NEXT: v_mov_b32_e32 v1, s213345; VGPR-NEXT: v_mov_b32_e32 v2, s223346; VGPR-NEXT: v_mov_b32_e32 v3, s233347; VGPR-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc13348; VGPR-NEXT: s_waitcnt vmcnt(0)3349; VGPR-NEXT: s_nop 03350; VGPR-NEXT: v_mov_b32_e32 v0, s83351; VGPR-NEXT: v_mov_b32_e32 v1, s93352; VGPR-NEXT: v_mov_b32_e32 v2, s103353; VGPR-NEXT: v_mov_b32_e32 v3, s113354; VGPR-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc13355; VGPR-NEXT: s_waitcnt vmcnt(0)3356; VGPR-NEXT: s_nop 03357; VGPR-NEXT: v_mov_b32_e32 v0, s123358; VGPR-NEXT: v_mov_b32_e32 v1, s133359; VGPR-NEXT: v_mov_b32_e32 v2, s143360; VGPR-NEXT: v_mov_b32_e32 v3, s153361; VGPR-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc13362; VGPR-NEXT: s_waitcnt vmcnt(0)3363; VGPR-NEXT: s_endpgm3364 %result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 0, i32 0, i32 0)3365 store volatile <16 x i32> %result, ptr addrspace(1) null3366 store volatile <16 x i32> %arg2, ptr addrspace(1) null3367 ret void3368}3369 3370define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2) #1 {3371; SDAG-LABEL: test_mfma_i32_32x32x32_i8__flags:3372; SDAG: ; %bb.0:3373; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x243374; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x643375; SDAG-NEXT: v_mov_b64_e32 v[0:1], 483376; SDAG-NEXT: v_mov_b64_e32 v[2:3], 323377; SDAG-NEXT: s_waitcnt lgkmcnt(0)3378; SDAG-NEXT: v_mov_b32_e32 v4, s243379; SDAG-NEXT: v_mov_b32_e32 v5, s253380; SDAG-NEXT: v_mov_b32_e32 v6, s263381; SDAG-NEXT: v_mov_b32_e32 v7, s273382; SDAG-NEXT: v_accvgpr_write_b32 a31, s233383; SDAG-NEXT: v_mov_b32_e32 v8, s283384; SDAG-NEXT: v_mov_b32_e32 v9, s293385; SDAG-NEXT: v_mov_b32_e32 v10, s303386; SDAG-NEXT: v_mov_b32_e32 v11, s313387; SDAG-NEXT: v_accvgpr_write_b32 a30, s223388; SDAG-NEXT: v_accvgpr_write_b32 a29, s213389; SDAG-NEXT: v_accvgpr_write_b32 a28, s203390; SDAG-NEXT: v_accvgpr_write_b32 a27, s193391; SDAG-NEXT: v_accvgpr_write_b32 a26, s183392; SDAG-NEXT: v_accvgpr_write_b32 a25, s173393; SDAG-NEXT: v_accvgpr_write_b32 a24, s163394; SDAG-NEXT: v_accvgpr_write_b32 a23, s153395; SDAG-NEXT: v_accvgpr_write_b32 a22, s143396; SDAG-NEXT: v_accvgpr_write_b32 a21, s133397; SDAG-NEXT: v_accvgpr_write_b32 a20, s123398; SDAG-NEXT: v_accvgpr_write_b32 a19, s113399; SDAG-NEXT: v_accvgpr_write_b32 a18, s103400; SDAG-NEXT: v_accvgpr_write_b32 a17, s93401; SDAG-NEXT: v_accvgpr_write_b32 a16, s83402; SDAG-NEXT: s_nop 13403; SDAG-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31] cbsz:2 abid:3 blgp:13404; SDAG-NEXT: v_mov_b64_e32 v[4:5], 163405; SDAG-NEXT: v_mov_b64_e32 v[6:7], 03406; SDAG-NEXT: v_mov_b32_e32 v8, s163407; SDAG-NEXT: v_mov_b32_e32 v9, s173408; SDAG-NEXT: v_mov_b32_e32 v10, s183409; SDAG-NEXT: v_mov_b32_e32 v11, s193410; SDAG-NEXT: s_nop 53411; SDAG-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc13412; SDAG-NEXT: s_waitcnt vmcnt(0)3413; SDAG-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc13414; SDAG-NEXT: s_waitcnt vmcnt(0)3415; SDAG-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc13416; SDAG-NEXT: s_waitcnt vmcnt(0)3417; SDAG-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc13418; SDAG-NEXT: s_waitcnt vmcnt(0)3419; SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc13420; SDAG-NEXT: s_waitcnt vmcnt(0)3421; SDAG-NEXT: v_mov_b32_e32 v2, s103422; SDAG-NEXT: v_mov_b32_e32 v3, s113423; SDAG-NEXT: v_mov_b32_e32 v8, s203424; SDAG-NEXT: v_mov_b32_e32 v9, s213425; SDAG-NEXT: v_mov_b32_e32 v10, s223426; SDAG-NEXT: v_mov_b32_e32 v11, s233427; SDAG-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc13428; SDAG-NEXT: s_waitcnt vmcnt(0)3429; SDAG-NEXT: v_mov_b32_e32 v0, s83430; SDAG-NEXT: v_mov_b32_e32 v1, s93431; SDAG-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc13432; SDAG-NEXT: s_waitcnt vmcnt(0)3433; SDAG-NEXT: s_nop 03434; SDAG-NEXT: v_mov_b32_e32 v0, s123435; SDAG-NEXT: v_mov_b32_e32 v1, s133436; SDAG-NEXT: v_mov_b32_e32 v2, s143437; SDAG-NEXT: v_mov_b32_e32 v3, s153438; SDAG-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc13439; SDAG-NEXT: s_waitcnt vmcnt(0)3440; SDAG-NEXT: s_endpgm3441;3442; GISEL-LABEL: test_mfma_i32_32x32x32_i8__flags:3443; GISEL: ; %bb.0:3444; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x243445; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x643446; GISEL-NEXT: v_mov_b64_e32 v[12:13], 03447; GISEL-NEXT: v_mov_b64_e32 v[14:15], 163448; GISEL-NEXT: v_mov_b64_e32 v[16:17], 323449; GISEL-NEXT: s_waitcnt lgkmcnt(0)3450; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[24:25]3451; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[26:27]3452; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[28:29]3453; GISEL-NEXT: v_accvgpr_write_b32 a0, s83454; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[30:31]3455; GISEL-NEXT: v_accvgpr_write_b32 a1, s93456; GISEL-NEXT: v_accvgpr_write_b32 a2, s103457; GISEL-NEXT: v_accvgpr_write_b32 a3, s113458; GISEL-NEXT: v_accvgpr_write_b32 a4, s123459; GISEL-NEXT: v_accvgpr_write_b32 a5, s133460; GISEL-NEXT: v_accvgpr_write_b32 a6, s143461; GISEL-NEXT: v_accvgpr_write_b32 a7, s153462; GISEL-NEXT: v_accvgpr_write_b32 a8, s163463; GISEL-NEXT: v_accvgpr_write_b32 a9, s173464; GISEL-NEXT: v_accvgpr_write_b32 a10, s183465; GISEL-NEXT: v_accvgpr_write_b32 a11, s193466; GISEL-NEXT: v_accvgpr_write_b32 a12, s203467; GISEL-NEXT: v_accvgpr_write_b32 a13, s213468; GISEL-NEXT: v_accvgpr_write_b32 a14, s223469; GISEL-NEXT: v_accvgpr_write_b32 a15, s233470; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]3471; GISEL-NEXT: v_mov_b64_e32 v[18:19], 483472; GISEL-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:13473; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]3474; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]3475; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]3476; GISEL-NEXT: s_nop 83477; GISEL-NEXT: global_store_dwordx4 v[12:13], a[16:19], off sc0 sc13478; GISEL-NEXT: s_waitcnt vmcnt(0)3479; GISEL-NEXT: global_store_dwordx4 v[14:15], a[20:23], off sc0 sc13480; GISEL-NEXT: s_waitcnt vmcnt(0)3481; GISEL-NEXT: global_store_dwordx4 v[16:17], a[24:27], off sc0 sc13482; GISEL-NEXT: s_waitcnt vmcnt(0)3483; GISEL-NEXT: global_store_dwordx4 v[18:19], a[28:31], off sc0 sc13484; GISEL-NEXT: s_waitcnt vmcnt(0)3485; GISEL-NEXT: global_store_dwordx4 v[12:13], v[8:11], off sc0 sc13486; GISEL-NEXT: s_waitcnt vmcnt(0)3487; GISEL-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc13488; GISEL-NEXT: s_waitcnt vmcnt(0)3489; GISEL-NEXT: s_nop 03490; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[16:17]3491; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19]3492; GISEL-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc13493; GISEL-NEXT: s_waitcnt vmcnt(0)3494; GISEL-NEXT: s_nop 03495; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[20:21]3496; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[22:23]3497; GISEL-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc13498; GISEL-NEXT: s_waitcnt vmcnt(0)3499; GISEL-NEXT: s_endpgm3500;3501; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__flags:3502; HEURRC: ; %bb.0:3503; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x243504; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x643505; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 483506; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 323507; HEURRC-NEXT: s_waitcnt lgkmcnt(0)3508; HEURRC-NEXT: v_mov_b32_e32 v4, s243509; HEURRC-NEXT: v_mov_b32_e32 v5, s253510; HEURRC-NEXT: v_mov_b32_e32 v6, s263511; HEURRC-NEXT: v_mov_b32_e32 v7, s273512; HEURRC-NEXT: v_accvgpr_write_b32 a31, s233513; HEURRC-NEXT: v_mov_b32_e32 v8, s283514; HEURRC-NEXT: v_mov_b32_e32 v9, s293515; HEURRC-NEXT: v_mov_b32_e32 v10, s303516; HEURRC-NEXT: v_mov_b32_e32 v11, s313517; HEURRC-NEXT: v_accvgpr_write_b32 a30, s223518; HEURRC-NEXT: v_accvgpr_write_b32 a29, s213519; HEURRC-NEXT: v_accvgpr_write_b32 a28, s203520; HEURRC-NEXT: v_accvgpr_write_b32 a27, s193521; HEURRC-NEXT: v_accvgpr_write_b32 a26, s183522; HEURRC-NEXT: v_accvgpr_write_b32 a25, s173523; HEURRC-NEXT: v_accvgpr_write_b32 a24, s163524; HEURRC-NEXT: v_accvgpr_write_b32 a23, s153525; HEURRC-NEXT: v_accvgpr_write_b32 a22, s143526; HEURRC-NEXT: v_accvgpr_write_b32 a21, s133527; HEURRC-NEXT: v_accvgpr_write_b32 a20, s123528; HEURRC-NEXT: v_accvgpr_write_b32 a19, s113529; HEURRC-NEXT: v_accvgpr_write_b32 a18, s103530; HEURRC-NEXT: v_accvgpr_write_b32 a17, s93531; HEURRC-NEXT: v_accvgpr_write_b32 a16, s83532; HEURRC-NEXT: s_nop 13533; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31] cbsz:2 abid:3 blgp:13534; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 163535; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 03536; HEURRC-NEXT: v_mov_b32_e32 v8, s163537; HEURRC-NEXT: v_mov_b32_e32 v9, s173538; HEURRC-NEXT: v_mov_b32_e32 v10, s183539; HEURRC-NEXT: v_mov_b32_e32 v11, s193540; HEURRC-NEXT: s_nop 53541; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc13542; HEURRC-NEXT: s_waitcnt vmcnt(0)3543; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc13544; HEURRC-NEXT: s_waitcnt vmcnt(0)3545; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc13546; HEURRC-NEXT: s_waitcnt vmcnt(0)3547; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc13548; HEURRC-NEXT: s_waitcnt vmcnt(0)3549; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc13550; HEURRC-NEXT: s_waitcnt vmcnt(0)3551; HEURRC-NEXT: v_mov_b32_e32 v2, s103552; HEURRC-NEXT: v_mov_b32_e32 v3, s113553; HEURRC-NEXT: v_mov_b32_e32 v8, s203554; HEURRC-NEXT: v_mov_b32_e32 v9, s213555; HEURRC-NEXT: v_mov_b32_e32 v10, s223556; HEURRC-NEXT: v_mov_b32_e32 v11, s233557; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc13558; HEURRC-NEXT: s_waitcnt vmcnt(0)3559; HEURRC-NEXT: v_mov_b32_e32 v0, s83560; HEURRC-NEXT: v_mov_b32_e32 v1, s93561; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc13562; HEURRC-NEXT: s_waitcnt vmcnt(0)3563; HEURRC-NEXT: s_nop 03564; HEURRC-NEXT: v_mov_b32_e32 v0, s123565; HEURRC-NEXT: v_mov_b32_e32 v1, s133566; HEURRC-NEXT: v_mov_b32_e32 v2, s143567; HEURRC-NEXT: v_mov_b32_e32 v3, s153568; HEURRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc13569; HEURRC-NEXT: s_waitcnt vmcnt(0)3570; HEURRC-NEXT: s_endpgm3571;3572; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__flags:3573; VGPRRC: ; %bb.0:3574; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x243575; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x643576; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 483577; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 323578; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)3579; VGPRRC-NEXT: v_mov_b32_e32 v36, s243580; VGPRRC-NEXT: v_mov_b32_e32 v37, s253581; VGPRRC-NEXT: v_mov_b32_e32 v38, s263582; VGPRRC-NEXT: v_mov_b32_e32 v39, s273583; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]3584; VGPRRC-NEXT: v_mov_b32_e32 v40, s283585; VGPRRC-NEXT: v_mov_b32_e32 v41, s293586; VGPRRC-NEXT: v_mov_b32_e32 v42, s303587; VGPRRC-NEXT: v_mov_b32_e32 v43, s313588; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]3589; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]3590; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]3591; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]3592; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]3593; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]3594; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]3595; VGPRRC-NEXT: s_nop 13596; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31] cbsz:2 abid:3 blgp:13597; VGPRRC-NEXT: s_nop 113598; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc13599; VGPRRC-NEXT: s_waitcnt vmcnt(0)3600; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc13601; VGPRRC-NEXT: s_waitcnt vmcnt(0)3602; VGPRRC-NEXT: s_nop 03603; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], 163604; VGPRRC-NEXT: global_store_dwordx4 v[8:9], v[4:7], off sc0 sc13605; VGPRRC-NEXT: s_waitcnt vmcnt(0)3606; VGPRRC-NEXT: s_nop 03607; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], 03608; VGPRRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc13609; VGPRRC-NEXT: s_waitcnt vmcnt(0)3610; VGPRRC-NEXT: s_nop 03611; VGPRRC-NEXT: v_mov_b32_e32 v0, s163612; VGPRRC-NEXT: v_mov_b32_e32 v1, s173613; VGPRRC-NEXT: v_mov_b32_e32 v2, s183614; VGPRRC-NEXT: v_mov_b32_e32 v3, s193615; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc13616; VGPRRC-NEXT: s_waitcnt vmcnt(0)3617; VGPRRC-NEXT: s_nop 03618; VGPRRC-NEXT: v_mov_b32_e32 v0, s203619; VGPRRC-NEXT: v_mov_b32_e32 v1, s213620; VGPRRC-NEXT: v_mov_b32_e32 v2, s223621; VGPRRC-NEXT: v_mov_b32_e32 v3, s233622; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc13623; VGPRRC-NEXT: s_waitcnt vmcnt(0)3624; VGPRRC-NEXT: s_nop 03625; VGPRRC-NEXT: v_mov_b32_e32 v0, s83626; VGPRRC-NEXT: v_mov_b32_e32 v1, s93627; VGPRRC-NEXT: v_mov_b32_e32 v2, s103628; VGPRRC-NEXT: v_mov_b32_e32 v3, s113629; VGPRRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc13630; VGPRRC-NEXT: s_waitcnt vmcnt(0)3631; VGPRRC-NEXT: s_nop 03632; VGPRRC-NEXT: v_mov_b32_e32 v0, s123633; VGPRRC-NEXT: v_mov_b32_e32 v1, s133634; VGPRRC-NEXT: v_mov_b32_e32 v2, s143635; VGPRRC-NEXT: v_mov_b32_e32 v3, s153636; VGPRRC-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc13637; VGPRRC-NEXT: s_waitcnt vmcnt(0)3638; VGPRRC-NEXT: s_endpgm3639; AGPR-LABEL: test_mfma_i32_32x32x32_i8__flags:3640; AGPR: ; %bb.0:3641; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x243642; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x643643; AGPR-NEXT: v_mov_b64_e32 v[8:9], 483644; AGPR-NEXT: v_mov_b64_e32 v[10:11], 323645; AGPR-NEXT: v_mov_b64_e32 v[12:13], 163646; AGPR-NEXT: s_waitcnt lgkmcnt(0)3647; AGPR-NEXT: v_mov_b32_e32 v0, s243648; AGPR-NEXT: v_mov_b32_e32 v1, s253649; AGPR-NEXT: v_mov_b32_e32 v2, s263650; AGPR-NEXT: v_mov_b32_e32 v3, s273651; AGPR-NEXT: v_accvgpr_write_b32 a0, s83652; AGPR-NEXT: v_mov_b32_e32 v4, s283653; AGPR-NEXT: v_mov_b32_e32 v5, s293654; AGPR-NEXT: v_mov_b32_e32 v6, s303655; AGPR-NEXT: v_mov_b32_e32 v7, s313656; AGPR-NEXT: v_accvgpr_write_b32 a1, s93657; AGPR-NEXT: v_accvgpr_write_b32 a2, s103658; AGPR-NEXT: v_accvgpr_write_b32 a3, s113659; AGPR-NEXT: v_accvgpr_write_b32 a4, s123660; AGPR-NEXT: v_accvgpr_write_b32 a5, s133661; AGPR-NEXT: v_accvgpr_write_b32 a6, s143662; AGPR-NEXT: v_accvgpr_write_b32 a7, s153663; AGPR-NEXT: v_accvgpr_write_b32 a8, s163664; AGPR-NEXT: v_accvgpr_write_b32 a9, s173665; AGPR-NEXT: v_accvgpr_write_b32 a10, s183666; AGPR-NEXT: v_accvgpr_write_b32 a11, s193667; AGPR-NEXT: v_accvgpr_write_b32 a12, s203668; AGPR-NEXT: v_accvgpr_write_b32 a13, s213669; AGPR-NEXT: v_accvgpr_write_b32 a14, s223670; AGPR-NEXT: v_accvgpr_write_b32 a15, s233671; AGPR-NEXT: v_mov_b64_e32 v[14:15], 03672; AGPR-NEXT: s_nop 03673; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:13674; AGPR-NEXT: v_mov_b32_e32 v0, s163675; AGPR-NEXT: v_mov_b32_e32 v1, s173676; AGPR-NEXT: v_mov_b32_e32 v2, s183677; AGPR-NEXT: v_mov_b32_e32 v3, s193678; AGPR-NEXT: s_nop 73679; AGPR-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc13680; AGPR-NEXT: s_waitcnt vmcnt(0)3681; AGPR-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc13682; AGPR-NEXT: s_waitcnt vmcnt(0)3683; AGPR-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc13684; AGPR-NEXT: s_waitcnt vmcnt(0)3685; AGPR-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc13686; AGPR-NEXT: s_waitcnt vmcnt(0)3687; AGPR-NEXT: global_store_dwordx4 v[10:11], v[0:3], off sc0 sc13688; AGPR-NEXT: s_waitcnt vmcnt(0)3689; AGPR-NEXT: s_nop 03690; AGPR-NEXT: v_mov_b32_e32 v0, s203691; AGPR-NEXT: v_mov_b32_e32 v1, s213692; AGPR-NEXT: v_mov_b32_e32 v2, s223693; AGPR-NEXT: v_mov_b32_e32 v3, s233694; AGPR-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc13695; AGPR-NEXT: s_waitcnt vmcnt(0)3696; AGPR-NEXT: s_nop 03697; AGPR-NEXT: v_mov_b32_e32 v0, s83698; AGPR-NEXT: v_mov_b32_e32 v1, s93699; AGPR-NEXT: v_mov_b32_e32 v2, s103700; AGPR-NEXT: v_mov_b32_e32 v3, s113701; AGPR-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc13702; AGPR-NEXT: s_waitcnt vmcnt(0)3703; AGPR-NEXT: s_nop 03704; AGPR-NEXT: v_mov_b32_e32 v0, s123705; AGPR-NEXT: v_mov_b32_e32 v1, s133706; AGPR-NEXT: v_mov_b32_e32 v2, s143707; AGPR-NEXT: v_mov_b32_e32 v3, s153708; AGPR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc13709; AGPR-NEXT: s_waitcnt vmcnt(0)3710; AGPR-NEXT: s_endpgm3711; VGPR-LABEL: test_mfma_i32_32x32x32_i8__flags:3712; VGPR: ; %bb.0:3713; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x243714; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x643715; VGPR-NEXT: v_mov_b64_e32 v[40:41], 483716; VGPR-NEXT: v_mov_b64_e32 v[42:43], 323717; VGPR-NEXT: v_mov_b64_e32 v[44:45], 163718; VGPR-NEXT: s_waitcnt lgkmcnt(0)3719; VGPR-NEXT: v_mov_b32_e32 v32, s243720; VGPR-NEXT: v_mov_b32_e32 v33, s253721; VGPR-NEXT: v_mov_b32_e32 v34, s263722; VGPR-NEXT: v_mov_b32_e32 v35, s273723; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]3724; VGPR-NEXT: v_mov_b32_e32 v36, s283725; VGPR-NEXT: v_mov_b32_e32 v37, s293726; VGPR-NEXT: v_mov_b32_e32 v38, s303727; VGPR-NEXT: v_mov_b32_e32 v39, s313728; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]3729; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]3730; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]3731; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]3732; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]3733; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]3734; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]3735; VGPR-NEXT: v_mov_b64_e32 v[46:47], 03736; VGPR-NEXT: s_nop 03737; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:13738; VGPR-NEXT: s_nop 73739; VGPR-NEXT: s_nop 33740; VGPR-NEXT: global_store_dwordx4 v[40:41], v[28:31], off sc0 sc13741; VGPR-NEXT: s_waitcnt vmcnt(0)3742; VGPR-NEXT: global_store_dwordx4 v[42:43], v[24:27], off sc0 sc13743; VGPR-NEXT: s_waitcnt vmcnt(0)3744; VGPR-NEXT: global_store_dwordx4 v[44:45], v[20:23], off sc0 sc13745; VGPR-NEXT: s_waitcnt vmcnt(0)3746; VGPR-NEXT: global_store_dwordx4 v[46:47], v[16:19], off sc0 sc13747; VGPR-NEXT: s_waitcnt vmcnt(0)3748; VGPR-NEXT: v_mov_b32_e32 v0, s163749; VGPR-NEXT: v_mov_b32_e32 v1, s173750; VGPR-NEXT: v_mov_b32_e32 v2, s183751; VGPR-NEXT: v_mov_b32_e32 v3, s193752; VGPR-NEXT: global_store_dwordx4 v[42:43], v[0:3], off sc0 sc13753; VGPR-NEXT: s_waitcnt vmcnt(0)3754; VGPR-NEXT: s_nop 03755; VGPR-NEXT: v_mov_b32_e32 v0, s203756; VGPR-NEXT: v_mov_b32_e32 v1, s213757; VGPR-NEXT: v_mov_b32_e32 v2, s223758; VGPR-NEXT: v_mov_b32_e32 v3, s233759; VGPR-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc13760; VGPR-NEXT: s_waitcnt vmcnt(0)3761; VGPR-NEXT: s_nop 03762; VGPR-NEXT: v_mov_b32_e32 v0, s83763; VGPR-NEXT: v_mov_b32_e32 v1, s93764; VGPR-NEXT: v_mov_b32_e32 v2, s103765; VGPR-NEXT: v_mov_b32_e32 v3, s113766; VGPR-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc13767; VGPR-NEXT: s_waitcnt vmcnt(0)3768; VGPR-NEXT: s_nop 03769; VGPR-NEXT: v_mov_b32_e32 v0, s123770; VGPR-NEXT: v_mov_b32_e32 v1, s133771; VGPR-NEXT: v_mov_b32_e32 v2, s143772; VGPR-NEXT: v_mov_b32_e32 v3, s153773; VGPR-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc13774; VGPR-NEXT: s_waitcnt vmcnt(0)3775; VGPR-NEXT: s_endpgm3776 %result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 2, i32 3, i32 1)3777 store volatile <16 x i32> %result, ptr addrspace(1) null3778 store volatile <16 x i32> %arg2, ptr addrspace(1) null3779 ret void3780}3781 3782define <16 x i32> @test_mfma_i32_32x32x32_i8__mac(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2) {3783; GCN-LABEL: test_mfma_i32_32x32x32_i8__mac:3784; GCN: ; %bb.0:3785; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3786; GCN-NEXT: v_accvgpr_write_b32 a0, v83787; GCN-NEXT: v_accvgpr_write_b32 a1, v93788; GCN-NEXT: v_accvgpr_write_b32 a2, v103789; GCN-NEXT: v_accvgpr_write_b32 a3, v113790; GCN-NEXT: v_accvgpr_write_b32 a4, v123791; GCN-NEXT: v_accvgpr_write_b32 a5, v133792; GCN-NEXT: v_accvgpr_write_b32 a6, v143793; GCN-NEXT: v_accvgpr_write_b32 a7, v153794; GCN-NEXT: v_accvgpr_write_b32 a8, v163795; GCN-NEXT: v_accvgpr_write_b32 a9, v173796; GCN-NEXT: v_accvgpr_write_b32 a10, v183797; GCN-NEXT: v_accvgpr_write_b32 a11, v193798; GCN-NEXT: v_accvgpr_write_b32 a12, v203799; GCN-NEXT: v_accvgpr_write_b32 a13, v213800; GCN-NEXT: v_accvgpr_write_b32 a14, v223801; GCN-NEXT: v_accvgpr_write_b32 a15, v233802; GCN-NEXT: s_nop 13803; GCN-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15]3804; GCN-NEXT: s_nop 113805; GCN-NEXT: v_accvgpr_read_b32 v0, a03806; GCN-NEXT: v_accvgpr_read_b32 v1, a13807; GCN-NEXT: v_accvgpr_read_b32 v2, a23808; GCN-NEXT: v_accvgpr_read_b32 v3, a33809; GCN-NEXT: v_accvgpr_read_b32 v4, a43810; GCN-NEXT: v_accvgpr_read_b32 v5, a53811; GCN-NEXT: v_accvgpr_read_b32 v6, a63812; GCN-NEXT: v_accvgpr_read_b32 v7, a73813; GCN-NEXT: v_accvgpr_read_b32 v8, a83814; GCN-NEXT: v_accvgpr_read_b32 v9, a93815; GCN-NEXT: v_accvgpr_read_b32 v10, a103816; GCN-NEXT: v_accvgpr_read_b32 v11, a113817; GCN-NEXT: v_accvgpr_read_b32 v12, a123818; GCN-NEXT: v_accvgpr_read_b32 v13, a133819; GCN-NEXT: v_accvgpr_read_b32 v14, a143820; GCN-NEXT: v_accvgpr_read_b32 v15, a153821; GCN-NEXT: s_setpc_b64 s[30:31]3822;3823; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__mac:3824; HEURRC: ; %bb.0:3825; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3826; HEURRC-NEXT: v_accvgpr_write_b32 a0, v83827; HEURRC-NEXT: v_accvgpr_write_b32 a1, v93828; HEURRC-NEXT: v_accvgpr_write_b32 a2, v103829; HEURRC-NEXT: v_accvgpr_write_b32 a3, v113830; HEURRC-NEXT: v_accvgpr_write_b32 a4, v123831; HEURRC-NEXT: v_accvgpr_write_b32 a5, v133832; HEURRC-NEXT: v_accvgpr_write_b32 a6, v143833; HEURRC-NEXT: v_accvgpr_write_b32 a7, v153834; HEURRC-NEXT: v_accvgpr_write_b32 a8, v163835; HEURRC-NEXT: v_accvgpr_write_b32 a9, v173836; HEURRC-NEXT: v_accvgpr_write_b32 a10, v183837; HEURRC-NEXT: v_accvgpr_write_b32 a11, v193838; HEURRC-NEXT: v_accvgpr_write_b32 a12, v203839; HEURRC-NEXT: v_accvgpr_write_b32 a13, v213840; HEURRC-NEXT: v_accvgpr_write_b32 a14, v223841; HEURRC-NEXT: v_accvgpr_write_b32 a15, v233842; HEURRC-NEXT: s_nop 13843; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15]3844; HEURRC-NEXT: s_nop 113845; HEURRC-NEXT: v_accvgpr_read_b32 v0, a03846; HEURRC-NEXT: v_accvgpr_read_b32 v1, a13847; HEURRC-NEXT: v_accvgpr_read_b32 v2, a23848; HEURRC-NEXT: v_accvgpr_read_b32 v3, a33849; HEURRC-NEXT: v_accvgpr_read_b32 v4, a43850; HEURRC-NEXT: v_accvgpr_read_b32 v5, a53851; HEURRC-NEXT: v_accvgpr_read_b32 v6, a63852; HEURRC-NEXT: v_accvgpr_read_b32 v7, a73853; HEURRC-NEXT: v_accvgpr_read_b32 v8, a83854; HEURRC-NEXT: v_accvgpr_read_b32 v9, a93855; HEURRC-NEXT: v_accvgpr_read_b32 v10, a103856; HEURRC-NEXT: v_accvgpr_read_b32 v11, a113857; HEURRC-NEXT: v_accvgpr_read_b32 v12, a123858; HEURRC-NEXT: v_accvgpr_read_b32 v13, a133859; HEURRC-NEXT: v_accvgpr_read_b32 v14, a143860; HEURRC-NEXT: v_accvgpr_read_b32 v15, a153861; HEURRC-NEXT: s_setpc_b64 s[30:31]3862;3863; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__mac:3864; VGPRRC: ; %bb.0:3865; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3866; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[8:23], v[0:3], v[4:7], v[8:23]3867; VGPRRC-NEXT: s_nop 113868; VGPRRC-NEXT: v_mov_b32_e32 v0, v83869; VGPRRC-NEXT: v_mov_b32_e32 v1, v93870; VGPRRC-NEXT: v_mov_b32_e32 v2, v103871; VGPRRC-NEXT: v_mov_b32_e32 v3, v113872; VGPRRC-NEXT: v_mov_b32_e32 v4, v123873; VGPRRC-NEXT: v_mov_b32_e32 v5, v133874; VGPRRC-NEXT: v_mov_b32_e32 v6, v143875; VGPRRC-NEXT: v_mov_b32_e32 v7, v153876; VGPRRC-NEXT: v_mov_b32_e32 v8, v163877; VGPRRC-NEXT: v_mov_b32_e32 v9, v173878; VGPRRC-NEXT: v_mov_b32_e32 v10, v183879; VGPRRC-NEXT: v_mov_b32_e32 v11, v193880; VGPRRC-NEXT: v_mov_b32_e32 v12, v203881; VGPRRC-NEXT: v_mov_b32_e32 v13, v213882; VGPRRC-NEXT: v_mov_b32_e32 v14, v223883; VGPRRC-NEXT: v_mov_b32_e32 v15, v233884; VGPRRC-NEXT: s_setpc_b64 s[30:31]3885; AGPR-LABEL: test_mfma_i32_32x32x32_i8__mac:3886; AGPR: ; %bb.0:3887; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3888; AGPR-NEXT: v_accvgpr_write_b32 a0, v83889; AGPR-NEXT: v_accvgpr_write_b32 a1, v93890; AGPR-NEXT: v_accvgpr_write_b32 a2, v103891; AGPR-NEXT: v_accvgpr_write_b32 a3, v113892; AGPR-NEXT: v_accvgpr_write_b32 a4, v123893; AGPR-NEXT: v_accvgpr_write_b32 a5, v133894; AGPR-NEXT: v_accvgpr_write_b32 a6, v143895; AGPR-NEXT: v_accvgpr_write_b32 a7, v153896; AGPR-NEXT: v_accvgpr_write_b32 a8, v163897; AGPR-NEXT: v_accvgpr_write_b32 a9, v173898; AGPR-NEXT: v_accvgpr_write_b32 a10, v183899; AGPR-NEXT: v_accvgpr_write_b32 a11, v193900; AGPR-NEXT: v_accvgpr_write_b32 a12, v203901; AGPR-NEXT: v_accvgpr_write_b32 a13, v213902; AGPR-NEXT: v_accvgpr_write_b32 a14, v223903; AGPR-NEXT: v_accvgpr_write_b32 a15, v233904; AGPR-NEXT: s_nop 13905; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15]3906; AGPR-NEXT: s_nop 73907; AGPR-NEXT: s_nop 33908; AGPR-NEXT: v_accvgpr_read_b32 v0, a03909; AGPR-NEXT: v_accvgpr_read_b32 v1, a13910; AGPR-NEXT: v_accvgpr_read_b32 v2, a23911; AGPR-NEXT: v_accvgpr_read_b32 v3, a33912; AGPR-NEXT: v_accvgpr_read_b32 v4, a43913; AGPR-NEXT: v_accvgpr_read_b32 v5, a53914; AGPR-NEXT: v_accvgpr_read_b32 v6, a63915; AGPR-NEXT: v_accvgpr_read_b32 v7, a73916; AGPR-NEXT: v_accvgpr_read_b32 v8, a83917; AGPR-NEXT: v_accvgpr_read_b32 v9, a93918; AGPR-NEXT: v_accvgpr_read_b32 v10, a103919; AGPR-NEXT: v_accvgpr_read_b32 v11, a113920; AGPR-NEXT: v_accvgpr_read_b32 v12, a123921; AGPR-NEXT: v_accvgpr_read_b32 v13, a133922; AGPR-NEXT: v_accvgpr_read_b32 v14, a143923; AGPR-NEXT: v_accvgpr_read_b32 v15, a153924; AGPR-NEXT: s_setpc_b64 s[30:31]3925; VGPR-LABEL: test_mfma_i32_32x32x32_i8__mac:3926; VGPR: ; %bb.0:3927; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3928; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[8:23], v[0:3], v[4:7], v[8:23]3929; VGPR-NEXT: s_nop 73930; VGPR-NEXT: s_nop 33931; VGPR-NEXT: v_mov_b32_e32 v0, v83932; VGPR-NEXT: v_mov_b32_e32 v1, v93933; VGPR-NEXT: v_mov_b32_e32 v2, v103934; VGPR-NEXT: v_mov_b32_e32 v3, v113935; VGPR-NEXT: v_mov_b32_e32 v4, v123936; VGPR-NEXT: v_mov_b32_e32 v5, v133937; VGPR-NEXT: v_mov_b32_e32 v6, v143938; VGPR-NEXT: v_mov_b32_e32 v7, v153939; VGPR-NEXT: v_mov_b32_e32 v8, v163940; VGPR-NEXT: v_mov_b32_e32 v9, v173941; VGPR-NEXT: v_mov_b32_e32 v10, v183942; VGPR-NEXT: v_mov_b32_e32 v11, v193943; VGPR-NEXT: v_mov_b32_e32 v12, v203944; VGPR-NEXT: v_mov_b32_e32 v13, v213945; VGPR-NEXT: v_mov_b32_e32 v14, v223946; VGPR-NEXT: v_mov_b32_e32 v15, v233947; VGPR-NEXT: s_setpc_b64 s[30:31]3948 %result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 0, i32 0, i32 0)3949 ret <16 x i32> %result3950}3951 3952define <16 x i32> @test_mfma_i32_32x32x32_i8__mac__flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2) {3953; GCN-LABEL: test_mfma_i32_32x32x32_i8__mac__flags:3954; GCN: ; %bb.0:3955; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3956; GCN-NEXT: v_accvgpr_write_b32 a0, v83957; GCN-NEXT: v_accvgpr_write_b32 a1, v93958; GCN-NEXT: v_accvgpr_write_b32 a2, v103959; GCN-NEXT: v_accvgpr_write_b32 a3, v113960; GCN-NEXT: v_accvgpr_write_b32 a4, v123961; GCN-NEXT: v_accvgpr_write_b32 a5, v133962; GCN-NEXT: v_accvgpr_write_b32 a6, v143963; GCN-NEXT: v_accvgpr_write_b32 a7, v153964; GCN-NEXT: v_accvgpr_write_b32 a8, v163965; GCN-NEXT: v_accvgpr_write_b32 a9, v173966; GCN-NEXT: v_accvgpr_write_b32 a10, v183967; GCN-NEXT: v_accvgpr_write_b32 a11, v193968; GCN-NEXT: v_accvgpr_write_b32 a12, v203969; GCN-NEXT: v_accvgpr_write_b32 a13, v213970; GCN-NEXT: v_accvgpr_write_b32 a14, v223971; GCN-NEXT: v_accvgpr_write_b32 a15, v233972; GCN-NEXT: s_nop 13973; GCN-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:13974; GCN-NEXT: s_nop 113975; GCN-NEXT: v_accvgpr_read_b32 v0, a03976; GCN-NEXT: v_accvgpr_read_b32 v1, a13977; GCN-NEXT: v_accvgpr_read_b32 v2, a23978; GCN-NEXT: v_accvgpr_read_b32 v3, a33979; GCN-NEXT: v_accvgpr_read_b32 v4, a43980; GCN-NEXT: v_accvgpr_read_b32 v5, a53981; GCN-NEXT: v_accvgpr_read_b32 v6, a63982; GCN-NEXT: v_accvgpr_read_b32 v7, a73983; GCN-NEXT: v_accvgpr_read_b32 v8, a83984; GCN-NEXT: v_accvgpr_read_b32 v9, a93985; GCN-NEXT: v_accvgpr_read_b32 v10, a103986; GCN-NEXT: v_accvgpr_read_b32 v11, a113987; GCN-NEXT: v_accvgpr_read_b32 v12, a123988; GCN-NEXT: v_accvgpr_read_b32 v13, a133989; GCN-NEXT: v_accvgpr_read_b32 v14, a143990; GCN-NEXT: v_accvgpr_read_b32 v15, a153991; GCN-NEXT: s_setpc_b64 s[30:31]3992;3993; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__mac__flags:3994; HEURRC: ; %bb.0:3995; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3996; HEURRC-NEXT: v_accvgpr_write_b32 a0, v83997; HEURRC-NEXT: v_accvgpr_write_b32 a1, v93998; HEURRC-NEXT: v_accvgpr_write_b32 a2, v103999; HEURRC-NEXT: v_accvgpr_write_b32 a3, v114000; HEURRC-NEXT: v_accvgpr_write_b32 a4, v124001; HEURRC-NEXT: v_accvgpr_write_b32 a5, v134002; HEURRC-NEXT: v_accvgpr_write_b32 a6, v144003; HEURRC-NEXT: v_accvgpr_write_b32 a7, v154004; HEURRC-NEXT: v_accvgpr_write_b32 a8, v164005; HEURRC-NEXT: v_accvgpr_write_b32 a9, v174006; HEURRC-NEXT: v_accvgpr_write_b32 a10, v184007; HEURRC-NEXT: v_accvgpr_write_b32 a11, v194008; HEURRC-NEXT: v_accvgpr_write_b32 a12, v204009; HEURRC-NEXT: v_accvgpr_write_b32 a13, v214010; HEURRC-NEXT: v_accvgpr_write_b32 a14, v224011; HEURRC-NEXT: v_accvgpr_write_b32 a15, v234012; HEURRC-NEXT: s_nop 14013; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:14014; HEURRC-NEXT: s_nop 114015; HEURRC-NEXT: v_accvgpr_read_b32 v0, a04016; HEURRC-NEXT: v_accvgpr_read_b32 v1, a14017; HEURRC-NEXT: v_accvgpr_read_b32 v2, a24018; HEURRC-NEXT: v_accvgpr_read_b32 v3, a34019; HEURRC-NEXT: v_accvgpr_read_b32 v4, a44020; HEURRC-NEXT: v_accvgpr_read_b32 v5, a54021; HEURRC-NEXT: v_accvgpr_read_b32 v6, a64022; HEURRC-NEXT: v_accvgpr_read_b32 v7, a74023; HEURRC-NEXT: v_accvgpr_read_b32 v8, a84024; HEURRC-NEXT: v_accvgpr_read_b32 v9, a94025; HEURRC-NEXT: v_accvgpr_read_b32 v10, a104026; HEURRC-NEXT: v_accvgpr_read_b32 v11, a114027; HEURRC-NEXT: v_accvgpr_read_b32 v12, a124028; HEURRC-NEXT: v_accvgpr_read_b32 v13, a134029; HEURRC-NEXT: v_accvgpr_read_b32 v14, a144030; HEURRC-NEXT: v_accvgpr_read_b32 v15, a154031; HEURRC-NEXT: s_setpc_b64 s[30:31]4032;4033; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__mac__flags:4034; VGPRRC: ; %bb.0:4035; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4036; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[8:23], v[0:3], v[4:7], v[8:23] cbsz:1 abid:1 blgp:14037; VGPRRC-NEXT: s_nop 114038; VGPRRC-NEXT: v_mov_b32_e32 v0, v84039; VGPRRC-NEXT: v_mov_b32_e32 v1, v94040; VGPRRC-NEXT: v_mov_b32_e32 v2, v104041; VGPRRC-NEXT: v_mov_b32_e32 v3, v114042; VGPRRC-NEXT: v_mov_b32_e32 v4, v124043; VGPRRC-NEXT: v_mov_b32_e32 v5, v134044; VGPRRC-NEXT: v_mov_b32_e32 v6, v144045; VGPRRC-NEXT: v_mov_b32_e32 v7, v154046; VGPRRC-NEXT: v_mov_b32_e32 v8, v164047; VGPRRC-NEXT: v_mov_b32_e32 v9, v174048; VGPRRC-NEXT: v_mov_b32_e32 v10, v184049; VGPRRC-NEXT: v_mov_b32_e32 v11, v194050; VGPRRC-NEXT: v_mov_b32_e32 v12, v204051; VGPRRC-NEXT: v_mov_b32_e32 v13, v214052; VGPRRC-NEXT: v_mov_b32_e32 v14, v224053; VGPRRC-NEXT: v_mov_b32_e32 v15, v234054; VGPRRC-NEXT: s_setpc_b64 s[30:31]4055; AGPR-LABEL: test_mfma_i32_32x32x32_i8__mac__flags:4056; AGPR: ; %bb.0:4057; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4058; AGPR-NEXT: v_accvgpr_write_b32 a0, v84059; AGPR-NEXT: v_accvgpr_write_b32 a1, v94060; AGPR-NEXT: v_accvgpr_write_b32 a2, v104061; AGPR-NEXT: v_accvgpr_write_b32 a3, v114062; AGPR-NEXT: v_accvgpr_write_b32 a4, v124063; AGPR-NEXT: v_accvgpr_write_b32 a5, v134064; AGPR-NEXT: v_accvgpr_write_b32 a6, v144065; AGPR-NEXT: v_accvgpr_write_b32 a7, v154066; AGPR-NEXT: v_accvgpr_write_b32 a8, v164067; AGPR-NEXT: v_accvgpr_write_b32 a9, v174068; AGPR-NEXT: v_accvgpr_write_b32 a10, v184069; AGPR-NEXT: v_accvgpr_write_b32 a11, v194070; AGPR-NEXT: v_accvgpr_write_b32 a12, v204071; AGPR-NEXT: v_accvgpr_write_b32 a13, v214072; AGPR-NEXT: v_accvgpr_write_b32 a14, v224073; AGPR-NEXT: v_accvgpr_write_b32 a15, v234074; AGPR-NEXT: s_nop 14075; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:14076; AGPR-NEXT: s_nop 74077; AGPR-NEXT: s_nop 34078; AGPR-NEXT: v_accvgpr_read_b32 v0, a04079; AGPR-NEXT: v_accvgpr_read_b32 v1, a14080; AGPR-NEXT: v_accvgpr_read_b32 v2, a24081; AGPR-NEXT: v_accvgpr_read_b32 v3, a34082; AGPR-NEXT: v_accvgpr_read_b32 v4, a44083; AGPR-NEXT: v_accvgpr_read_b32 v5, a54084; AGPR-NEXT: v_accvgpr_read_b32 v6, a64085; AGPR-NEXT: v_accvgpr_read_b32 v7, a74086; AGPR-NEXT: v_accvgpr_read_b32 v8, a84087; AGPR-NEXT: v_accvgpr_read_b32 v9, a94088; AGPR-NEXT: v_accvgpr_read_b32 v10, a104089; AGPR-NEXT: v_accvgpr_read_b32 v11, a114090; AGPR-NEXT: v_accvgpr_read_b32 v12, a124091; AGPR-NEXT: v_accvgpr_read_b32 v13, a134092; AGPR-NEXT: v_accvgpr_read_b32 v14, a144093; AGPR-NEXT: v_accvgpr_read_b32 v15, a154094; AGPR-NEXT: s_setpc_b64 s[30:31]4095; VGPR-LABEL: test_mfma_i32_32x32x32_i8__mac__flags:4096; VGPR: ; %bb.0:4097; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4098; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[8:23], v[0:3], v[4:7], v[8:23] cbsz:1 abid:1 blgp:14099; VGPR-NEXT: s_nop 74100; VGPR-NEXT: s_nop 34101; VGPR-NEXT: v_mov_b32_e32 v0, v84102; VGPR-NEXT: v_mov_b32_e32 v1, v94103; VGPR-NEXT: v_mov_b32_e32 v2, v104104; VGPR-NEXT: v_mov_b32_e32 v3, v114105; VGPR-NEXT: v_mov_b32_e32 v4, v124106; VGPR-NEXT: v_mov_b32_e32 v5, v134107; VGPR-NEXT: v_mov_b32_e32 v6, v144108; VGPR-NEXT: v_mov_b32_e32 v7, v154109; VGPR-NEXT: v_mov_b32_e32 v8, v164110; VGPR-NEXT: v_mov_b32_e32 v9, v174111; VGPR-NEXT: v_mov_b32_e32 v10, v184112; VGPR-NEXT: v_mov_b32_e32 v11, v194113; VGPR-NEXT: v_mov_b32_e32 v12, v204114; VGPR-NEXT: v_mov_b32_e32 v13, v214115; VGPR-NEXT: v_mov_b32_e32 v14, v224116; VGPR-NEXT: v_mov_b32_e32 v15, v234117; VGPR-NEXT: s_setpc_b64 s[30:31]4118 %result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 1, i32 1, i32 1)4119 ret <16 x i32> %result4120}4121 4122define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {4123; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:4124; SDAG: ; %bb.0:4125; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x244126; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa44127; SDAG-NEXT: v_mov_b32_e32 v40, 04128; SDAG-NEXT: s_waitcnt lgkmcnt(0)4129; SDAG-NEXT: v_mov_b32_e32 v32, s204130; SDAG-NEXT: v_mov_b32_e32 v33, s214131; SDAG-NEXT: v_mov_b32_e32 v34, s224132; SDAG-NEXT: v_mov_b32_e32 v35, s234133; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x644134; SDAG-NEXT: v_mov_b32_e32 v36, s244135; SDAG-NEXT: v_mov_b32_e32 v37, s254136; SDAG-NEXT: v_mov_b32_e32 v38, s264137; SDAG-NEXT: v_mov_b32_e32 v39, s274138; SDAG-NEXT: s_waitcnt lgkmcnt(0)4139; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]4140; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]4141; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]4142; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]4143; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]4144; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]4145; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]4146; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]4147; SDAG-NEXT: s_nop 14148; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]4149; SDAG-NEXT: s_nop 64150; SDAG-NEXT: v_mov_b32_e32 v16, s204151; SDAG-NEXT: v_mov_b32_e32 v17, s214152; SDAG-NEXT: v_mov_b32_e32 v18, s224153; SDAG-NEXT: v_mov_b32_e32 v19, s234154; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc14155; SDAG-NEXT: s_waitcnt vmcnt(0)4156; SDAG-NEXT: s_nop 04157; SDAG-NEXT: v_mov_b32_e32 v16, s164158; SDAG-NEXT: v_mov_b32_e32 v17, s174159; SDAG-NEXT: v_mov_b32_e32 v18, s184160; SDAG-NEXT: v_mov_b32_e32 v19, s194161; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc14162; SDAG-NEXT: s_waitcnt vmcnt(0)4163; SDAG-NEXT: s_nop 04164; SDAG-NEXT: v_mov_b32_e32 v16, s124165; SDAG-NEXT: v_mov_b32_e32 v17, s134166; SDAG-NEXT: v_mov_b32_e32 v18, s144167; SDAG-NEXT: v_mov_b32_e32 v19, s154168; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc14169; SDAG-NEXT: s_waitcnt vmcnt(0)4170; SDAG-NEXT: s_nop 04171; SDAG-NEXT: v_mov_b32_e32 v16, s84172; SDAG-NEXT: v_mov_b32_e32 v17, s94173; SDAG-NEXT: v_mov_b32_e32 v18, s104174; SDAG-NEXT: v_mov_b32_e32 v19, s114175; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc14176; SDAG-NEXT: s_waitcnt vmcnt(0)4177; SDAG-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc14178; SDAG-NEXT: s_waitcnt vmcnt(0)4179; SDAG-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc14180; SDAG-NEXT: s_waitcnt vmcnt(0)4181; SDAG-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc14182; SDAG-NEXT: s_waitcnt vmcnt(0)4183; SDAG-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc14184; SDAG-NEXT: s_waitcnt vmcnt(0)4185; SDAG-NEXT: s_endpgm4186;4187; GISEL-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:4188; GISEL: ; %bb.0:4189; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x244190; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x644191; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa44192; GISEL-NEXT: v_mov_b32_e32 v56, 04193; GISEL-NEXT: s_waitcnt lgkmcnt(0)4194; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]4195; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]4196; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]4197; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]4198; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]4199; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]4200; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]4201; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]4202; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]4203; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]4204; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]4205; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]4206; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]4207; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]4208; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15]4209; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[14:15]4210; GISEL-NEXT: v_mov_b64_e32 v[50:51], s[18:19]4211; GISEL-NEXT: v_mov_b64_e32 v[54:55], s[22:23]4212; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[12:13]4213; GISEL-NEXT: v_mov_b64_e32 v[48:49], s[16:17]4214; GISEL-NEXT: v_mov_b64_e32 v[52:53], s[20:21]4215; GISEL-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc14216; GISEL-NEXT: s_waitcnt vmcnt(0)4217; GISEL-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc14218; GISEL-NEXT: s_waitcnt vmcnt(0)4219; GISEL-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc14220; GISEL-NEXT: s_waitcnt vmcnt(0)4221; GISEL-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc14222; GISEL-NEXT: s_waitcnt vmcnt(0)4223; GISEL-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc14224; GISEL-NEXT: s_waitcnt vmcnt(0)4225; GISEL-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc14226; GISEL-NEXT: s_waitcnt vmcnt(0)4227; GISEL-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc14228; GISEL-NEXT: s_waitcnt vmcnt(0)4229; GISEL-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc14230; GISEL-NEXT: s_waitcnt vmcnt(0)4231; GISEL-NEXT: s_endpgm4232;4233; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:4234; HEURRC: ; %bb.0:4235; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x244236; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa44237; HEURRC-NEXT: v_mov_b32_e32 v40, 04238; HEURRC-NEXT: s_waitcnt lgkmcnt(0)4239; HEURRC-NEXT: v_mov_b32_e32 v32, s204240; HEURRC-NEXT: v_mov_b32_e32 v33, s214241; HEURRC-NEXT: v_mov_b32_e32 v34, s224242; HEURRC-NEXT: v_mov_b32_e32 v35, s234243; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x644244; HEURRC-NEXT: v_mov_b32_e32 v36, s244245; HEURRC-NEXT: v_mov_b32_e32 v37, s254246; HEURRC-NEXT: v_mov_b32_e32 v38, s264247; HEURRC-NEXT: v_mov_b32_e32 v39, s274248; HEURRC-NEXT: s_waitcnt lgkmcnt(0)4249; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]4250; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]4251; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]4252; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]4253; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]4254; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]4255; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]4256; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]4257; HEURRC-NEXT: s_nop 14258; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]4259; HEURRC-NEXT: s_nop 64260; HEURRC-NEXT: v_mov_b32_e32 v16, s204261; HEURRC-NEXT: v_mov_b32_e32 v17, s214262; HEURRC-NEXT: v_mov_b32_e32 v18, s224263; HEURRC-NEXT: v_mov_b32_e32 v19, s234264; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc14265; HEURRC-NEXT: s_waitcnt vmcnt(0)4266; HEURRC-NEXT: s_nop 04267; HEURRC-NEXT: v_mov_b32_e32 v16, s164268; HEURRC-NEXT: v_mov_b32_e32 v17, s174269; HEURRC-NEXT: v_mov_b32_e32 v18, s184270; HEURRC-NEXT: v_mov_b32_e32 v19, s194271; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc14272; HEURRC-NEXT: s_waitcnt vmcnt(0)4273; HEURRC-NEXT: s_nop 04274; HEURRC-NEXT: v_mov_b32_e32 v16, s124275; HEURRC-NEXT: v_mov_b32_e32 v17, s134276; HEURRC-NEXT: v_mov_b32_e32 v18, s144277; HEURRC-NEXT: v_mov_b32_e32 v19, s154278; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc14279; HEURRC-NEXT: s_waitcnt vmcnt(0)4280; HEURRC-NEXT: s_nop 04281; HEURRC-NEXT: v_mov_b32_e32 v16, s84282; HEURRC-NEXT: v_mov_b32_e32 v17, s94283; HEURRC-NEXT: v_mov_b32_e32 v18, s104284; HEURRC-NEXT: v_mov_b32_e32 v19, s114285; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc14286; HEURRC-NEXT: s_waitcnt vmcnt(0)4287; HEURRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc14288; HEURRC-NEXT: s_waitcnt vmcnt(0)4289; HEURRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc14290; HEURRC-NEXT: s_waitcnt vmcnt(0)4291; HEURRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc14292; HEURRC-NEXT: s_waitcnt vmcnt(0)4293; HEURRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc14294; HEURRC-NEXT: s_waitcnt vmcnt(0)4295; HEURRC-NEXT: s_endpgm4296;4297; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:4298; VGPRRC: ; %bb.0:4299; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x244300; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa44301; VGPRRC-NEXT: v_mov_b32_e32 v40, 04302; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)4303; VGPRRC-NEXT: v_mov_b32_e32 v32, s204304; VGPRRC-NEXT: v_mov_b32_e32 v33, s214305; VGPRRC-NEXT: v_mov_b32_e32 v34, s224306; VGPRRC-NEXT: v_mov_b32_e32 v35, s234307; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x644308; VGPRRC-NEXT: v_mov_b32_e32 v36, s244309; VGPRRC-NEXT: v_mov_b32_e32 v37, s254310; VGPRRC-NEXT: v_mov_b32_e32 v38, s264311; VGPRRC-NEXT: v_mov_b32_e32 v39, s274312; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)4313; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]4314; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]4315; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]4316; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]4317; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]4318; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]4319; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]4320; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]4321; VGPRRC-NEXT: s_nop 14322; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]4323; VGPRRC-NEXT: s_nop 64324; VGPRRC-NEXT: v_mov_b32_e32 v16, s204325; VGPRRC-NEXT: v_mov_b32_e32 v17, s214326; VGPRRC-NEXT: v_mov_b32_e32 v18, s224327; VGPRRC-NEXT: v_mov_b32_e32 v19, s234328; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc14329; VGPRRC-NEXT: s_waitcnt vmcnt(0)4330; VGPRRC-NEXT: s_nop 04331; VGPRRC-NEXT: v_mov_b32_e32 v16, s164332; VGPRRC-NEXT: v_mov_b32_e32 v17, s174333; VGPRRC-NEXT: v_mov_b32_e32 v18, s184334; VGPRRC-NEXT: v_mov_b32_e32 v19, s194335; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc14336; VGPRRC-NEXT: s_waitcnt vmcnt(0)4337; VGPRRC-NEXT: s_nop 04338; VGPRRC-NEXT: v_mov_b32_e32 v16, s124339; VGPRRC-NEXT: v_mov_b32_e32 v17, s134340; VGPRRC-NEXT: v_mov_b32_e32 v18, s144341; VGPRRC-NEXT: v_mov_b32_e32 v19, s154342; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc14343; VGPRRC-NEXT: s_waitcnt vmcnt(0)4344; VGPRRC-NEXT: s_nop 04345; VGPRRC-NEXT: v_mov_b32_e32 v16, s84346; VGPRRC-NEXT: v_mov_b32_e32 v17, s94347; VGPRRC-NEXT: v_mov_b32_e32 v18, s104348; VGPRRC-NEXT: v_mov_b32_e32 v19, s114349; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc14350; VGPRRC-NEXT: s_waitcnt vmcnt(0)4351; VGPRRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc14352; VGPRRC-NEXT: s_waitcnt vmcnt(0)4353; VGPRRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc14354; VGPRRC-NEXT: s_waitcnt vmcnt(0)4355; VGPRRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc14356; VGPRRC-NEXT: s_waitcnt vmcnt(0)4357; VGPRRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc14358; VGPRRC-NEXT: s_waitcnt vmcnt(0)4359; VGPRRC-NEXT: s_endpgm4360; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:4361; AGPR: ; %bb.0:4362; AGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x244363; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa44364; AGPR-NEXT: v_mov_b32_e32 v8, 04365; AGPR-NEXT: s_waitcnt lgkmcnt(0)4366; AGPR-NEXT: v_mov_b32_e32 v0, s204367; AGPR-NEXT: v_mov_b32_e32 v1, s214368; AGPR-NEXT: v_mov_b32_e32 v2, s224369; AGPR-NEXT: v_mov_b32_e32 v3, s234370; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x644371; AGPR-NEXT: v_mov_b32_e32 v4, s244372; AGPR-NEXT: v_mov_b32_e32 v5, s254373; AGPR-NEXT: v_mov_b32_e32 v6, s264374; AGPR-NEXT: v_mov_b32_e32 v7, s274375; AGPR-NEXT: s_waitcnt lgkmcnt(0)4376; AGPR-NEXT: v_accvgpr_write_b32 a31, s234377; AGPR-NEXT: v_accvgpr_write_b32 a30, s224378; AGPR-NEXT: v_accvgpr_write_b32 a29, s214379; AGPR-NEXT: v_accvgpr_write_b32 a28, s204380; AGPR-NEXT: v_accvgpr_write_b32 a27, s194381; AGPR-NEXT: v_accvgpr_write_b32 a26, s184382; AGPR-NEXT: v_accvgpr_write_b32 a25, s174383; AGPR-NEXT: v_accvgpr_write_b32 a24, s164384; AGPR-NEXT: v_accvgpr_write_b32 a23, s154385; AGPR-NEXT: v_accvgpr_write_b32 a22, s144386; AGPR-NEXT: v_accvgpr_write_b32 a21, s134387; AGPR-NEXT: v_accvgpr_write_b32 a20, s124388; AGPR-NEXT: v_accvgpr_write_b32 a19, s114389; AGPR-NEXT: v_accvgpr_write_b32 a18, s104390; AGPR-NEXT: v_accvgpr_write_b32 a17, s94391; AGPR-NEXT: v_accvgpr_write_b32 a16, s84392; AGPR-NEXT: s_nop 14393; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[16:31]4394; AGPR-NEXT: v_mov_b32_e32 v0, s204395; AGPR-NEXT: v_mov_b32_e32 v1, s214396; AGPR-NEXT: v_mov_b32_e32 v2, s224397; AGPR-NEXT: v_mov_b32_e32 v3, s234398; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48 sc0 sc14399; AGPR-NEXT: s_waitcnt vmcnt(0)4400; AGPR-NEXT: s_nop 04401; AGPR-NEXT: v_mov_b32_e32 v0, s164402; AGPR-NEXT: v_mov_b32_e32 v1, s174403; AGPR-NEXT: v_mov_b32_e32 v2, s184404; AGPR-NEXT: v_mov_b32_e32 v3, s194405; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc14406; AGPR-NEXT: s_waitcnt vmcnt(0)4407; AGPR-NEXT: s_nop 04408; AGPR-NEXT: v_mov_b32_e32 v0, s124409; AGPR-NEXT: v_mov_b32_e32 v1, s134410; AGPR-NEXT: v_mov_b32_e32 v2, s144411; AGPR-NEXT: v_mov_b32_e32 v3, s154412; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc14413; AGPR-NEXT: s_waitcnt vmcnt(0)4414; AGPR-NEXT: s_nop 04415; AGPR-NEXT: v_mov_b32_e32 v0, s84416; AGPR-NEXT: v_mov_b32_e32 v1, s94417; AGPR-NEXT: v_mov_b32_e32 v2, s104418; AGPR-NEXT: v_mov_b32_e32 v3, s114419; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc14420; AGPR-NEXT: s_waitcnt vmcnt(0)4421; AGPR-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc14422; AGPR-NEXT: s_waitcnt vmcnt(0)4423; AGPR-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc14424; AGPR-NEXT: s_waitcnt vmcnt(0)4425; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc14426; AGPR-NEXT: s_waitcnt vmcnt(0)4427; AGPR-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc14428; AGPR-NEXT: s_waitcnt vmcnt(0)4429; AGPR-NEXT: s_endpgm4430; VGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:4431; VGPR: ; %bb.0:4432; VGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x244433; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa44434; VGPR-NEXT: v_mov_b32_e32 v40, 04435; VGPR-NEXT: s_waitcnt lgkmcnt(0)4436; VGPR-NEXT: v_mov_b32_e32 v32, s204437; VGPR-NEXT: v_mov_b32_e32 v33, s214438; VGPR-NEXT: v_mov_b32_e32 v34, s224439; VGPR-NEXT: v_mov_b32_e32 v35, s234440; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x644441; VGPR-NEXT: v_mov_b32_e32 v36, s244442; VGPR-NEXT: v_mov_b32_e32 v37, s254443; VGPR-NEXT: v_mov_b32_e32 v38, s264444; VGPR-NEXT: v_mov_b32_e32 v39, s274445; VGPR-NEXT: s_waitcnt lgkmcnt(0)4446; VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]4447; VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]4448; VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]4449; VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]4450; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]4451; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]4452; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]4453; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]4454; VGPR-NEXT: s_nop 14455; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]4456; VGPR-NEXT: s_nop 64457; VGPR-NEXT: v_mov_b32_e32 v16, s204458; VGPR-NEXT: v_mov_b32_e32 v17, s214459; VGPR-NEXT: v_mov_b32_e32 v18, s224460; VGPR-NEXT: v_mov_b32_e32 v19, s234461; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc14462; VGPR-NEXT: s_waitcnt vmcnt(0)4463; VGPR-NEXT: s_nop 04464; VGPR-NEXT: v_mov_b32_e32 v16, s164465; VGPR-NEXT: v_mov_b32_e32 v17, s174466; VGPR-NEXT: v_mov_b32_e32 v18, s184467; VGPR-NEXT: v_mov_b32_e32 v19, s194468; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc14469; VGPR-NEXT: s_waitcnt vmcnt(0)4470; VGPR-NEXT: s_nop 04471; VGPR-NEXT: v_mov_b32_e32 v16, s124472; VGPR-NEXT: v_mov_b32_e32 v17, s134473; VGPR-NEXT: v_mov_b32_e32 v18, s144474; VGPR-NEXT: v_mov_b32_e32 v19, s154475; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc14476; VGPR-NEXT: s_waitcnt vmcnt(0)4477; VGPR-NEXT: s_nop 04478; VGPR-NEXT: v_mov_b32_e32 v16, s84479; VGPR-NEXT: v_mov_b32_e32 v17, s94480; VGPR-NEXT: v_mov_b32_e32 v18, s104481; VGPR-NEXT: v_mov_b32_e32 v19, s114482; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc14483; VGPR-NEXT: s_waitcnt vmcnt(0)4484; VGPR-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc14485; VGPR-NEXT: s_waitcnt vmcnt(0)4486; VGPR-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc14487; VGPR-NEXT: s_waitcnt vmcnt(0)4488; VGPR-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc14489; VGPR-NEXT: s_waitcnt vmcnt(0)4490; VGPR-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc14491; VGPR-NEXT: s_waitcnt vmcnt(0)4492; VGPR-NEXT: s_endpgm4493 %result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 0, i32 0, i32 0)4494 store volatile <16 x i32> %arg2, ptr addrspace(1) %out4495 store volatile <16 x i32> %result, ptr addrspace(1) %out4496 ret void4497}4498 4499define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {4500; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:4501; SDAG: ; %bb.0:4502; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x244503; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa44504; SDAG-NEXT: v_mov_b32_e32 v40, 04505; SDAG-NEXT: s_waitcnt lgkmcnt(0)4506; SDAG-NEXT: v_mov_b32_e32 v32, s204507; SDAG-NEXT: v_mov_b32_e32 v33, s214508; SDAG-NEXT: v_mov_b32_e32 v34, s224509; SDAG-NEXT: v_mov_b32_e32 v35, s234510; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x644511; SDAG-NEXT: v_mov_b32_e32 v36, s244512; SDAG-NEXT: v_mov_b32_e32 v37, s254513; SDAG-NEXT: v_mov_b32_e32 v38, s264514; SDAG-NEXT: v_mov_b32_e32 v39, s274515; SDAG-NEXT: s_waitcnt lgkmcnt(0)4516; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]4517; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]4518; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]4519; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]4520; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]4521; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]4522; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]4523; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]4524; SDAG-NEXT: s_nop 14525; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:34526; SDAG-NEXT: s_nop 64527; SDAG-NEXT: v_mov_b32_e32 v16, s204528; SDAG-NEXT: v_mov_b32_e32 v17, s214529; SDAG-NEXT: v_mov_b32_e32 v18, s224530; SDAG-NEXT: v_mov_b32_e32 v19, s234531; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc14532; SDAG-NEXT: s_waitcnt vmcnt(0)4533; SDAG-NEXT: s_nop 04534; SDAG-NEXT: v_mov_b32_e32 v16, s164535; SDAG-NEXT: v_mov_b32_e32 v17, s174536; SDAG-NEXT: v_mov_b32_e32 v18, s184537; SDAG-NEXT: v_mov_b32_e32 v19, s194538; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc14539; SDAG-NEXT: s_waitcnt vmcnt(0)4540; SDAG-NEXT: s_nop 04541; SDAG-NEXT: v_mov_b32_e32 v16, s124542; SDAG-NEXT: v_mov_b32_e32 v17, s134543; SDAG-NEXT: v_mov_b32_e32 v18, s144544; SDAG-NEXT: v_mov_b32_e32 v19, s154545; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc14546; SDAG-NEXT: s_waitcnt vmcnt(0)4547; SDAG-NEXT: s_nop 04548; SDAG-NEXT: v_mov_b32_e32 v16, s84549; SDAG-NEXT: v_mov_b32_e32 v17, s94550; SDAG-NEXT: v_mov_b32_e32 v18, s104551; SDAG-NEXT: v_mov_b32_e32 v19, s114552; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc14553; SDAG-NEXT: s_waitcnt vmcnt(0)4554; SDAG-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc14555; SDAG-NEXT: s_waitcnt vmcnt(0)4556; SDAG-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc14557; SDAG-NEXT: s_waitcnt vmcnt(0)4558; SDAG-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc14559; SDAG-NEXT: s_waitcnt vmcnt(0)4560; SDAG-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc14561; SDAG-NEXT: s_waitcnt vmcnt(0)4562; SDAG-NEXT: s_endpgm4563;4564; GISEL-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:4565; GISEL: ; %bb.0:4566; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x244567; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x644568; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa44569; GISEL-NEXT: v_mov_b32_e32 v56, 04570; GISEL-NEXT: s_waitcnt lgkmcnt(0)4571; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]4572; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]4573; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]4574; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]4575; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]4576; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]4577; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]4578; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]4579; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]4580; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]4581; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]4582; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]4583; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]4584; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]4585; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:1 abid:2 blgp:34586; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[14:15]4587; GISEL-NEXT: v_mov_b64_e32 v[50:51], s[18:19]4588; GISEL-NEXT: v_mov_b64_e32 v[54:55], s[22:23]4589; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[12:13]4590; GISEL-NEXT: v_mov_b64_e32 v[48:49], s[16:17]4591; GISEL-NEXT: v_mov_b64_e32 v[52:53], s[20:21]4592; GISEL-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc14593; GISEL-NEXT: s_waitcnt vmcnt(0)4594; GISEL-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc14595; GISEL-NEXT: s_waitcnt vmcnt(0)4596; GISEL-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc14597; GISEL-NEXT: s_waitcnt vmcnt(0)4598; GISEL-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc14599; GISEL-NEXT: s_waitcnt vmcnt(0)4600; GISEL-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc14601; GISEL-NEXT: s_waitcnt vmcnt(0)4602; GISEL-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc14603; GISEL-NEXT: s_waitcnt vmcnt(0)4604; GISEL-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc14605; GISEL-NEXT: s_waitcnt vmcnt(0)4606; GISEL-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc14607; GISEL-NEXT: s_waitcnt vmcnt(0)4608; GISEL-NEXT: s_endpgm4609;4610; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:4611; HEURRC: ; %bb.0:4612; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x244613; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa44614; HEURRC-NEXT: v_mov_b32_e32 v40, 04615; HEURRC-NEXT: s_waitcnt lgkmcnt(0)4616; HEURRC-NEXT: v_mov_b32_e32 v32, s204617; HEURRC-NEXT: v_mov_b32_e32 v33, s214618; HEURRC-NEXT: v_mov_b32_e32 v34, s224619; HEURRC-NEXT: v_mov_b32_e32 v35, s234620; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x644621; HEURRC-NEXT: v_mov_b32_e32 v36, s244622; HEURRC-NEXT: v_mov_b32_e32 v37, s254623; HEURRC-NEXT: v_mov_b32_e32 v38, s264624; HEURRC-NEXT: v_mov_b32_e32 v39, s274625; HEURRC-NEXT: s_waitcnt lgkmcnt(0)4626; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]4627; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]4628; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]4629; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]4630; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]4631; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]4632; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]4633; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]4634; HEURRC-NEXT: s_nop 14635; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:34636; HEURRC-NEXT: s_nop 64637; HEURRC-NEXT: v_mov_b32_e32 v16, s204638; HEURRC-NEXT: v_mov_b32_e32 v17, s214639; HEURRC-NEXT: v_mov_b32_e32 v18, s224640; HEURRC-NEXT: v_mov_b32_e32 v19, s234641; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc14642; HEURRC-NEXT: s_waitcnt vmcnt(0)4643; HEURRC-NEXT: s_nop 04644; HEURRC-NEXT: v_mov_b32_e32 v16, s164645; HEURRC-NEXT: v_mov_b32_e32 v17, s174646; HEURRC-NEXT: v_mov_b32_e32 v18, s184647; HEURRC-NEXT: v_mov_b32_e32 v19, s194648; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc14649; HEURRC-NEXT: s_waitcnt vmcnt(0)4650; HEURRC-NEXT: s_nop 04651; HEURRC-NEXT: v_mov_b32_e32 v16, s124652; HEURRC-NEXT: v_mov_b32_e32 v17, s134653; HEURRC-NEXT: v_mov_b32_e32 v18, s144654; HEURRC-NEXT: v_mov_b32_e32 v19, s154655; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc14656; HEURRC-NEXT: s_waitcnt vmcnt(0)4657; HEURRC-NEXT: s_nop 04658; HEURRC-NEXT: v_mov_b32_e32 v16, s84659; HEURRC-NEXT: v_mov_b32_e32 v17, s94660; HEURRC-NEXT: v_mov_b32_e32 v18, s104661; HEURRC-NEXT: v_mov_b32_e32 v19, s114662; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc14663; HEURRC-NEXT: s_waitcnt vmcnt(0)4664; HEURRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc14665; HEURRC-NEXT: s_waitcnt vmcnt(0)4666; HEURRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc14667; HEURRC-NEXT: s_waitcnt vmcnt(0)4668; HEURRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc14669; HEURRC-NEXT: s_waitcnt vmcnt(0)4670; HEURRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc14671; HEURRC-NEXT: s_waitcnt vmcnt(0)4672; HEURRC-NEXT: s_endpgm4673;4674; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:4675; VGPRRC: ; %bb.0:4676; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x244677; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa44678; VGPRRC-NEXT: v_mov_b32_e32 v40, 04679; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)4680; VGPRRC-NEXT: v_mov_b32_e32 v32, s204681; VGPRRC-NEXT: v_mov_b32_e32 v33, s214682; VGPRRC-NEXT: v_mov_b32_e32 v34, s224683; VGPRRC-NEXT: v_mov_b32_e32 v35, s234684; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x644685; VGPRRC-NEXT: v_mov_b32_e32 v36, s244686; VGPRRC-NEXT: v_mov_b32_e32 v37, s254687; VGPRRC-NEXT: v_mov_b32_e32 v38, s264688; VGPRRC-NEXT: v_mov_b32_e32 v39, s274689; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)4690; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]4691; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]4692; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]4693; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]4694; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]4695; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]4696; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]4697; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]4698; VGPRRC-NEXT: s_nop 14699; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:34700; VGPRRC-NEXT: s_nop 64701; VGPRRC-NEXT: v_mov_b32_e32 v16, s204702; VGPRRC-NEXT: v_mov_b32_e32 v17, s214703; VGPRRC-NEXT: v_mov_b32_e32 v18, s224704; VGPRRC-NEXT: v_mov_b32_e32 v19, s234705; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc14706; VGPRRC-NEXT: s_waitcnt vmcnt(0)4707; VGPRRC-NEXT: s_nop 04708; VGPRRC-NEXT: v_mov_b32_e32 v16, s164709; VGPRRC-NEXT: v_mov_b32_e32 v17, s174710; VGPRRC-NEXT: v_mov_b32_e32 v18, s184711; VGPRRC-NEXT: v_mov_b32_e32 v19, s194712; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc14713; VGPRRC-NEXT: s_waitcnt vmcnt(0)4714; VGPRRC-NEXT: s_nop 04715; VGPRRC-NEXT: v_mov_b32_e32 v16, s124716; VGPRRC-NEXT: v_mov_b32_e32 v17, s134717; VGPRRC-NEXT: v_mov_b32_e32 v18, s144718; VGPRRC-NEXT: v_mov_b32_e32 v19, s154719; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc14720; VGPRRC-NEXT: s_waitcnt vmcnt(0)4721; VGPRRC-NEXT: s_nop 04722; VGPRRC-NEXT: v_mov_b32_e32 v16, s84723; VGPRRC-NEXT: v_mov_b32_e32 v17, s94724; VGPRRC-NEXT: v_mov_b32_e32 v18, s104725; VGPRRC-NEXT: v_mov_b32_e32 v19, s114726; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc14727; VGPRRC-NEXT: s_waitcnt vmcnt(0)4728; VGPRRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc14729; VGPRRC-NEXT: s_waitcnt vmcnt(0)4730; VGPRRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc14731; VGPRRC-NEXT: s_waitcnt vmcnt(0)4732; VGPRRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc14733; VGPRRC-NEXT: s_waitcnt vmcnt(0)4734; VGPRRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc14735; VGPRRC-NEXT: s_waitcnt vmcnt(0)4736; VGPRRC-NEXT: s_endpgm4737; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:4738; AGPR: ; %bb.0:4739; AGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x244740; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa44741; AGPR-NEXT: v_mov_b32_e32 v8, 04742; AGPR-NEXT: s_waitcnt lgkmcnt(0)4743; AGPR-NEXT: v_mov_b32_e32 v0, s204744; AGPR-NEXT: v_mov_b32_e32 v1, s214745; AGPR-NEXT: v_mov_b32_e32 v2, s224746; AGPR-NEXT: v_mov_b32_e32 v3, s234747; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x644748; AGPR-NEXT: v_mov_b32_e32 v4, s244749; AGPR-NEXT: v_mov_b32_e32 v5, s254750; AGPR-NEXT: v_mov_b32_e32 v6, s264751; AGPR-NEXT: v_mov_b32_e32 v7, s274752; AGPR-NEXT: s_waitcnt lgkmcnt(0)4753; AGPR-NEXT: v_accvgpr_write_b32 a31, s234754; AGPR-NEXT: v_accvgpr_write_b32 a30, s224755; AGPR-NEXT: v_accvgpr_write_b32 a29, s214756; AGPR-NEXT: v_accvgpr_write_b32 a28, s204757; AGPR-NEXT: v_accvgpr_write_b32 a27, s194758; AGPR-NEXT: v_accvgpr_write_b32 a26, s184759; AGPR-NEXT: v_accvgpr_write_b32 a25, s174760; AGPR-NEXT: v_accvgpr_write_b32 a24, s164761; AGPR-NEXT: v_accvgpr_write_b32 a23, s154762; AGPR-NEXT: v_accvgpr_write_b32 a22, s144763; AGPR-NEXT: v_accvgpr_write_b32 a21, s134764; AGPR-NEXT: v_accvgpr_write_b32 a20, s124765; AGPR-NEXT: v_accvgpr_write_b32 a19, s114766; AGPR-NEXT: v_accvgpr_write_b32 a18, s104767; AGPR-NEXT: v_accvgpr_write_b32 a17, s94768; AGPR-NEXT: v_accvgpr_write_b32 a16, s84769; AGPR-NEXT: s_nop 14770; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[16:31] cbsz:1 abid:2 blgp:34771; AGPR-NEXT: v_mov_b32_e32 v0, s204772; AGPR-NEXT: v_mov_b32_e32 v1, s214773; AGPR-NEXT: v_mov_b32_e32 v2, s224774; AGPR-NEXT: v_mov_b32_e32 v3, s234775; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48 sc0 sc14776; AGPR-NEXT: s_waitcnt vmcnt(0)4777; AGPR-NEXT: s_nop 04778; AGPR-NEXT: v_mov_b32_e32 v0, s164779; AGPR-NEXT: v_mov_b32_e32 v1, s174780; AGPR-NEXT: v_mov_b32_e32 v2, s184781; AGPR-NEXT: v_mov_b32_e32 v3, s194782; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc14783; AGPR-NEXT: s_waitcnt vmcnt(0)4784; AGPR-NEXT: s_nop 04785; AGPR-NEXT: v_mov_b32_e32 v0, s124786; AGPR-NEXT: v_mov_b32_e32 v1, s134787; AGPR-NEXT: v_mov_b32_e32 v2, s144788; AGPR-NEXT: v_mov_b32_e32 v3, s154789; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc14790; AGPR-NEXT: s_waitcnt vmcnt(0)4791; AGPR-NEXT: s_nop 04792; AGPR-NEXT: v_mov_b32_e32 v0, s84793; AGPR-NEXT: v_mov_b32_e32 v1, s94794; AGPR-NEXT: v_mov_b32_e32 v2, s104795; AGPR-NEXT: v_mov_b32_e32 v3, s114796; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc14797; AGPR-NEXT: s_waitcnt vmcnt(0)4798; AGPR-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc14799; AGPR-NEXT: s_waitcnt vmcnt(0)4800; AGPR-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc14801; AGPR-NEXT: s_waitcnt vmcnt(0)4802; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc14803; AGPR-NEXT: s_waitcnt vmcnt(0)4804; AGPR-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc14805; AGPR-NEXT: s_waitcnt vmcnt(0)4806; AGPR-NEXT: s_endpgm4807; VGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:4808; VGPR: ; %bb.0:4809; VGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x244810; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa44811; VGPR-NEXT: v_mov_b32_e32 v40, 04812; VGPR-NEXT: s_waitcnt lgkmcnt(0)4813; VGPR-NEXT: v_mov_b32_e32 v32, s204814; VGPR-NEXT: v_mov_b32_e32 v33, s214815; VGPR-NEXT: v_mov_b32_e32 v34, s224816; VGPR-NEXT: v_mov_b32_e32 v35, s234817; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x644818; VGPR-NEXT: v_mov_b32_e32 v36, s244819; VGPR-NEXT: v_mov_b32_e32 v37, s254820; VGPR-NEXT: v_mov_b32_e32 v38, s264821; VGPR-NEXT: v_mov_b32_e32 v39, s274822; VGPR-NEXT: s_waitcnt lgkmcnt(0)4823; VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]4824; VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]4825; VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]4826; VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]4827; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]4828; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]4829; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]4830; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]4831; VGPR-NEXT: s_nop 14832; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:34833; VGPR-NEXT: s_nop 64834; VGPR-NEXT: v_mov_b32_e32 v16, s204835; VGPR-NEXT: v_mov_b32_e32 v17, s214836; VGPR-NEXT: v_mov_b32_e32 v18, s224837; VGPR-NEXT: v_mov_b32_e32 v19, s234838; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc14839; VGPR-NEXT: s_waitcnt vmcnt(0)4840; VGPR-NEXT: s_nop 04841; VGPR-NEXT: v_mov_b32_e32 v16, s164842; VGPR-NEXT: v_mov_b32_e32 v17, s174843; VGPR-NEXT: v_mov_b32_e32 v18, s184844; VGPR-NEXT: v_mov_b32_e32 v19, s194845; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc14846; VGPR-NEXT: s_waitcnt vmcnt(0)4847; VGPR-NEXT: s_nop 04848; VGPR-NEXT: v_mov_b32_e32 v16, s124849; VGPR-NEXT: v_mov_b32_e32 v17, s134850; VGPR-NEXT: v_mov_b32_e32 v18, s144851; VGPR-NEXT: v_mov_b32_e32 v19, s154852; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc14853; VGPR-NEXT: s_waitcnt vmcnt(0)4854; VGPR-NEXT: s_nop 04855; VGPR-NEXT: v_mov_b32_e32 v16, s84856; VGPR-NEXT: v_mov_b32_e32 v17, s94857; VGPR-NEXT: v_mov_b32_e32 v18, s104858; VGPR-NEXT: v_mov_b32_e32 v19, s114859; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc14860; VGPR-NEXT: s_waitcnt vmcnt(0)4861; VGPR-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc14862; VGPR-NEXT: s_waitcnt vmcnt(0)4863; VGPR-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc14864; VGPR-NEXT: s_waitcnt vmcnt(0)4865; VGPR-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc14866; VGPR-NEXT: s_waitcnt vmcnt(0)4867; VGPR-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc14868; VGPR-NEXT: s_waitcnt vmcnt(0)4869; VGPR-NEXT: s_endpgm4870 %result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 1, i32 2, i32 3)4871 store volatile <16 x i32> %arg2, ptr addrspace(1) %out4872 store volatile <16 x i32> %result, ptr addrspace(1) %out4873 ret void4874}4875 4876define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {4877; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:4878; SDAG: ; %bb.0:4879; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x244880; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa44881; SDAG-NEXT: s_waitcnt lgkmcnt(0)4882; SDAG-NEXT: v_mov_b32_e32 v16, s204883; SDAG-NEXT: v_mov_b32_e32 v17, s214884; SDAG-NEXT: v_mov_b32_e32 v18, s224885; SDAG-NEXT: v_mov_b32_e32 v19, s234886; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x644887; SDAG-NEXT: v_mov_b32_e32 v20, s244888; SDAG-NEXT: v_mov_b32_e32 v21, s254889; SDAG-NEXT: v_mov_b32_e32 v22, s264890; SDAG-NEXT: v_mov_b32_e32 v23, s274891; SDAG-NEXT: s_waitcnt lgkmcnt(0)4892; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]4893; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]4894; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]4895; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]4896; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]4897; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]4898; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]4899; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]4900; SDAG-NEXT: s_nop 14901; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]4902; SDAG-NEXT: v_mov_b32_e32 v16, 04903; SDAG-NEXT: s_nop 104904; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:484905; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:324906; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:164907; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]4908; SDAG-NEXT: s_endpgm4909;4910; GISEL-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:4911; GISEL: ; %bb.0:4912; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x244913; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x644914; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa44915; GISEL-NEXT: s_waitcnt lgkmcnt(0)4916; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[24:25]4917; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[26:27]4918; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[28:29]4919; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]4920; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[30:31]4921; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]4922; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]4923; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]4924; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]4925; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]4926; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]4927; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]4928; GISEL-NEXT: s_nop 14929; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]4930; GISEL-NEXT: v_mov_b32_e32 v16, 04931; GISEL-NEXT: s_nop 104932; GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]4933; GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:164934; GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:324935; GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:484936; GISEL-NEXT: s_endpgm4937;4938; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:4939; HEURRC: ; %bb.0:4940; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x244941; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa44942; HEURRC-NEXT: s_waitcnt lgkmcnt(0)4943; HEURRC-NEXT: v_mov_b32_e32 v16, s204944; HEURRC-NEXT: v_mov_b32_e32 v17, s214945; HEURRC-NEXT: v_mov_b32_e32 v18, s224946; HEURRC-NEXT: v_mov_b32_e32 v19, s234947; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x644948; HEURRC-NEXT: v_mov_b32_e32 v20, s244949; HEURRC-NEXT: v_mov_b32_e32 v21, s254950; HEURRC-NEXT: v_mov_b32_e32 v22, s264951; HEURRC-NEXT: v_mov_b32_e32 v23, s274952; HEURRC-NEXT: s_waitcnt lgkmcnt(0)4953; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]4954; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]4955; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]4956; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]4957; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]4958; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]4959; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]4960; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]4961; HEURRC-NEXT: s_nop 14962; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]4963; HEURRC-NEXT: v_mov_b32_e32 v16, 04964; HEURRC-NEXT: s_nop 104965; HEURRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:484966; HEURRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:324967; HEURRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:164968; HEURRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]4969; HEURRC-NEXT: s_endpgm4970;4971; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:4972; VGPRRC: ; %bb.0:4973; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x244974; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa44975; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)4976; VGPRRC-NEXT: v_mov_b32_e32 v16, s204977; VGPRRC-NEXT: v_mov_b32_e32 v17, s214978; VGPRRC-NEXT: v_mov_b32_e32 v18, s224979; VGPRRC-NEXT: v_mov_b32_e32 v19, s234980; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x644981; VGPRRC-NEXT: v_mov_b32_e32 v20, s244982; VGPRRC-NEXT: v_mov_b32_e32 v21, s254983; VGPRRC-NEXT: v_mov_b32_e32 v22, s264984; VGPRRC-NEXT: v_mov_b32_e32 v23, s274985; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)4986; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]4987; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]4988; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]4989; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]4990; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]4991; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]4992; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]4993; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]4994; VGPRRC-NEXT: s_nop 14995; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]4996; VGPRRC-NEXT: v_mov_b32_e32 v16, 04997; VGPRRC-NEXT: s_nop 104998; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:484999; VGPRRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:325000; VGPRRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:165001; VGPRRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]5002; VGPRRC-NEXT: s_endpgm5003; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:5004; AGPR: ; %bb.0:5005; AGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x245006; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa45007; AGPR-NEXT: s_waitcnt lgkmcnt(0)5008; AGPR-NEXT: v_mov_b32_e32 v0, s205009; AGPR-NEXT: v_mov_b32_e32 v1, s215010; AGPR-NEXT: v_mov_b32_e32 v2, s225011; AGPR-NEXT: v_mov_b32_e32 v3, s235012; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x645013; AGPR-NEXT: v_mov_b32_e32 v4, s245014; AGPR-NEXT: v_mov_b32_e32 v5, s255015; AGPR-NEXT: v_mov_b32_e32 v6, s265016; AGPR-NEXT: v_mov_b32_e32 v7, s275017; AGPR-NEXT: s_waitcnt lgkmcnt(0)5018; AGPR-NEXT: v_accvgpr_write_b32 a0, s85019; AGPR-NEXT: v_accvgpr_write_b32 a1, s95020; AGPR-NEXT: v_accvgpr_write_b32 a2, s105021; AGPR-NEXT: v_accvgpr_write_b32 a3, s115022; AGPR-NEXT: v_accvgpr_write_b32 a4, s125023; AGPR-NEXT: v_accvgpr_write_b32 a5, s135024; AGPR-NEXT: v_accvgpr_write_b32 a6, s145025; AGPR-NEXT: v_accvgpr_write_b32 a7, s155026; AGPR-NEXT: v_accvgpr_write_b32 a8, s165027; AGPR-NEXT: v_accvgpr_write_b32 a9, s175028; AGPR-NEXT: v_accvgpr_write_b32 a10, s185029; AGPR-NEXT: v_accvgpr_write_b32 a11, s195030; AGPR-NEXT: v_accvgpr_write_b32 a12, s205031; AGPR-NEXT: v_accvgpr_write_b32 a13, s215032; AGPR-NEXT: v_accvgpr_write_b32 a14, s225033; AGPR-NEXT: v_accvgpr_write_b32 a15, s235034; AGPR-NEXT: s_nop 15035; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15]5036; AGPR-NEXT: v_mov_b32_e32 v0, 05037; AGPR-NEXT: s_nop 75038; AGPR-NEXT: s_nop 25039; AGPR-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:485040; AGPR-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:325041; AGPR-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:165042; AGPR-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1]5043; AGPR-NEXT: s_endpgm5044; VGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:5045; VGPR: ; %bb.0:5046; VGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x245047; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa45048; VGPR-NEXT: s_waitcnt lgkmcnt(0)5049; VGPR-NEXT: v_mov_b32_e32 v16, s205050; VGPR-NEXT: v_mov_b32_e32 v17, s215051; VGPR-NEXT: v_mov_b32_e32 v18, s225052; VGPR-NEXT: v_mov_b32_e32 v19, s235053; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x645054; VGPR-NEXT: v_mov_b32_e32 v20, s245055; VGPR-NEXT: v_mov_b32_e32 v21, s255056; VGPR-NEXT: v_mov_b32_e32 v22, s265057; VGPR-NEXT: v_mov_b32_e32 v23, s275058; VGPR-NEXT: s_waitcnt lgkmcnt(0)5059; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]5060; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]5061; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]5062; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]5063; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]5064; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]5065; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]5066; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]5067; VGPR-NEXT: s_nop 15068; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]5069; VGPR-NEXT: v_mov_b32_e32 v16, 05070; VGPR-NEXT: s_nop 75071; VGPR-NEXT: s_nop 25072; VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:485073; VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:325074; VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:165075; VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]5076; VGPR-NEXT: s_endpgm5077 %result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 0, i32 0, i32 0)5078 store <16 x i32> %result, ptr addrspace(1) %out5079 ret void5080}5081 5082define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {5083; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:5084; SDAG: ; %bb.0:5085; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x245086; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa45087; SDAG-NEXT: s_waitcnt lgkmcnt(0)5088; SDAG-NEXT: v_mov_b32_e32 v16, s205089; SDAG-NEXT: v_mov_b32_e32 v17, s215090; SDAG-NEXT: v_mov_b32_e32 v18, s225091; SDAG-NEXT: v_mov_b32_e32 v19, s235092; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x645093; SDAG-NEXT: v_mov_b32_e32 v20, s245094; SDAG-NEXT: v_mov_b32_e32 v21, s255095; SDAG-NEXT: v_mov_b32_e32 v22, s265096; SDAG-NEXT: v_mov_b32_e32 v23, s275097; SDAG-NEXT: s_waitcnt lgkmcnt(0)5098; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]5099; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]5100; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]5101; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]5102; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]5103; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]5104; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]5105; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]5106; SDAG-NEXT: s_nop 15107; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:15108; SDAG-NEXT: v_mov_b32_e32 v16, 05109; SDAG-NEXT: s_nop 105110; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:485111; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:325112; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:165113; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]5114; SDAG-NEXT: s_endpgm5115;5116; GISEL-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:5117; GISEL: ; %bb.0:5118; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x245119; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x645120; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa45121; GISEL-NEXT: s_waitcnt lgkmcnt(0)5122; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[24:25]5123; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[26:27]5124; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[28:29]5125; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]5126; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[30:31]5127; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]5128; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]5129; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]5130; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]5131; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]5132; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]5133; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]5134; GISEL-NEXT: s_nop 15135; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:15136; GISEL-NEXT: v_mov_b32_e32 v16, 05137; GISEL-NEXT: s_nop 105138; GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]5139; GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:165140; GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:325141; GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:485142; GISEL-NEXT: s_endpgm5143;5144; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:5145; HEURRC: ; %bb.0:5146; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x245147; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa45148; HEURRC-NEXT: s_waitcnt lgkmcnt(0)5149; HEURRC-NEXT: v_mov_b32_e32 v16, s205150; HEURRC-NEXT: v_mov_b32_e32 v17, s215151; HEURRC-NEXT: v_mov_b32_e32 v18, s225152; HEURRC-NEXT: v_mov_b32_e32 v19, s235153; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x645154; HEURRC-NEXT: v_mov_b32_e32 v20, s245155; HEURRC-NEXT: v_mov_b32_e32 v21, s255156; HEURRC-NEXT: v_mov_b32_e32 v22, s265157; HEURRC-NEXT: v_mov_b32_e32 v23, s275158; HEURRC-NEXT: s_waitcnt lgkmcnt(0)5159; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]5160; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]5161; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]5162; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]5163; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]5164; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]5165; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]5166; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]5167; HEURRC-NEXT: s_nop 15168; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:15169; HEURRC-NEXT: v_mov_b32_e32 v16, 05170; HEURRC-NEXT: s_nop 105171; HEURRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:485172; HEURRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:325173; HEURRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:165174; HEURRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]5175; HEURRC-NEXT: s_endpgm5176;5177; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:5178; VGPRRC: ; %bb.0:5179; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x245180; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa45181; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)5182; VGPRRC-NEXT: v_mov_b32_e32 v16, s205183; VGPRRC-NEXT: v_mov_b32_e32 v17, s215184; VGPRRC-NEXT: v_mov_b32_e32 v18, s225185; VGPRRC-NEXT: v_mov_b32_e32 v19, s235186; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x645187; VGPRRC-NEXT: v_mov_b32_e32 v20, s245188; VGPRRC-NEXT: v_mov_b32_e32 v21, s255189; VGPRRC-NEXT: v_mov_b32_e32 v22, s265190; VGPRRC-NEXT: v_mov_b32_e32 v23, s275191; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)5192; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]5193; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]5194; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]5195; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]5196; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]5197; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]5198; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]5199; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]5200; VGPRRC-NEXT: s_nop 15201; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:15202; VGPRRC-NEXT: v_mov_b32_e32 v16, 05203; VGPRRC-NEXT: s_nop 105204; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:485205; VGPRRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:325206; VGPRRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:165207; VGPRRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]5208; VGPRRC-NEXT: s_endpgm5209; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:5210; AGPR: ; %bb.0:5211; AGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x245212; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa45213; AGPR-NEXT: s_waitcnt lgkmcnt(0)5214; AGPR-NEXT: v_mov_b32_e32 v0, s205215; AGPR-NEXT: v_mov_b32_e32 v1, s215216; AGPR-NEXT: v_mov_b32_e32 v2, s225217; AGPR-NEXT: v_mov_b32_e32 v3, s235218; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x645219; AGPR-NEXT: v_mov_b32_e32 v4, s245220; AGPR-NEXT: v_mov_b32_e32 v5, s255221; AGPR-NEXT: v_mov_b32_e32 v6, s265222; AGPR-NEXT: v_mov_b32_e32 v7, s275223; AGPR-NEXT: s_waitcnt lgkmcnt(0)5224; AGPR-NEXT: v_accvgpr_write_b32 a0, s85225; AGPR-NEXT: v_accvgpr_write_b32 a1, s95226; AGPR-NEXT: v_accvgpr_write_b32 a2, s105227; AGPR-NEXT: v_accvgpr_write_b32 a3, s115228; AGPR-NEXT: v_accvgpr_write_b32 a4, s125229; AGPR-NEXT: v_accvgpr_write_b32 a5, s135230; AGPR-NEXT: v_accvgpr_write_b32 a6, s145231; AGPR-NEXT: v_accvgpr_write_b32 a7, s155232; AGPR-NEXT: v_accvgpr_write_b32 a8, s165233; AGPR-NEXT: v_accvgpr_write_b32 a9, s175234; AGPR-NEXT: v_accvgpr_write_b32 a10, s185235; AGPR-NEXT: v_accvgpr_write_b32 a11, s195236; AGPR-NEXT: v_accvgpr_write_b32 a12, s205237; AGPR-NEXT: v_accvgpr_write_b32 a13, s215238; AGPR-NEXT: v_accvgpr_write_b32 a14, s225239; AGPR-NEXT: v_accvgpr_write_b32 a15, s235240; AGPR-NEXT: s_nop 15241; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:3 abid:2 blgp:15242; AGPR-NEXT: v_mov_b32_e32 v0, 05243; AGPR-NEXT: s_nop 75244; AGPR-NEXT: s_nop 25245; AGPR-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:485246; AGPR-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:325247; AGPR-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:165248; AGPR-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1]5249; AGPR-NEXT: s_endpgm5250; VGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:5251; VGPR: ; %bb.0:5252; VGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x245253; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa45254; VGPR-NEXT: s_waitcnt lgkmcnt(0)5255; VGPR-NEXT: v_mov_b32_e32 v16, s205256; VGPR-NEXT: v_mov_b32_e32 v17, s215257; VGPR-NEXT: v_mov_b32_e32 v18, s225258; VGPR-NEXT: v_mov_b32_e32 v19, s235259; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x645260; VGPR-NEXT: v_mov_b32_e32 v20, s245261; VGPR-NEXT: v_mov_b32_e32 v21, s255262; VGPR-NEXT: v_mov_b32_e32 v22, s265263; VGPR-NEXT: v_mov_b32_e32 v23, s275264; VGPR-NEXT: s_waitcnt lgkmcnt(0)5265; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]5266; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]5267; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]5268; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]5269; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]5270; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]5271; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]5272; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]5273; VGPR-NEXT: s_nop 15274; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:15275; VGPR-NEXT: v_mov_b32_e32 v16, 05276; VGPR-NEXT: s_nop 75277; VGPR-NEXT: s_nop 25278; VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:485279; VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:325280; VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:165281; VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]5282; VGPR-NEXT: s_endpgm5283 %result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 3, i32 2, i32 1)5284 store <16 x i32> %result, ptr addrspace(1) %out5285 ret void5286}5287 5288; --------------------------------------------------------------------5289; llvm.amdgcn.mfma.f32.16x16x32.bf165290; --------------------------------------------------------------------5291 5292declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat>, <8 x bfloat>, <4 x float>, i32 immarg, i32 immarg, i32 immarg)5293 5294define <4 x float> @test_mfma_f32_16x16x32_bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2) {5295; GCN-LABEL: test_mfma_f32_16x16x32_bf16:5296; GCN: ; %bb.0:5297; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5298; GCN-NEXT: v_accvgpr_write_b32 a0, v85299; GCN-NEXT: v_accvgpr_write_b32 a1, v95300; GCN-NEXT: v_accvgpr_write_b32 a2, v105301; GCN-NEXT: v_accvgpr_write_b32 a3, v115302; GCN-NEXT: s_nop 15303; GCN-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3]5304; GCN-NEXT: s_nop 75305; GCN-NEXT: v_accvgpr_read_b32 v0, a05306; GCN-NEXT: v_accvgpr_read_b32 v1, a15307; GCN-NEXT: v_accvgpr_read_b32 v2, a25308; GCN-NEXT: v_accvgpr_read_b32 v3, a35309; GCN-NEXT: s_setpc_b64 s[30:31]5310;5311; HEURRC-LABEL: test_mfma_f32_16x16x32_bf16:5312; HEURRC: ; %bb.0:5313; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5314; HEURRC-NEXT: v_accvgpr_write_b32 a0, v85315; HEURRC-NEXT: v_accvgpr_write_b32 a1, v95316; HEURRC-NEXT: v_accvgpr_write_b32 a2, v105317; HEURRC-NEXT: v_accvgpr_write_b32 a3, v115318; HEURRC-NEXT: s_nop 15319; HEURRC-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3]5320; HEURRC-NEXT: s_nop 75321; HEURRC-NEXT: v_accvgpr_read_b32 v0, a05322; HEURRC-NEXT: v_accvgpr_read_b32 v1, a15323; HEURRC-NEXT: v_accvgpr_read_b32 v2, a25324; HEURRC-NEXT: v_accvgpr_read_b32 v3, a35325; HEURRC-NEXT: s_setpc_b64 s[30:31]5326;5327; VGPRRC-LABEL: test_mfma_f32_16x16x32_bf16:5328; VGPRRC: ; %bb.0:5329; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5330; VGPRRC-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11]5331; VGPRRC-NEXT: s_setpc_b64 s[30:31]5332; AGPR-LABEL: test_mfma_f32_16x16x32_bf16:5333; AGPR: ; %bb.0:5334; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5335; AGPR-NEXT: v_accvgpr_write_b32 a0, v85336; AGPR-NEXT: v_accvgpr_write_b32 a1, v95337; AGPR-NEXT: v_accvgpr_write_b32 a2, v105338; AGPR-NEXT: v_accvgpr_write_b32 a3, v115339; AGPR-NEXT: s_nop 15340; AGPR-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3]5341; AGPR-NEXT: s_nop 75342; AGPR-NEXT: v_accvgpr_read_b32 v0, a05343; AGPR-NEXT: v_accvgpr_read_b32 v1, a15344; AGPR-NEXT: v_accvgpr_read_b32 v2, a25345; AGPR-NEXT: v_accvgpr_read_b32 v3, a35346; AGPR-NEXT: s_setpc_b64 s[30:31]5347; VGPR-LABEL: test_mfma_f32_16x16x32_bf16:5348; VGPR: ; %bb.0:5349; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5350; VGPR-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11]5351; VGPR-NEXT: s_setpc_b64 s[30:31]5352 %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0)5353 ret <4 x float> %result5354}5355 5356define <4 x float> @test_mfma_f32_16x16x32_bf16__flags(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2) {5357; GCN-LABEL: test_mfma_f32_16x16x32_bf16__flags:5358; GCN: ; %bb.0:5359; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5360; GCN-NEXT: v_accvgpr_write_b32 a0, v85361; GCN-NEXT: v_accvgpr_write_b32 a1, v95362; GCN-NEXT: v_accvgpr_write_b32 a2, v105363; GCN-NEXT: v_accvgpr_write_b32 a3, v115364; GCN-NEXT: s_nop 15365; GCN-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:15366; GCN-NEXT: s_nop 75367; GCN-NEXT: v_accvgpr_read_b32 v0, a05368; GCN-NEXT: v_accvgpr_read_b32 v1, a15369; GCN-NEXT: v_accvgpr_read_b32 v2, a25370; GCN-NEXT: v_accvgpr_read_b32 v3, a35371; GCN-NEXT: s_setpc_b64 s[30:31]5372;5373; HEURRC-LABEL: test_mfma_f32_16x16x32_bf16__flags:5374; HEURRC: ; %bb.0:5375; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5376; HEURRC-NEXT: v_accvgpr_write_b32 a0, v85377; HEURRC-NEXT: v_accvgpr_write_b32 a1, v95378; HEURRC-NEXT: v_accvgpr_write_b32 a2, v105379; HEURRC-NEXT: v_accvgpr_write_b32 a3, v115380; HEURRC-NEXT: s_nop 15381; HEURRC-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:15382; HEURRC-NEXT: s_nop 75383; HEURRC-NEXT: v_accvgpr_read_b32 v0, a05384; HEURRC-NEXT: v_accvgpr_read_b32 v1, a15385; HEURRC-NEXT: v_accvgpr_read_b32 v2, a25386; HEURRC-NEXT: v_accvgpr_read_b32 v3, a35387; HEURRC-NEXT: s_setpc_b64 s[30:31]5388;5389; VGPRRC-LABEL: test_mfma_f32_16x16x32_bf16__flags:5390; VGPRRC: ; %bb.0:5391; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5392; VGPRRC-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:15393; VGPRRC-NEXT: s_setpc_b64 s[30:31]5394; AGPR-LABEL: test_mfma_f32_16x16x32_bf16__flags:5395; AGPR: ; %bb.0:5396; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5397; AGPR-NEXT: v_accvgpr_write_b32 a0, v85398; AGPR-NEXT: v_accvgpr_write_b32 a1, v95399; AGPR-NEXT: v_accvgpr_write_b32 a2, v105400; AGPR-NEXT: v_accvgpr_write_b32 a3, v115401; AGPR-NEXT: s_nop 15402; AGPR-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:15403; AGPR-NEXT: s_nop 75404; AGPR-NEXT: v_accvgpr_read_b32 v0, a05405; AGPR-NEXT: v_accvgpr_read_b32 v1, a15406; AGPR-NEXT: v_accvgpr_read_b32 v2, a25407; AGPR-NEXT: v_accvgpr_read_b32 v3, a35408; AGPR-NEXT: s_setpc_b64 s[30:31]5409; VGPR-LABEL: test_mfma_f32_16x16x32_bf16__flags:5410; VGPR: ; %bb.0:5411; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5412; VGPR-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:15413; VGPR-NEXT: s_setpc_b64 s[30:31]5414 %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2, i32 1, i32 1, i32 1)5415 ret <4 x float> %result5416}5417 5418define amdgpu_kernel void @test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd(ptr addrspace(1) %out, <8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2) #0 {5419; GCN-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:5420; GCN: ; %bb.0:5421; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x345422; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x545423; GCN-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x245424; GCN-NEXT: v_mov_b32_e32 v4, 05425; GCN-NEXT: s_waitcnt lgkmcnt(0)5426; GCN-NEXT: v_mov_b64_e32 v[6:7], s[8:9]5427; GCN-NEXT: v_mov_b64_e32 v[8:9], s[10:11]5428; GCN-NEXT: v_mov_b64_e32 v[10:11], s[12:13]5429; GCN-NEXT: v_mov_b64_e32 v[0:1], s[0:1]5430; GCN-NEXT: v_mov_b64_e32 v[12:13], s[14:15]5431; GCN-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5432; GCN-NEXT: s_nop 15433; GCN-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[6:9], v[10:13], v[0:3]5434; GCN-NEXT: s_nop 75435; GCN-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]5436; GCN-NEXT: s_endpgm5437;5438; HEURRC-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:5439; HEURRC: ; %bb.0:5440; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x345441; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x545442; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x245443; HEURRC-NEXT: v_mov_b32_e32 v4, 05444; HEURRC-NEXT: s_waitcnt lgkmcnt(0)5445; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[8:9]5446; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[10:11]5447; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[12:13]5448; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[0:1]5449; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[14:15]5450; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5451; HEURRC-NEXT: s_nop 15452; HEURRC-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[6:9], v[10:13], v[0:3]5453; HEURRC-NEXT: s_nop 75454; HEURRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]5455; HEURRC-NEXT: s_endpgm5456;5457; VGPRRC-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:5458; VGPRRC: ; %bb.0:5459; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x345460; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x545461; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x245462; VGPRRC-NEXT: v_mov_b32_e32 v4, 05463; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)5464; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[8:9]5465; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[10:11]5466; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[12:13]5467; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[0:1]5468; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[14:15]5469; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5470; VGPRRC-NEXT: s_nop 15471; VGPRRC-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[6:9], v[10:13], v[0:3]5472; VGPRRC-NEXT: s_nop 75473; VGPRRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]5474; VGPRRC-NEXT: s_endpgm5475; AGPR-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:5476; AGPR: ; %bb.0:5477; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x345478; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x545479; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x245480; AGPR-NEXT: v_mov_b32_e32 v8, 05481; AGPR-NEXT: s_waitcnt lgkmcnt(0)5482; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]5483; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]5484; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]5485; AGPR-NEXT: v_accvgpr_write_b32 a0, s05486; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]5487; AGPR-NEXT: v_accvgpr_write_b32 a1, s15488; AGPR-NEXT: v_accvgpr_write_b32 a2, s25489; AGPR-NEXT: v_accvgpr_write_b32 a3, s35490; AGPR-NEXT: s_nop 15491; AGPR-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3]5492; AGPR-NEXT: s_nop 75493; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7]5494; AGPR-NEXT: s_endpgm5495; VGPR-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:5496; VGPR: ; %bb.0:5497; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x345498; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x545499; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x245500; VGPR-NEXT: v_mov_b32_e32 v12, 05501; VGPR-NEXT: s_waitcnt lgkmcnt(0)5502; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]5503; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]5504; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]5505; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3]5506; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]5507; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[0:1]5508; VGPR-NEXT: s_nop 15509; VGPR-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11]5510; VGPR-NEXT: s_nop 75511; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]5512; VGPR-NEXT: s_endpgm5513 %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0)5514 store <4 x float> %result, ptr addrspace(1) %out5515 ret void5516}5517 5518define amdgpu_kernel void @test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags(ptr addrspace(1) %out, <8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2) #0 {5519; GCN-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:5520; GCN: ; %bb.0:5521; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x345522; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x545523; GCN-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x245524; GCN-NEXT: v_mov_b32_e32 v4, 05525; GCN-NEXT: s_waitcnt lgkmcnt(0)5526; GCN-NEXT: v_mov_b64_e32 v[6:7], s[8:9]5527; GCN-NEXT: v_mov_b64_e32 v[8:9], s[10:11]5528; GCN-NEXT: v_mov_b64_e32 v[10:11], s[12:13]5529; GCN-NEXT: v_mov_b64_e32 v[0:1], s[0:1]5530; GCN-NEXT: v_mov_b64_e32 v[12:13], s[14:15]5531; GCN-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5532; GCN-NEXT: s_nop 15533; GCN-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:15534; GCN-NEXT: s_nop 75535; GCN-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]5536; GCN-NEXT: s_endpgm5537;5538; HEURRC-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:5539; HEURRC: ; %bb.0:5540; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x345541; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x545542; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x245543; HEURRC-NEXT: v_mov_b32_e32 v4, 05544; HEURRC-NEXT: s_waitcnt lgkmcnt(0)5545; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[8:9]5546; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[10:11]5547; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[12:13]5548; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[0:1]5549; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[14:15]5550; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5551; HEURRC-NEXT: s_nop 15552; HEURRC-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:15553; HEURRC-NEXT: s_nop 75554; HEURRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]5555; HEURRC-NEXT: s_endpgm5556;5557; VGPRRC-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:5558; VGPRRC: ; %bb.0:5559; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x345560; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x545561; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x245562; VGPRRC-NEXT: v_mov_b32_e32 v4, 05563; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)5564; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[8:9]5565; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[10:11]5566; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[12:13]5567; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[0:1]5568; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[14:15]5569; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5570; VGPRRC-NEXT: s_nop 15571; VGPRRC-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:15572; VGPRRC-NEXT: s_nop 75573; VGPRRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]5574; VGPRRC-NEXT: s_endpgm5575; AGPR-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:5576; AGPR: ; %bb.0:5577; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x345578; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x545579; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x245580; AGPR-NEXT: v_mov_b32_e32 v8, 05581; AGPR-NEXT: s_waitcnt lgkmcnt(0)5582; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]5583; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]5584; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]5585; AGPR-NEXT: v_accvgpr_write_b32 a0, s05586; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]5587; AGPR-NEXT: v_accvgpr_write_b32 a1, s15588; AGPR-NEXT: v_accvgpr_write_b32 a2, s25589; AGPR-NEXT: v_accvgpr_write_b32 a3, s35590; AGPR-NEXT: s_nop 15591; AGPR-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:15592; AGPR-NEXT: s_nop 75593; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7]5594; AGPR-NEXT: s_endpgm5595; VGPR-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:5596; VGPR: ; %bb.0:5597; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x345598; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x545599; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x245600; VGPR-NEXT: v_mov_b32_e32 v12, 05601; VGPR-NEXT: s_waitcnt lgkmcnt(0)5602; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]5603; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]5604; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]5605; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3]5606; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]5607; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[0:1]5608; VGPR-NEXT: s_nop 15609; VGPR-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:15610; VGPR-NEXT: s_nop 75611; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]5612; VGPR-NEXT: s_endpgm5613 %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2, i32 3, i32 2, i32 1)5614 store <4 x float> %result, ptr addrspace(1) %out5615 ret void5616}5617 5618attributes #0 = { "amdgpu-flat-work-group-size"="512,512" "amdgpu-agpr-alloc"="0,0" }5619attributes #1 = { "amdgpu-flat-work-group-size"="1,64" }5620