brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 42; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s --check-prefix=GFX12503; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -global-isel -global-isel-abort=2 < %s | FileCheck %s --check-prefix=GISEL4 5define amdgpu_ps void @test_wmma_f32_16x16x4_f32(<2 x float> %A, <2 x float> %B, <8 x float> %C, ptr addrspace(1) %out) {6; GFX1250-LABEL: test_wmma_f32_16x16x4_f32:7; GFX1250:       ; %bb.0: ; %bb8; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19; GFX1250-NEXT:    v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] matrix_b_reuse10; GFX1250-NEXT:    s_clause 0x111; GFX1250-NEXT:    global_store_b128 v[12:13], v[8:11], off offset:1612; GFX1250-NEXT:    global_store_b128 v[12:13], v[4:7], off13; GFX1250-NEXT:    s_endpgm14;15; GISEL-LABEL: test_wmma_f32_16x16x4_f32:16; GISEL:       ; %bb.0: ; %bb17; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 118; GISEL-NEXT:    v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] matrix_b_reuse19; GISEL-NEXT:    s_clause 0x120; GISEL-NEXT:    global_store_b128 v[12:13], v[4:7], off21; GISEL-NEXT:    global_store_b128 v[12:13], v[8:11], off offset:1622; GISEL-NEXT:    s_endpgm23bb:24  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x4.f32.v8f32.v2f32(i1 0, <2 x float> %A, i1 0, <2 x float> %B, i16 0, <8 x float> %C, i1 false, i1 true)25  store <8 x float> %res, ptr addrspace(1) %out26  ret void27}28 29define amdgpu_ps void @test_wmma_f32_16x16x32_bf16(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {30; GFX1250-LABEL: test_wmma_f32_16x16x32_bf16:31; GFX1250:       ; %bb.0: ; %bb32; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 133; GFX1250-NEXT:    v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse34; GFX1250-NEXT:    s_clause 0x135; GFX1250-NEXT:    global_store_b128 v[24:25], v[20:23], off offset:1636; GFX1250-NEXT:    global_store_b128 v[24:25], v[16:19], off37; GFX1250-NEXT:    s_endpgm38;39; GISEL-LABEL: test_wmma_f32_16x16x32_bf16:40; GISEL:       ; %bb.0: ; %bb41; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 142; GISEL-NEXT:    v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse43; GISEL-NEXT:    s_clause 0x144; GISEL-NEXT:    global_store_b128 v[24:25], v[20:23], off offset:1645; GISEL-NEXT:    global_store_b128 v[24:25], v[16:19], off46; GISEL-NEXT:    s_endpgm47bb:48  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.bf16.v8f32.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 0, <8 x float> %C, i1 false, i1 true)49  store <8 x float> %res, ptr addrspace(1) %out50  ret void51}52 53define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16(<16 x bfloat> %A, <16 x bfloat> %B, <8 x bfloat> %C, ptr addrspace(1) %out) {54; GFX1250-LABEL: test_wmma_bf16_16x16x32_bf16:55; GFX1250:       ; %bb.0: ; %bb56; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 157; GFX1250-NEXT:    v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse58; GFX1250-NEXT:    global_store_b128 v[20:21], v[16:19], off59; GFX1250-NEXT:    s_endpgm60;61; GISEL-LABEL: test_wmma_bf16_16x16x32_bf16:62; GISEL:       ; %bb.0: ; %bb63; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 164; GISEL-NEXT:    v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse65; GISEL-NEXT:    global_store_b128 v[20:21], v[16:19], off66; GISEL-NEXT:    s_endpgm67bb:68  %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16.16x16x32.bf16.v8bf16.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 0, <8 x bfloat> %C, i1 false, i1 true)69  store <8 x bfloat> %res, ptr addrspace(1) %out70  ret void71}72 73define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {74; GFX1250-LABEL: test_wmma_bf16f32_16x16x32_bf16:75; GFX1250:       ; %bb.0: ; %bb76; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 177; GFX1250-NEXT:    v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] matrix_b_reuse78; GFX1250-NEXT:    global_store_b128 v[24:25], v[26:29], off79; GFX1250-NEXT:    s_endpgm80;81; GISEL-LABEL: test_wmma_bf16f32_16x16x32_bf16:82; GISEL:       ; %bb.0: ; %bb83; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 184; GISEL-NEXT:    v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] matrix_b_reuse85; GISEL-NEXT:    global_store_b128 v[24:25], v[26:29], off86; GISEL-NEXT:    s_endpgm87bb:88  %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16f32.16x16x32.bf16.v8bf16.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 0, <8 x float> %C, i1 false, i1 true)89  store <8 x bfloat> %res, ptr addrspace(1) %out90  ret void91}92 93define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {94; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_fp8:95; GFX1250:       ; %bb.0: ; %bb96; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 197; GFX1250-NEXT:    v_wmma_f32_16x16x64_fp8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse98; GFX1250-NEXT:    s_clause 0x199; GFX1250-NEXT:    global_store_b128 v[24:25], v[20:23], off offset:16100; GFX1250-NEXT:    global_store_b128 v[24:25], v[16:19], off101; GFX1250-NEXT:    s_endpgm102;103; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_fp8:104; GISEL:       ; %bb.0: ; %bb105; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1106; GISEL-NEXT:    v_wmma_f32_16x16x64_fp8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse107; GISEL-NEXT:    s_clause 0x1108; GISEL-NEXT:    global_store_b128 v[24:25], v[16:19], off109; GISEL-NEXT:    global_store_b128 v[24:25], v[20:23], off offset:16110; GISEL-NEXT:    s_endpgm111bb:112  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.fp8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x float> %C, i1 false, i1 true)113  store <8 x float> %res, ptr addrspace(1) %out114  ret void115}116 117define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {118; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_bf8:119; GFX1250:       ; %bb.0: ; %bb120; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1121; GFX1250-NEXT:    v_wmma_f32_16x16x64_fp8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse122; GFX1250-NEXT:    s_clause 0x1123; GFX1250-NEXT:    global_store_b128 v[24:25], v[20:23], off offset:16124; GFX1250-NEXT:    global_store_b128 v[24:25], v[16:19], off125; GFX1250-NEXT:    s_endpgm126;127; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_bf8:128; GISEL:       ; %bb.0: ; %bb129; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1130; GISEL-NEXT:    v_wmma_f32_16x16x64_fp8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse131; GISEL-NEXT:    s_clause 0x1132; GISEL-NEXT:    global_store_b128 v[24:25], v[16:19], off133; GISEL-NEXT:    global_store_b128 v[24:25], v[20:23], off offset:16134; GISEL-NEXT:    s_endpgm135bb:136  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.bf8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x float> %C, i1 false, i1 true)137  store <8 x float> %res, ptr addrspace(1) %out138  ret void139}140 141define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {142; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_fp8:143; GFX1250:       ; %bb.0: ; %bb144; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1145; GFX1250-NEXT:    v_wmma_f32_16x16x64_bf8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse146; GFX1250-NEXT:    s_clause 0x1147; GFX1250-NEXT:    global_store_b128 v[24:25], v[20:23], off offset:16148; GFX1250-NEXT:    global_store_b128 v[24:25], v[16:19], off149; GFX1250-NEXT:    s_endpgm150;151; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_fp8:152; GISEL:       ; %bb.0: ; %bb153; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1154; GISEL-NEXT:    v_wmma_f32_16x16x64_bf8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse155; GISEL-NEXT:    s_clause 0x1156; GISEL-NEXT:    global_store_b128 v[24:25], v[16:19], off157; GISEL-NEXT:    global_store_b128 v[24:25], v[20:23], off offset:16158; GISEL-NEXT:    s_endpgm159bb:160  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.fp8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x float> %C, i1 false, i1 true)161  store <8 x float> %res, ptr addrspace(1) %out162  ret void163}164 165define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {166; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_bf8:167; GFX1250:       ; %bb.0: ; %bb168; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1169; GFX1250-NEXT:    v_wmma_f32_16x16x64_bf8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse170; GFX1250-NEXT:    s_clause 0x1171; GFX1250-NEXT:    global_store_b128 v[24:25], v[20:23], off offset:16172; GFX1250-NEXT:    global_store_b128 v[24:25], v[16:19], off173; GFX1250-NEXT:    s_endpgm174;175; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_bf8:176; GISEL:       ; %bb.0: ; %bb177; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1178; GISEL-NEXT:    v_wmma_f32_16x16x64_bf8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse179; GISEL-NEXT:    s_clause 0x1180; GISEL-NEXT:    global_store_b128 v[24:25], v[16:19], off181; GISEL-NEXT:    global_store_b128 v[24:25], v[20:23], off offset:16182; GISEL-NEXT:    s_endpgm183bb:184  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.bf8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x float> %C, i1 false, i1 true)185  store <8 x float> %res, ptr addrspace(1) %out186  ret void187}188 189define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {190; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_fp8:191; GFX1250:       ; %bb.0: ; %bb192; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1193; GFX1250-NEXT:    v_wmma_f16_16x16x64_fp8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse194; GFX1250-NEXT:    global_store_b128 v[20:21], v[16:19], off195; GFX1250-NEXT:    s_endpgm196;197; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_fp8:198; GISEL:       ; %bb.0: ; %bb199; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1200; GISEL-NEXT:    v_wmma_f16_16x16x64_fp8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse201; GISEL-NEXT:    global_store_b128 v[20:21], v[16:19], off202; GISEL-NEXT:    s_endpgm203bb:204  %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.fp8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x half> %C, i1 false, i1 true)205  store <8 x half> %res, ptr addrspace(1) %out206  ret void207}208 209define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {210; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_bf8:211; GFX1250:       ; %bb.0: ; %bb212; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1213; GFX1250-NEXT:    v_wmma_f16_16x16x64_fp8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse214; GFX1250-NEXT:    global_store_b128 v[20:21], v[16:19], off215; GFX1250-NEXT:    s_endpgm216;217; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_bf8:218; GISEL:       ; %bb.0: ; %bb219; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1220; GISEL-NEXT:    v_wmma_f16_16x16x64_fp8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse221; GISEL-NEXT:    global_store_b128 v[20:21], v[16:19], off222; GISEL-NEXT:    s_endpgm223bb:224  %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.bf8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x half> %C, i1 false, i1 true)225  store <8 x half> %res, ptr addrspace(1) %out226  ret void227}228 229define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {230; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_fp8:231; GFX1250:       ; %bb.0: ; %bb232; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1233; GFX1250-NEXT:    v_wmma_f16_16x16x64_bf8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse234; GFX1250-NEXT:    global_store_b128 v[20:21], v[16:19], off235; GFX1250-NEXT:    s_endpgm236;237; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_fp8:238; GISEL:       ; %bb.0: ; %bb239; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1240; GISEL-NEXT:    v_wmma_f16_16x16x64_bf8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse241; GISEL-NEXT:    global_store_b128 v[20:21], v[16:19], off242; GISEL-NEXT:    s_endpgm243bb:244  %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.fp8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x half> %C, i1 false, i1 true)245  store <8 x half> %res, ptr addrspace(1) %out246  ret void247}248 249define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {250; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_bf8:251; GFX1250:       ; %bb.0: ; %bb252; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1253; GFX1250-NEXT:    v_wmma_f16_16x16x64_bf8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse254; GFX1250-NEXT:    global_store_b128 v[20:21], v[16:19], off255; GFX1250-NEXT:    s_endpgm256;257; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_bf8:258; GISEL:       ; %bb.0: ; %bb259; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1260; GISEL-NEXT:    v_wmma_f16_16x16x64_bf8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse261; GISEL-NEXT:    global_store_b128 v[20:21], v[16:19], off262; GISEL-NEXT:    s_endpgm263bb:264  %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.bf8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x half> %C, i1 false, i1 true)265  store <8 x half> %res, ptr addrspace(1) %out266  ret void267}268 269define amdgpu_ps void @test_wmma_i32_16x16x64_iu8(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out) {270; GFX1250-LABEL: test_wmma_i32_16x16x64_iu8:271; GFX1250:       ; %bb.0: ; %bb272; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1273; GFX1250-NEXT:    v_wmma_i32_16x16x64_iu8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse274; GFX1250-NEXT:    s_clause 0x1275; GFX1250-NEXT:    global_store_b128 v[24:25], v[20:23], off offset:16276; GFX1250-NEXT:    global_store_b128 v[24:25], v[16:19], off277; GFX1250-NEXT:    s_endpgm278;279; GISEL-LABEL: test_wmma_i32_16x16x64_iu8:280; GISEL:       ; %bb.0: ; %bb281; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1282; GISEL-NEXT:    v_wmma_i32_16x16x64_iu8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse283; GISEL-NEXT:    s_clause 0x1284; GISEL-NEXT:    global_store_b128 v[24:25], v[16:19], off285; GISEL-NEXT:    global_store_b128 v[24:25], v[20:23], off offset:16286; GISEL-NEXT:    s_endpgm287bb:288  %res = call <8 x i32> @llvm.amdgcn.wmma.i32.16x16x64.iu8.v8i32.v8i32(i1 0, <8 x i32> %A, i1 0, <8 x i32> %B, <8 x i32> %C, i1 false, i1 true)289  store <8 x i32> %res, ptr addrspace(1) %out290  ret void291}292 293define amdgpu_ps void @test_wmma_f32_16x16x32_f16(<16 x half> %A, <16 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {294; GFX1250-LABEL: test_wmma_f32_16x16x32_f16:295; GFX1250:       ; %bb.0: ; %bb296; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1297; GFX1250-NEXT:    v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse298; GFX1250-NEXT:    s_clause 0x1299; GFX1250-NEXT:    global_store_b128 v[24:25], v[20:23], off offset:16300; GFX1250-NEXT:    global_store_b128 v[24:25], v[16:19], off301; GFX1250-NEXT:    s_endpgm302;303; GISEL-LABEL: test_wmma_f32_16x16x32_f16:304; GISEL:       ; %bb.0: ; %bb305; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1306; GISEL-NEXT:    v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse307; GISEL-NEXT:    s_clause 0x1308; GISEL-NEXT:    global_store_b128 v[24:25], v[16:19], off309; GISEL-NEXT:    global_store_b128 v[24:25], v[20:23], off offset:16310; GISEL-NEXT:    s_endpgm311bb:312  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 0, <16 x half> %A, i1 0, <16 x half> %B, i16 0, <8 x float> %C, i1 false, i1 true)313  store <8 x float> %res, ptr addrspace(1) %out314  ret void315}316 317define amdgpu_ps void @test_wmma_f16_16x16x32_f16(<16 x half> %A, <16 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {318; GFX1250-LABEL: test_wmma_f16_16x16x32_f16:319; GFX1250:       ; %bb.0: ; %bb320; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1321; GFX1250-NEXT:    v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse322; GFX1250-NEXT:    global_store_b128 v[20:21], v[16:19], off323; GFX1250-NEXT:    s_endpgm324;325; GISEL-LABEL: test_wmma_f16_16x16x32_f16:326; GISEL:       ; %bb.0: ; %bb327; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1328; GISEL-NEXT:    v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse329; GISEL-NEXT:    global_store_b128 v[20:21], v[16:19], off330; GISEL-NEXT:    s_endpgm331bb:332  %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x32.f16.v8f16.v16f16(i1 0, <16 x half> %A, i1 0, <16 x half> %B, i16 0, <8 x half> %C, i1 false, i1 true)333  store <8 x half> %res, ptr addrspace(1) %out334  ret void335}336 337define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {338; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4:339; GFX1250:       ; %bb.0: ; %bb340; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1341; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6342; GFX1250-NEXT:    s_clause 0x1343; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:16344; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off345; GFX1250-NEXT:    s_endpgm346;347; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4:348; GISEL:       ; %bb.0: ; %bb349; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1350; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6351; GISEL-NEXT:    s_clause 0x1352; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off353; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:16354; GISEL-NEXT:    s_endpgm355bb:356  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 1, <16 x i32> %A, i32 2, <16 x i32> %B, i16 0, <8 x float> %C)357  store <8 x float> %res, ptr addrspace(1) %out358  ret void359}360 361define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {362; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp8_bf8:363; GFX1250:       ; %bb.0: ; %bb364; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1365; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_fmt:MATRIX_FMT_BF8366; GFX1250-NEXT:    s_clause 0x1367; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:16368; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off369; GFX1250-NEXT:    s_endpgm370;371; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp8_bf8:372; GISEL:       ; %bb.0: ; %bb373; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1374; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_fmt:MATRIX_FMT_BF8375; GISEL-NEXT:    s_clause 0x1376; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off377; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:16378; GISEL-NEXT:    s_endpgm379bb:380  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 1, <16 x i32> %B, i16 0, <8 x float> %C)381  store <8 x float> %res, ptr addrspace(1) %out382  ret void383}384 385define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp8_fp6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {386; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp8_fp6:387; GFX1250:       ; %bb.0: ; %bb388; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1389; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35] matrix_b_fmt:MATRIX_FMT_FP6390; GFX1250-NEXT:    s_clause 0x1391; GFX1250-NEXT:    global_store_b128 v[36:37], v[32:35], off offset:16392; GFX1250-NEXT:    global_store_b128 v[36:37], v[28:31], off393; GFX1250-NEXT:    s_endpgm394;395; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp8_fp6:396; GISEL:       ; %bb.0: ; %bb397; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1398; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35] matrix_b_fmt:MATRIX_FMT_FP6399; GISEL-NEXT:    s_clause 0x1400; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off401; GISEL-NEXT:    global_store_b128 v[36:37], v[32:35], off offset:16402; GISEL-NEXT:    s_endpgm403bb:404  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v12i32(i32 0, <16 x i32> %A, i32 2, <12 x i32> %B, i16 0, <8 x float> %C)405  store <8 x float> %res, ptr addrspace(1) %out406  ret void407}408 409define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp8_bf6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {410; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp8_bf6:411; GFX1250:       ; %bb.0: ; %bb412; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1413; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35] matrix_b_fmt:MATRIX_FMT_BF6414; GFX1250-NEXT:    s_clause 0x1415; GFX1250-NEXT:    global_store_b128 v[36:37], v[32:35], off offset:16416; GFX1250-NEXT:    global_store_b128 v[36:37], v[28:31], off417; GFX1250-NEXT:    s_endpgm418;419; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp8_bf6:420; GISEL:       ; %bb.0: ; %bb421; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1422; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35] matrix_b_fmt:MATRIX_FMT_BF6423; GISEL-NEXT:    s_clause 0x1424; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off425; GISEL-NEXT:    global_store_b128 v[36:37], v[32:35], off offset:16426; GISEL-NEXT:    s_endpgm427bb:428  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v12i32(i32 0, <16 x i32> %A, i32 3, <12 x i32> %B, i16 0, <8 x float> %C)429  store <8 x float> %res, ptr addrspace(1) %out430  ret void431}432 433define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp8_fp4(<16 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {434; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp8_fp4:435; GFX1250:       ; %bb.0: ; %bb436; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1437; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31] matrix_b_fmt:MATRIX_FMT_FP4438; GFX1250-NEXT:    s_clause 0x1439; GFX1250-NEXT:    global_store_b128 v[32:33], v[28:31], off offset:16440; GFX1250-NEXT:    global_store_b128 v[32:33], v[24:27], off441; GFX1250-NEXT:    s_endpgm442;443; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp8_fp4:444; GISEL:       ; %bb.0: ; %bb445; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1446; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31] matrix_b_fmt:MATRIX_FMT_FP4447; GISEL-NEXT:    s_clause 0x1448; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off449; GISEL-NEXT:    global_store_b128 v[32:33], v[28:31], off offset:16450; GISEL-NEXT:    s_endpgm451bb:452  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v8i32(i32 0, <16 x i32> %A, i32 4, <8 x i32> %B, i16 0, <8 x float> %C)453  store <8 x float> %res, ptr addrspace(1) %out454  ret void455}456 457define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf8_fp8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {458; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_fp8:459; GFX1250:       ; %bb.0: ; %bb460; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1461; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] matrix_a_fmt:MATRIX_FMT_BF8462; GFX1250-NEXT:    s_clause 0x1463; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:16464; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off465; GFX1250-NEXT:    s_endpgm466;467; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_fp8:468; GISEL:       ; %bb.0: ; %bb469; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1470; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] matrix_a_fmt:MATRIX_FMT_BF8471; GISEL-NEXT:    s_clause 0x1472; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off473; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:16474; GISEL-NEXT:    s_endpgm475bb:476  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 1, <16 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> %C)477  store <8 x float> %res, ptr addrspace(1) %out478  ret void479}480 481define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {482; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_bf8:483; GFX1250:       ; %bb.0: ; %bb484; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1485; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF8486; GFX1250-NEXT:    s_clause 0x1487; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:16488; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off489; GFX1250-NEXT:    s_endpgm490;491; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_bf8:492; GISEL:       ; %bb.0: ; %bb493; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1494; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF8495; GISEL-NEXT:    s_clause 0x1496; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off497; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:16498; GISEL-NEXT:    s_endpgm499bb:500  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 1, <16 x i32> %A, i32 1, <16 x i32> %B, i16 0, <8 x float> %C)501  store <8 x float> %res, ptr addrspace(1) %out502  ret void503}504 505define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf8_fp6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {506; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_fp6:507; GFX1250:       ; %bb.0: ; %bb508; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1509; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6510; GFX1250-NEXT:    s_clause 0x1511; GFX1250-NEXT:    global_store_b128 v[36:37], v[32:35], off offset:16512; GFX1250-NEXT:    global_store_b128 v[36:37], v[28:31], off513; GFX1250-NEXT:    s_endpgm514;515; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_fp6:516; GISEL:       ; %bb.0: ; %bb517; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1518; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6519; GISEL-NEXT:    s_clause 0x1520; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off521; GISEL-NEXT:    global_store_b128 v[36:37], v[32:35], off offset:16522; GISEL-NEXT:    s_endpgm523bb:524  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v12i32(i32 1, <16 x i32> %A, i32 2, <12 x i32> %B, i16 0, <8 x float> %C)525  store <8 x float> %res, ptr addrspace(1) %out526  ret void527}528 529define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf8_bf6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {530; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_bf6:531; GFX1250:       ; %bb.0: ; %bb532; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1533; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF6534; GFX1250-NEXT:    s_clause 0x1535; GFX1250-NEXT:    global_store_b128 v[36:37], v[32:35], off offset:16536; GFX1250-NEXT:    global_store_b128 v[36:37], v[28:31], off537; GFX1250-NEXT:    s_endpgm538;539; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_bf6:540; GISEL:       ; %bb.0: ; %bb541; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1542; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF6543; GISEL-NEXT:    s_clause 0x1544; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off545; GISEL-NEXT:    global_store_b128 v[36:37], v[32:35], off offset:16546; GISEL-NEXT:    s_endpgm547bb:548  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v12i32(i32 1, <16 x i32> %A, i32 3, <12 x i32> %B, i16 0, <8 x float> %C)549  store <8 x float> %res, ptr addrspace(1) %out550  ret void551}552 553define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf8_fp4(<16 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {554; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_fp4:555; GFX1250:       ; %bb.0: ; %bb556; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1557; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP4558; GFX1250-NEXT:    s_clause 0x1559; GFX1250-NEXT:    global_store_b128 v[32:33], v[28:31], off offset:16560; GFX1250-NEXT:    global_store_b128 v[32:33], v[24:27], off561; GFX1250-NEXT:    s_endpgm562;563; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_fp4:564; GISEL:       ; %bb.0: ; %bb565; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1566; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP4567; GISEL-NEXT:    s_clause 0x1568; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off569; GISEL-NEXT:    global_store_b128 v[32:33], v[28:31], off offset:16570; GISEL-NEXT:    s_endpgm571bb:572  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v8i32(i32 1, <16 x i32> %A, i32 4, <8 x i32> %B, i16 0, <8 x float> %C)573  store <8 x float> %res, ptr addrspace(1) %out574  ret void575}576 577define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp6_fp8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {578; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_fp8:579; GFX1250:       ; %bb.0: ; %bb580; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1581; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35] matrix_a_fmt:MATRIX_FMT_FP6582; GFX1250-NEXT:    s_clause 0x1583; GFX1250-NEXT:    global_store_b128 v[36:37], v[32:35], off offset:16584; GFX1250-NEXT:    global_store_b128 v[36:37], v[28:31], off585; GFX1250-NEXT:    s_endpgm586;587; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_fp8:588; GISEL:       ; %bb.0: ; %bb589; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1590; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35] matrix_a_fmt:MATRIX_FMT_FP6591; GISEL-NEXT:    s_clause 0x1592; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off593; GISEL-NEXT:    global_store_b128 v[36:37], v[32:35], off offset:16594; GISEL-NEXT:    s_endpgm595bb:596  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v12i32.v16i32(i32 2, <12 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> %C)597  store <8 x float> %res, ptr addrspace(1) %out598  ret void599}600 601define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp6_bf8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {602; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_bf8:603; GFX1250:       ; %bb.0: ; %bb604; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1605; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_BF8606; GFX1250-NEXT:    s_clause 0x1607; GFX1250-NEXT:    global_store_b128 v[36:37], v[32:35], off offset:16608; GFX1250-NEXT:    global_store_b128 v[36:37], v[28:31], off609; GFX1250-NEXT:    s_endpgm610;611; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_bf8:612; GISEL:       ; %bb.0: ; %bb613; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1614; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_BF8615; GISEL-NEXT:    s_clause 0x1616; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off617; GISEL-NEXT:    global_store_b128 v[36:37], v[32:35], off offset:16618; GISEL-NEXT:    s_endpgm619bb:620  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v12i32.v16i32(i32 2, <12 x i32> %A, i32 1, <16 x i32> %B, i16 0, <8 x float> %C)621  store <8 x float> %res, ptr addrspace(1) %out622  ret void623}624 625define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp6_fp6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {626; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_fp6:627; GFX1250:       ; %bb.0: ; %bb628; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1629; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP6630; GFX1250-NEXT:    s_clause 0x1631; GFX1250-NEXT:    global_store_b128 v[32:33], v[28:31], off offset:16632; GFX1250-NEXT:    global_store_b128 v[32:33], v[24:27], off633; GFX1250-NEXT:    s_endpgm634;635; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_fp6:636; GISEL:       ; %bb.0: ; %bb637; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1638; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP6639; GISEL-NEXT:    s_clause 0x1640; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off641; GISEL-NEXT:    global_store_b128 v[32:33], v[28:31], off offset:16642; GISEL-NEXT:    s_endpgm643bb:644  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v12i32.v12i32(i32 2, <12 x i32> %A, i32 2, <12 x i32> %B, i16 0, <8 x float> %C)645  store <8 x float> %res, ptr addrspace(1) %out646  ret void647}648 649define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp6_bf6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {650; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_bf6:651; GFX1250:       ; %bb.0: ; %bb652; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1653; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP4654; GFX1250-NEXT:    s_clause 0x1655; GFX1250-NEXT:    global_store_b128 v[32:33], v[28:31], off offset:16656; GFX1250-NEXT:    global_store_b128 v[32:33], v[24:27], off657; GFX1250-NEXT:    s_endpgm658;659; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_bf6:660; GISEL:       ; %bb.0: ; %bb661; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1662; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP4663; GISEL-NEXT:    s_clause 0x1664; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off665; GISEL-NEXT:    global_store_b128 v[32:33], v[28:31], off offset:16666; GISEL-NEXT:    s_endpgm667bb:668  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v12i32.v12i32(i32 2, <12 x i32> %A, i32 4, <12 x i32> %B, i16 0, <8 x float> %C)669  store <8 x float> %res, ptr addrspace(1) %out670  ret void671}672 673define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp6_fp4(<12 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {674; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_fp4:675; GFX1250:       ; %bb.0: ; %bb676; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1677; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP4678; GFX1250-NEXT:    s_clause 0x1679; GFX1250-NEXT:    global_store_b128 v[28:29], v[24:27], off offset:16680; GFX1250-NEXT:    global_store_b128 v[28:29], v[20:23], off681; GFX1250-NEXT:    s_endpgm682;683; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_fp4:684; GISEL:       ; %bb.0: ; %bb685; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1686; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP4687; GISEL-NEXT:    s_clause 0x1688; GISEL-NEXT:    global_store_b128 v[28:29], v[20:23], off689; GISEL-NEXT:    global_store_b128 v[28:29], v[24:27], off offset:16690; GISEL-NEXT:    s_endpgm691bb:692  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v12i32.v8i32(i32 2, <12 x i32> %A, i32 4, <8 x i32> %B, i16 0, <8 x float> %C)693  store <8 x float> %res, ptr addrspace(1) %out694  ret void695}696 697define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf6_fp8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {698; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_fp8:699; GFX1250:       ; %bb.0: ; %bb700; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1701; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35] matrix_a_fmt:MATRIX_FMT_BF6702; GFX1250-NEXT:    s_clause 0x1703; GFX1250-NEXT:    global_store_b128 v[36:37], v[32:35], off offset:16704; GFX1250-NEXT:    global_store_b128 v[36:37], v[28:31], off705; GFX1250-NEXT:    s_endpgm706;707; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_fp8:708; GISEL:       ; %bb.0: ; %bb709; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1710; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35] matrix_a_fmt:MATRIX_FMT_BF6711; GISEL-NEXT:    s_clause 0x1712; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off713; GISEL-NEXT:    global_store_b128 v[36:37], v[32:35], off offset:16714; GISEL-NEXT:    s_endpgm715bb:716  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v12i32.v16i32(i32 3, <12 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> %C)717  store <8 x float> %res, ptr addrspace(1) %out718  ret void719}720 721define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf6_bf8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {722; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_bf8:723; GFX1250:       ; %bb.0: ; %bb724; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1725; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_BF8726; GFX1250-NEXT:    s_clause 0x1727; GFX1250-NEXT:    global_store_b128 v[36:37], v[32:35], off offset:16728; GFX1250-NEXT:    global_store_b128 v[36:37], v[28:31], off729; GFX1250-NEXT:    s_endpgm730;731; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_bf8:732; GISEL:       ; %bb.0: ; %bb733; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1734; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_BF8735; GISEL-NEXT:    s_clause 0x1736; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off737; GISEL-NEXT:    global_store_b128 v[36:37], v[32:35], off offset:16738; GISEL-NEXT:    s_endpgm739bb:740  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v12i32.v16i32(i32 3, <12 x i32> %A, i32 1, <16 x i32> %B, i16 0, <8 x float> %C)741  store <8 x float> %res, ptr addrspace(1) %out742  ret void743}744 745define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf6_fp6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {746; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_fp6:747; GFX1250:       ; %bb.0: ; %bb748; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1749; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP6750; GFX1250-NEXT:    s_clause 0x1751; GFX1250-NEXT:    global_store_b128 v[32:33], v[28:31], off offset:16752; GFX1250-NEXT:    global_store_b128 v[32:33], v[24:27], off753; GFX1250-NEXT:    s_endpgm754;755; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_fp6:756; GISEL:       ; %bb.0: ; %bb757; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1758; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP6759; GISEL-NEXT:    s_clause 0x1760; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off761; GISEL-NEXT:    global_store_b128 v[32:33], v[28:31], off offset:16762; GISEL-NEXT:    s_endpgm763bb:764  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v12i32.v12i32(i32 3, <12 x i32> %A, i32 2, <12 x i32> %B, i16 0, <8 x float> %C)765  store <8 x float> %res, ptr addrspace(1) %out766  ret void767}768 769define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf6_bf6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {770; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_bf6:771; GFX1250:       ; %bb.0: ; %bb772; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1773; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP4774; GFX1250-NEXT:    s_clause 0x1775; GFX1250-NEXT:    global_store_b128 v[32:33], v[28:31], off offset:16776; GFX1250-NEXT:    global_store_b128 v[32:33], v[24:27], off777; GFX1250-NEXT:    s_endpgm778;779; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_bf6:780; GISEL:       ; %bb.0: ; %bb781; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1782; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP4783; GISEL-NEXT:    s_clause 0x1784; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off785; GISEL-NEXT:    global_store_b128 v[32:33], v[28:31], off offset:16786; GISEL-NEXT:    s_endpgm787bb:788  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v12i32.v12i32(i32 3, <12 x i32> %A, i32 4, <12 x i32> %B, i16 0, <8 x float> %C)789  store <8 x float> %res, ptr addrspace(1) %out790  ret void791}792 793define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf6_fp4(<12 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {794; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_fp4:795; GFX1250:       ; %bb.0: ; %bb796; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1797; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP4798; GFX1250-NEXT:    s_clause 0x1799; GFX1250-NEXT:    global_store_b128 v[28:29], v[24:27], off offset:16800; GFX1250-NEXT:    global_store_b128 v[28:29], v[20:23], off801; GFX1250-NEXT:    s_endpgm802;803; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_fp4:804; GISEL:       ; %bb.0: ; %bb805; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1806; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP4807; GISEL-NEXT:    s_clause 0x1808; GISEL-NEXT:    global_store_b128 v[28:29], v[20:23], off809; GISEL-NEXT:    global_store_b128 v[28:29], v[24:27], off offset:16810; GISEL-NEXT:    s_endpgm811bb:812  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v12i32.v8i32(i32 3, <12 x i32> %A, i32 4, <8 x i32> %B, i16 0, <8 x float> %C)813  store <8 x float> %res, ptr addrspace(1) %out814  ret void815}816 817define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp4_fp8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {818; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_fp8:819; GFX1250:       ; %bb.0: ; %bb820; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1821; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31] matrix_a_fmt:MATRIX_FMT_FP4822; GFX1250-NEXT:    s_clause 0x1823; GFX1250-NEXT:    global_store_b128 v[32:33], v[28:31], off offset:16824; GFX1250-NEXT:    global_store_b128 v[32:33], v[24:27], off825; GFX1250-NEXT:    s_endpgm826;827; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_fp8:828; GISEL:       ; %bb.0: ; %bb829; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1830; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31] matrix_a_fmt:MATRIX_FMT_FP4831; GISEL-NEXT:    s_clause 0x1832; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off833; GISEL-NEXT:    global_store_b128 v[32:33], v[28:31], off offset:16834; GISEL-NEXT:    s_endpgm835bb:836  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v8i32.v16i32(i32 4, <8 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> %C)837  store <8 x float> %res, ptr addrspace(1) %out838  ret void839}840 841define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp4_bf8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {842; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_bf8:843; GFX1250:       ; %bb.0: ; %bb844; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1845; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_BF8846; GFX1250-NEXT:    s_clause 0x1847; GFX1250-NEXT:    global_store_b128 v[32:33], v[28:31], off offset:16848; GFX1250-NEXT:    global_store_b128 v[32:33], v[24:27], off849; GFX1250-NEXT:    s_endpgm850;851; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_bf8:852; GISEL:       ; %bb.0: ; %bb853; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1854; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_BF8855; GISEL-NEXT:    s_clause 0x1856; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off857; GISEL-NEXT:    global_store_b128 v[32:33], v[28:31], off offset:16858; GISEL-NEXT:    s_endpgm859bb:860  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v8i32.v16i32(i32 4, <8 x i32> %A, i32 1, <16 x i32> %B, i16 0, <8 x float> %C)861  store <8 x float> %res, ptr addrspace(1) %out862  ret void863}864 865define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp4_fp6(<8 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {866; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_fp6:867; GFX1250:       ; %bb.0: ; %bb868; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1869; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP6870; GFX1250-NEXT:    s_clause 0x1871; GFX1250-NEXT:    global_store_b128 v[28:29], v[24:27], off offset:16872; GFX1250-NEXT:    global_store_b128 v[28:29], v[20:23], off873; GFX1250-NEXT:    s_endpgm874;875; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_fp6:876; GISEL:       ; %bb.0: ; %bb877; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1878; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP6879; GISEL-NEXT:    s_clause 0x1880; GISEL-NEXT:    global_store_b128 v[28:29], v[20:23], off881; GISEL-NEXT:    global_store_b128 v[28:29], v[24:27], off offset:16882; GISEL-NEXT:    s_endpgm883bb:884  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v8i32.v12i32(i32 4, <8 x i32> %A, i32 2, <12 x i32> %B, i16 0, <8 x float> %C)885  store <8 x float> %res, ptr addrspace(1) %out886  ret void887}888 889define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp4_bf6(<8 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {890; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_bf6:891; GFX1250:       ; %bb.0: ; %bb892; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1893; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4894; GFX1250-NEXT:    s_clause 0x1895; GFX1250-NEXT:    global_store_b128 v[28:29], v[24:27], off offset:16896; GFX1250-NEXT:    global_store_b128 v[28:29], v[20:23], off897; GFX1250-NEXT:    s_endpgm898;899; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_bf6:900; GISEL:       ; %bb.0: ; %bb901; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1902; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4903; GISEL-NEXT:    s_clause 0x1904; GISEL-NEXT:    global_store_b128 v[28:29], v[20:23], off905; GISEL-NEXT:    global_store_b128 v[28:29], v[24:27], off offset:16906; GISEL-NEXT:    s_endpgm907bb:908  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v8i32.v12i32(i32 4, <8 x i32> %A, i32 4, <12 x i32> %B, i16 0, <8 x float> %C)909  store <8 x float> %res, ptr addrspace(1) %out910  ret void911}912 913define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp4_fp4(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {914; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_fp4:915; GFX1250:       ; %bb.0: ; %bb916; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1917; GFX1250-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[16:23], v[0:7], v[8:15], v[16:23] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4918; GFX1250-NEXT:    s_clause 0x1919; GFX1250-NEXT:    global_store_b128 v[24:25], v[20:23], off offset:16920; GFX1250-NEXT:    global_store_b128 v[24:25], v[16:19], off921; GFX1250-NEXT:    s_endpgm922;923; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_fp4:924; GISEL:       ; %bb.0: ; %bb925; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1926; GISEL-NEXT:    v_wmma_f32_16x16x128_f8f6f4 v[16:23], v[0:7], v[8:15], v[16:23] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4927; GISEL-NEXT:    s_clause 0x1928; GISEL-NEXT:    global_store_b128 v[24:25], v[16:19], off929; GISEL-NEXT:    global_store_b128 v[24:25], v[20:23], off offset:16930; GISEL-NEXT:    s_endpgm931bb:932  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v8i32.v8i32(i32 4, <8 x i32> %A, i32 4, <8 x i32> %B, i16 0, <8 x float> %C)933  store <8 x float> %res, ptr addrspace(1) %out934  ret void935}936 937define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {938; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4:939; GFX1250:       ; %bb.0: ; %bb940; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1941; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v40, v41 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1942; GFX1250-NEXT:    s_clause 0x1943; GFX1250-NEXT:    global_store_b128 v[42:43], v[36:39], off offset:16944; GFX1250-NEXT:    global_store_b128 v[42:43], v[32:35], off945; GFX1250-NEXT:    s_endpgm946;947; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4:948; GISEL:       ; %bb.0: ; %bb949; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1950; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v40, v41 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1951; GISEL-NEXT:    s_clause 0x1952; GISEL-NEXT:    global_store_b128 v[42:43], v[32:35], off953; GISEL-NEXT:    global_store_b128 v[42:43], v[36:39], off offset:16954; GISEL-NEXT:    s_endpgm955bb:956  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 1, <16 x i32> %A, i32 2, <16 x i32> %B, i16 0, <8 x float> %C, i32 1, i32 0, i32 %scale_src0, i32 1, i32 0, i32 %scale_src1, i1 false, i1 false)957  store <8 x float> %res, ptr addrspace(1) %out958  ret void959}960 961define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_ss(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 inreg %scale_src0, i32 inreg %scale_src1, ptr addrspace(1) %out) {962; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_ss:963; GFX1250:       ; %bb.0: ; %bb964; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1965; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], s0, s1 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_a_reuse966; GFX1250-NEXT:    s_clause 0x1967; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:16968; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off969; GFX1250-NEXT:    s_endpgm970;971; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_ss:972; GISEL:       ; %bb.0: ; %bb973; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1974; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], s0, s1 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_a_reuse975; GISEL-NEXT:    s_clause 0x1976; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off977; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:16978; GISEL-NEXT:    s_endpgm979bb:980  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 1, <16 x i32> %A, i32 2, <16 x i32> %B, i16 0, <8 x float> %C, i32 2, i32 1, i32 %scale_src0, i32 1, i32 2, i32 %scale_src1, i1 true, i1 false)981  store <8 x float> %res, ptr addrspace(1) %out982  ret void983}984 985define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_si_scale(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 inreg %scale_src0, ptr addrspace(1) %out) {986; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_si_scale:987; GFX1250:       ; %bb.0: ; %bb988; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1989; GFX1250-NEXT:    v_mov_b32_e32 v42, 0x64990; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)991; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], s0, v42 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_reuse992; GFX1250-NEXT:    s_clause 0x1993; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:16994; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off995; GFX1250-NEXT:    s_endpgm996;997; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_si_scale:998; GISEL:       ; %bb.0: ; %bb999; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11000; GISEL-NEXT:    v_mov_b32_e32 v42, 0x641001; GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)1002; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], s0, v42 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_reuse1003; GISEL-NEXT:    s_clause 0x11004; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off1005; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:161006; GISEL-NEXT:    s_endpgm1007bb:1008  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32i.v16i32(i32 1, <16 x i32> %A, i32 2, <16 x i32> %B, i16 0, <8 x float> %C, i32 3, i32 2, i32 %scale_src0, i32 0, i32 1, i32 100, i1 false, i1 true)1009  store <8 x float> %res, ptr addrspace(1) %out1010  ret void1011}1012 1013define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1014; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp8_bf8:1015; GFX1250:       ; %bb.0: ; %bb1016; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11017; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v40, v41 matrix_b_fmt:MATRIX_FMT_BF81018; GFX1250-NEXT:    s_clause 0x11019; GFX1250-NEXT:    global_store_b128 v[42:43], v[36:39], off offset:161020; GFX1250-NEXT:    global_store_b128 v[42:43], v[32:35], off1021; GFX1250-NEXT:    s_endpgm1022;1023; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp8_bf8:1024; GISEL:       ; %bb.0: ; %bb1025; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11026; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v40, v41 matrix_b_fmt:MATRIX_FMT_BF81027; GISEL-NEXT:    s_clause 0x11028; GISEL-NEXT:    global_store_b128 v[42:43], v[32:35], off1029; GISEL-NEXT:    global_store_b128 v[42:43], v[36:39], off offset:161030; GISEL-NEXT:    s_endpgm1031bb:1032  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 1, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1033  store <8 x float> %res, ptr addrspace(1) %out1034  ret void1035}1036 1037define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp8_fp6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1038; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp8_fp6:1039; GFX1250:       ; %bb.0: ; %bb1040; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11041; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v36, v37 matrix_b_fmt:MATRIX_FMT_FP61042; GFX1250-NEXT:    s_clause 0x11043; GFX1250-NEXT:    global_store_b128 v[38:39], v[32:35], off offset:161044; GFX1250-NEXT:    global_store_b128 v[38:39], v[28:31], off1045; GFX1250-NEXT:    s_endpgm1046;1047; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp8_fp6:1048; GISEL:       ; %bb.0: ; %bb1049; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11050; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v36, v37 matrix_b_fmt:MATRIX_FMT_FP61051; GISEL-NEXT:    s_clause 0x11052; GISEL-NEXT:    global_store_b128 v[38:39], v[28:31], off1053; GISEL-NEXT:    global_store_b128 v[38:39], v[32:35], off offset:161054; GISEL-NEXT:    s_endpgm1055bb:1056  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v12i32(i32 0, <16 x i32> %A, i32 2, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1057  store <8 x float> %res, ptr addrspace(1) %out1058  ret void1059}1060 1061define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp8_bf6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1062; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp8_bf6:1063; GFX1250:       ; %bb.0: ; %bb1064; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11065; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v36, v37 matrix_b_fmt:MATRIX_FMT_BF61066; GFX1250-NEXT:    s_clause 0x11067; GFX1250-NEXT:    global_store_b128 v[38:39], v[32:35], off offset:161068; GFX1250-NEXT:    global_store_b128 v[38:39], v[28:31], off1069; GFX1250-NEXT:    s_endpgm1070;1071; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp8_bf6:1072; GISEL:       ; %bb.0: ; %bb1073; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11074; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v36, v37 matrix_b_fmt:MATRIX_FMT_BF61075; GISEL-NEXT:    s_clause 0x11076; GISEL-NEXT:    global_store_b128 v[38:39], v[28:31], off1077; GISEL-NEXT:    global_store_b128 v[38:39], v[32:35], off offset:161078; GISEL-NEXT:    s_endpgm1079bb:1080  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v12i32(i32 0, <16 x i32> %A, i32 3, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1081  store <8 x float> %res, ptr addrspace(1) %out1082  ret void1083}1084 1085define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp8_fp4(<16 x i32> %A, <8 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1086; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp8_fp4:1087; GFX1250:       ; %bb.0: ; %bb1088; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11089; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31], v32, v33 matrix_b_fmt:MATRIX_FMT_FP41090; GFX1250-NEXT:    s_clause 0x11091; GFX1250-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:161092; GFX1250-NEXT:    global_store_b128 v[34:35], v[24:27], off1093; GFX1250-NEXT:    s_endpgm1094;1095; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp8_fp4:1096; GISEL:       ; %bb.0: ; %bb1097; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11098; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31], v32, v33 matrix_b_fmt:MATRIX_FMT_FP41099; GISEL-NEXT:    s_clause 0x11100; GISEL-NEXT:    global_store_b128 v[34:35], v[24:27], off1101; GISEL-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:161102; GISEL-NEXT:    s_endpgm1103bb:1104  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v8i32(i32 0, <16 x i32> %A, i32 4, <8 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1105  store <8 x float> %res, ptr addrspace(1) %out1106  ret void1107}1108 1109define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1110; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp8:1111; GFX1250:       ; %bb.0: ; %bb1112; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11113; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v40, v41 matrix_a_fmt:MATRIX_FMT_BF81114; GFX1250-NEXT:    s_clause 0x11115; GFX1250-NEXT:    global_store_b128 v[42:43], v[36:39], off offset:161116; GFX1250-NEXT:    global_store_b128 v[42:43], v[32:35], off1117; GFX1250-NEXT:    s_endpgm1118;1119; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp8:1120; GISEL:       ; %bb.0: ; %bb1121; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11122; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v40, v41 matrix_a_fmt:MATRIX_FMT_BF81123; GISEL-NEXT:    s_clause 0x11124; GISEL-NEXT:    global_store_b128 v[42:43], v[32:35], off1125; GISEL-NEXT:    global_store_b128 v[42:43], v[36:39], off offset:161126; GISEL-NEXT:    s_endpgm1127bb:1128  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 1, <16 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1129  store <8 x float> %res, ptr addrspace(1) %out1130  ret void1131}1132 1133define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1134; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_bf8:1135; GFX1250:       ; %bb.0: ; %bb1136; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11137; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v40, v41 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF81138; GFX1250-NEXT:    s_clause 0x11139; GFX1250-NEXT:    global_store_b128 v[42:43], v[36:39], off offset:161140; GFX1250-NEXT:    global_store_b128 v[42:43], v[32:35], off1141; GFX1250-NEXT:    s_endpgm1142;1143; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_bf8:1144; GISEL:       ; %bb.0: ; %bb1145; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11146; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v40, v41 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF81147; GISEL-NEXT:    s_clause 0x11148; GISEL-NEXT:    global_store_b128 v[42:43], v[32:35], off1149; GISEL-NEXT:    global_store_b128 v[42:43], v[36:39], off offset:161150; GISEL-NEXT:    s_endpgm1151bb:1152  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 1, <16 x i32> %A, i32 1, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1153  store <8 x float> %res, ptr addrspace(1) %out1154  ret void1155}1156 1157define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1158; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp6:1159; GFX1250:       ; %bb.0: ; %bb1160; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11161; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP61162; GFX1250-NEXT:    s_clause 0x11163; GFX1250-NEXT:    global_store_b128 v[38:39], v[32:35], off offset:161164; GFX1250-NEXT:    global_store_b128 v[38:39], v[28:31], off1165; GFX1250-NEXT:    s_endpgm1166;1167; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp6:1168; GISEL:       ; %bb.0: ; %bb1169; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11170; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP61171; GISEL-NEXT:    s_clause 0x11172; GISEL-NEXT:    global_store_b128 v[38:39], v[28:31], off1173; GISEL-NEXT:    global_store_b128 v[38:39], v[32:35], off offset:161174; GISEL-NEXT:    s_endpgm1175bb:1176  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v12i32(i32 1, <16 x i32> %A, i32 2, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1177  store <8 x float> %res, ptr addrspace(1) %out1178  ret void1179}1180 1181define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf8_bf6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1182; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_bf6:1183; GFX1250:       ; %bb.0: ; %bb1184; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11185; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF61186; GFX1250-NEXT:    s_clause 0x11187; GFX1250-NEXT:    global_store_b128 v[38:39], v[32:35], off offset:161188; GFX1250-NEXT:    global_store_b128 v[38:39], v[28:31], off1189; GFX1250-NEXT:    s_endpgm1190;1191; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_bf6:1192; GISEL:       ; %bb.0: ; %bb1193; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11194; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF61195; GISEL-NEXT:    s_clause 0x11196; GISEL-NEXT:    global_store_b128 v[38:39], v[28:31], off1197; GISEL-NEXT:    global_store_b128 v[38:39], v[32:35], off offset:161198; GISEL-NEXT:    s_endpgm1199bb:1200  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v12i32(i32 1, <16 x i32> %A, i32 3, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1201  store <8 x float> %res, ptr addrspace(1) %out1202  ret void1203}1204 1205define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp4(<16 x i32> %A, <8 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1206; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp4:1207; GFX1250:       ; %bb.0: ; %bb1208; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11209; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP41210; GFX1250-NEXT:    s_clause 0x11211; GFX1250-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:161212; GFX1250-NEXT:    global_store_b128 v[34:35], v[24:27], off1213; GFX1250-NEXT:    s_endpgm1214;1215; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp4:1216; GISEL:       ; %bb.0: ; %bb1217; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11218; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP41219; GISEL-NEXT:    s_clause 0x11220; GISEL-NEXT:    global_store_b128 v[34:35], v[24:27], off1221; GISEL-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:161222; GISEL-NEXT:    s_endpgm1223bb:1224  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v8i32(i32 1, <16 x i32> %A, i32 4, <8 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1225  store <8 x float> %res, ptr addrspace(1) %out1226  ret void1227}1228 1229define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1230; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp8:1231; GFX1250:       ; %bb.0: ; %bb1232; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11233; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_FP61234; GFX1250-NEXT:    s_clause 0x11235; GFX1250-NEXT:    global_store_b128 v[38:39], v[32:35], off offset:161236; GFX1250-NEXT:    global_store_b128 v[38:39], v[28:31], off1237; GFX1250-NEXT:    s_endpgm1238;1239; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp8:1240; GISEL:       ; %bb.0: ; %bb1241; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11242; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_FP61243; GISEL-NEXT:    s_clause 0x11244; GISEL-NEXT:    global_store_b128 v[38:39], v[28:31], off1245; GISEL-NEXT:    global_store_b128 v[38:39], v[32:35], off offset:161246; GISEL-NEXT:    s_endpgm1247bb:1248  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v12i32.v16i32(i32 2, <12 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1249  store <8 x float> %res, ptr addrspace(1) %out1250  ret void1251}1252 1253define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp6_bf8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1254; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_bf8:1255; GFX1250:       ; %bb.0: ; %bb1256; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11257; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_BF81258; GFX1250-NEXT:    s_clause 0x11259; GFX1250-NEXT:    global_store_b128 v[38:39], v[32:35], off offset:161260; GFX1250-NEXT:    global_store_b128 v[38:39], v[28:31], off1261; GFX1250-NEXT:    s_endpgm1262;1263; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_bf8:1264; GISEL:       ; %bb.0: ; %bb1265; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11266; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_BF81267; GISEL-NEXT:    s_clause 0x11268; GISEL-NEXT:    global_store_b128 v[38:39], v[28:31], off1269; GISEL-NEXT:    global_store_b128 v[38:39], v[32:35], off offset:161270; GISEL-NEXT:    s_endpgm1271bb:1272  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v12i32.v16i32(i32 2, <12 x i32> %A, i32 1, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1273  store <8 x float> %res, ptr addrspace(1) %out1274  ret void1275}1276 1277define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1278; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp6:1279; GFX1250:       ; %bb.0: ; %bb1280; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11281; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP61282; GFX1250-NEXT:    s_clause 0x11283; GFX1250-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:161284; GFX1250-NEXT:    global_store_b128 v[34:35], v[24:27], off1285; GFX1250-NEXT:    s_endpgm1286;1287; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp6:1288; GISEL:       ; %bb.0: ; %bb1289; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11290; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP61291; GISEL-NEXT:    s_clause 0x11292; GISEL-NEXT:    global_store_b128 v[34:35], v[24:27], off1293; GISEL-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:161294; GISEL-NEXT:    s_endpgm1295bb:1296  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v12i32.v12i32(i32 2, <12 x i32> %A, i32 2, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1297  store <8 x float> %res, ptr addrspace(1) %out1298  ret void1299}1300 1301define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp6_bf6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1302; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_bf6:1303; GFX1250:       ; %bb.0: ; %bb1304; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11305; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP41306; GFX1250-NEXT:    s_clause 0x11307; GFX1250-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:161308; GFX1250-NEXT:    global_store_b128 v[34:35], v[24:27], off1309; GFX1250-NEXT:    s_endpgm1310;1311; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_bf6:1312; GISEL:       ; %bb.0: ; %bb1313; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11314; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP41315; GISEL-NEXT:    s_clause 0x11316; GISEL-NEXT:    global_store_b128 v[34:35], v[24:27], off1317; GISEL-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:161318; GISEL-NEXT:    s_endpgm1319bb:1320  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v12i32.v12i32(i32 2, <12 x i32> %A, i32 4, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1321  store <8 x float> %res, ptr addrspace(1) %out1322  ret void1323}1324 1325define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp4(<12 x i32> %A, <8 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1326; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp4:1327; GFX1250:       ; %bb.0: ; %bb1328; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11329; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27], v28, v29 matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP41330; GFX1250-NEXT:    s_clause 0x11331; GFX1250-NEXT:    global_store_b128 v[30:31], v[24:27], off offset:161332; GFX1250-NEXT:    global_store_b128 v[30:31], v[20:23], off1333; GFX1250-NEXT:    s_endpgm1334;1335; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp4:1336; GISEL:       ; %bb.0: ; %bb1337; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11338; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27], v28, v29 matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP41339; GISEL-NEXT:    s_clause 0x11340; GISEL-NEXT:    global_store_b128 v[30:31], v[20:23], off1341; GISEL-NEXT:    global_store_b128 v[30:31], v[24:27], off offset:161342; GISEL-NEXT:    s_endpgm1343bb:1344  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v12i32.v8i32(i32 2, <12 x i32> %A, i32 4, <8 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1345  store <8 x float> %res, ptr addrspace(1) %out1346  ret void1347}1348 1349define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1350; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp8:1351; GFX1250:       ; %bb.0: ; %bb1352; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11353; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_BF61354; GFX1250-NEXT:    s_clause 0x11355; GFX1250-NEXT:    global_store_b128 v[38:39], v[32:35], off offset:161356; GFX1250-NEXT:    global_store_b128 v[38:39], v[28:31], off1357; GFX1250-NEXT:    s_endpgm1358;1359; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp8:1360; GISEL:       ; %bb.0: ; %bb1361; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11362; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_BF61363; GISEL-NEXT:    s_clause 0x11364; GISEL-NEXT:    global_store_b128 v[38:39], v[28:31], off1365; GISEL-NEXT:    global_store_b128 v[38:39], v[32:35], off offset:161366; GISEL-NEXT:    s_endpgm1367bb:1368  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v12i32.v16i32(i32 3, <12 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1369  store <8 x float> %res, ptr addrspace(1) %out1370  ret void1371}1372 1373define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf6_bf8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1374; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_bf8:1375; GFX1250:       ; %bb.0: ; %bb1376; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11377; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_BF81378; GFX1250-NEXT:    s_clause 0x11379; GFX1250-NEXT:    global_store_b128 v[38:39], v[32:35], off offset:161380; GFX1250-NEXT:    global_store_b128 v[38:39], v[28:31], off1381; GFX1250-NEXT:    s_endpgm1382;1383; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_bf8:1384; GISEL:       ; %bb.0: ; %bb1385; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11386; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_BF81387; GISEL-NEXT:    s_clause 0x11388; GISEL-NEXT:    global_store_b128 v[38:39], v[28:31], off1389; GISEL-NEXT:    global_store_b128 v[38:39], v[32:35], off offset:161390; GISEL-NEXT:    s_endpgm1391bb:1392  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v12i32.v16i32(i32 3, <12 x i32> %A, i32 1, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1393  store <8 x float> %res, ptr addrspace(1) %out1394  ret void1395}1396 1397define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1398; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp6:1399; GFX1250:       ; %bb.0: ; %bb1400; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11401; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP61402; GFX1250-NEXT:    s_clause 0x11403; GFX1250-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:161404; GFX1250-NEXT:    global_store_b128 v[34:35], v[24:27], off1405; GFX1250-NEXT:    s_endpgm1406;1407; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp6:1408; GISEL:       ; %bb.0: ; %bb1409; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11410; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP61411; GISEL-NEXT:    s_clause 0x11412; GISEL-NEXT:    global_store_b128 v[34:35], v[24:27], off1413; GISEL-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:161414; GISEL-NEXT:    s_endpgm1415bb:1416  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v12i32.v12i32(i32 3, <12 x i32> %A, i32 2, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1417  store <8 x float> %res, ptr addrspace(1) %out1418  ret void1419}1420 1421define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf6_bf6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1422; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_bf6:1423; GFX1250:       ; %bb.0: ; %bb1424; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11425; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP41426; GFX1250-NEXT:    s_clause 0x11427; GFX1250-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:161428; GFX1250-NEXT:    global_store_b128 v[34:35], v[24:27], off1429; GFX1250-NEXT:    s_endpgm1430;1431; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_bf6:1432; GISEL:       ; %bb.0: ; %bb1433; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11434; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP41435; GISEL-NEXT:    s_clause 0x11436; GISEL-NEXT:    global_store_b128 v[34:35], v[24:27], off1437; GISEL-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:161438; GISEL-NEXT:    s_endpgm1439bb:1440  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v12i32.v12i32(i32 3, <12 x i32> %A, i32 4, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1441  store <8 x float> %res, ptr addrspace(1) %out1442  ret void1443}1444 1445define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp4(<12 x i32> %A, <8 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1446; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp4:1447; GFX1250:       ; %bb.0: ; %bb1448; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11449; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27], v28, v29 matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP41450; GFX1250-NEXT:    s_clause 0x11451; GFX1250-NEXT:    global_store_b128 v[30:31], v[24:27], off offset:161452; GFX1250-NEXT:    global_store_b128 v[30:31], v[20:23], off1453; GFX1250-NEXT:    s_endpgm1454;1455; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp4:1456; GISEL:       ; %bb.0: ; %bb1457; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11458; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27], v28, v29 matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP41459; GISEL-NEXT:    s_clause 0x11460; GISEL-NEXT:    global_store_b128 v[30:31], v[20:23], off1461; GISEL-NEXT:    global_store_b128 v[30:31], v[24:27], off offset:161462; GISEL-NEXT:    s_endpgm1463bb:1464  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v12i32.v8i32(i32 3, <12 x i32> %A, i32 4, <8 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1465  store <8 x float> %res, ptr addrspace(1) %out1466  ret void1467}1468 1469define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1470; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp8:1471; GFX1250:       ; %bb.0: ; %bb1472; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11473; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_FP41474; GFX1250-NEXT:    s_clause 0x11475; GFX1250-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:161476; GFX1250-NEXT:    global_store_b128 v[34:35], v[24:27], off1477; GFX1250-NEXT:    s_endpgm1478;1479; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp8:1480; GISEL:       ; %bb.0: ; %bb1481; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11482; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_FP41483; GISEL-NEXT:    s_clause 0x11484; GISEL-NEXT:    global_store_b128 v[34:35], v[24:27], off1485; GISEL-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:161486; GISEL-NEXT:    s_endpgm1487bb:1488  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v8i32.v16i32(i32 4, <8 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1489  store <8 x float> %res, ptr addrspace(1) %out1490  ret void1491}1492 1493define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp4_bf8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1494; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_bf8:1495; GFX1250:       ; %bb.0: ; %bb1496; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11497; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_BF81498; GFX1250-NEXT:    s_clause 0x11499; GFX1250-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:161500; GFX1250-NEXT:    global_store_b128 v[34:35], v[24:27], off1501; GFX1250-NEXT:    s_endpgm1502;1503; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_bf8:1504; GISEL:       ; %bb.0: ; %bb1505; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11506; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_BF81507; GISEL-NEXT:    s_clause 0x11508; GISEL-NEXT:    global_store_b128 v[34:35], v[24:27], off1509; GISEL-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:161510; GISEL-NEXT:    s_endpgm1511bb:1512  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v8i32.v16i32(i32 4, <8 x i32> %A, i32 1, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1513  store <8 x float> %res, ptr addrspace(1) %out1514  ret void1515}1516 1517define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp6(<8 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1518; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp6:1519; GFX1250:       ; %bb.0: ; %bb1520; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11521; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27], v28, v29 matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP61522; GFX1250-NEXT:    s_clause 0x11523; GFX1250-NEXT:    global_store_b128 v[30:31], v[24:27], off offset:161524; GFX1250-NEXT:    global_store_b128 v[30:31], v[20:23], off1525; GFX1250-NEXT:    s_endpgm1526;1527; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp6:1528; GISEL:       ; %bb.0: ; %bb1529; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11530; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27], v28, v29 matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP61531; GISEL-NEXT:    s_clause 0x11532; GISEL-NEXT:    global_store_b128 v[30:31], v[20:23], off1533; GISEL-NEXT:    global_store_b128 v[30:31], v[24:27], off offset:161534; GISEL-NEXT:    s_endpgm1535bb:1536  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v8i32.v12i32(i32 4, <8 x i32> %A, i32 2, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1537  store <8 x float> %res, ptr addrspace(1) %out1538  ret void1539}1540 1541define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp4_bf6(<8 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1542; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_bf6:1543; GFX1250:       ; %bb.0: ; %bb1544; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11545; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27], v28, v29 matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP41546; GFX1250-NEXT:    s_clause 0x11547; GFX1250-NEXT:    global_store_b128 v[30:31], v[24:27], off offset:161548; GFX1250-NEXT:    global_store_b128 v[30:31], v[20:23], off1549; GFX1250-NEXT:    s_endpgm1550;1551; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_bf6:1552; GISEL:       ; %bb.0: ; %bb1553; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11554; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27], v28, v29 matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP41555; GISEL-NEXT:    s_clause 0x11556; GISEL-NEXT:    global_store_b128 v[30:31], v[20:23], off1557; GISEL-NEXT:    global_store_b128 v[30:31], v[24:27], off offset:161558; GISEL-NEXT:    s_endpgm1559bb:1560  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v8i32.v12i32(i32 4, <8 x i32> %A, i32 4, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1561  store <8 x float> %res, ptr addrspace(1) %out1562  ret void1563}1564 1565define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp4(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {1566; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp4:1567; GFX1250:       ; %bb.0: ; %bb1568; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11569; GFX1250-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[16:23], v[0:7], v[8:15], v[16:23], v24, v25 matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP41570; GFX1250-NEXT:    s_clause 0x11571; GFX1250-NEXT:    global_store_b128 v[26:27], v[20:23], off offset:161572; GFX1250-NEXT:    global_store_b128 v[26:27], v[16:19], off1573; GFX1250-NEXT:    s_endpgm1574;1575; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp4:1576; GISEL:       ; %bb.0: ; %bb1577; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11578; GISEL-NEXT:    v_wmma_scale_f32_16x16x128_f8f6f4 v[16:23], v[0:7], v[8:15], v[16:23], v24, v25 matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP41579; GISEL-NEXT:    s_clause 0x11580; GISEL-NEXT:    global_store_b128 v[26:27], v[16:19], off1581; GISEL-NEXT:    global_store_b128 v[26:27], v[20:23], off offset:161582; GISEL-NEXT:    s_endpgm1583bb:1584  %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v8i32.v8i32(i32 4, <8 x i32> %A, i32 4, <8 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i32 %scale_src0, i32 0, i32 0, i32 %scale_src1, i1 false, i1 false)1585  store <8 x float> %res, ptr addrspace(1) %out1586  ret void1587}1588 1589define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {1590; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4:1591; GFX1250:       ; %bb.0: ; %bb1592; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11593; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v[40:41], v[42:43] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW11594; GFX1250-NEXT:    s_clause 0x11595; GFX1250-NEXT:    global_store_b128 v[44:45], v[36:39], off offset:161596; GFX1250-NEXT:    global_store_b128 v[44:45], v[32:35], off1597; GFX1250-NEXT:    s_endpgm1598;1599; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4:1600; GISEL:       ; %bb.0: ; %bb1601; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11602; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v[40:41], v[42:43] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW11603; GISEL-NEXT:    s_clause 0x11604; GISEL-NEXT:    global_store_b128 v[44:45], v[32:35], off1605; GISEL-NEXT:    global_store_b128 v[44:45], v[36:39], off offset:161606; GISEL-NEXT:    s_endpgm1607bb:1608  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32i.v16i32(i32 1, <16 x i32> %A, i32 2, <16 x i32> %B, i16 0, <8 x float> %C, i32 1, i32 0, i64 %scale_src0, i32 1, i32 0, i64 %scale_src1, i1 false, i1 false)1609  store <8 x float> %res, ptr addrspace(1) %out1610  ret void1611}1612 1613define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_ss(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 inreg %scale_src0, i64 inreg %scale_src1, ptr addrspace(1) %out) {1614; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_ss:1615; GFX1250:       ; %bb.0: ; %bb1616; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11617; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], s[0:1], s[2:3] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_a_reuse1618; GFX1250-NEXT:    s_clause 0x11619; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:161620; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off1621; GFX1250-NEXT:    s_endpgm1622;1623; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_ss:1624; GISEL:       ; %bb.0: ; %bb1625; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11626; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], s[0:1], s[2:3] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_a_reuse1627; GISEL-NEXT:    s_clause 0x11628; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off1629; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:161630; GISEL-NEXT:    s_endpgm1631bb:1632  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32i.v16i32(i32 1, <16 x i32> %A, i32 2, <16 x i32> %B, i16 0, <8 x float> %C, i32 2, i32 1, i64 %scale_src0, i32 1, i32 2, i64 %scale_src1, i1 true, i1 false)1633  store <8 x float> %res, ptr addrspace(1) %out1634  ret void1635}1636 1637define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_si_scale(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 inreg %scale_src0, ptr addrspace(1) %out) {1638; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_si_scale:1639; GFX1250:       ; %bb.0: ; %bb1640; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11641; GFX1250-NEXT:    v_mov_b64_e32 v[42:43], 0x641642; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)1643; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], s[0:1], v[42:43] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_reuse1644; GFX1250-NEXT:    s_clause 0x11645; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:161646; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off1647; GFX1250-NEXT:    s_endpgm1648;1649; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_si_scale:1650; GISEL:       ; %bb.0: ; %bb1651; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11652; GISEL-NEXT:    v_mov_b64_e32 v[42:43], 0x641653; GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)1654; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], s[0:1], v[42:43] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_reuse1655; GISEL-NEXT:    s_clause 0x11656; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off1657; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:161658; GISEL-NEXT:    s_endpgm1659bb:1660  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32i.v16i32(i32 1, <16 x i32> %A, i32 2, <16 x i32> %B, i16 0, <8 x float> %C, i32 3, i32 2, i64 %scale_src0, i32 0, i32 1, i64 100, i1 false, i1 true)1661  store <8 x float> %res, ptr addrspace(1) %out1662  ret void1663}1664 1665define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {1666; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_bf8:1667; GFX1250:       ; %bb.0: ; %bb1668; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11669; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v[40:41], v[42:43] matrix_b_fmt:MATRIX_FMT_BF81670; GFX1250-NEXT:    s_clause 0x11671; GFX1250-NEXT:    global_store_b128 v[44:45], v[36:39], off offset:161672; GFX1250-NEXT:    global_store_b128 v[44:45], v[32:35], off1673; GFX1250-NEXT:    s_endpgm1674;1675; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_bf8:1676; GISEL:       ; %bb.0: ; %bb1677; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11678; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v[40:41], v[42:43] matrix_b_fmt:MATRIX_FMT_BF81679; GISEL-NEXT:    s_clause 0x11680; GISEL-NEXT:    global_store_b128 v[44:45], v[32:35], off1681; GISEL-NEXT:    global_store_b128 v[44:45], v[36:39], off offset:161682; GISEL-NEXT:    s_endpgm1683bb:1684  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 1, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)1685  store <8 x float> %res, ptr addrspace(1) %out1686  ret void1687}1688 1689define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_fp6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {1690; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_fp6:1691; GFX1250:       ; %bb.0: ; %bb1692; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11693; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v[36:37], v[38:39] matrix_b_fmt:MATRIX_FMT_FP61694; GFX1250-NEXT:    s_clause 0x11695; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:161696; GFX1250-NEXT:    global_store_b128 v[40:41], v[28:31], off1697; GFX1250-NEXT:    s_endpgm1698;1699; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_fp6:1700; GISEL:       ; %bb.0: ; %bb1701; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11702; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v[36:37], v[38:39] matrix_b_fmt:MATRIX_FMT_FP61703; GISEL-NEXT:    s_clause 0x11704; GISEL-NEXT:    global_store_b128 v[40:41], v[28:31], off1705; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:161706; GISEL-NEXT:    s_endpgm1707bb:1708  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v12i32(i32 0, <16 x i32> %A, i32 2, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)1709  store <8 x float> %res, ptr addrspace(1) %out1710  ret void1711}1712 1713define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_bf6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {1714; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_bf6:1715; GFX1250:       ; %bb.0: ; %bb1716; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11717; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v[36:37], v[38:39] matrix_b_fmt:MATRIX_FMT_BF61718; GFX1250-NEXT:    s_clause 0x11719; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:161720; GFX1250-NEXT:    global_store_b128 v[40:41], v[28:31], off1721; GFX1250-NEXT:    s_endpgm1722;1723; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_bf6:1724; GISEL:       ; %bb.0: ; %bb1725; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11726; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v[36:37], v[38:39] matrix_b_fmt:MATRIX_FMT_BF61727; GISEL-NEXT:    s_clause 0x11728; GISEL-NEXT:    global_store_b128 v[40:41], v[28:31], off1729; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:161730; GISEL-NEXT:    s_endpgm1731bb:1732  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v12i32(i32 0, <16 x i32> %A, i32 3, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)1733  store <8 x float> %res, ptr addrspace(1) %out1734  ret void1735}1736 1737define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_fp4(<16 x i32> %A, <8 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {1738; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_fp4:1739; GFX1250:       ; %bb.0: ; %bb1740; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11741; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31], v[32:33], v[34:35] matrix_b_fmt:MATRIX_FMT_FP41742; GFX1250-NEXT:    s_clause 0x11743; GFX1250-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:161744; GFX1250-NEXT:    global_store_b128 v[36:37], v[24:27], off1745; GFX1250-NEXT:    s_endpgm1746;1747; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_fp4:1748; GISEL:       ; %bb.0: ; %bb1749; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11750; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31], v[32:33], v[34:35] matrix_b_fmt:MATRIX_FMT_FP41751; GISEL-NEXT:    s_clause 0x11752; GISEL-NEXT:    global_store_b128 v[36:37], v[24:27], off1753; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:161754; GISEL-NEXT:    s_endpgm1755bb:1756  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v8i32(i32 0, <16 x i32> %A, i32 4, <8 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)1757  store <8 x float> %res, ptr addrspace(1) %out1758  ret void1759}1760 1761define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {1762; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp8:1763; GFX1250:       ; %bb.0: ; %bb1764; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11765; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v[40:41], v[42:43] matrix_a_fmt:MATRIX_FMT_BF81766; GFX1250-NEXT:    s_clause 0x11767; GFX1250-NEXT:    global_store_b128 v[44:45], v[36:39], off offset:161768; GFX1250-NEXT:    global_store_b128 v[44:45], v[32:35], off1769; GFX1250-NEXT:    s_endpgm1770;1771; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp8:1772; GISEL:       ; %bb.0: ; %bb1773; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11774; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v[40:41], v[42:43] matrix_a_fmt:MATRIX_FMT_BF81775; GISEL-NEXT:    s_clause 0x11776; GISEL-NEXT:    global_store_b128 v[44:45], v[32:35], off1777; GISEL-NEXT:    global_store_b128 v[44:45], v[36:39], off offset:161778; GISEL-NEXT:    s_endpgm1779bb:1780  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 1, <16 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)1781  store <8 x float> %res, ptr addrspace(1) %out1782  ret void1783}1784 1785define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {1786; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_bf8:1787; GFX1250:       ; %bb.0: ; %bb1788; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11789; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v[40:41], v[42:43] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF81790; GFX1250-NEXT:    s_clause 0x11791; GFX1250-NEXT:    global_store_b128 v[44:45], v[36:39], off offset:161792; GFX1250-NEXT:    global_store_b128 v[44:45], v[32:35], off1793; GFX1250-NEXT:    s_endpgm1794;1795; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_bf8:1796; GISEL:       ; %bb.0: ; %bb1797; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11798; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v[40:41], v[42:43] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF81799; GISEL-NEXT:    s_clause 0x11800; GISEL-NEXT:    global_store_b128 v[44:45], v[32:35], off1801; GISEL-NEXT:    global_store_b128 v[44:45], v[36:39], off offset:161802; GISEL-NEXT:    s_endpgm1803bb:1804  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 1, <16 x i32> %A, i32 1, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)1805  store <8 x float> %res, ptr addrspace(1) %out1806  ret void1807}1808 1809define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {1810; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp6:1811; GFX1250:       ; %bb.0: ; %bb1812; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11813; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP61814; GFX1250-NEXT:    s_clause 0x11815; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:161816; GFX1250-NEXT:    global_store_b128 v[40:41], v[28:31], off1817; GFX1250-NEXT:    s_endpgm1818;1819; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp6:1820; GISEL:       ; %bb.0: ; %bb1821; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11822; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP61823; GISEL-NEXT:    s_clause 0x11824; GISEL-NEXT:    global_store_b128 v[40:41], v[28:31], off1825; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:161826; GISEL-NEXT:    s_endpgm1827bb:1828  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v12i32(i32 1, <16 x i32> %A, i32 2, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)1829  store <8 x float> %res, ptr addrspace(1) %out1830  ret void1831}1832 1833define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_bf6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {1834; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_bf6:1835; GFX1250:       ; %bb.0: ; %bb1836; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11837; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF61838; GFX1250-NEXT:    s_clause 0x11839; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:161840; GFX1250-NEXT:    global_store_b128 v[40:41], v[28:31], off1841; GFX1250-NEXT:    s_endpgm1842;1843; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_bf6:1844; GISEL:       ; %bb.0: ; %bb1845; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11846; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF61847; GISEL-NEXT:    s_clause 0x11848; GISEL-NEXT:    global_store_b128 v[40:41], v[28:31], off1849; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:161850; GISEL-NEXT:    s_endpgm1851bb:1852  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v12i32(i32 1, <16 x i32> %A, i32 3, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)1853  store <8 x float> %res, ptr addrspace(1) %out1854  ret void1855}1856 1857define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp4(<16 x i32> %A, <8 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {1858; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp4:1859; GFX1250:       ; %bb.0: ; %bb1860; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11861; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP41862; GFX1250-NEXT:    s_clause 0x11863; GFX1250-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:161864; GFX1250-NEXT:    global_store_b128 v[36:37], v[24:27], off1865; GFX1250-NEXT:    s_endpgm1866;1867; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp4:1868; GISEL:       ; %bb.0: ; %bb1869; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11870; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP41871; GISEL-NEXT:    s_clause 0x11872; GISEL-NEXT:    global_store_b128 v[36:37], v[24:27], off1873; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:161874; GISEL-NEXT:    s_endpgm1875bb:1876  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v8i32(i32 1, <16 x i32> %A, i32 4, <8 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)1877  store <8 x float> %res, ptr addrspace(1) %out1878  ret void1879}1880 1881define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {1882; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp8:1883; GFX1250:       ; %bb.0: ; %bb1884; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11885; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_FP61886; GFX1250-NEXT:    s_clause 0x11887; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:161888; GFX1250-NEXT:    global_store_b128 v[40:41], v[28:31], off1889; GFX1250-NEXT:    s_endpgm1890;1891; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp8:1892; GISEL:       ; %bb.0: ; %bb1893; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11894; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_FP61895; GISEL-NEXT:    s_clause 0x11896; GISEL-NEXT:    global_store_b128 v[40:41], v[28:31], off1897; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:161898; GISEL-NEXT:    s_endpgm1899bb:1900  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v12i32.v16i32(i32 2, <12 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)1901  store <8 x float> %res, ptr addrspace(1) %out1902  ret void1903}1904 1905define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_bf8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {1906; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_bf8:1907; GFX1250:       ; %bb.0: ; %bb1908; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11909; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_BF81910; GFX1250-NEXT:    s_clause 0x11911; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:161912; GFX1250-NEXT:    global_store_b128 v[40:41], v[28:31], off1913; GFX1250-NEXT:    s_endpgm1914;1915; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_bf8:1916; GISEL:       ; %bb.0: ; %bb1917; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11918; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_BF81919; GISEL-NEXT:    s_clause 0x11920; GISEL-NEXT:    global_store_b128 v[40:41], v[28:31], off1921; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:161922; GISEL-NEXT:    s_endpgm1923bb:1924  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v12i32.v16i32(i32 2, <12 x i32> %A, i32 1, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)1925  store <8 x float> %res, ptr addrspace(1) %out1926  ret void1927}1928 1929define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {1930; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp6:1931; GFX1250:       ; %bb.0: ; %bb1932; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11933; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP61934; GFX1250-NEXT:    s_clause 0x11935; GFX1250-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:161936; GFX1250-NEXT:    global_store_b128 v[36:37], v[24:27], off1937; GFX1250-NEXT:    s_endpgm1938;1939; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp6:1940; GISEL:       ; %bb.0: ; %bb1941; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11942; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP61943; GISEL-NEXT:    s_clause 0x11944; GISEL-NEXT:    global_store_b128 v[36:37], v[24:27], off1945; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:161946; GISEL-NEXT:    s_endpgm1947bb:1948  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v12i32.v12i32(i32 2, <12 x i32> %A, i32 2, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)1949  store <8 x float> %res, ptr addrspace(1) %out1950  ret void1951}1952 1953define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_bf6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {1954; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_bf6:1955; GFX1250:       ; %bb.0: ; %bb1956; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11957; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP41958; GFX1250-NEXT:    s_clause 0x11959; GFX1250-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:161960; GFX1250-NEXT:    global_store_b128 v[36:37], v[24:27], off1961; GFX1250-NEXT:    s_endpgm1962;1963; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_bf6:1964; GISEL:       ; %bb.0: ; %bb1965; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11966; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP41967; GISEL-NEXT:    s_clause 0x11968; GISEL-NEXT:    global_store_b128 v[36:37], v[24:27], off1969; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:161970; GISEL-NEXT:    s_endpgm1971bb:1972  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v12i32.v12i32(i32 2, <12 x i32> %A, i32 4, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)1973  store <8 x float> %res, ptr addrspace(1) %out1974  ret void1975}1976 1977define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp4(<12 x i32> %A, <8 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {1978; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp4:1979; GFX1250:       ; %bb.0: ; %bb1980; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11981; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27], v[28:29], v[30:31] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP41982; GFX1250-NEXT:    s_clause 0x11983; GFX1250-NEXT:    global_store_b128 v[32:33], v[24:27], off offset:161984; GFX1250-NEXT:    global_store_b128 v[32:33], v[20:23], off1985; GFX1250-NEXT:    s_endpgm1986;1987; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp4:1988; GISEL:       ; %bb.0: ; %bb1989; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11990; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27], v[28:29], v[30:31] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP41991; GISEL-NEXT:    s_clause 0x11992; GISEL-NEXT:    global_store_b128 v[32:33], v[20:23], off1993; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off offset:161994; GISEL-NEXT:    s_endpgm1995bb:1996  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v12i32.v8i32(i32 2, <12 x i32> %A, i32 4, <8 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)1997  store <8 x float> %res, ptr addrspace(1) %out1998  ret void1999}2000 2001define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {2002; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp8:2003; GFX1250:       ; %bb.0: ; %bb2004; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12005; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_BF62006; GFX1250-NEXT:    s_clause 0x12007; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:162008; GFX1250-NEXT:    global_store_b128 v[40:41], v[28:31], off2009; GFX1250-NEXT:    s_endpgm2010;2011; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp8:2012; GISEL:       ; %bb.0: ; %bb2013; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12014; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_BF62015; GISEL-NEXT:    s_clause 0x12016; GISEL-NEXT:    global_store_b128 v[40:41], v[28:31], off2017; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:162018; GISEL-NEXT:    s_endpgm2019bb:2020  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v12i32.v16i32(i32 3, <12 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)2021  store <8 x float> %res, ptr addrspace(1) %out2022  ret void2023}2024 2025define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_bf8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {2026; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_bf8:2027; GFX1250:       ; %bb.0: ; %bb2028; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12029; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_BF82030; GFX1250-NEXT:    s_clause 0x12031; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:162032; GFX1250-NEXT:    global_store_b128 v[40:41], v[28:31], off2033; GFX1250-NEXT:    s_endpgm2034;2035; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_bf8:2036; GISEL:       ; %bb.0: ; %bb2037; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12038; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_BF82039; GISEL-NEXT:    s_clause 0x12040; GISEL-NEXT:    global_store_b128 v[40:41], v[28:31], off2041; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:162042; GISEL-NEXT:    s_endpgm2043bb:2044  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v12i32.v16i32(i32 3, <12 x i32> %A, i32 1, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)2045  store <8 x float> %res, ptr addrspace(1) %out2046  ret void2047}2048 2049define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {2050; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp6:2051; GFX1250:       ; %bb.0: ; %bb2052; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12053; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP62054; GFX1250-NEXT:    s_clause 0x12055; GFX1250-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:162056; GFX1250-NEXT:    global_store_b128 v[36:37], v[24:27], off2057; GFX1250-NEXT:    s_endpgm2058;2059; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp6:2060; GISEL:       ; %bb.0: ; %bb2061; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12062; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP62063; GISEL-NEXT:    s_clause 0x12064; GISEL-NEXT:    global_store_b128 v[36:37], v[24:27], off2065; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:162066; GISEL-NEXT:    s_endpgm2067bb:2068  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v12i32.v12i32(i32 3, <12 x i32> %A, i32 2, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)2069  store <8 x float> %res, ptr addrspace(1) %out2070  ret void2071}2072 2073define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_bf6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {2074; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_bf6:2075; GFX1250:       ; %bb.0: ; %bb2076; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12077; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP42078; GFX1250-NEXT:    s_clause 0x12079; GFX1250-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:162080; GFX1250-NEXT:    global_store_b128 v[36:37], v[24:27], off2081; GFX1250-NEXT:    s_endpgm2082;2083; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_bf6:2084; GISEL:       ; %bb.0: ; %bb2085; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12086; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP42087; GISEL-NEXT:    s_clause 0x12088; GISEL-NEXT:    global_store_b128 v[36:37], v[24:27], off2089; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:162090; GISEL-NEXT:    s_endpgm2091bb:2092  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v12i32.v12i32(i32 3, <12 x i32> %A, i32 4, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)2093  store <8 x float> %res, ptr addrspace(1) %out2094  ret void2095}2096 2097define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp4(<12 x i32> %A, <8 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {2098; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp4:2099; GFX1250:       ; %bb.0: ; %bb2100; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12101; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27], v[28:29], v[30:31] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP42102; GFX1250-NEXT:    s_clause 0x12103; GFX1250-NEXT:    global_store_b128 v[32:33], v[24:27], off offset:162104; GFX1250-NEXT:    global_store_b128 v[32:33], v[20:23], off2105; GFX1250-NEXT:    s_endpgm2106;2107; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp4:2108; GISEL:       ; %bb.0: ; %bb2109; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12110; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27], v[28:29], v[30:31] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP42111; GISEL-NEXT:    s_clause 0x12112; GISEL-NEXT:    global_store_b128 v[32:33], v[20:23], off2113; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off offset:162114; GISEL-NEXT:    s_endpgm2115bb:2116  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v12i32.v8i32(i32 3, <12 x i32> %A, i32 4, <8 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)2117  store <8 x float> %res, ptr addrspace(1) %out2118  ret void2119}2120 2121define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {2122; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp8:2123; GFX1250:       ; %bb.0: ; %bb2124; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12125; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_FP42126; GFX1250-NEXT:    s_clause 0x12127; GFX1250-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:162128; GFX1250-NEXT:    global_store_b128 v[36:37], v[24:27], off2129; GFX1250-NEXT:    s_endpgm2130;2131; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp8:2132; GISEL:       ; %bb.0: ; %bb2133; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12134; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_FP42135; GISEL-NEXT:    s_clause 0x12136; GISEL-NEXT:    global_store_b128 v[36:37], v[24:27], off2137; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:162138; GISEL-NEXT:    s_endpgm2139bb:2140  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v8i32.v16i32(i32 4, <8 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)2141  store <8 x float> %res, ptr addrspace(1) %out2142  ret void2143}2144 2145define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_bf8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {2146; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_bf8:2147; GFX1250:       ; %bb.0: ; %bb2148; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12149; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_BF82150; GFX1250-NEXT:    s_clause 0x12151; GFX1250-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:162152; GFX1250-NEXT:    global_store_b128 v[36:37], v[24:27], off2153; GFX1250-NEXT:    s_endpgm2154;2155; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_bf8:2156; GISEL:       ; %bb.0: ; %bb2157; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12158; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_BF82159; GISEL-NEXT:    s_clause 0x12160; GISEL-NEXT:    global_store_b128 v[36:37], v[24:27], off2161; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:162162; GISEL-NEXT:    s_endpgm2163bb:2164  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v8i32.v16i32(i32 4, <8 x i32> %A, i32 1, <16 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)2165  store <8 x float> %res, ptr addrspace(1) %out2166  ret void2167}2168 2169define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp6(<8 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {2170; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp6:2171; GFX1250:       ; %bb.0: ; %bb2172; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12173; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27], v[28:29], v[30:31] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP62174; GFX1250-NEXT:    s_clause 0x12175; GFX1250-NEXT:    global_store_b128 v[32:33], v[24:27], off offset:162176; GFX1250-NEXT:    global_store_b128 v[32:33], v[20:23], off2177; GFX1250-NEXT:    s_endpgm2178;2179; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp6:2180; GISEL:       ; %bb.0: ; %bb2181; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12182; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27], v[28:29], v[30:31] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP62183; GISEL-NEXT:    s_clause 0x12184; GISEL-NEXT:    global_store_b128 v[32:33], v[20:23], off2185; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off offset:162186; GISEL-NEXT:    s_endpgm2187bb:2188  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v8i32.v12i32(i32 4, <8 x i32> %A, i32 2, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)2189  store <8 x float> %res, ptr addrspace(1) %out2190  ret void2191}2192 2193define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_bf6(<8 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {2194; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_bf6:2195; GFX1250:       ; %bb.0: ; %bb2196; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12197; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27], v[28:29], v[30:31] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP42198; GFX1250-NEXT:    s_clause 0x12199; GFX1250-NEXT:    global_store_b128 v[32:33], v[24:27], off offset:162200; GFX1250-NEXT:    global_store_b128 v[32:33], v[20:23], off2201; GFX1250-NEXT:    s_endpgm2202;2203; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_bf6:2204; GISEL:       ; %bb.0: ; %bb2205; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12206; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27], v[28:29], v[30:31] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP42207; GISEL-NEXT:    s_clause 0x12208; GISEL-NEXT:    global_store_b128 v[32:33], v[20:23], off2209; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off offset:162210; GISEL-NEXT:    s_endpgm2211bb:2212  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v8i32.v12i32(i32 4, <8 x i32> %A, i32 4, <12 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)2213  store <8 x float> %res, ptr addrspace(1) %out2214  ret void2215}2216 2217define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp4(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {2218; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp4:2219; GFX1250:       ; %bb.0: ; %bb2220; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12221; GFX1250-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[16:23], v[0:7], v[8:15], v[16:23], v[24:25], v[26:27] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP42222; GFX1250-NEXT:    s_clause 0x12223; GFX1250-NEXT:    global_store_b128 v[28:29], v[20:23], off offset:162224; GFX1250-NEXT:    global_store_b128 v[28:29], v[16:19], off2225; GFX1250-NEXT:    s_endpgm2226;2227; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp4:2228; GISEL:       ; %bb.0: ; %bb2229; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12230; GISEL-NEXT:    v_wmma_scale16_f32_16x16x128_f8f6f4 v[16:23], v[0:7], v[8:15], v[16:23], v[24:25], v[26:27] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP42231; GISEL-NEXT:    s_clause 0x12232; GISEL-NEXT:    global_store_b128 v[28:29], v[16:19], off2233; GISEL-NEXT:    global_store_b128 v[28:29], v[20:23], off offset:162234; GISEL-NEXT:    s_endpgm2235bb:2236  %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v8i32.v8i32(i32 4, <8 x i32> %A, i32 4, <8 x i32> %B, i16 0, <8 x float> %C, i32 0, i32 0, i64 %scale_src0, i32 0, i32 0, i64 %scale_src1, i1 false, i1 false)2237  store <8 x float> %res, ptr addrspace(1) %out2238  ret void2239}2240 2241define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {2242; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_fp8:2243; GFX1250:       ; %bb.0: ; %bb2244; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12245; GFX1250-NEXT:    v_wmma_f16_16x16x128_fp8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] matrix_b_reuse2246; GFX1250-NEXT:    global_store_b128 v[36:37], v[32:35], off2247; GFX1250-NEXT:    s_endpgm2248;2249; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_fp8:2250; GISEL:       ; %bb.0: ; %bb2251; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12252; GISEL-NEXT:    v_wmma_f16_16x16x128_fp8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] matrix_b_reuse2253; GISEL-NEXT:    global_store_b128 v[36:37], v[32:35], off2254; GISEL-NEXT:    s_endpgm2255bb:2256  %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.fp8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x half> %C, i1 false, i1 true)2257  store <8 x half> %res, ptr addrspace(1) %out2258  ret void2259}2260 2261define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {2262; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_bf8:2263; GFX1250:       ; %bb.0: ; %bb2264; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12265; GFX1250-NEXT:    v_wmma_f16_16x16x128_fp8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] matrix_b_reuse2266; GFX1250-NEXT:    global_store_b128 v[36:37], v[32:35], off2267; GFX1250-NEXT:    s_endpgm2268;2269; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_bf8:2270; GISEL:       ; %bb.0: ; %bb2271; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12272; GISEL-NEXT:    v_wmma_f16_16x16x128_fp8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] matrix_b_reuse2273; GISEL-NEXT:    global_store_b128 v[36:37], v[32:35], off2274; GISEL-NEXT:    s_endpgm2275bb:2276  %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.bf8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x half> %C, i1 false, i1 true)2277  store <8 x half> %res, ptr addrspace(1) %out2278  ret void2279}2280 2281define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {2282; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_fp8:2283; GFX1250:       ; %bb.0: ; %bb2284; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12285; GFX1250-NEXT:    v_wmma_f16_16x16x128_bf8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] matrix_b_reuse2286; GFX1250-NEXT:    global_store_b128 v[36:37], v[32:35], off2287; GFX1250-NEXT:    s_endpgm2288;2289; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_fp8:2290; GISEL:       ; %bb.0: ; %bb2291; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12292; GISEL-NEXT:    v_wmma_f16_16x16x128_bf8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] matrix_b_reuse2293; GISEL-NEXT:    global_store_b128 v[36:37], v[32:35], off2294; GISEL-NEXT:    s_endpgm2295bb:2296  %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.fp8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x half> %C, i1 false, i1 true)2297  store <8 x half> %res, ptr addrspace(1) %out2298  ret void2299}2300 2301define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {2302; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_bf8:2303; GFX1250:       ; %bb.0: ; %bb2304; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12305; GFX1250-NEXT:    v_wmma_f16_16x16x128_bf8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] matrix_b_reuse2306; GFX1250-NEXT:    global_store_b128 v[36:37], v[32:35], off2307; GFX1250-NEXT:    s_endpgm2308;2309; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_bf8:2310; GISEL:       ; %bb.0: ; %bb2311; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12312; GISEL-NEXT:    v_wmma_f16_16x16x128_bf8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] matrix_b_reuse2313; GISEL-NEXT:    global_store_b128 v[36:37], v[32:35], off2314; GISEL-NEXT:    s_endpgm2315bb:2316  %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.bf8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x half> %C, i1 false, i1 true)2317  store <8 x half> %res, ptr addrspace(1) %out2318  ret void2319}2320 2321define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {2322; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_fp8:2323; GFX1250:       ; %bb.0: ; %bb2324; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12325; GFX1250-NEXT:    v_wmma_f32_16x16x128_fp8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_reuse2326; GFX1250-NEXT:    s_clause 0x12327; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:162328; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off2329; GFX1250-NEXT:    s_endpgm2330;2331; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_fp8:2332; GISEL:       ; %bb.0: ; %bb2333; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12334; GISEL-NEXT:    v_wmma_f32_16x16x128_fp8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_reuse2335; GISEL-NEXT:    s_clause 0x12336; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off2337; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:162338; GISEL-NEXT:    s_endpgm2339bb:2340  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.fp8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x float> %C, i1 false, i1 true)2341  store <8 x float> %res, ptr addrspace(1) %out2342  ret void2343}2344 2345define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {2346; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_bf8:2347; GFX1250:       ; %bb.0: ; %bb2348; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12349; GFX1250-NEXT:    v_wmma_f32_16x16x128_fp8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_reuse2350; GFX1250-NEXT:    s_clause 0x12351; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:162352; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off2353; GFX1250-NEXT:    s_endpgm2354;2355; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_bf8:2356; GISEL:       ; %bb.0: ; %bb2357; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12358; GISEL-NEXT:    v_wmma_f32_16x16x128_fp8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_reuse2359; GISEL-NEXT:    s_clause 0x12360; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off2361; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:162362; GISEL-NEXT:    s_endpgm2363bb:2364  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.bf8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x float> %C, i1 false, i1 true)2365  store <8 x float> %res, ptr addrspace(1) %out2366  ret void2367}2368 2369define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {2370; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_fp8:2371; GFX1250:       ; %bb.0: ; %bb2372; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12373; GFX1250-NEXT:    v_wmma_f32_16x16x128_bf8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_reuse2374; GFX1250-NEXT:    s_clause 0x12375; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:162376; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off2377; GFX1250-NEXT:    s_endpgm2378;2379; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_fp8:2380; GISEL:       ; %bb.0: ; %bb2381; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12382; GISEL-NEXT:    v_wmma_f32_16x16x128_bf8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_reuse2383; GISEL-NEXT:    s_clause 0x12384; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off2385; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:162386; GISEL-NEXT:    s_endpgm2387bb:2388  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.fp8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x float> %C, i1 false, i1 true)2389  store <8 x float> %res, ptr addrspace(1) %out2390  ret void2391}2392 2393define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {2394; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_bf8:2395; GFX1250:       ; %bb.0: ; %bb2396; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12397; GFX1250-NEXT:    v_wmma_f32_16x16x128_bf8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_reuse2398; GFX1250-NEXT:    s_clause 0x12399; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:162400; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off2401; GFX1250-NEXT:    s_endpgm2402;2403; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_bf8:2404; GISEL:       ; %bb.0: ; %bb2405; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12406; GISEL-NEXT:    v_wmma_f32_16x16x128_bf8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_reuse2407; GISEL-NEXT:    s_clause 0x12408; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off2409; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:162410; GISEL-NEXT:    s_endpgm2411bb:2412  %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.bf8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x float> %C, i1 false, i1 true)2413  store <8 x float> %res, ptr addrspace(1) %out2414  ret void2415}2416 2417define amdgpu_ps void @test_wmma_f32_32x16x128_f4(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {2418; GFX1250-LABEL: test_wmma_f32_32x16x128_f4:2419; GFX1250:       ; %bb.0: ; %bb2420; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12421; GFX1250-NEXT:    v_wmma_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39]2422; GFX1250-NEXT:    s_clause 0x32423; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:482424; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:322425; GFX1250-NEXT:    global_store_b128 v[40:41], v[28:31], off offset:162426; GFX1250-NEXT:    global_store_b128 v[40:41], v[24:27], off2427; GFX1250-NEXT:    s_endpgm2428;2429; GISEL-LABEL: test_wmma_f32_32x16x128_f4:2430; GISEL:       ; %bb.0: ; %bb2431; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12432; GISEL-NEXT:    v_wmma_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39]2433; GISEL-NEXT:    s_clause 0x32434; GISEL-NEXT:    global_store_b128 v[40:41], v[24:27], off2435; GISEL-NEXT:    global_store_b128 v[40:41], v[28:31], off offset:162436; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:322437; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:482438; GISEL-NEXT:    s_endpgm2439bb:2440  %res = call <16 x float> @llvm.amdgcn.wmma.f32.32x16x128.f4.v16i32.v8i32.v16f32(<16 x i32> %A, <8 x i32> %B, i16 0, <16 x float> %C)2441  store <16 x float> %res, ptr addrspace(1) %out2442  ret void2443}2444 2445define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {2446; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4:2447; GFX1250:       ; %bb.0: ; %bb2448; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12449; GFX1250-NEXT:    v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], v40, v41 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW12450; GFX1250-NEXT:    s_clause 0x32451; GFX1250-NEXT:    global_store_b128 v[42:43], v[36:39], off offset:482452; GFX1250-NEXT:    global_store_b128 v[42:43], v[32:35], off offset:322453; GFX1250-NEXT:    global_store_b128 v[42:43], v[28:31], off offset:162454; GFX1250-NEXT:    global_store_b128 v[42:43], v[24:27], off2455; GFX1250-NEXT:    s_endpgm2456;2457; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4:2458; GISEL:       ; %bb.0: ; %bb2459; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12460; GISEL-NEXT:    v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], v40, v41 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW12461; GISEL-NEXT:    s_clause 0x32462; GISEL-NEXT:    global_store_b128 v[42:43], v[24:27], off2463; GISEL-NEXT:    global_store_b128 v[42:43], v[28:31], off offset:162464; GISEL-NEXT:    global_store_b128 v[42:43], v[32:35], off offset:322465; GISEL-NEXT:    global_store_b128 v[42:43], v[36:39], off offset:482466; GISEL-NEXT:    s_endpgm2467bb:2468  %res = call <16 x float> @llvm.amdgcn.wmma.scale.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 0, <16 x float> %C, i32 1, i32 0, i32 %scale_src0, i32 1, i32 0, i32 %scale_src1, i1 false, i1 false)2469  store <16 x float> %res, ptr addrspace(1) %out2470  ret void2471}2472 2473define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_ss(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, i32 inreg %scale_src0, i32 inreg %scale_src1, ptr addrspace(1) %out) {2474; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4_ss:2475; GFX1250:       ; %bb.0: ; %bb2476; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12477; GFX1250-NEXT:    v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], s0, s1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_a_reuse2478; GFX1250-NEXT:    s_clause 0x32479; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:482480; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:322481; GFX1250-NEXT:    global_store_b128 v[40:41], v[28:31], off offset:162482; GFX1250-NEXT:    global_store_b128 v[40:41], v[24:27], off2483; GFX1250-NEXT:    s_endpgm2484;2485; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4_ss:2486; GISEL:       ; %bb.0: ; %bb2487; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12488; GISEL-NEXT:    v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], s0, s1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_a_reuse2489; GISEL-NEXT:    s_clause 0x32490; GISEL-NEXT:    global_store_b128 v[40:41], v[24:27], off2491; GISEL-NEXT:    global_store_b128 v[40:41], v[28:31], off offset:162492; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:322493; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:482494; GISEL-NEXT:    s_endpgm2495bb:2496  %res = call <16 x float> @llvm.amdgcn.wmma.scale.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 0, <16 x float> %C, i32 2, i32 1, i32 %scale_src0, i32 1, i32 2, i32 %scale_src1, i1 true, i1 false)2497  store <16 x float> %res, ptr addrspace(1) %out2498  ret void2499}2500 2501define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_si_scale(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, i32 inreg %scale_src0, ptr addrspace(1) %out) {2502; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4_si_scale:2503; GFX1250:       ; %bb.0: ; %bb2504; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12505; GFX1250-NEXT:    v_mov_b32_e32 v42, 0x642506; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)2507; GFX1250-NEXT:    v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], s0, v42 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_reuse2508; GFX1250-NEXT:    s_clause 0x32509; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:482510; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:322511; GFX1250-NEXT:    global_store_b128 v[40:41], v[28:31], off offset:162512; GFX1250-NEXT:    global_store_b128 v[40:41], v[24:27], off2513; GFX1250-NEXT:    s_endpgm2514;2515; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4_si_scale:2516; GISEL:       ; %bb.0: ; %bb2517; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12518; GISEL-NEXT:    v_mov_b32_e32 v42, 0x642519; GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)2520; GISEL-NEXT:    v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], s0, v42 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_reuse2521; GISEL-NEXT:    s_clause 0x32522; GISEL-NEXT:    global_store_b128 v[40:41], v[24:27], off2523; GISEL-NEXT:    global_store_b128 v[40:41], v[28:31], off offset:162524; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:322525; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:482526; GISEL-NEXT:    s_endpgm2527bb:2528  %res = call <16 x float> @llvm.amdgcn.wmma.scale.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 0, <16 x float> %C, i32 3, i32 2, i32 %scale_src0, i32 0, i32 1, i32 100, i1 false, i1 true)2529  store <16 x float> %res, ptr addrspace(1) %out2530  ret void2531}2532 2533define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {2534; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4:2535; GFX1250:       ; %bb.0: ; %bb2536; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12537; GFX1250-NEXT:    v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], v[40:41], v[42:43] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW12538; GFX1250-NEXT:    s_clause 0x32539; GFX1250-NEXT:    global_store_b128 v[44:45], v[36:39], off offset:482540; GFX1250-NEXT:    global_store_b128 v[44:45], v[32:35], off offset:322541; GFX1250-NEXT:    global_store_b128 v[44:45], v[28:31], off offset:162542; GFX1250-NEXT:    global_store_b128 v[44:45], v[24:27], off2543; GFX1250-NEXT:    s_endpgm2544;2545; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4:2546; GISEL:       ; %bb.0: ; %bb2547; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12548; GISEL-NEXT:    v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], v[40:41], v[42:43] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW12549; GISEL-NEXT:    s_clause 0x32550; GISEL-NEXT:    global_store_b128 v[44:45], v[24:27], off2551; GISEL-NEXT:    global_store_b128 v[44:45], v[28:31], off offset:162552; GISEL-NEXT:    global_store_b128 v[44:45], v[32:35], off offset:322553; GISEL-NEXT:    global_store_b128 v[44:45], v[36:39], off offset:482554; GISEL-NEXT:    s_endpgm2555bb:2556  %res = call <16 x float> @llvm.amdgcn.wmma.scale16.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 0, <16 x float> %C, i32 1, i32 0, i64 %scale_src0, i32 1, i32 0, i64 %scale_src1, i1 false, i1 false)2557  store <16 x float> %res, ptr addrspace(1) %out2558  ret void2559}2560 2561define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_ss(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, i64 inreg %scale_src0, i64 inreg %scale_src1, ptr addrspace(1) %out) {2562; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4_ss:2563; GFX1250:       ; %bb.0: ; %bb2564; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12565; GFX1250-NEXT:    v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], s[0:1], s[2:3] matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_a_reuse2566; GFX1250-NEXT:    s_clause 0x32567; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:482568; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:322569; GFX1250-NEXT:    global_store_b128 v[40:41], v[28:31], off offset:162570; GFX1250-NEXT:    global_store_b128 v[40:41], v[24:27], off2571; GFX1250-NEXT:    s_endpgm2572;2573; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4_ss:2574; GISEL:       ; %bb.0: ; %bb2575; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12576; GISEL-NEXT:    v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], s[0:1], s[2:3] matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_a_reuse2577; GISEL-NEXT:    s_clause 0x32578; GISEL-NEXT:    global_store_b128 v[40:41], v[24:27], off2579; GISEL-NEXT:    global_store_b128 v[40:41], v[28:31], off offset:162580; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:322581; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:482582; GISEL-NEXT:    s_endpgm2583bb:2584  %res = call <16 x float> @llvm.amdgcn.wmma.scale16.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 0, <16 x float> %C, i32 2, i32 1, i64 %scale_src0, i32 1, i32 2, i64 %scale_src1, i1 true, i1 false)2585  store <16 x float> %res, ptr addrspace(1) %out2586  ret void2587}2588 2589define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_si_scale(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, i64 inreg %scale_src0, ptr addrspace(1) %out) {2590; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4_si_scale:2591; GFX1250:       ; %bb.0: ; %bb2592; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12593; GFX1250-NEXT:    v_mov_b64_e32 v[42:43], 0x642594; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)2595; GFX1250-NEXT:    v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], s[0:1], v[42:43] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_reuse2596; GFX1250-NEXT:    s_clause 0x32597; GFX1250-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:482598; GFX1250-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:322599; GFX1250-NEXT:    global_store_b128 v[40:41], v[28:31], off offset:162600; GFX1250-NEXT:    global_store_b128 v[40:41], v[24:27], off2601; GFX1250-NEXT:    s_endpgm2602;2603; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4_si_scale:2604; GISEL:       ; %bb.0: ; %bb2605; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12606; GISEL-NEXT:    v_mov_b64_e32 v[42:43], 0x642607; GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)2608; GISEL-NEXT:    v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], s[0:1], v[42:43] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_reuse2609; GISEL-NEXT:    s_clause 0x32610; GISEL-NEXT:    global_store_b128 v[40:41], v[24:27], off2611; GISEL-NEXT:    global_store_b128 v[40:41], v[28:31], off offset:162612; GISEL-NEXT:    global_store_b128 v[40:41], v[32:35], off offset:322613; GISEL-NEXT:    global_store_b128 v[40:41], v[36:39], off offset:482614; GISEL-NEXT:    s_endpgm2615bb:2616  %res = call <16 x float> @llvm.amdgcn.wmma.scale16.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 0, <16 x float> %C, i32 3, i32 2, i64 %scale_src0, i32 0, i32 1, i64 100, i1 false, i1 true)2617  store <16 x float> %res, ptr addrspace(1) %out2618  ret void2619}2620 2621define amdgpu_ps void @test_swmmac_f32_16x16x64_bf16(<16 x bfloat> %A, <32 x bfloat> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {2622; GFX1250-LABEL: test_swmmac_f32_16x16x64_bf16:2623; GFX1250:       ; %bb.0: ; %bb2624; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12625; GFX1250-NEXT:    v_swmmac_f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 matrix_b_reuse2626; GFX1250-NEXT:    v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v332627; GFX1250-NEXT:    s_clause 0x12628; GFX1250-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:162629; GFX1250-NEXT:    global_store_b128 v[34:35], v[24:27], off2630; GFX1250-NEXT:    s_endpgm2631;2632; GISEL-LABEL: test_swmmac_f32_16x16x64_bf16:2633; GISEL:       ; %bb.0: ; %bb2634; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12635; GISEL-NEXT:    v_swmmac_f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 matrix_b_reuse2636; GISEL-NEXT:    v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v332637; GISEL-NEXT:    s_clause 0x12638; GISEL-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:162639; GISEL-NEXT:    global_store_b128 v[34:35], v[24:27], off2640; GISEL-NEXT:    s_endpgm2641bb:2642  %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x64.bf16.v8f32.v16bf16.v32bf16.i16(i1 0, <16 x bfloat> %A, i1 0, <32 x bfloat> %B, <8 x float> %C, i16 %Index, i1 false, i1 true)2643  store <8 x float> %res, ptr addrspace(1) %out2644  ret void2645}2646 2647define amdgpu_ps void @test_swmmac_bf16_16x16x64_bf16(<16 x bfloat> %A, <32 x bfloat> %B, <8 x bfloat> %C, i16 %Index, ptr addrspace(1) %out) {2648; GFX1250-LABEL: test_swmmac_bf16_16x16x64_bf16:2649; GFX1250:       ; %bb.0: ; %bb2650; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12651; GFX1250-NEXT:    v_swmmac_bf16_16x16x64_bf16 v[24:27], v[0:7], v[8:23], v28 matrix_b_reuse2652; GFX1250-NEXT:    v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v292653; GFX1250-NEXT:    global_store_b128 v[30:31], v[24:27], off2654; GFX1250-NEXT:    s_endpgm2655;2656; GISEL-LABEL: test_swmmac_bf16_16x16x64_bf16:2657; GISEL:       ; %bb.0: ; %bb2658; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12659; GISEL-NEXT:    v_swmmac_bf16_16x16x64_bf16 v[24:27], v[0:7], v[8:23], v28 matrix_b_reuse2660; GISEL-NEXT:    v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v292661; GISEL-NEXT:    global_store_b128 v[30:31], v[24:27], off2662; GISEL-NEXT:    s_endpgm2663bb:2664  %res = call <8 x bfloat> @llvm.amdgcn.swmmac.bf16.16x16x64.bf16.v8bf16.v16bf16.v32bf16.i16(i1 0, <16 x bfloat> %A, i1 0, <32 x bfloat> %B, <8 x bfloat> %C, i16 %Index, i1 false, i1 true)2665  store <8 x bfloat> %res, ptr addrspace(1) %out2666  ret void2667}2668 2669define amdgpu_ps void @test_swmmac_bf16f32_16x16x64_bf16(<16 x bfloat> %A, <32 x bfloat> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {2670; GFX1250-LABEL: test_swmmac_bf16f32_16x16x64_bf16:2671; GFX1250:       ; %bb.0: ; %bb2672; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12673; GFX1250-NEXT:    v_swmmac_bf16f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 matrix_b_reuse2674; GFX1250-NEXT:    v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v332675; GFX1250-NEXT:    s_clause 0x12676; GFX1250-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:162677; GFX1250-NEXT:    global_store_b128 v[34:35], v[24:27], off2678; GFX1250-NEXT:    s_endpgm2679;2680; GISEL-LABEL: test_swmmac_bf16f32_16x16x64_bf16:2681; GISEL:       ; %bb.0: ; %bb2682; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12683; GISEL-NEXT:    v_swmmac_bf16f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 matrix_b_reuse2684; GISEL-NEXT:    v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v332685; GISEL-NEXT:    s_clause 0x12686; GISEL-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:162687; GISEL-NEXT:    global_store_b128 v[34:35], v[24:27], off2688; GISEL-NEXT:    s_endpgm2689bb:2690  %res = call <8 x float> @llvm.amdgcn.swmmac.bf16f32.16x16x64.bf16.v8f32.v16bf16.v32bf16.i16(i1 0, <16 x bfloat> %A, i1 0, <32 x bfloat> %B, <8 x float> %C, i16 %Index, i1 false, i1 true)2691  store <8 x float> %res, ptr addrspace(1) %out2692  ret void2693}2694 2695define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_fp8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %Index, ptr addrspace(1) %out) {2696; GFX1250-LABEL: test_swmmac_f32_16x16x128_fp8_fp8:2697; GFX1250:       ; %bb.0: ; %bb2698; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12699; GFX1250-NEXT:    v_swmmac_f32_16x16x128_fp8_fp8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse2700; GFX1250-NEXT:    s_clause 0x12701; GFX1250-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:162702; GFX1250-NEXT:    global_store_b128 v[34:35], v[24:27], off2703; GFX1250-NEXT:    s_endpgm2704;2705; GISEL-LABEL: test_swmmac_f32_16x16x128_fp8_fp8:2706; GISEL:       ; %bb.0: ; %bb2707; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12708; GISEL-NEXT:    v_swmmac_f32_16x16x128_fp8_fp8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse2709; GISEL-NEXT:    s_clause 0x12710; GISEL-NEXT:    global_store_b128 v[34:35], v[24:27], off2711; GISEL-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:162712; GISEL-NEXT:    s_endpgm2713bb:2714  %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.fp8.fp8.v8f32.v8i32.v16i32.i64(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %Index, i1 false, i1 true)2715  store <8 x float> %res, ptr addrspace(1) %out2716  ret void2717}2718 2719define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_bf8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %Index, ptr addrspace(1) %out) {2720; GFX1250-LABEL: test_swmmac_f32_16x16x128_fp8_bf8:2721; GFX1250:       ; %bb.0: ; %bb2722; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12723; GFX1250-NEXT:    v_swmmac_f32_16x16x128_fp8_bf8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse2724; GFX1250-NEXT:    s_clause 0x12725; GFX1250-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:162726; GFX1250-NEXT:    global_store_b128 v[34:35], v[24:27], off2727; GFX1250-NEXT:    s_endpgm2728;2729; GISEL-LABEL: test_swmmac_f32_16x16x128_fp8_bf8:2730; GISEL:       ; %bb.0: ; %bb2731; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12732; GISEL-NEXT:    v_swmmac_f32_16x16x128_fp8_bf8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse2733; GISEL-NEXT:    s_clause 0x12734; GISEL-NEXT:    global_store_b128 v[34:35], v[24:27], off2735; GISEL-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:162736; GISEL-NEXT:    s_endpgm2737bb:2738  %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.fp8.bf8.v8f32.v8i32.v16i32.i64(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %Index, i1 false, i1 true)2739  store <8 x float> %res, ptr addrspace(1) %out2740  ret void2741}2742 2743define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_fp8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %Index, ptr addrspace(1) %out) {2744; GFX1250-LABEL: test_swmmac_f32_16x16x128_bf8_fp8:2745; GFX1250:       ; %bb.0: ; %bb2746; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12747; GFX1250-NEXT:    v_swmmac_f32_16x16x128_bf8_fp8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse2748; GFX1250-NEXT:    s_clause 0x12749; GFX1250-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:162750; GFX1250-NEXT:    global_store_b128 v[34:35], v[24:27], off2751; GFX1250-NEXT:    s_endpgm2752;2753; GISEL-LABEL: test_swmmac_f32_16x16x128_bf8_fp8:2754; GISEL:       ; %bb.0: ; %bb2755; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12756; GISEL-NEXT:    v_swmmac_f32_16x16x128_bf8_fp8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse2757; GISEL-NEXT:    s_clause 0x12758; GISEL-NEXT:    global_store_b128 v[34:35], v[24:27], off2759; GISEL-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:162760; GISEL-NEXT:    s_endpgm2761bb:2762  %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.bf8.fp8.v8f32.v8i32.v16i32.i64(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %Index, i1 false, i1 true)2763  store <8 x float> %res, ptr addrspace(1) %out2764  ret void2765}2766 2767define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_bf8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %Index, ptr addrspace(1) %out) {2768; GFX1250-LABEL: test_swmmac_f32_16x16x128_bf8_bf8:2769; GFX1250:       ; %bb.0: ; %bb2770; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12771; GFX1250-NEXT:    v_swmmac_f32_16x16x128_bf8_bf8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse2772; GFX1250-NEXT:    s_clause 0x12773; GFX1250-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:162774; GFX1250-NEXT:    global_store_b128 v[34:35], v[24:27], off2775; GFX1250-NEXT:    s_endpgm2776;2777; GISEL-LABEL: test_swmmac_f32_16x16x128_bf8_bf8:2778; GISEL:       ; %bb.0: ; %bb2779; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12780; GISEL-NEXT:    v_swmmac_f32_16x16x128_bf8_bf8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse2781; GISEL-NEXT:    s_clause 0x12782; GISEL-NEXT:    global_store_b128 v[34:35], v[24:27], off2783; GISEL-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:162784; GISEL-NEXT:    s_endpgm2785bb:2786  %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.bf8.bf8.v8f32.v8i32.v16i32.i64(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %Index, i1 false, i1 true)2787  store <8 x float> %res, ptr addrspace(1) %out2788  ret void2789}2790 2791define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_fp8(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i16 %Index, ptr addrspace(1) %out) {2792; GFX1250-LABEL: test_swmmac_f16_16x16x128_fp8_fp8:2793; GFX1250:       ; %bb.0: ; %bb2794; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12795; GFX1250-NEXT:    v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v292796; GFX1250-NEXT:    v_mov_b32_e32 v29, 02797; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)2798; GFX1250-NEXT:    v_swmmac_f16_16x16x128_fp8_fp8 v[24:27], v[0:7], v[8:23], v[28:29] matrix_b_reuse2799; GFX1250-NEXT:    global_store_b128 v[30:31], v[24:27], off2800; GFX1250-NEXT:    s_endpgm2801;2802; GISEL-LABEL: test_swmmac_f16_16x16x128_fp8_fp8:2803; GISEL:       ; %bb.0: ; %bb2804; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12805; GISEL-NEXT:    v_swmmac_f16_16x16x128_fp8_fp8 v[24:27], v[0:7], v[8:23], v[28:29] matrix_b_reuse2806; GISEL-NEXT:    v_dual_mov_b32 v32, v29 :: v_dual_mov_b32 v33, v302807; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off2808; GISEL-NEXT:    s_endpgm2809bb:2810  %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.fp8.fp8.v8f16.v8i32.v16i32.i16(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i16 %Index, i1 false, i1 true)2811  store <8 x half> %res, ptr addrspace(1) %out2812  ret void2813}2814 2815define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_bf8(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i16 %Index, ptr addrspace(1) %out) {2816; GFX1250-LABEL: test_swmmac_f16_16x16x128_fp8_bf8:2817; GFX1250:       ; %bb.0: ; %bb2818; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12819; GFX1250-NEXT:    v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v292820; GFX1250-NEXT:    v_mov_b32_e32 v29, 02821; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)2822; GFX1250-NEXT:    v_swmmac_f16_16x16x128_fp8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] matrix_b_reuse2823; GFX1250-NEXT:    global_store_b128 v[30:31], v[24:27], off2824; GFX1250-NEXT:    s_endpgm2825;2826; GISEL-LABEL: test_swmmac_f16_16x16x128_fp8_bf8:2827; GISEL:       ; %bb.0: ; %bb2828; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12829; GISEL-NEXT:    v_swmmac_f16_16x16x128_fp8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] matrix_b_reuse2830; GISEL-NEXT:    v_dual_mov_b32 v32, v29 :: v_dual_mov_b32 v33, v302831; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off2832; GISEL-NEXT:    s_endpgm2833bb:2834  %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.fp8.bf8.v8f16.v8i32.v16i32.i16(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i16 %Index, i1 false, i1 true)2835  store <8 x half> %res, ptr addrspace(1) %out2836  ret void2837}2838 2839define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_fp8(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i16 %Index, ptr addrspace(1) %out) {2840; GFX1250-LABEL: test_swmmac_f16_16x16x128_bf8_fp8:2841; GFX1250:       ; %bb.0: ; %bb2842; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12843; GFX1250-NEXT:    v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v292844; GFX1250-NEXT:    v_mov_b32_e32 v29, 02845; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)2846; GFX1250-NEXT:    v_swmmac_f16_16x16x128_bf8_fp8 v[24:27], v[0:7], v[8:23], v[28:29] matrix_b_reuse2847; GFX1250-NEXT:    global_store_b128 v[30:31], v[24:27], off2848; GFX1250-NEXT:    s_endpgm2849;2850; GISEL-LABEL: test_swmmac_f16_16x16x128_bf8_fp8:2851; GISEL:       ; %bb.0: ; %bb2852; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12853; GISEL-NEXT:    v_swmmac_f16_16x16x128_bf8_fp8 v[24:27], v[0:7], v[8:23], v[28:29] matrix_b_reuse2854; GISEL-NEXT:    v_dual_mov_b32 v32, v29 :: v_dual_mov_b32 v33, v302855; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off2856; GISEL-NEXT:    s_endpgm2857bb:2858  %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.bf8.fp8.v8f16.v8i32.v16i32.i16(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i16 %Index, i1 false, i1 true)2859  store <8 x half> %res, ptr addrspace(1) %out2860  ret void2861}2862 2863define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_bf8(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i16 %Index, ptr addrspace(1) %out) {2864; GFX1250-LABEL: test_swmmac_f16_16x16x128_bf8_bf8:2865; GFX1250:       ; %bb.0: ; %bb2866; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12867; GFX1250-NEXT:    v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v292868; GFX1250-NEXT:    v_mov_b32_e32 v29, 02869; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)2870; GFX1250-NEXT:    v_swmmac_f16_16x16x128_bf8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] matrix_b_reuse2871; GFX1250-NEXT:    global_store_b128 v[30:31], v[24:27], off2872; GFX1250-NEXT:    s_endpgm2873;2874; GISEL-LABEL: test_swmmac_f16_16x16x128_bf8_bf8:2875; GISEL:       ; %bb.0: ; %bb2876; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12877; GISEL-NEXT:    v_swmmac_f16_16x16x128_bf8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] matrix_b_reuse2878; GISEL-NEXT:    v_dual_mov_b32 v32, v29 :: v_dual_mov_b32 v33, v302879; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off2880; GISEL-NEXT:    s_endpgm2881bb:2882  %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.bf8.bf8.v8f16.v8i32.v16i32.i16(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i16 %Index, i1 false, i1 true)2883  store <8 x half> %res, ptr addrspace(1) %out2884  ret void2885}2886 2887define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8(<8 x i32> %A, <16 x i32> %B, <8 x i32> %C, i64 %Index, ptr addrspace(1) %out) {2888; GFX1250-LABEL: test_swmmac_i32_16x16x128_iu8:2889; GFX1250:       ; %bb.0: ; %bb2890; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12891; GFX1250-NEXT:    v_swmmac_i32_16x16x128_iu8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse2892; GFX1250-NEXT:    s_clause 0x12893; GFX1250-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:162894; GFX1250-NEXT:    global_store_b128 v[34:35], v[24:27], off2895; GFX1250-NEXT:    s_endpgm2896;2897; GISEL-LABEL: test_swmmac_i32_16x16x128_iu8:2898; GISEL:       ; %bb.0: ; %bb2899; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12900; GISEL-NEXT:    v_swmmac_i32_16x16x128_iu8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse2901; GISEL-NEXT:    s_clause 0x12902; GISEL-NEXT:    global_store_b128 v[34:35], v[24:27], off2903; GISEL-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:162904; GISEL-NEXT:    s_endpgm2905bb:2906  %res = call <8 x i32> @llvm.amdgcn.swmmac.i32.16x16x128.iu8.v8i32.v8i32.v16i32.i64(i1 0, <8 x i32> %A, i1 0, <16 x i32> %B, <8 x i32> %C, i64 %Index, i1 false, i1 true)2907  store <8 x i32> %res, ptr addrspace(1) %out2908  ret void2909}2910 2911define amdgpu_ps void @test_swmmac_f32_16x16x64_f16(<16 x half> %A, <32 x half> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {2912; GFX1250-LABEL: test_swmmac_f32_16x16x64_f16:2913; GFX1250:       ; %bb.0: ; %bb2914; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12915; GFX1250-NEXT:    v_swmmac_f32_16x16x64_f16 v[24:31], v[0:7], v[8:23], v32 matrix_b_reuse2916; GFX1250-NEXT:    v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v332917; GFX1250-NEXT:    s_clause 0x12918; GFX1250-NEXT:    global_store_b128 v[34:35], v[28:31], off offset:162919; GFX1250-NEXT:    global_store_b128 v[34:35], v[24:27], off2920; GFX1250-NEXT:    s_endpgm2921;2922; GISEL-LABEL: test_swmmac_f32_16x16x64_f16:2923; GISEL:       ; %bb.0: ; %bb2924; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12925; GISEL-NEXT:    v_swmmac_f32_16x16x64_f16 v[24:31], v[0:7], v[8:23], v32 matrix_b_reuse2926; GISEL-NEXT:    v_dual_mov_b32 v36, v33 :: v_dual_mov_b32 v37, v342927; GISEL-NEXT:    s_clause 0x12928; GISEL-NEXT:    global_store_b128 v[36:37], v[24:27], off2929; GISEL-NEXT:    global_store_b128 v[36:37], v[28:31], off offset:162930; GISEL-NEXT:    s_endpgm2931bb:2932  %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x64.f16.v8f32.v16f16.v32f16.i16(i1 0, <16 x half> %A, i1 0, <32 x half> %B, <8 x float> %C, i16 %Index, i1 false, i1 true)2933  store <8 x float> %res, ptr addrspace(1) %out2934  ret void2935}2936 2937define amdgpu_ps void @test_swmmac_f16_16x16x64_f16(<16 x half> %A, <32 x half> %B, <8 x half> %C, i16 %Index, ptr addrspace(1) %out) {2938; GFX1250-LABEL: test_swmmac_f16_16x16x64_f16:2939; GFX1250:       ; %bb.0: ; %bb2940; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12941; GFX1250-NEXT:    v_swmmac_f16_16x16x64_f16 v[24:27], v[0:7], v[8:23], v28 matrix_b_reuse2942; GFX1250-NEXT:    v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v292943; GFX1250-NEXT:    global_store_b128 v[30:31], v[24:27], off2944; GFX1250-NEXT:    s_endpgm2945;2946; GISEL-LABEL: test_swmmac_f16_16x16x64_f16:2947; GISEL:       ; %bb.0: ; %bb2948; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12949; GISEL-NEXT:    v_swmmac_f16_16x16x64_f16 v[24:27], v[0:7], v[8:23], v28 matrix_b_reuse2950; GISEL-NEXT:    v_dual_mov_b32 v32, v29 :: v_dual_mov_b32 v33, v302951; GISEL-NEXT:    global_store_b128 v[32:33], v[24:27], off2952; GISEL-NEXT:    s_endpgm2953bb:2954  %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x64.f16.v8f16.v16f16.v32f16.i16(i1 0, <16 x half> %A, i1 0, <32 x half> %B, <8 x half> %C, i16 %Index, i1 false, i1 true)2955  store <8 x half> %res, ptr addrspace(1) %out2956  ret void2957}2958 2959declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x4.f32.v8f32.v2f32(i1, <2 x float>, i1, <2 x float>, i16, <8 x float>, i1, i1)2960declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.bf16.v8f32.v16bf16(i1, <16 x bfloat>, i1, <16 x bfloat>, i16, <8 x float>, i1, i1)2961declare <8 x bfloat> @llvm.amdgcn.wmma.bf16.16x16x32.bf16.v8bf16.v16bf16(i1, <16 x bfloat>, i1, <16 x bfloat>, i16, <8 x bfloat>, i1, i1)2962declare <8 x bfloat> @llvm.amdgcn.wmma.bf16f32.16x16x32.bf16.v8bf16.v16bf16(i1, <16 x bfloat>, i1, <16 x bfloat>, i16, <8 x float>, i1, i1)2963declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.fp8.v8f32.v8i32(<8 x i32>, <8 x i32>, i16, <8 x float>, i1, i1)2964declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.bf8.v8f32.v8i32(<8 x i32>, <8 x i32>, i16, <8 x float>, i1, i1)2965declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.fp8.v8f32.v8i32(<8 x i32>, <8 x i32>, i16, <8 x float>, i1, i1)2966declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.bf8.v8f32.v8i32(<8 x i32>, <8 x i32>, i16, <8 x float>, i1, i1)2967declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.fp8.v8f16.v8i32(<8 x i32>, <8 x i32>, i16, <8 x half>, i1, i1)2968declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.bf8.v8f16.v8i32(<8 x i32>, <8 x i32>, i16, <8 x half>, i1, i1)2969declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.fp8.v8f16.v8i32(<8 x i32>, <8 x i32>, i16, <8 x half>, i1, i1)2970declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.bf8.v8f16.v8i32(<8 x i32>, <8 x i32>, i16, <8 x half>, i1, i1)2971declare <8 x i32> @llvm.amdgcn.wmma.i32.16x16x64.iu8.v8i32.v8i32(i1 immarg, <8 x i32>, i1 immarg, <8 x i32>, <8 x i32>, i1, i1)2972declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1, <16 x half>, i1, <16 x half>, i16, <8 x float>, i1, i1)2973declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x32.f16.v8f16.v16f16(i1, <16 x half>, i1, <16 x half>, i16, <8 x half>, i1, i1)2974declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32, <16 x i32>, i32, <16 x i32>, i16, <8 x float>)2975declare <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32, <16 x i32>, i32, <16 x i32>, i16, <8 x float>, i32, i32, i32, i32, i32, i32, i1, i1)2976declare <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32, <16 x i32>, i32, <16 x i32>, i16, <8 x float>, i32, i32, i64, i32, i32, i64, i1, i1)2977declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.fp8.v8f16.v16i32(<16 x i32>, <16 x i32>, i16, <8 x half>, i1, i1)2978declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.bf8.v8f16.v16i32(<16 x i32>, <16 x i32>, i16, <8 x half>, i1, i1)2979declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.fp8.v8f16.v16i32(<16 x i32>, <16 x i32>, i16, <8 x half>, i1, i1)2980declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.bf8.v8f16.v16i32(<16 x i32>, <16 x i32>, i16, <8 x half>, i1, i1)2981declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.fp8.v8f32.v16i32(<16 x i32>, <16 x i32>, i16, <8 x float>, i1, i1)2982declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.bf8.v8f32.v16i32(<16 x i32>, <16 x i32>, i16, <8 x float>, i1, i1)2983declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.fp8.v8f32.v16i32(<16 x i32>, <16 x i32>, i16, <8 x float>, i1, i1)2984declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.bf8.v8f32.v16i32(<16 x i32>, <16 x i32>, i16, <8 x float>, i1, i1)2985declare <16 x float> @llvm.amdgcn.wmma.f32.32x16x128.f4.v16i32.v8i32.v16f32(<16 x i32>, <8 x i32>, i16, <16 x float>)2986declare <16 x float> @llvm.amdgcn.wmma.scale.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32>, <8 x i32>, i16, <16 x float>, i32, i32, i32, i32, i32, i32, i1, i1)2987declare <16 x float> @llvm.amdgcn.wmma.scale16.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32>, <8 x i32>, i16, <16 x float>, i32, i32, i64, i32, i32, i64, i1, i1)2988 2989declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x64.bf16.v8f32.v16bf16.v32bf16.i16(i1, <16 x bfloat>, i1, <32 x bfloat>, <8 x float>, i16, i1, i1)2990declare <8 x bfloat> @llvm.amdgcn.swmmac.bf16.16x16x64.bf16.v8bf16.v16bf16.v32bf16.i16(i1, <16 x bfloat>, i1, <32 x bfloat>, <8 x bfloat>, i16, i1, i1)2991declare <8 x float> @llvm.amdgcn.swmmac.bf16f32.16x16x64.bf16.v8f32.v16bf16.v32bf16.i16(i1, <16 x bfloat>, i1, <32 x bfloat>, <8 x float>, i16, i1, i1)2992declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.fp8.fp8.v8f32.v8i32.v16i32.i64(<8 x i32>, <16 x i32>, <8 x float>, i64, i1, i1)2993declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.fp8.bf8.v8f32.v8i32.v16i32.i64(<8 x i32>, <16 x i32>, <8 x float>, i64, i1, i1)2994declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.bf8.fp8.v8f32.v8i32.v16i32.i64(<8 x i32>, <16 x i32>, <8 x float>, i64, i1, i1)2995declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.bf8.bf8.v8f32.v8i32.v16i32.i64(<8 x i32>, <16 x i32>, <8 x float>, i64, i1, i1)2996declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.fp8.fp8.v8f16.v8i32.v16i32.i64(<8 x i32>, <16 x i32>, <8 x half>, i64, i1, i1)2997declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.fp8.bf8.v8f16.v8i32.v16i32.i64(<8 x i32>, <16 x i32>, <8 x half>, i64, i1, i1)2998declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.bf8.fp8.v8f16.v8i32.v16i32.i64(<8 x i32>, <16 x i32>, <8 x half>, i64, i1, i1)2999declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.bf8.bf8.v8f16.v8i32.v16i32.i64(<8 x i32>, <16 x i32>, <8 x half>, i64, i1, i1)3000declare <8 x i32> @llvm.amdgcn.swmmac.i32.16x16x128.iu8.v8i32.v8i32.v16i32.i64(i1 immarg, <8 x i32>, i1 immarg, <16 x i32>, <8 x i32>, i64 %Index, i1, i1)3001declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x64.f16.v8f32.v16f16.v32f16.i16(i1, <16 x half>, i1, <32 x half>, <8 x float>, i16, i1, i1)3002declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x64.f16.v8f16.v16f16.v32f16.i16(i1, <16 x half>, i1, <32 x half>, <8 x half>, i16, i1, i1)3003