3040 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 42; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s --check-prefix=GFX12503; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -global-isel -global-isel-abort=2 < %s | FileCheck %s --check-prefix=GISEL4 5define amdgpu_ps void @test_wmma_f32_16x16x4_f32(<2 x float> %A, <2 x float> %B, ptr addrspace(1) %out) {6; GFX1250-LABEL: test_wmma_f32_16x16x4_f32:7; GFX1250: ; %bb.0: ; %bb8; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19; GFX1250-NEXT: v_wmma_f32_16x16x4_f32 v[6:13], v[0:1], v[2:3], 1.010; GFX1250-NEXT: s_clause 0x111; GFX1250-NEXT: global_store_b128 v[4:5], v[10:13], off offset:1612; GFX1250-NEXT: global_store_b128 v[4:5], v[6:9], off13; GFX1250-NEXT: s_endpgm14;15; GISEL-LABEL: test_wmma_f32_16x16x4_f32:16; GISEL: ; %bb.0: ; %bb17; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 118; GISEL-NEXT: v_wmma_f32_16x16x4_f32 v[6:13], v[0:1], v[2:3], 1.019; GISEL-NEXT: s_clause 0x120; GISEL-NEXT: global_store_b128 v[4:5], v[6:9], off21; GISEL-NEXT: global_store_b128 v[4:5], v[10:13], off offset:1622; GISEL-NEXT: s_endpgm23bb:24 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x4.f32.v8f32.v2f32(i1 0, <2 x float> %A, i1 0, <2 x float> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)25 store <8 x float> %res, ptr addrspace(1) %out26 ret void27}28 29define amdgpu_ps void @test_wmma_f32_16x16x4_f32_non_splat(<2 x float> %A, <2 x float> %B, ptr addrspace(1) %out) {30; GFX1250-LABEL: test_wmma_f32_16x16x4_f32_non_splat:31; GFX1250: ; %bb.0: ; %bb32; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 133; GFX1250-NEXT: v_dual_mov_b32 v6, 1.0 :: v_dual_mov_b32 v8, 2.034; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)35; GFX1250-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v9, v636; GFX1250-NEXT: v_dual_mov_b32 v10, v6 :: v_dual_mov_b32 v11, v637; GFX1250-NEXT: v_dual_mov_b32 v12, v6 :: v_dual_mov_b32 v13, v638; GFX1250-NEXT: v_wmma_f32_16x16x4_f32 v[6:13], v[0:1], v[2:3], v[6:13]39; GFX1250-NEXT: s_clause 0x140; GFX1250-NEXT: global_store_b128 v[4:5], v[10:13], off offset:1641; GFX1250-NEXT: global_store_b128 v[4:5], v[6:9], off42; GFX1250-NEXT: s_endpgm43;44; GISEL-LABEL: test_wmma_f32_16x16x4_f32_non_splat:45; GISEL: ; %bb.0: ; %bb46; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 147; GISEL-NEXT: s_mov_b32 s0, 1.048; GISEL-NEXT: s_mov_b32 s2, 2.049; GISEL-NEXT: s_mov_b32 s6, s050; GISEL-NEXT: s_mov_b32 s7, s051; GISEL-NEXT: s_mov_b32 s1, s052; GISEL-NEXT: s_mov_b32 s3, s053; GISEL-NEXT: s_mov_b32 s4, s054; GISEL-NEXT: s_mov_b32 s5, s055; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[6:7]56; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[4:5]57; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]58; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[0:1]59; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)60; GISEL-NEXT: v_wmma_f32_16x16x4_f32 v[6:13], v[0:1], v[2:3], v[6:13]61; GISEL-NEXT: s_clause 0x162; GISEL-NEXT: global_store_b128 v[4:5], v[6:9], off63; GISEL-NEXT: global_store_b128 v[4:5], v[10:13], off offset:1664; GISEL-NEXT: s_endpgm65bb:66 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x4.f32.v8f32.v2f32(i1 0, <2 x float> %A, i1 0, <2 x float> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)67 store <8 x float> %res, ptr addrspace(1) %out68 ret void69}70 71define amdgpu_ps void @test_wmma_f32_16x16x4_f32_non_inlineable(<2 x float> %A, <2 x float> %B, ptr addrspace(1) %out) {72; GFX1250-LABEL: test_wmma_f32_16x16x4_f32_non_inlineable:73; GFX1250: ; %bb.0: ; %bb74; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 175; GFX1250-NEXT: v_mov_b32_e32 v6, 0x4040000076; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)77; GFX1250-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v8, v678; GFX1250-NEXT: v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v10, v679; GFX1250-NEXT: v_dual_mov_b32 v11, v6 :: v_dual_mov_b32 v12, v680; GFX1250-NEXT: v_mov_b32_e32 v13, v681; GFX1250-NEXT: v_wmma_f32_16x16x4_f32 v[6:13], v[0:1], v[2:3], v[6:13]82; GFX1250-NEXT: s_clause 0x183; GFX1250-NEXT: global_store_b128 v[4:5], v[10:13], off offset:1684; GFX1250-NEXT: global_store_b128 v[4:5], v[6:9], off85; GFX1250-NEXT: s_endpgm86;87; GISEL-LABEL: test_wmma_f32_16x16x4_f32_non_inlineable:88; GISEL: ; %bb.0: ; %bb89; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 190; GISEL-NEXT: s_mov_b32 s0, 0x4040000091; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)92; GISEL-NEXT: s_mov_b32 s6, s093; GISEL-NEXT: s_mov_b32 s7, s094; GISEL-NEXT: s_mov_b32 s1, s095; GISEL-NEXT: s_mov_b32 s2, s096; GISEL-NEXT: s_mov_b32 s3, s097; GISEL-NEXT: s_mov_b32 s4, s098; GISEL-NEXT: s_mov_b32 s5, s099; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[6:7]100; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[4:5]101; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]102; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[0:1]103; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)104; GISEL-NEXT: v_wmma_f32_16x16x4_f32 v[6:13], v[0:1], v[2:3], v[6:13]105; GISEL-NEXT: s_clause 0x1106; GISEL-NEXT: global_store_b128 v[4:5], v[6:9], off107; GISEL-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16108; GISEL-NEXT: s_endpgm109bb:110 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x4.f32.v8f32.v2f32(i1 0, <2 x float> %A, i1 0, <2 x float> %B, i16 0, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>, i1 false, i1 false)111 store <8 x float> %res, ptr addrspace(1) %out112 ret void113}114 115define amdgpu_ps void @test_wmma_f32_16x16x32_bf16(<16 x bfloat> %A, <16 x bfloat> %B, ptr addrspace(1) %out) {116; GFX1250-LABEL: test_wmma_f32_16x16x32_bf16:117; GFX1250: ; %bb.0: ; %bb118; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1119; GFX1250-NEXT: v_wmma_f32_16x16x32_bf16 v[18:25], v[0:7], v[8:15], 1.0120; GFX1250-NEXT: s_clause 0x1121; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16122; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off123; GFX1250-NEXT: s_endpgm124;125; GISEL-LABEL: test_wmma_f32_16x16x32_bf16:126; GISEL: ; %bb.0: ; %bb127; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1128; GISEL-NEXT: v_wmma_f32_16x16x32_bf16 v[18:25], v[0:7], v[8:15], 1.0129; GISEL-NEXT: s_clause 0x1130; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16131; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off132; GISEL-NEXT: s_endpgm133bb:134 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.bf16.v8f32.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)135 store <8 x float> %res, ptr addrspace(1) %out136 ret void137}138 139define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_non_splat(<16 x bfloat> %A, <16 x bfloat> %B, ptr addrspace(1) %out) {140; GFX1250-LABEL: test_wmma_f32_16x16x32_bf16_non_splat:141; GFX1250: ; %bb.0: ; %bb142; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1143; GFX1250-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0144; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)145; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v21, v18146; GFX1250-NEXT: v_dual_mov_b32 v22, v18 :: v_dual_mov_b32 v23, v18147; GFX1250-NEXT: v_dual_mov_b32 v24, v18 :: v_dual_mov_b32 v25, v18148; GFX1250-NEXT: v_wmma_f32_16x16x32_bf16 v[18:25], v[0:7], v[8:15], v[18:25]149; GFX1250-NEXT: s_clause 0x1150; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16151; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off152; GFX1250-NEXT: s_endpgm153;154; GISEL-LABEL: test_wmma_f32_16x16x32_bf16_non_splat:155; GISEL: ; %bb.0: ; %bb156; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1157; GISEL-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0158; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)159; GISEL-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v21, v18160; GISEL-NEXT: v_dual_mov_b32 v22, v18 :: v_dual_mov_b32 v23, v18161; GISEL-NEXT: v_dual_mov_b32 v24, v18 :: v_dual_mov_b32 v25, v18162; GISEL-NEXT: v_wmma_f32_16x16x32_bf16 v[18:25], v[0:7], v[8:15], v[18:25]163; GISEL-NEXT: s_clause 0x1164; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16165; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off166; GISEL-NEXT: s_endpgm167bb:168 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.bf16.v8f32.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)169 store <8 x float> %res, ptr addrspace(1) %out170 ret void171}172 173define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_non_inlineable(<16 x bfloat> %A, <16 x bfloat> %B, ptr addrspace(1) %out) {174; GFX1250-LABEL: test_wmma_f32_16x16x32_bf16_non_inlineable:175; GFX1250: ; %bb.0: ; %bb176; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1177; GFX1250-NEXT: v_mov_b32_e32 v18, 0x40400000178; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)179; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v18180; GFX1250-NEXT: v_dual_mov_b32 v21, v18 :: v_dual_mov_b32 v22, v18181; GFX1250-NEXT: v_dual_mov_b32 v23, v18 :: v_dual_mov_b32 v24, v18182; GFX1250-NEXT: v_mov_b32_e32 v25, v18183; GFX1250-NEXT: v_wmma_f32_16x16x32_bf16 v[18:25], v[0:7], v[8:15], v[18:25]184; GFX1250-NEXT: s_clause 0x1185; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16186; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off187; GFX1250-NEXT: s_endpgm188;189; GISEL-LABEL: test_wmma_f32_16x16x32_bf16_non_inlineable:190; GISEL: ; %bb.0: ; %bb191; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1192; GISEL-NEXT: v_mov_b32_e32 v18, 0x40400000193; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)194; GISEL-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v18195; GISEL-NEXT: v_dual_mov_b32 v21, v18 :: v_dual_mov_b32 v22, v18196; GISEL-NEXT: v_dual_mov_b32 v23, v18 :: v_dual_mov_b32 v24, v18197; GISEL-NEXT: v_mov_b32_e32 v25, v18198; GISEL-NEXT: v_wmma_f32_16x16x32_bf16 v[18:25], v[0:7], v[8:15], v[18:25]199; GISEL-NEXT: s_clause 0x1200; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16201; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off202; GISEL-NEXT: s_endpgm203bb:204 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.bf16.v8f32.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 0, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>, i1 false, i1 false)205 store <8 x float> %res, ptr addrspace(1) %out206 ret void207}208 209define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16(<16 x bfloat> %A, <16 x bfloat> %B, ptr addrspace(1) %out) {210; GFX1250-LABEL: test_wmma_bf16_16x16x32_bf16:211; GFX1250: ; %bb.0: ; %bb212; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1213; GFX1250-NEXT: v_wmma_bf16_16x16x32_bf16 v[18:21], v[0:7], v[8:15], 1.0214; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off215; GFX1250-NEXT: s_endpgm216;217; GISEL-LABEL: test_wmma_bf16_16x16x32_bf16:218; GISEL: ; %bb.0: ; %bb219; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1220; GISEL-NEXT: v_wmma_bf16_16x16x32_bf16 v[18:21], v[0:7], v[8:15], 1.0221; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off222; GISEL-NEXT: s_endpgm223bb:224 %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16.16x16x32.bf16.v8bf16.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 0, <8 x bfloat> <bfloat 1.0, bfloat 1.0, bfloat 1.0, bfloat 1.0, bfloat 1.0, bfloat 1.0, bfloat 1.0, bfloat 1.0>, i1 false, i1 false)225 store <8 x bfloat> %res, ptr addrspace(1) %out226 ret void227}228 229define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_non_splat(<16 x bfloat> %A, <16 x bfloat> %B, ptr addrspace(1) %out) {230; GFX1250-LABEL: test_wmma_bf16_16x16x32_bf16_non_splat:231; GFX1250: ; %bb.0: ; %bb232; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1233; GFX1250-NEXT: v_dual_mov_b32 v18, 0x3f803f80 :: v_dual_mov_b32 v19, 1.0234; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)235; GFX1250-NEXT: v_dual_mov_b32 v20, v18 :: v_dual_mov_b32 v21, v18236; GFX1250-NEXT: v_wmma_bf16_16x16x32_bf16 v[18:21], v[0:7], v[8:15], v[18:21]237; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off238; GFX1250-NEXT: s_endpgm239;240; GISEL-LABEL: test_wmma_bf16_16x16x32_bf16_non_splat:241; GISEL: ; %bb.0: ; %bb242; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1243; GISEL-NEXT: v_dual_mov_b32 v18, 0x3f803f80 :: v_dual_mov_b32 v19, 1.0244; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)245; GISEL-NEXT: v_dual_mov_b32 v20, v18 :: v_dual_mov_b32 v21, v18246; GISEL-NEXT: v_wmma_bf16_16x16x32_bf16 v[18:21], v[0:7], v[8:15], v[18:21]247; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off248; GISEL-NEXT: s_endpgm249bb:250 %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16.16x16x32.bf16.v8bf16.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 0, <8 x bfloat> <bfloat 1.0, bfloat 1.0, bfloat 0.0, bfloat 1.0, bfloat 1.0, bfloat 1.0, bfloat 1.0, bfloat 1.0>, i1 false, i1 false)251 store <8 x bfloat> %res, ptr addrspace(1) %out252 ret void253}254 255define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_non_inlineable(<16 x bfloat> %A, <16 x bfloat> %B, ptr addrspace(1) %out) {256; GFX1250-LABEL: test_wmma_bf16_16x16x32_bf16_non_inlineable:257; GFX1250: ; %bb.0: ; %bb258; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1259; GFX1250-NEXT: v_mov_b32_e32 v18, 0x3fc03fc0260; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)261; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v18262; GFX1250-NEXT: v_mov_b32_e32 v21, v18263; GFX1250-NEXT: v_wmma_bf16_16x16x32_bf16 v[18:21], v[0:7], v[8:15], v[18:21]264; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off265; GFX1250-NEXT: s_endpgm266;267; GISEL-LABEL: test_wmma_bf16_16x16x32_bf16_non_inlineable:268; GISEL: ; %bb.0: ; %bb269; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1270; GISEL-NEXT: v_mov_b32_e32 v18, 0x3fc03fc0271; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)272; GISEL-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v18273; GISEL-NEXT: v_mov_b32_e32 v21, v18274; GISEL-NEXT: v_wmma_bf16_16x16x32_bf16 v[18:21], v[0:7], v[8:15], v[18:21]275; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off276; GISEL-NEXT: s_endpgm277bb:278 %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16.16x16x32.bf16.v8bf16.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 0, <8 x bfloat> <bfloat 1.5, bfloat 1.5, bfloat 1.5, bfloat 1.5, bfloat 1.5, bfloat 1.5, bfloat 1.5, bfloat 1.5>, i1 false, i1 false)279 store <8 x bfloat> %res, ptr addrspace(1) %out280 ret void281}282 283define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {284; GFX1250-LABEL: test_wmma_bf16f32_16x16x32_bf16:285; GFX1250: ; %bb.0: ; %bb286; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1287; GFX1250-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[18:21], v[0:7], v[8:15], 1.0288; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off289; GFX1250-NEXT: s_endpgm290;291; GISEL-LABEL: test_wmma_bf16f32_16x16x32_bf16:292; GISEL: ; %bb.0: ; %bb293; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1294; GISEL-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[18:21], v[0:7], v[8:15], 1.0295; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off296; GISEL-NEXT: s_endpgm297bb:298 %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16f32.16x16x32.bf16.v8bf16.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)299 store <8 x bfloat> %res, ptr addrspace(1) %out300 ret void301}302 303define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_non_splat(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {304; GFX1250-LABEL: test_wmma_bf16f32_16x16x32_bf16_non_splat:305; GFX1250: ; %bb.0: ; %bb306; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1307; GFX1250-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0308; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)309; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v21, v18310; GFX1250-NEXT: v_dual_mov_b32 v22, v18 :: v_dual_mov_b32 v23, v18311; GFX1250-NEXT: v_dual_mov_b32 v24, v18 :: v_dual_mov_b32 v25, v18312; GFX1250-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[18:25]313; GFX1250-NEXT: global_store_b128 v[16:17], v[26:29], off314; GFX1250-NEXT: s_endpgm315;316; GISEL-LABEL: test_wmma_bf16f32_16x16x32_bf16_non_splat:317; GISEL: ; %bb.0: ; %bb318; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1319; GISEL-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0320; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)321; GISEL-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v21, v18322; GISEL-NEXT: v_dual_mov_b32 v22, v18 :: v_dual_mov_b32 v23, v18323; GISEL-NEXT: v_dual_mov_b32 v24, v18 :: v_dual_mov_b32 v25, v18324; GISEL-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[18:25]325; GISEL-NEXT: global_store_b128 v[16:17], v[26:29], off326; GISEL-NEXT: s_endpgm327bb:328 %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16f32.16x16x32.bf16.v8bf16.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)329 store <8 x bfloat> %res, ptr addrspace(1) %out330 ret void331}332 333define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_non_inlinable(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {334; GFX1250-LABEL: test_wmma_bf16f32_16x16x32_bf16_non_inlinable:335; GFX1250: ; %bb.0: ; %bb336; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1337; GFX1250-NEXT: v_mov_b32_e32 v18, 0x40400000338; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)339; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v18340; GFX1250-NEXT: v_dual_mov_b32 v21, v18 :: v_dual_mov_b32 v22, v18341; GFX1250-NEXT: v_dual_mov_b32 v23, v18 :: v_dual_mov_b32 v24, v18342; GFX1250-NEXT: v_mov_b32_e32 v25, v18343; GFX1250-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[18:25]344; GFX1250-NEXT: global_store_b128 v[16:17], v[26:29], off345; GFX1250-NEXT: s_endpgm346;347; GISEL-LABEL: test_wmma_bf16f32_16x16x32_bf16_non_inlinable:348; GISEL: ; %bb.0: ; %bb349; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1350; GISEL-NEXT: v_mov_b32_e32 v18, 0x40400000351; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)352; GISEL-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v18353; GISEL-NEXT: v_dual_mov_b32 v21, v18 :: v_dual_mov_b32 v22, v18354; GISEL-NEXT: v_dual_mov_b32 v23, v18 :: v_dual_mov_b32 v24, v18355; GISEL-NEXT: v_mov_b32_e32 v25, v18356; GISEL-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[18:25]357; GISEL-NEXT: global_store_b128 v[16:17], v[26:29], off358; GISEL-NEXT: s_endpgm359bb:360 %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16f32.16x16x32.bf16.v8bf16.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 0, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>, i1 false, i1 false)361 store <8 x bfloat> %res, ptr addrspace(1) %out362 ret void363}364 365define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {366; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_fp8:367; GFX1250: ; %bb.0: ; %bb368; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1369; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[18:25], v[0:7], v[8:15], 1.0370; GFX1250-NEXT: s_clause 0x1371; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16372; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off373; GFX1250-NEXT: s_endpgm374;375; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_fp8:376; GISEL: ; %bb.0: ; %bb377; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1378; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[18:25], v[0:7], v[8:15], 1.0379; GISEL-NEXT: s_clause 0x1380; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off381; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16382; GISEL-NEXT: s_endpgm383bb:384 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.fp8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)385 store <8 x float> %res, ptr addrspace(1) %out386 ret void387}388 389define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {390; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_fp8_non_splat:391; GFX1250: ; %bb.0: ; %bb392; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1393; GFX1250-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0394; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)395; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v21, v18396; GFX1250-NEXT: v_dual_mov_b32 v22, v18 :: v_dual_mov_b32 v23, v18397; GFX1250-NEXT: v_dual_mov_b32 v24, v18 :: v_dual_mov_b32 v25, v18398; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[18:25], v[0:7], v[8:15], v[18:25]399; GFX1250-NEXT: s_clause 0x1400; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16401; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off402; GFX1250-NEXT: s_endpgm403;404; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_fp8_non_splat:405; GISEL: ; %bb.0: ; %bb406; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1407; GISEL-NEXT: s_mov_b32 s0, 1.0408; GISEL-NEXT: s_mov_b32 s2, 2.0409; GISEL-NEXT: s_mov_b32 s6, s0410; GISEL-NEXT: s_mov_b32 s7, s0411; GISEL-NEXT: s_mov_b32 s1, s0412; GISEL-NEXT: s_mov_b32 s3, s0413; GISEL-NEXT: s_mov_b32 s4, s0414; GISEL-NEXT: s_mov_b32 s5, s0415; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[6:7]416; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[4:5]417; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]418; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]419; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)420; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[18:25], v[0:7], v[8:15], v[18:25]421; GISEL-NEXT: s_clause 0x1422; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off423; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16424; GISEL-NEXT: s_endpgm425bb:426 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.fp8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)427 store <8 x float> %res, ptr addrspace(1) %out428 ret void429}430 431define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {432; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_fp8_non_inlineable:433; GFX1250: ; %bb.0: ; %bb434; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1435; GFX1250-NEXT: v_mov_b32_e32 v18, 0x40400000436; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)437; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v18438; GFX1250-NEXT: v_dual_mov_b32 v21, v18 :: v_dual_mov_b32 v22, v18439; GFX1250-NEXT: v_dual_mov_b32 v23, v18 :: v_dual_mov_b32 v24, v18440; GFX1250-NEXT: v_mov_b32_e32 v25, v18441; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[18:25], v[0:7], v[8:15], v[18:25]442; GFX1250-NEXT: s_clause 0x1443; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16444; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off445; GFX1250-NEXT: s_endpgm446;447; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_fp8_non_inlineable:448; GISEL: ; %bb.0: ; %bb449; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1450; GISEL-NEXT: s_mov_b32 s0, 0x40400000451; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)452; GISEL-NEXT: s_mov_b32 s6, s0453; GISEL-NEXT: s_mov_b32 s7, s0454; GISEL-NEXT: s_mov_b32 s1, s0455; GISEL-NEXT: s_mov_b32 s2, s0456; GISEL-NEXT: s_mov_b32 s3, s0457; GISEL-NEXT: s_mov_b32 s4, s0458; GISEL-NEXT: s_mov_b32 s5, s0459; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[6:7]460; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[4:5]461; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]462; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]463; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)464; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[18:25], v[0:7], v[8:15], v[18:25]465; GISEL-NEXT: s_clause 0x1466; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off467; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16468; GISEL-NEXT: s_endpgm469bb:470 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.fp8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>, i1 false, i1 false)471 store <8 x float> %res, ptr addrspace(1) %out472 ret void473}474 475define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {476; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_bf8:477; GFX1250: ; %bb.0: ; %bb478; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1479; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[18:25], v[0:7], v[8:15], 1.0480; GFX1250-NEXT: s_clause 0x1481; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16482; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off483; GFX1250-NEXT: s_endpgm484;485; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_bf8:486; GISEL: ; %bb.0: ; %bb487; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1488; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[18:25], v[0:7], v[8:15], 1.0489; GISEL-NEXT: s_clause 0x1490; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off491; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16492; GISEL-NEXT: s_endpgm493bb:494 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.bf8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)495 store <8 x float> %res, ptr addrspace(1) %out496 ret void497}498 499define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {500; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_bf8_non_splat:501; GFX1250: ; %bb.0: ; %bb502; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1503; GFX1250-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0504; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)505; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v21, v18506; GFX1250-NEXT: v_dual_mov_b32 v22, v18 :: v_dual_mov_b32 v23, v18507; GFX1250-NEXT: v_dual_mov_b32 v24, v18 :: v_dual_mov_b32 v25, v18508; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[18:25], v[0:7], v[8:15], v[18:25]509; GFX1250-NEXT: s_clause 0x1510; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16511; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off512; GFX1250-NEXT: s_endpgm513;514; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_bf8_non_splat:515; GISEL: ; %bb.0: ; %bb516; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1517; GISEL-NEXT: s_mov_b32 s0, 1.0518; GISEL-NEXT: s_mov_b32 s2, 2.0519; GISEL-NEXT: s_mov_b32 s6, s0520; GISEL-NEXT: s_mov_b32 s7, s0521; GISEL-NEXT: s_mov_b32 s1, s0522; GISEL-NEXT: s_mov_b32 s3, s0523; GISEL-NEXT: s_mov_b32 s4, s0524; GISEL-NEXT: s_mov_b32 s5, s0525; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[6:7]526; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[4:5]527; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]528; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]529; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)530; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[18:25], v[0:7], v[8:15], v[18:25]531; GISEL-NEXT: s_clause 0x1532; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off533; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16534; GISEL-NEXT: s_endpgm535bb:536 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.bf8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)537 store <8 x float> %res, ptr addrspace(1) %out538 ret void539}540 541define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {542; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_bf8_non_inlineable:543; GFX1250: ; %bb.0: ; %bb544; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1545; GFX1250-NEXT: v_mov_b32_e32 v18, 0x40400000546; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)547; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v18548; GFX1250-NEXT: v_dual_mov_b32 v21, v18 :: v_dual_mov_b32 v22, v18549; GFX1250-NEXT: v_dual_mov_b32 v23, v18 :: v_dual_mov_b32 v24, v18550; GFX1250-NEXT: v_mov_b32_e32 v25, v18551; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[18:25], v[0:7], v[8:15], v[18:25]552; GFX1250-NEXT: s_clause 0x1553; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16554; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off555; GFX1250-NEXT: s_endpgm556;557; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_bf8_non_inlineable:558; GISEL: ; %bb.0: ; %bb559; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1560; GISEL-NEXT: s_mov_b32 s0, 0x40400000561; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)562; GISEL-NEXT: s_mov_b32 s6, s0563; GISEL-NEXT: s_mov_b32 s7, s0564; GISEL-NEXT: s_mov_b32 s1, s0565; GISEL-NEXT: s_mov_b32 s2, s0566; GISEL-NEXT: s_mov_b32 s3, s0567; GISEL-NEXT: s_mov_b32 s4, s0568; GISEL-NEXT: s_mov_b32 s5, s0569; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[6:7]570; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[4:5]571; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]572; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]573; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)574; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[18:25], v[0:7], v[8:15], v[18:25]575; GISEL-NEXT: s_clause 0x1576; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off577; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16578; GISEL-NEXT: s_endpgm579bb:580 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.bf8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>, i1 false, i1 false)581 store <8 x float> %res, ptr addrspace(1) %out582 ret void583}584 585define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {586; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_fp8:587; GFX1250: ; %bb.0: ; %bb588; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1589; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[18:25], v[0:7], v[8:15], 1.0590; GFX1250-NEXT: s_clause 0x1591; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16592; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off593; GFX1250-NEXT: s_endpgm594;595; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_fp8:596; GISEL: ; %bb.0: ; %bb597; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1598; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[18:25], v[0:7], v[8:15], 1.0599; GISEL-NEXT: s_clause 0x1600; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off601; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16602; GISEL-NEXT: s_endpgm603bb:604 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.fp8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)605 store <8 x float> %res, ptr addrspace(1) %out606 ret void607}608 609define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {610; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_fp8_non_splat:611; GFX1250: ; %bb.0: ; %bb612; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1613; GFX1250-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0614; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)615; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v21, v18616; GFX1250-NEXT: v_dual_mov_b32 v22, v18 :: v_dual_mov_b32 v23, v18617; GFX1250-NEXT: v_dual_mov_b32 v24, v18 :: v_dual_mov_b32 v25, v18618; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[18:25], v[0:7], v[8:15], v[18:25]619; GFX1250-NEXT: s_clause 0x1620; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16621; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off622; GFX1250-NEXT: s_endpgm623;624; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_fp8_non_splat:625; GISEL: ; %bb.0: ; %bb626; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1627; GISEL-NEXT: s_mov_b32 s0, 1.0628; GISEL-NEXT: s_mov_b32 s2, 2.0629; GISEL-NEXT: s_mov_b32 s6, s0630; GISEL-NEXT: s_mov_b32 s7, s0631; GISEL-NEXT: s_mov_b32 s1, s0632; GISEL-NEXT: s_mov_b32 s3, s0633; GISEL-NEXT: s_mov_b32 s4, s0634; GISEL-NEXT: s_mov_b32 s5, s0635; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[6:7]636; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[4:5]637; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]638; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]639; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)640; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[18:25], v[0:7], v[8:15], v[18:25]641; GISEL-NEXT: s_clause 0x1642; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off643; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16644; GISEL-NEXT: s_endpgm645bb:646 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.fp8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)647 store <8 x float> %res, ptr addrspace(1) %out648 ret void649}650 651define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {652; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_fp8_non_inlineable:653; GFX1250: ; %bb.0: ; %bb654; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1655; GFX1250-NEXT: v_mov_b32_e32 v18, 0x40400000656; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)657; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v18658; GFX1250-NEXT: v_dual_mov_b32 v21, v18 :: v_dual_mov_b32 v22, v18659; GFX1250-NEXT: v_dual_mov_b32 v23, v18 :: v_dual_mov_b32 v24, v18660; GFX1250-NEXT: v_mov_b32_e32 v25, v18661; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[18:25], v[0:7], v[8:15], v[18:25]662; GFX1250-NEXT: s_clause 0x1663; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16664; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off665; GFX1250-NEXT: s_endpgm666;667; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_fp8_non_inlineable:668; GISEL: ; %bb.0: ; %bb669; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1670; GISEL-NEXT: s_mov_b32 s0, 0x40400000671; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)672; GISEL-NEXT: s_mov_b32 s6, s0673; GISEL-NEXT: s_mov_b32 s7, s0674; GISEL-NEXT: s_mov_b32 s1, s0675; GISEL-NEXT: s_mov_b32 s2, s0676; GISEL-NEXT: s_mov_b32 s3, s0677; GISEL-NEXT: s_mov_b32 s4, s0678; GISEL-NEXT: s_mov_b32 s5, s0679; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[6:7]680; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[4:5]681; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]682; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]683; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)684; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[18:25], v[0:7], v[8:15], v[18:25]685; GISEL-NEXT: s_clause 0x1686; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off687; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16688; GISEL-NEXT: s_endpgm689bb:690 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.fp8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>, i1 false, i1 false)691 store <8 x float> %res, ptr addrspace(1) %out692 ret void693}694 695define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {696; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_bf8:697; GFX1250: ; %bb.0: ; %bb698; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1699; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[18:25], v[0:7], v[8:15], 1.0700; GFX1250-NEXT: s_clause 0x1701; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16702; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off703; GFX1250-NEXT: s_endpgm704;705; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_bf8:706; GISEL: ; %bb.0: ; %bb707; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1708; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[18:25], v[0:7], v[8:15], 1.0709; GISEL-NEXT: s_clause 0x1710; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off711; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16712; GISEL-NEXT: s_endpgm713bb:714 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.bf8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)715 store <8 x float> %res, ptr addrspace(1) %out716 ret void717}718 719define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {720; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_bf8_non_splat:721; GFX1250: ; %bb.0: ; %bb722; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1723; GFX1250-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0724; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)725; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v21, v18726; GFX1250-NEXT: v_dual_mov_b32 v22, v18 :: v_dual_mov_b32 v23, v18727; GFX1250-NEXT: v_dual_mov_b32 v24, v18 :: v_dual_mov_b32 v25, v18728; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[18:25], v[0:7], v[8:15], v[18:25]729; GFX1250-NEXT: s_clause 0x1730; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16731; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off732; GFX1250-NEXT: s_endpgm733;734; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_bf8_non_splat:735; GISEL: ; %bb.0: ; %bb736; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1737; GISEL-NEXT: s_mov_b32 s0, 1.0738; GISEL-NEXT: s_mov_b32 s2, 2.0739; GISEL-NEXT: s_mov_b32 s6, s0740; GISEL-NEXT: s_mov_b32 s7, s0741; GISEL-NEXT: s_mov_b32 s1, s0742; GISEL-NEXT: s_mov_b32 s3, s0743; GISEL-NEXT: s_mov_b32 s4, s0744; GISEL-NEXT: s_mov_b32 s5, s0745; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[6:7]746; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[4:5]747; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]748; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]749; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)750; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[18:25], v[0:7], v[8:15], v[18:25]751; GISEL-NEXT: s_clause 0x1752; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off753; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16754; GISEL-NEXT: s_endpgm755bb:756 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.bf8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)757 store <8 x float> %res, ptr addrspace(1) %out758 ret void759}760 761define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {762; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_bf8_non_inlineable:763; GFX1250: ; %bb.0: ; %bb764; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1765; GFX1250-NEXT: v_mov_b32_e32 v18, 0x40400000766; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)767; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v18768; GFX1250-NEXT: v_dual_mov_b32 v21, v18 :: v_dual_mov_b32 v22, v18769; GFX1250-NEXT: v_dual_mov_b32 v23, v18 :: v_dual_mov_b32 v24, v18770; GFX1250-NEXT: v_mov_b32_e32 v25, v18771; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[18:25], v[0:7], v[8:15], v[18:25]772; GFX1250-NEXT: s_clause 0x1773; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16774; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off775; GFX1250-NEXT: s_endpgm776;777; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_bf8_non_inlineable:778; GISEL: ; %bb.0: ; %bb779; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1780; GISEL-NEXT: s_mov_b32 s0, 0x40400000781; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)782; GISEL-NEXT: s_mov_b32 s6, s0783; GISEL-NEXT: s_mov_b32 s7, s0784; GISEL-NEXT: s_mov_b32 s1, s0785; GISEL-NEXT: s_mov_b32 s2, s0786; GISEL-NEXT: s_mov_b32 s3, s0787; GISEL-NEXT: s_mov_b32 s4, s0788; GISEL-NEXT: s_mov_b32 s5, s0789; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[6:7]790; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[4:5]791; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]792; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]793; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)794; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[18:25], v[0:7], v[8:15], v[18:25]795; GISEL-NEXT: s_clause 0x1796; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off797; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:16798; GISEL-NEXT: s_endpgm799bb:800 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.bf8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>, i1 false, i1 false)801 store <8 x float> %res, ptr addrspace(1) %out802 ret void803}804 805define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {806; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_fp8:807; GFX1250: ; %bb.0: ; %bb808; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1809; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[18:21], v[0:7], v[8:15], 1.0810; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off811; GFX1250-NEXT: s_endpgm812;813; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_fp8:814; GISEL: ; %bb.0: ; %bb815; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1816; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[18:21], v[0:7], v[8:15], 1.0817; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off818; GISEL-NEXT: s_endpgm819bb:820 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.fp8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x half> <half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)821 store <8 x half> %res, ptr addrspace(1) %out822 ret void823}824 825define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {826; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_fp8_non_splat:827; GFX1250: ; %bb.0: ; %bb828; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1829; GFX1250-NEXT: v_mov_b32_e32 v18, 0x3c003c00830; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)831; GFX1250-NEXT: v_dual_mov_b32 v19, 0x3c004000 :: v_dual_mov_b32 v20, v18832; GFX1250-NEXT: v_mov_b32_e32 v21, v18833; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[18:21], v[0:7], v[8:15], v[18:21]834; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off835; GFX1250-NEXT: s_endpgm836;837; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_fp8_non_splat:838; GISEL: ; %bb.0: ; %bb839; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1840; GISEL-NEXT: s_mov_b32 s0, 0x3c003c00841; GISEL-NEXT: s_mov_b32 s1, 0x3c004000842; GISEL-NEXT: s_mov_b32 s2, s0843; GISEL-NEXT: s_mov_b32 s3, s0844; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)845; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]846; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]847; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[18:21], v[0:7], v[8:15], v[18:21]848; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off849; GISEL-NEXT: s_endpgm850bb:851 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.fp8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x half> <half 1.0, half 1.0, half 2.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)852 store <8 x half> %res, ptr addrspace(1) %out853 ret void854}855 856define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {857; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_fp8_non_inlineable:858; GFX1250: ; %bb.0: ; %bb859; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1860; GFX1250-NEXT: v_mov_b32_e32 v18, 0x42004200861; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)862; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v18863; GFX1250-NEXT: v_mov_b32_e32 v21, v18864; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[18:21], v[0:7], v[8:15], v[18:21]865; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off866; GFX1250-NEXT: s_endpgm867;868; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_fp8_non_inlineable:869; GISEL: ; %bb.0: ; %bb870; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1871; GISEL-NEXT: s_mov_b32 s0, 0x42004200872; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)873; GISEL-NEXT: s_mov_b32 s2, s0874; GISEL-NEXT: s_mov_b32 s3, s0875; GISEL-NEXT: s_mov_b32 s1, s0876; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]877; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]878; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[18:21], v[0:7], v[8:15], v[18:21]879; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off880; GISEL-NEXT: s_endpgm881bb:882 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.fp8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x half> <half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0>, i1 false, i1 false)883 store <8 x half> %res, ptr addrspace(1) %out884 ret void885}886 887define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {888; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_bf8:889; GFX1250: ; %bb.0: ; %bb890; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1891; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[18:21], v[0:7], v[8:15], 1.0 neg_hi:[0,0,1]892; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off893; GFX1250-NEXT: s_endpgm894;895; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_bf8:896; GISEL: ; %bb.0: ; %bb897; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1898; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[18:21], v[0:7], v[8:15], 1.0 neg_hi:[0,0,1]899; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off900; GISEL-NEXT: s_endpgm901bb:902 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.bf8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 2, <8 x half> <half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)903 store <8 x half> %res, ptr addrspace(1) %out904 ret void905}906 907define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {908; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_bf8_non_splat:909; GFX1250: ; %bb.0: ; %bb910; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1911; GFX1250-NEXT: v_mov_b32_e32 v18, 0x3c003c00912; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)913; GFX1250-NEXT: v_dual_mov_b32 v19, 0x3c004000 :: v_dual_mov_b32 v20, v18914; GFX1250-NEXT: v_mov_b32_e32 v21, v18915; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[18:21], v[0:7], v[8:15], v[18:21] neg_hi:[0,0,1]916; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off917; GFX1250-NEXT: s_endpgm918;919; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_bf8_non_splat:920; GISEL: ; %bb.0: ; %bb921; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1922; GISEL-NEXT: s_mov_b32 s0, 0x3c003c00923; GISEL-NEXT: s_mov_b32 s1, 0x3c004000924; GISEL-NEXT: s_mov_b32 s2, s0925; GISEL-NEXT: s_mov_b32 s3, s0926; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)927; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]928; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]929; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[18:21], v[0:7], v[8:15], v[18:21] neg_hi:[0,0,1]930; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off931; GISEL-NEXT: s_endpgm932bb:933 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.bf8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 2, <8 x half> <half 1.0, half 1.0, half 2.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)934 store <8 x half> %res, ptr addrspace(1) %out935 ret void936}937 938define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {939; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_bf8_non_inlineable:940; GFX1250: ; %bb.0: ; %bb941; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1942; GFX1250-NEXT: v_mov_b32_e32 v18, 0x42004200943; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)944; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v18945; GFX1250-NEXT: v_mov_b32_e32 v21, v18946; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[18:21], v[0:7], v[8:15], v[18:21] neg_hi:[0,0,1]947; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off948; GFX1250-NEXT: s_endpgm949;950; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_bf8_non_inlineable:951; GISEL: ; %bb.0: ; %bb952; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1953; GISEL-NEXT: s_mov_b32 s0, 0x42004200954; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)955; GISEL-NEXT: s_mov_b32 s2, s0956; GISEL-NEXT: s_mov_b32 s3, s0957; GISEL-NEXT: s_mov_b32 s1, s0958; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]959; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]960; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[18:21], v[0:7], v[8:15], v[18:21] neg_hi:[0,0,1]961; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off962; GISEL-NEXT: s_endpgm963bb:964 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.bf8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 2, <8 x half> <half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0>, i1 false, i1 false)965 store <8 x half> %res, ptr addrspace(1) %out966 ret void967}968 969define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {970; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_fp8:971; GFX1250: ; %bb.0: ; %bb972; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1973; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[18:21], v[0:7], v[8:15], 1.0974; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off975; GFX1250-NEXT: s_endpgm976;977; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_fp8:978; GISEL: ; %bb.0: ; %bb979; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1980; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[18:21], v[0:7], v[8:15], 1.0981; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off982; GISEL-NEXT: s_endpgm983bb:984 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.fp8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x half> <half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)985 store <8 x half> %res, ptr addrspace(1) %out986 ret void987}988 989define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {990; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_fp8_non_splat:991; GFX1250: ; %bb.0: ; %bb992; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1993; GFX1250-NEXT: v_mov_b32_e32 v18, 0x3c003c00994; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)995; GFX1250-NEXT: v_dual_mov_b32 v19, 0x3c004000 :: v_dual_mov_b32 v20, v18996; GFX1250-NEXT: v_mov_b32_e32 v21, v18997; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[18:21], v[0:7], v[8:15], v[18:21]998; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off999; GFX1250-NEXT: s_endpgm1000;1001; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_fp8_non_splat:1002; GISEL: ; %bb.0: ; %bb1003; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11004; GISEL-NEXT: s_mov_b32 s0, 0x3c003c001005; GISEL-NEXT: s_mov_b32 s1, 0x3c0040001006; GISEL-NEXT: s_mov_b32 s2, s01007; GISEL-NEXT: s_mov_b32 s3, s01008; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1009; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]1010; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]1011; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[18:21], v[0:7], v[8:15], v[18:21]1012; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off1013; GISEL-NEXT: s_endpgm1014bb:1015 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.fp8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x half> <half 1.0, half 1.0, half 2.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)1016 store <8 x half> %res, ptr addrspace(1) %out1017 ret void1018}1019 1020define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {1021; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_fp8_non_inlineable:1022; GFX1250: ; %bb.0: ; %bb1023; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11024; GFX1250-NEXT: v_mov_b32_e32 v18, 0x420042001025; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1026; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v181027; GFX1250-NEXT: v_mov_b32_e32 v21, v181028; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[18:21], v[0:7], v[8:15], v[18:21]1029; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off1030; GFX1250-NEXT: s_endpgm1031;1032; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_fp8_non_inlineable:1033; GISEL: ; %bb.0: ; %bb1034; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11035; GISEL-NEXT: s_mov_b32 s0, 0x420042001036; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)1037; GISEL-NEXT: s_mov_b32 s2, s01038; GISEL-NEXT: s_mov_b32 s3, s01039; GISEL-NEXT: s_mov_b32 s1, s01040; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]1041; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]1042; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[18:21], v[0:7], v[8:15], v[18:21]1043; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off1044; GISEL-NEXT: s_endpgm1045bb:1046 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.fp8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x half> <half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0>, i1 false, i1 false)1047 store <8 x half> %res, ptr addrspace(1) %out1048 ret void1049}1050 1051define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {1052; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_bf8:1053; GFX1250: ; %bb.0: ; %bb1054; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11055; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[18:21], v[0:7], v[8:15], 1.01056; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off1057; GFX1250-NEXT: s_endpgm1058;1059; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_bf8:1060; GISEL: ; %bb.0: ; %bb1061; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11062; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[18:21], v[0:7], v[8:15], 1.01063; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off1064; GISEL-NEXT: s_endpgm1065bb:1066 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.bf8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x half> <half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)1067 store <8 x half> %res, ptr addrspace(1) %out1068 ret void1069}1070 1071define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {1072; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_bf8_non_splat:1073; GFX1250: ; %bb.0: ; %bb1074; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11075; GFX1250-NEXT: v_mov_b32_e32 v18, 0x3c003c001076; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1077; GFX1250-NEXT: v_dual_mov_b32 v19, 0x3c004000 :: v_dual_mov_b32 v20, v181078; GFX1250-NEXT: v_mov_b32_e32 v21, v181079; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[18:21], v[0:7], v[8:15], v[18:21]1080; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off1081; GFX1250-NEXT: s_endpgm1082;1083; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_bf8_non_splat:1084; GISEL: ; %bb.0: ; %bb1085; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11086; GISEL-NEXT: s_mov_b32 s0, 0x3c003c001087; GISEL-NEXT: s_mov_b32 s1, 0x3c0040001088; GISEL-NEXT: s_mov_b32 s2, s01089; GISEL-NEXT: s_mov_b32 s3, s01090; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1091; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]1092; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]1093; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[18:21], v[0:7], v[8:15], v[18:21]1094; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off1095; GISEL-NEXT: s_endpgm1096bb:1097 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.bf8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x half> <half 1.0, half 1.0, half 2.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)1098 store <8 x half> %res, ptr addrspace(1) %out1099 ret void1100}1101 1102define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {1103; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_bf8_non_inlineable:1104; GFX1250: ; %bb.0: ; %bb1105; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11106; GFX1250-NEXT: v_mov_b32_e32 v18, 0x420042001107; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1108; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v181109; GFX1250-NEXT: v_mov_b32_e32 v21, v181110; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[18:21], v[0:7], v[8:15], v[18:21]1111; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off1112; GFX1250-NEXT: s_endpgm1113;1114; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_bf8_non_inlineable:1115; GISEL: ; %bb.0: ; %bb1116; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11117; GISEL-NEXT: s_mov_b32 s0, 0x420042001118; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)1119; GISEL-NEXT: s_mov_b32 s2, s01120; GISEL-NEXT: s_mov_b32 s3, s01121; GISEL-NEXT: s_mov_b32 s1, s01122; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]1123; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]1124; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[18:21], v[0:7], v[8:15], v[18:21]1125; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off1126; GISEL-NEXT: s_endpgm1127bb:1128 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.bf8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 0, <8 x half> <half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0>, i1 false, i1 false)1129 store <8 x half> %res, ptr addrspace(1) %out1130 ret void1131}1132 1133define amdgpu_ps void @test_wmma_i32_16x16x64_iu8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {1134; GFX1250-LABEL: test_wmma_i32_16x16x64_iu8:1135; GFX1250: ; %bb.0: ; %bb1136; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11137; GFX1250-NEXT: v_wmma_i32_16x16x64_iu8 v[18:25], v[0:7], v[8:15], 11138; GFX1250-NEXT: s_clause 0x11139; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:161140; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off1141; GFX1250-NEXT: s_endpgm1142;1143; GISEL-LABEL: test_wmma_i32_16x16x64_iu8:1144; GISEL: ; %bb.0: ; %bb1145; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11146; GISEL-NEXT: v_wmma_i32_16x16x64_iu8 v[18:25], v[0:7], v[8:15], 11147; GISEL-NEXT: s_clause 0x11148; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off1149; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:161150; GISEL-NEXT: s_endpgm1151bb:1152 %res = call <8 x i32> @llvm.amdgcn.wmma.i32.16x16x64.iu8.v8i32.v8i32(i1 0, <8 x i32> %A, i1 0, <8 x i32> %B, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, i1 false, i1 false)1153 store <8 x i32> %res, ptr addrspace(1) %out1154 ret void1155}1156 1157define amdgpu_ps void @test_wmma_i32_16x16x64_iu8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {1158; GFX1250-LABEL: test_wmma_i32_16x16x64_iu8_non_splat:1159; GFX1250: ; %bb.0: ; %bb1160; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11161; GFX1250-NEXT: v_dual_mov_b32 v18, 1 :: v_dual_mov_b32 v20, 21162; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)1163; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v21, v181164; GFX1250-NEXT: v_dual_mov_b32 v22, v18 :: v_dual_mov_b32 v23, v181165; GFX1250-NEXT: v_dual_mov_b32 v24, v18 :: v_dual_mov_b32 v25, v181166; GFX1250-NEXT: v_wmma_i32_16x16x64_iu8 v[18:25], v[0:7], v[8:15], v[18:25]1167; GFX1250-NEXT: s_clause 0x11168; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:161169; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off1170; GFX1250-NEXT: s_endpgm1171;1172; GISEL-LABEL: test_wmma_i32_16x16x64_iu8_non_splat:1173; GISEL: ; %bb.0: ; %bb1174; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11175; GISEL-NEXT: s_mov_b32 s0, 11176; GISEL-NEXT: s_mov_b32 s2, 21177; GISEL-NEXT: s_mov_b32 s6, s01178; GISEL-NEXT: s_mov_b32 s7, s01179; GISEL-NEXT: s_mov_b32 s1, s01180; GISEL-NEXT: s_mov_b32 s3, s01181; GISEL-NEXT: s_mov_b32 s4, s01182; GISEL-NEXT: s_mov_b32 s5, s01183; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[6:7]1184; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[4:5]1185; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]1186; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]1187; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)1188; GISEL-NEXT: v_wmma_i32_16x16x64_iu8 v[18:25], v[0:7], v[8:15], v[18:25]1189; GISEL-NEXT: s_clause 0x11190; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off1191; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:161192; GISEL-NEXT: s_endpgm1193bb:1194 %res = call <8 x i32> @llvm.amdgcn.wmma.i32.16x16x64.iu8.v8i32.v8i32(i1 0, <8 x i32> %A, i1 0, <8 x i32> %B, <8 x i32> <i32 1, i32 1, i32 2, i32 1, i32 1, i32 1, i32 1, i32 1>, i1 false, i1 false)1195 store <8 x i32> %res, ptr addrspace(1) %out1196 ret void1197}1198 1199define amdgpu_ps void @test_wmma_i32_16x16x64_iu8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {1200; GFX1250-LABEL: test_wmma_i32_16x16x64_iu8_non_inlineable:1201; GFX1250: ; %bb.0: ; %bb1202; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11203; GFX1250-NEXT: v_mov_b32_e32 v18, 0x801204; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)1205; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v181206; GFX1250-NEXT: v_dual_mov_b32 v21, v18 :: v_dual_mov_b32 v22, v181207; GFX1250-NEXT: v_dual_mov_b32 v23, v18 :: v_dual_mov_b32 v24, v181208; GFX1250-NEXT: v_mov_b32_e32 v25, v181209; GFX1250-NEXT: v_wmma_i32_16x16x64_iu8 v[18:25], v[0:7], v[8:15], v[18:25]1210; GFX1250-NEXT: s_clause 0x11211; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:161212; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off1213; GFX1250-NEXT: s_endpgm1214;1215; GISEL-LABEL: test_wmma_i32_16x16x64_iu8_non_inlineable:1216; GISEL: ; %bb.0: ; %bb1217; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11218; GISEL-NEXT: s_movk_i32 s0, 0x801219; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1220; GISEL-NEXT: s_mov_b32 s6, s01221; GISEL-NEXT: s_mov_b32 s7, s01222; GISEL-NEXT: s_mov_b32 s1, s01223; GISEL-NEXT: s_mov_b32 s2, s01224; GISEL-NEXT: s_mov_b32 s3, s01225; GISEL-NEXT: s_mov_b32 s4, s01226; GISEL-NEXT: s_mov_b32 s5, s01227; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[6:7]1228; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[4:5]1229; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]1230; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]1231; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)1232; GISEL-NEXT: v_wmma_i32_16x16x64_iu8 v[18:25], v[0:7], v[8:15], v[18:25]1233; GISEL-NEXT: s_clause 0x11234; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off1235; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:161236; GISEL-NEXT: s_endpgm1237bb:1238 %res = call <8 x i32> @llvm.amdgcn.wmma.i32.16x16x64.iu8.v8i32.v8i32(i1 0, <8 x i32> %A, i1 0, <8 x i32> %B, <8 x i32> <i32 128, i32 128, i32 128, i32 128, i32 128, i32 128, i32 128, i32 128>, i1 false, i1 false)1239 store <8 x i32> %res, ptr addrspace(1) %out1240 ret void1241}1242 1243define amdgpu_ps void @test_wmma_f32_16x16x32_f16(<16 x half> %A, <16 x half> %B, ptr addrspace(1) %out) {1244; GFX1250-LABEL: test_wmma_f32_16x16x32_f16:1245; GFX1250: ; %bb.0: ; %bb1246; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11247; GFX1250-NEXT: v_wmma_f32_16x16x32_f16 v[18:25], v[0:7], v[8:15], 1.01248; GFX1250-NEXT: s_clause 0x11249; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:161250; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off1251; GFX1250-NEXT: s_endpgm1252;1253; GISEL-LABEL: test_wmma_f32_16x16x32_f16:1254; GISEL: ; %bb.0: ; %bb1255; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11256; GISEL-NEXT: v_wmma_f32_16x16x32_f16 v[18:25], v[0:7], v[8:15], 1.01257; GISEL-NEXT: s_clause 0x11258; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off1259; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:161260; GISEL-NEXT: s_endpgm1261bb:1262 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 0, <16 x half> %A, i1 0, <16 x half> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)1263 store <8 x float> %res, ptr addrspace(1) %out1264 ret void1265}1266 1267define amdgpu_ps void @test_wmma_f32_16x16x32_f16_non_splat(<16 x half> %A, <16 x half> %B, ptr addrspace(1) %out) {1268; GFX1250-LABEL: test_wmma_f32_16x16x32_f16_non_splat:1269; GFX1250: ; %bb.0: ; %bb1270; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11271; GFX1250-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.01272; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)1273; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v21, v181274; GFX1250-NEXT: v_dual_mov_b32 v22, v18 :: v_dual_mov_b32 v23, v181275; GFX1250-NEXT: v_dual_mov_b32 v24, v18 :: v_dual_mov_b32 v25, v181276; GFX1250-NEXT: v_wmma_f32_16x16x32_f16 v[18:25], v[0:7], v[8:15], v[18:25]1277; GFX1250-NEXT: s_clause 0x11278; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:161279; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off1280; GFX1250-NEXT: s_endpgm1281;1282; GISEL-LABEL: test_wmma_f32_16x16x32_f16_non_splat:1283; GISEL: ; %bb.0: ; %bb1284; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11285; GISEL-NEXT: s_mov_b32 s0, 1.01286; GISEL-NEXT: s_mov_b32 s2, 2.01287; GISEL-NEXT: s_mov_b32 s6, s01288; GISEL-NEXT: s_mov_b32 s7, s01289; GISEL-NEXT: s_mov_b32 s1, s01290; GISEL-NEXT: s_mov_b32 s3, s01291; GISEL-NEXT: s_mov_b32 s4, s01292; GISEL-NEXT: s_mov_b32 s5, s01293; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[6:7]1294; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[4:5]1295; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]1296; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]1297; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)1298; GISEL-NEXT: v_wmma_f32_16x16x32_f16 v[18:25], v[0:7], v[8:15], v[18:25]1299; GISEL-NEXT: s_clause 0x11300; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off1301; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:161302; GISEL-NEXT: s_endpgm1303bb:1304 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 0, <16 x half> %A, i1 0, <16 x half> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)1305 store <8 x float> %res, ptr addrspace(1) %out1306 ret void1307}1308 1309define amdgpu_ps void @test_wmma_f32_16x16x32_f16_non_inlineable(<16 x half> %A, <16 x half> %B, ptr addrspace(1) %out) {1310; GFX1250-LABEL: test_wmma_f32_16x16x32_f16_non_inlineable:1311; GFX1250: ; %bb.0: ; %bb1312; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11313; GFX1250-NEXT: v_mov_b32_e32 v18, 0x404000001314; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)1315; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v181316; GFX1250-NEXT: v_dual_mov_b32 v21, v18 :: v_dual_mov_b32 v22, v181317; GFX1250-NEXT: v_dual_mov_b32 v23, v18 :: v_dual_mov_b32 v24, v181318; GFX1250-NEXT: v_mov_b32_e32 v25, v181319; GFX1250-NEXT: v_wmma_f32_16x16x32_f16 v[18:25], v[0:7], v[8:15], v[18:25]1320; GFX1250-NEXT: s_clause 0x11321; GFX1250-NEXT: global_store_b128 v[16:17], v[22:25], off offset:161322; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off1323; GFX1250-NEXT: s_endpgm1324;1325; GISEL-LABEL: test_wmma_f32_16x16x32_f16_non_inlineable:1326; GISEL: ; %bb.0: ; %bb1327; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11328; GISEL-NEXT: s_mov_b32 s0, 0x404000001329; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1330; GISEL-NEXT: s_mov_b32 s6, s01331; GISEL-NEXT: s_mov_b32 s7, s01332; GISEL-NEXT: s_mov_b32 s1, s01333; GISEL-NEXT: s_mov_b32 s2, s01334; GISEL-NEXT: s_mov_b32 s3, s01335; GISEL-NEXT: s_mov_b32 s4, s01336; GISEL-NEXT: s_mov_b32 s5, s01337; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[6:7]1338; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[4:5]1339; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]1340; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]1341; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)1342; GISEL-NEXT: v_wmma_f32_16x16x32_f16 v[18:25], v[0:7], v[8:15], v[18:25]1343; GISEL-NEXT: s_clause 0x11344; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off1345; GISEL-NEXT: global_store_b128 v[16:17], v[22:25], off offset:161346; GISEL-NEXT: s_endpgm1347bb:1348 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 0, <16 x half> %A, i1 0, <16 x half> %B, i16 0, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>, i1 false, i1 false)1349 store <8 x float> %res, ptr addrspace(1) %out1350 ret void1351}1352 1353define amdgpu_ps void @test_wmma_f16_16x16x32_f16(<16 x half> %A, <16 x half> %B, ptr addrspace(1) %out) {1354; GFX1250-LABEL: test_wmma_f16_16x16x32_f16:1355; GFX1250: ; %bb.0: ; %bb1356; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11357; GFX1250-NEXT: v_wmma_f16_16x16x32_f16 v[18:21], v[0:7], v[8:15], 1.01358; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off1359; GFX1250-NEXT: s_endpgm1360;1361; GISEL-LABEL: test_wmma_f16_16x16x32_f16:1362; GISEL: ; %bb.0: ; %bb1363; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11364; GISEL-NEXT: v_wmma_f16_16x16x32_f16 v[18:21], v[0:7], v[8:15], 1.01365; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off1366; GISEL-NEXT: s_endpgm1367bb:1368 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x32.f16.v8f16.v16f16(i1 0, <16 x half> %A, i1 0, <16 x half> %B, i16 0, <8 x half> <half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)1369 store <8 x half> %res, ptr addrspace(1) %out1370 ret void1371}1372 1373define amdgpu_ps void @test_wmma_f16_16x16x32_f16_non_splat(<16 x half> %A, <16 x half> %B, ptr addrspace(1) %out) {1374; GFX1250-LABEL: test_wmma_f16_16x16x32_f16_non_splat:1375; GFX1250: ; %bb.0: ; %bb1376; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11377; GFX1250-NEXT: v_mov_b32_e32 v18, 0x3c003c001378; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1379; GFX1250-NEXT: v_dual_mov_b32 v19, 0x3c004000 :: v_dual_mov_b32 v20, v181380; GFX1250-NEXT: v_mov_b32_e32 v21, v181381; GFX1250-NEXT: v_wmma_f16_16x16x32_f16 v[18:21], v[0:7], v[8:15], v[18:21]1382; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off1383; GFX1250-NEXT: s_endpgm1384;1385; GISEL-LABEL: test_wmma_f16_16x16x32_f16_non_splat:1386; GISEL: ; %bb.0: ; %bb1387; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11388; GISEL-NEXT: s_mov_b32 s0, 0x3c003c001389; GISEL-NEXT: s_mov_b32 s1, 0x3c0040001390; GISEL-NEXT: s_mov_b32 s2, s01391; GISEL-NEXT: s_mov_b32 s3, s01392; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1393; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]1394; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]1395; GISEL-NEXT: v_wmma_f16_16x16x32_f16 v[18:21], v[0:7], v[8:15], v[18:21]1396; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off1397; GISEL-NEXT: s_endpgm1398bb:1399 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x32.f16.v8f16.v16f16(i1 0, <16 x half> %A, i1 0, <16 x half> %B, i16 0, <8 x half> <half 1.0, half 1.0, half 2.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)1400 store <8 x half> %res, ptr addrspace(1) %out1401 ret void1402}1403 1404define amdgpu_ps void @test_wmma_f16_16x16x32_f16_non_inlineable(<16 x half> %A, <16 x half> %B, ptr addrspace(1) %out) {1405; GFX1250-LABEL: test_wmma_f16_16x16x32_f16_non_inlineable:1406; GFX1250: ; %bb.0: ; %bb1407; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11408; GFX1250-NEXT: v_mov_b32_e32 v18, 0x420042001409; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1410; GFX1250-NEXT: v_dual_mov_b32 v19, v18 :: v_dual_mov_b32 v20, v181411; GFX1250-NEXT: v_mov_b32_e32 v21, v181412; GFX1250-NEXT: v_wmma_f16_16x16x32_f16 v[18:21], v[0:7], v[8:15], v[18:21]1413; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off1414; GFX1250-NEXT: s_endpgm1415;1416; GISEL-LABEL: test_wmma_f16_16x16x32_f16_non_inlineable:1417; GISEL: ; %bb.0: ; %bb1418; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11419; GISEL-NEXT: s_mov_b32 s0, 0x420042001420; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)1421; GISEL-NEXT: s_mov_b32 s2, s01422; GISEL-NEXT: s_mov_b32 s3, s01423; GISEL-NEXT: s_mov_b32 s1, s01424; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[2:3]1425; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[0:1]1426; GISEL-NEXT: v_wmma_f16_16x16x32_f16 v[18:21], v[0:7], v[8:15], v[18:21]1427; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off1428; GISEL-NEXT: s_endpgm1429bb:1430 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x32.f16.v8f16.v16f16(i1 0, <16 x half> %A, i1 0, <16 x half> %B, i16 0, <8 x half> <half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0>, i1 false, i1 false)1431 store <8 x half> %res, ptr addrspace(1) %out1432 ret void1433}1434 1435define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {1436; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4:1437; GFX1250: ; %bb.0: ; %bb1438; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11439; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], 1.01440; GFX1250-NEXT: s_clause 0x11441; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161442; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1443; GFX1250-NEXT: s_endpgm1444;1445; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4:1446; GISEL: ; %bb.0: ; %bb1447; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11448; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], 1.01449; GISEL-NEXT: s_clause 0x11450; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1451; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161452; GISEL-NEXT: s_endpgm1453bb:1454 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>)1455 store <8 x float> %res, ptr addrspace(1) %out1456 ret void1457}1458 1459define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {1460; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_non_splat:1461; GFX1250: ; %bb.0: ; %bb1462; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11463; GFX1250-NEXT: v_dual_mov_b32 v34, 1.0 :: v_dual_mov_b32 v35, 2.01464; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)1465; GFX1250-NEXT: v_dual_mov_b32 v36, v34 :: v_dual_mov_b32 v37, v341466; GFX1250-NEXT: v_dual_mov_b32 v38, v34 :: v_dual_mov_b32 v39, v341467; GFX1250-NEXT: v_dual_mov_b32 v40, v34 :: v_dual_mov_b32 v41, v341468; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], v[34:41]1469; GFX1250-NEXT: s_clause 0x11470; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161471; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1472; GFX1250-NEXT: s_endpgm1473;1474; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_non_splat:1475; GISEL: ; %bb.0: ; %bb1476; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11477; GISEL-NEXT: s_mov_b32 s0, 1.01478; GISEL-NEXT: s_mov_b32 s1, 2.01479; GISEL-NEXT: s_mov_b32 s6, s01480; GISEL-NEXT: s_mov_b32 s7, s01481; GISEL-NEXT: s_mov_b32 s2, s01482; GISEL-NEXT: s_mov_b32 s3, s01483; GISEL-NEXT: s_mov_b32 s4, s01484; GISEL-NEXT: s_mov_b32 s5, s01485; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[6:7]1486; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[4:5]1487; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]1488; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]1489; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)1490; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], v[34:41]1491; GISEL-NEXT: s_clause 0x11492; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1493; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161494; GISEL-NEXT: s_endpgm1495bb:1496 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> <float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>)1497 store <8 x float> %res, ptr addrspace(1) %out1498 ret void1499}1500 1501define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {1502; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_non_inlineable:1503; GFX1250: ; %bb.0: ; %bb1504; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11505; GFX1250-NEXT: v_mov_b32_e32 v34, 0x404000001506; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)1507; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v36, v341508; GFX1250-NEXT: v_dual_mov_b32 v37, v34 :: v_dual_mov_b32 v38, v341509; GFX1250-NEXT: v_dual_mov_b32 v39, v34 :: v_dual_mov_b32 v40, v341510; GFX1250-NEXT: v_mov_b32_e32 v41, v341511; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], v[34:41]1512; GFX1250-NEXT: s_clause 0x11513; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161514; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1515; GFX1250-NEXT: s_endpgm1516;1517; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_non_inlineable:1518; GISEL: ; %bb.0: ; %bb1519; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11520; GISEL-NEXT: s_mov_b32 s0, 0x404000001521; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1522; GISEL-NEXT: s_mov_b32 s6, s01523; GISEL-NEXT: s_mov_b32 s7, s01524; GISEL-NEXT: s_mov_b32 s1, s01525; GISEL-NEXT: s_mov_b32 s2, s01526; GISEL-NEXT: s_mov_b32 s3, s01527; GISEL-NEXT: s_mov_b32 s4, s01528; GISEL-NEXT: s_mov_b32 s5, s01529; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[6:7]1530; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[4:5]1531; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]1532; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]1533; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)1534; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], v[34:41]1535; GISEL-NEXT: s_clause 0x11536; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1537; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161538; GISEL-NEXT: s_endpgm1539bb:1540 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>)1541 store <8 x float> %res, ptr addrspace(1) %out1542 ret void1543}1544 1545define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4(<16 x i32> %A, <16 x i32> %B, i32 inreg %scale_src0, i32 inreg %scale_src1, ptr addrspace(1) %out) {1546; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4:1547; GFX1250: ; %bb.0: ; %bb1548; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11549; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], 1.0, s0, s1 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse1550; GFX1250-NEXT: s_clause 0x11551; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161552; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1553; GFX1250-NEXT: s_endpgm1554;1555; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4:1556; GISEL: ; %bb.0: ; %bb1557; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11558; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], 1.0, s0, s1 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse1559; GISEL-NEXT: s_clause 0x11560; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1561; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161562; GISEL-NEXT: s_endpgm1563bb:1564 %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i32 1, i32 0, i32 %scale_src0, i32 1, i32 0, i32 %scale_src1, i1 true, i1 false)1565 store <8 x float> %res, ptr addrspace(1) %out1566 ret void1567}1568 1569define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {1570; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_non_splat:1571; GFX1250: ; %bb.0: ; %bb1572; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11573; GFX1250-NEXT: v_dual_mov_b32 v34, 1.0 :: v_dual_mov_b32 v35, 2.01574; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)1575; GFX1250-NEXT: v_dual_mov_b32 v36, v34 :: v_dual_mov_b32 v37, v341576; GFX1250-NEXT: v_dual_mov_b32 v38, v34 :: v_dual_mov_b32 v39, v341577; GFX1250-NEXT: v_dual_mov_b32 v40, v34 :: v_dual_mov_b32 v41, v341578; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], v[34:41], 1, 2 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW11579; GFX1250-NEXT: s_clause 0x11580; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161581; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1582; GFX1250-NEXT: s_endpgm1583;1584; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_non_splat:1585; GISEL: ; %bb.0: ; %bb1586; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11587; GISEL-NEXT: s_mov_b32 s0, 1.01588; GISEL-NEXT: s_mov_b32 s1, 2.01589; GISEL-NEXT: s_mov_b32 s6, s01590; GISEL-NEXT: s_mov_b32 s7, s01591; GISEL-NEXT: s_mov_b32 s2, s01592; GISEL-NEXT: s_mov_b32 s3, s01593; GISEL-NEXT: s_mov_b32 s4, s01594; GISEL-NEXT: s_mov_b32 s5, s01595; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[6:7]1596; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[4:5]1597; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]1598; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]1599; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)1600; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], v[34:41], 1, 2 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW11601; GISEL-NEXT: s_clause 0x11602; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1603; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161604; GISEL-NEXT: s_endpgm1605bb:1606 %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> <float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i32 1, i32 0, i32 1, i32 1, i32 0, i32 2, i1 false, i1 false)1607 store <8 x float> %res, ptr addrspace(1) %out1608 ret void1609}1610 1611define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {1612; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_non_inlineable:1613; GFX1250: ; %bb.0: ; %bb1614; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11615; GFX1250-NEXT: v_mov_b32_e32 v34, 0x404000001616; GFX1250-NEXT: v_mov_b32_e32 v43, 0x641617; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)1618; GFX1250-NEXT: v_dual_mov_b32 v42, 0x65 :: v_dual_mov_b32 v41, v341619; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v36, v341620; GFX1250-NEXT: v_dual_mov_b32 v37, v34 :: v_dual_mov_b32 v38, v341621; GFX1250-NEXT: v_dual_mov_b32 v39, v34 :: v_dual_mov_b32 v40, v341622; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], v[34:41], v43, v42 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse1623; GFX1250-NEXT: s_clause 0x11624; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161625; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1626; GFX1250-NEXT: s_endpgm1627;1628; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_non_inlineable:1629; GISEL: ; %bb.0: ; %bb1630; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11631; GISEL-NEXT: s_mov_b32 s0, 0x404000001632; GISEL-NEXT: v_mov_b32_e32 v42, 0x641633; GISEL-NEXT: s_mov_b32 s6, s01634; GISEL-NEXT: s_mov_b32 s7, s01635; GISEL-NEXT: s_mov_b32 s1, s01636; GISEL-NEXT: s_mov_b32 s2, s01637; GISEL-NEXT: s_mov_b32 s3, s01638; GISEL-NEXT: s_mov_b32 s4, s01639; GISEL-NEXT: s_mov_b32 s5, s01640; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[6:7]1641; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[4:5]1642; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]1643; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]1644; GISEL-NEXT: v_mov_b32_e32 v43, 0x651645; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)1646; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], v[34:41], v42, v43 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse1647; GISEL-NEXT: s_clause 0x11648; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1649; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161650; GISEL-NEXT: s_endpgm1651bb:1652 %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>, i32 1, i32 0, i32 100, i32 1, i32 0, i32 101, i1 true, i1 false)1653 store <8 x float> %res, ptr addrspace(1) %out1654 ret void1655}1656 1657define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4(<16 x i32> %A, <16 x i32> %B, i64 inreg %scale_src0, i64 inreg %scale_src1, ptr addrspace(1) %out) {1658; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4:1659; GFX1250: ; %bb.0: ; %bb1660; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11661; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], 1.0, s[0:1], s[2:3] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse1662; GFX1250-NEXT: s_clause 0x11663; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161664; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1665; GFX1250-NEXT: s_endpgm1666;1667; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4:1668; GISEL: ; %bb.0: ; %bb1669; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11670; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], 1.0, s[0:1], s[2:3] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse1671; GISEL-NEXT: s_clause 0x11672; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1673; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161674; GISEL-NEXT: s_endpgm1675bb:1676 %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i32 1, i32 0, i64 %scale_src0, i32 1, i32 0, i64 %scale_src1, i1 true, i1 false)1677 store <8 x float> %res, ptr addrspace(1) %out1678 ret void1679}1680 1681define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {1682; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_non_splat:1683; GFX1250: ; %bb.0: ; %bb1684; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11685; GFX1250-NEXT: v_dual_mov_b32 v34, 1.0 :: v_dual_mov_b32 v35, 2.01686; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)1687; GFX1250-NEXT: v_dual_mov_b32 v36, v34 :: v_dual_mov_b32 v37, v341688; GFX1250-NEXT: v_dual_mov_b32 v38, v34 :: v_dual_mov_b32 v39, v341689; GFX1250-NEXT: v_dual_mov_b32 v40, v34 :: v_dual_mov_b32 v41, v341690; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], v[34:41], 1, 2 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW11691; GFX1250-NEXT: s_clause 0x11692; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161693; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1694; GFX1250-NEXT: s_endpgm1695;1696; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_non_splat:1697; GISEL: ; %bb.0: ; %bb1698; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11699; GISEL-NEXT: s_mov_b32 s0, 1.01700; GISEL-NEXT: s_mov_b32 s1, 2.01701; GISEL-NEXT: s_mov_b32 s6, s01702; GISEL-NEXT: s_mov_b32 s7, s01703; GISEL-NEXT: s_mov_b32 s2, s01704; GISEL-NEXT: s_mov_b32 s3, s01705; GISEL-NEXT: s_mov_b32 s4, s01706; GISEL-NEXT: s_mov_b32 s5, s01707; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[6:7]1708; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[4:5]1709; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]1710; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]1711; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)1712; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], v[34:41], 1, 2 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW11713; GISEL-NEXT: s_clause 0x11714; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1715; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161716; GISEL-NEXT: s_endpgm1717bb:1718 %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> <float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i32 1, i32 0, i64 1, i32 1, i32 0, i64 2, i1 false, i1 false)1719 store <8 x float> %res, ptr addrspace(1) %out1720 ret void1721}1722 1723define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {1724; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_non_inlineable:1725; GFX1250: ; %bb.0: ; %bb1726; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11727; GFX1250-NEXT: v_mov_b32_e32 v34, 0x404000001728; GFX1250-NEXT: v_mov_b64_e32 v[42:43], 0x651729; GFX1250-NEXT: v_mov_b64_e32 v[44:45], 0x641730; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1)1731; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v36, v341732; GFX1250-NEXT: v_dual_mov_b32 v37, v34 :: v_dual_mov_b32 v38, v341733; GFX1250-NEXT: v_dual_mov_b32 v39, v34 :: v_dual_mov_b32 v40, v341734; GFX1250-NEXT: v_mov_b32_e32 v41, v341735; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], v[34:41], v[44:45], v[42:43] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse1736; GFX1250-NEXT: s_clause 0x11737; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161738; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1739; GFX1250-NEXT: s_endpgm1740;1741; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_non_inlineable:1742; GISEL: ; %bb.0: ; %bb1743; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11744; GISEL-NEXT: s_mov_b32 s0, 0x404000001745; GISEL-NEXT: v_mov_b64_e32 v[42:43], 0x641746; GISEL-NEXT: s_mov_b32 s6, s01747; GISEL-NEXT: s_mov_b32 s7, s01748; GISEL-NEXT: s_mov_b32 s1, s01749; GISEL-NEXT: s_mov_b32 s2, s01750; GISEL-NEXT: s_mov_b32 s3, s01751; GISEL-NEXT: s_mov_b32 s4, s01752; GISEL-NEXT: s_mov_b32 s5, s01753; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[6:7]1754; GISEL-NEXT: v_mov_b64_e32 v[44:45], 0x651755; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[4:5]1756; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]1757; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]1758; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)1759; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], v[34:41], v[42:43], v[44:45] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse1760; GISEL-NEXT: s_clause 0x11761; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1762; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:161763; GISEL-NEXT: s_endpgm1764bb:1765 %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>, i32 1, i32 0, i64 100, i32 1, i32 0, i64 101, i1 true, i1 false)1766 store <8 x float> %res, ptr addrspace(1) %out1767 ret void1768}1769 1770define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {1771; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_fp8:1772; GFX1250: ; %bb.0: ; %bb1773; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11774; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[34:37], v[0:15], v[16:31], 1.01775; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1776; GFX1250-NEXT: s_endpgm1777;1778; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_fp8:1779; GISEL: ; %bb.0: ; %bb1780; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11781; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[34:37], v[0:15], v[16:31], 1.01782; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1783; GISEL-NEXT: s_endpgm1784bb:1785 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.fp8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x half> <half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)1786 store <8 x half> %res, ptr addrspace(1) %out1787 ret void1788}1789 1790define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {1791; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_fp8_non_splat:1792; GFX1250: ; %bb.0: ; %bb1793; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11794; GFX1250-NEXT: v_mov_b32_e32 v34, 0x3c003c001795; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1796; GFX1250-NEXT: v_dual_mov_b32 v35, 0x3c004000 :: v_dual_mov_b32 v36, v341797; GFX1250-NEXT: v_mov_b32_e32 v37, v341798; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[34:37], v[0:15], v[16:31], v[34:37]1799; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1800; GFX1250-NEXT: s_endpgm1801;1802; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_fp8_non_splat:1803; GISEL: ; %bb.0: ; %bb1804; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11805; GISEL-NEXT: s_mov_b32 s0, 0x3c003c001806; GISEL-NEXT: s_mov_b32 s1, 0x3c0040001807; GISEL-NEXT: s_mov_b32 s2, s01808; GISEL-NEXT: s_mov_b32 s3, s01809; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1810; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]1811; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]1812; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[34:37], v[0:15], v[16:31], v[34:37]1813; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1814; GISEL-NEXT: s_endpgm1815bb:1816 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.fp8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x half> <half 1.0, half 1.0, half 2.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)1817 store <8 x half> %res, ptr addrspace(1) %out1818 ret void1819}1820 1821define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {1822; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_fp8_non_inlineable:1823; GFX1250: ; %bb.0: ; %bb1824; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11825; GFX1250-NEXT: v_mov_b32_e32 v34, 0x420042001826; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1827; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v36, v341828; GFX1250-NEXT: v_mov_b32_e32 v37, v341829; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[34:37], v[0:15], v[16:31], v[34:37]1830; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1831; GFX1250-NEXT: s_endpgm1832;1833; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_fp8_non_inlineable:1834; GISEL: ; %bb.0: ; %bb1835; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11836; GISEL-NEXT: s_mov_b32 s0, 0x420042001837; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)1838; GISEL-NEXT: s_mov_b32 s2, s01839; GISEL-NEXT: s_mov_b32 s3, s01840; GISEL-NEXT: s_mov_b32 s1, s01841; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]1842; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]1843; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[34:37], v[0:15], v[16:31], v[34:37]1844; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1845; GISEL-NEXT: s_endpgm1846bb:1847 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.fp8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x half> <half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0>, i1 false, i1 false)1848 store <8 x half> %res, ptr addrspace(1) %out1849 ret void1850}1851 1852define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {1853; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_bf8:1854; GFX1250: ; %bb.0: ; %bb1855; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11856; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[34:37], v[0:15], v[16:31], 1.0 neg_hi:[0,0,1]1857; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1858; GFX1250-NEXT: s_endpgm1859;1860; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_bf8:1861; GISEL: ; %bb.0: ; %bb1862; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11863; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[34:37], v[0:15], v[16:31], 1.0 neg_hi:[0,0,1]1864; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1865; GISEL-NEXT: s_endpgm1866bb:1867 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.bf8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 2, <8 x half> <half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)1868 store <8 x half> %res, ptr addrspace(1) %out1869 ret void1870}1871 1872define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {1873; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_bf8_non_splat:1874; GFX1250: ; %bb.0: ; %bb1875; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11876; GFX1250-NEXT: v_mov_b32_e32 v34, 0x3c003c001877; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1878; GFX1250-NEXT: v_dual_mov_b32 v35, 0x3c004000 :: v_dual_mov_b32 v36, v341879; GFX1250-NEXT: v_mov_b32_e32 v37, v341880; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[34:37], v[0:15], v[16:31], v[34:37] neg_hi:[0,0,1]1881; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1882; GFX1250-NEXT: s_endpgm1883;1884; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_bf8_non_splat:1885; GISEL: ; %bb.0: ; %bb1886; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11887; GISEL-NEXT: s_mov_b32 s0, 0x3c003c001888; GISEL-NEXT: s_mov_b32 s1, 0x3c0040001889; GISEL-NEXT: s_mov_b32 s2, s01890; GISEL-NEXT: s_mov_b32 s3, s01891; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1892; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]1893; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]1894; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[34:37], v[0:15], v[16:31], v[34:37] neg_hi:[0,0,1]1895; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1896; GISEL-NEXT: s_endpgm1897bb:1898 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.bf8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 2, <8 x half> <half 1.0, half 1.0, half 2.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)1899 store <8 x half> %res, ptr addrspace(1) %out1900 ret void1901}1902 1903define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {1904; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_bf8_non_inlineable:1905; GFX1250: ; %bb.0: ; %bb1906; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11907; GFX1250-NEXT: v_mov_b32_e32 v34, 0x420042001908; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1909; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v36, v341910; GFX1250-NEXT: v_mov_b32_e32 v37, v341911; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[34:37], v[0:15], v[16:31], v[34:37] neg_hi:[0,0,1]1912; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1913; GFX1250-NEXT: s_endpgm1914;1915; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_bf8_non_inlineable:1916; GISEL: ; %bb.0: ; %bb1917; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11918; GISEL-NEXT: s_mov_b32 s0, 0x420042001919; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)1920; GISEL-NEXT: s_mov_b32 s2, s01921; GISEL-NEXT: s_mov_b32 s3, s01922; GISEL-NEXT: s_mov_b32 s1, s01923; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]1924; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]1925; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[34:37], v[0:15], v[16:31], v[34:37] neg_hi:[0,0,1]1926; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1927; GISEL-NEXT: s_endpgm1928bb:1929 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.bf8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 2, <8 x half> <half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0>, i1 false, i1 false)1930 store <8 x half> %res, ptr addrspace(1) %out1931 ret void1932}1933 1934define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {1935; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_fp8:1936; GFX1250: ; %bb.0: ; %bb1937; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11938; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[34:37], v[0:15], v[16:31], 1.01939; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1940; GFX1250-NEXT: s_endpgm1941;1942; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_fp8:1943; GISEL: ; %bb.0: ; %bb1944; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11945; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[34:37], v[0:15], v[16:31], 1.01946; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1947; GISEL-NEXT: s_endpgm1948bb:1949 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.fp8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x half> <half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)1950 store <8 x half> %res, ptr addrspace(1) %out1951 ret void1952}1953 1954define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {1955; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_fp8_non_splat:1956; GFX1250: ; %bb.0: ; %bb1957; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11958; GFX1250-NEXT: v_mov_b32_e32 v34, 0x3c003c001959; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1960; GFX1250-NEXT: v_dual_mov_b32 v35, 0x3c004000 :: v_dual_mov_b32 v36, v341961; GFX1250-NEXT: v_mov_b32_e32 v37, v341962; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[34:37], v[0:15], v[16:31], v[34:37]1963; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1964; GFX1250-NEXT: s_endpgm1965;1966; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_fp8_non_splat:1967; GISEL: ; %bb.0: ; %bb1968; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11969; GISEL-NEXT: s_mov_b32 s0, 0x3c003c001970; GISEL-NEXT: s_mov_b32 s1, 0x3c0040001971; GISEL-NEXT: s_mov_b32 s2, s01972; GISEL-NEXT: s_mov_b32 s3, s01973; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1974; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]1975; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]1976; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[34:37], v[0:15], v[16:31], v[34:37]1977; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off1978; GISEL-NEXT: s_endpgm1979bb:1980 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.fp8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x half> <half 1.0, half 1.0, half 2.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)1981 store <8 x half> %res, ptr addrspace(1) %out1982 ret void1983}1984 1985define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {1986; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_fp8_non_inlineable:1987; GFX1250: ; %bb.0: ; %bb1988; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11989; GFX1250-NEXT: v_mov_b32_e32 v34, 0x420042001990; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)1991; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v36, v341992; GFX1250-NEXT: v_mov_b32_e32 v37, v341993; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[34:37], v[0:15], v[16:31], v[34:37]1994; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off1995; GFX1250-NEXT: s_endpgm1996;1997; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_fp8_non_inlineable:1998; GISEL: ; %bb.0: ; %bb1999; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12000; GISEL-NEXT: s_mov_b32 s0, 0x420042002001; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)2002; GISEL-NEXT: s_mov_b32 s2, s02003; GISEL-NEXT: s_mov_b32 s3, s02004; GISEL-NEXT: s_mov_b32 s1, s02005; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]2006; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]2007; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[34:37], v[0:15], v[16:31], v[34:37]2008; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off2009; GISEL-NEXT: s_endpgm2010bb:2011 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.fp8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x half> <half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0>, i1 false, i1 false)2012 store <8 x half> %res, ptr addrspace(1) %out2013 ret void2014}2015 2016define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {2017; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_bf8:2018; GFX1250: ; %bb.0: ; %bb2019; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12020; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[34:37], v[0:15], v[16:31], 1.02021; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off2022; GFX1250-NEXT: s_endpgm2023;2024; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_bf8:2025; GISEL: ; %bb.0: ; %bb2026; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12027; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[34:37], v[0:15], v[16:31], 1.02028; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off2029; GISEL-NEXT: s_endpgm2030bb:2031 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.bf8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x half> <half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)2032 store <8 x half> %res, ptr addrspace(1) %out2033 ret void2034}2035 2036define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {2037; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_bf8_non_splat:2038; GFX1250: ; %bb.0: ; %bb2039; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12040; GFX1250-NEXT: v_mov_b32_e32 v34, 0x3c003c002041; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)2042; GFX1250-NEXT: v_dual_mov_b32 v35, 0x3c004000 :: v_dual_mov_b32 v36, v342043; GFX1250-NEXT: v_mov_b32_e32 v37, v342044; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[34:37], v[0:15], v[16:31], v[34:37]2045; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off2046; GFX1250-NEXT: s_endpgm2047;2048; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_bf8_non_splat:2049; GISEL: ; %bb.0: ; %bb2050; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12051; GISEL-NEXT: s_mov_b32 s0, 0x3c003c002052; GISEL-NEXT: s_mov_b32 s1, 0x3c0040002053; GISEL-NEXT: s_mov_b32 s2, s02054; GISEL-NEXT: s_mov_b32 s3, s02055; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)2056; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]2057; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]2058; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[34:37], v[0:15], v[16:31], v[34:37]2059; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off2060; GISEL-NEXT: s_endpgm2061bb:2062 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.bf8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x half> <half 1.0, half 1.0, half 2.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 false, i1 false)2063 store <8 x half> %res, ptr addrspace(1) %out2064 ret void2065}2066 2067define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {2068; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_bf8_non_inlineable:2069; GFX1250: ; %bb.0: ; %bb2070; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12071; GFX1250-NEXT: v_mov_b32_e32 v34, 0x420042002072; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)2073; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v36, v342074; GFX1250-NEXT: v_mov_b32_e32 v37, v342075; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[34:37], v[0:15], v[16:31], v[34:37]2076; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off2077; GFX1250-NEXT: s_endpgm2078;2079; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_bf8_non_inlineable:2080; GISEL: ; %bb.0: ; %bb2081; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12082; GISEL-NEXT: s_mov_b32 s0, 0x420042002083; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)2084; GISEL-NEXT: s_mov_b32 s2, s02085; GISEL-NEXT: s_mov_b32 s3, s02086; GISEL-NEXT: s_mov_b32 s1, s02087; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]2088; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]2089; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[34:37], v[0:15], v[16:31], v[34:37]2090; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off2091; GISEL-NEXT: s_endpgm2092bb:2093 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.bf8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x half> <half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0>, i1 false, i1 false)2094 store <8 x half> %res, ptr addrspace(1) %out2095 ret void2096}2097 2098define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {2099; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_fp8:2100; GFX1250: ; %bb.0: ; %bb2101; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12102; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[34:41], v[0:15], v[16:31], 1.02103; GFX1250-NEXT: s_clause 0x12104; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162105; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off2106; GFX1250-NEXT: s_endpgm2107;2108; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_fp8:2109; GISEL: ; %bb.0: ; %bb2110; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12111; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[34:41], v[0:15], v[16:31], 1.02112; GISEL-NEXT: s_clause 0x12113; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off2114; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162115; GISEL-NEXT: s_endpgm2116bb:2117 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.fp8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)2118 store <8 x float> %res, ptr addrspace(1) %out2119 ret void2120}2121 2122define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {2123; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_fp8_non_splat:2124; GFX1250: ; %bb.0: ; %bb2125; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12126; GFX1250-NEXT: v_dual_mov_b32 v34, 1.0 :: v_dual_mov_b32 v36, 2.02127; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)2128; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v37, v342129; GFX1250-NEXT: v_dual_mov_b32 v38, v34 :: v_dual_mov_b32 v39, v342130; GFX1250-NEXT: v_dual_mov_b32 v40, v34 :: v_dual_mov_b32 v41, v342131; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[34:41], v[0:15], v[16:31], v[34:41]2132; GFX1250-NEXT: s_clause 0x12133; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162134; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off2135; GFX1250-NEXT: s_endpgm2136;2137; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_fp8_non_splat:2138; GISEL: ; %bb.0: ; %bb2139; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12140; GISEL-NEXT: s_mov_b32 s0, 1.02141; GISEL-NEXT: s_mov_b32 s2, 2.02142; GISEL-NEXT: s_mov_b32 s6, s02143; GISEL-NEXT: s_mov_b32 s7, s02144; GISEL-NEXT: s_mov_b32 s1, s02145; GISEL-NEXT: s_mov_b32 s3, s02146; GISEL-NEXT: s_mov_b32 s4, s02147; GISEL-NEXT: s_mov_b32 s5, s02148; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[6:7]2149; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[4:5]2150; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]2151; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]2152; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2153; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[34:41], v[0:15], v[16:31], v[34:41]2154; GISEL-NEXT: s_clause 0x12155; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off2156; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162157; GISEL-NEXT: s_endpgm2158bb:2159 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.fp8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)2160 store <8 x float> %res, ptr addrspace(1) %out2161 ret void2162}2163 2164define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {2165; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_fp8_non_inlineable:2166; GFX1250: ; %bb.0: ; %bb2167; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12168; GFX1250-NEXT: v_mov_b32_e32 v34, 0x404000002169; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)2170; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v36, v342171; GFX1250-NEXT: v_dual_mov_b32 v37, v34 :: v_dual_mov_b32 v38, v342172; GFX1250-NEXT: v_dual_mov_b32 v39, v34 :: v_dual_mov_b32 v40, v342173; GFX1250-NEXT: v_mov_b32_e32 v41, v342174; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[34:41], v[0:15], v[16:31], v[34:41]2175; GFX1250-NEXT: s_clause 0x12176; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162177; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off2178; GFX1250-NEXT: s_endpgm2179;2180; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_fp8_non_inlineable:2181; GISEL: ; %bb.0: ; %bb2182; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12183; GISEL-NEXT: s_mov_b32 s0, 0x404000002184; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2185; GISEL-NEXT: s_mov_b32 s6, s02186; GISEL-NEXT: s_mov_b32 s7, s02187; GISEL-NEXT: s_mov_b32 s1, s02188; GISEL-NEXT: s_mov_b32 s2, s02189; GISEL-NEXT: s_mov_b32 s3, s02190; GISEL-NEXT: s_mov_b32 s4, s02191; GISEL-NEXT: s_mov_b32 s5, s02192; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[6:7]2193; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[4:5]2194; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]2195; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]2196; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2197; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[34:41], v[0:15], v[16:31], v[34:41]2198; GISEL-NEXT: s_clause 0x12199; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off2200; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162201; GISEL-NEXT: s_endpgm2202bb:2203 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.fp8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>, i1 false, i1 false)2204 store <8 x float> %res, ptr addrspace(1) %out2205 ret void2206}2207 2208define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {2209; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_bf8:2210; GFX1250: ; %bb.0: ; %bb2211; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12212; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[34:41], v[0:15], v[16:31], 1.02213; GFX1250-NEXT: s_clause 0x12214; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162215; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off2216; GFX1250-NEXT: s_endpgm2217;2218; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_bf8:2219; GISEL: ; %bb.0: ; %bb2220; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12221; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[34:41], v[0:15], v[16:31], 1.02222; GISEL-NEXT: s_clause 0x12223; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off2224; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162225; GISEL-NEXT: s_endpgm2226bb:2227 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.bf8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)2228 store <8 x float> %res, ptr addrspace(1) %out2229 ret void2230}2231 2232define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {2233; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_bf8_non_splat:2234; GFX1250: ; %bb.0: ; %bb2235; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12236; GFX1250-NEXT: v_dual_mov_b32 v34, 1.0 :: v_dual_mov_b32 v36, 2.02237; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)2238; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v37, v342239; GFX1250-NEXT: v_dual_mov_b32 v38, v34 :: v_dual_mov_b32 v39, v342240; GFX1250-NEXT: v_dual_mov_b32 v40, v34 :: v_dual_mov_b32 v41, v342241; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[34:41], v[0:15], v[16:31], v[34:41]2242; GFX1250-NEXT: s_clause 0x12243; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162244; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off2245; GFX1250-NEXT: s_endpgm2246;2247; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_bf8_non_splat:2248; GISEL: ; %bb.0: ; %bb2249; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12250; GISEL-NEXT: s_mov_b32 s0, 1.02251; GISEL-NEXT: s_mov_b32 s2, 2.02252; GISEL-NEXT: s_mov_b32 s6, s02253; GISEL-NEXT: s_mov_b32 s7, s02254; GISEL-NEXT: s_mov_b32 s1, s02255; GISEL-NEXT: s_mov_b32 s3, s02256; GISEL-NEXT: s_mov_b32 s4, s02257; GISEL-NEXT: s_mov_b32 s5, s02258; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[6:7]2259; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[4:5]2260; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]2261; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]2262; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2263; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[34:41], v[0:15], v[16:31], v[34:41]2264; GISEL-NEXT: s_clause 0x12265; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off2266; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162267; GISEL-NEXT: s_endpgm2268bb:2269 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.bf8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)2270 store <8 x float> %res, ptr addrspace(1) %out2271 ret void2272}2273 2274define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {2275; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_bf8_non_inlineable:2276; GFX1250: ; %bb.0: ; %bb2277; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12278; GFX1250-NEXT: v_mov_b32_e32 v34, 0x404000002279; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)2280; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v36, v342281; GFX1250-NEXT: v_dual_mov_b32 v37, v34 :: v_dual_mov_b32 v38, v342282; GFX1250-NEXT: v_dual_mov_b32 v39, v34 :: v_dual_mov_b32 v40, v342283; GFX1250-NEXT: v_mov_b32_e32 v41, v342284; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[34:41], v[0:15], v[16:31], v[34:41]2285; GFX1250-NEXT: s_clause 0x12286; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162287; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off2288; GFX1250-NEXT: s_endpgm2289;2290; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_bf8_non_inlineable:2291; GISEL: ; %bb.0: ; %bb2292; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12293; GISEL-NEXT: s_mov_b32 s0, 0x404000002294; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2295; GISEL-NEXT: s_mov_b32 s6, s02296; GISEL-NEXT: s_mov_b32 s7, s02297; GISEL-NEXT: s_mov_b32 s1, s02298; GISEL-NEXT: s_mov_b32 s2, s02299; GISEL-NEXT: s_mov_b32 s3, s02300; GISEL-NEXT: s_mov_b32 s4, s02301; GISEL-NEXT: s_mov_b32 s5, s02302; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[6:7]2303; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[4:5]2304; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]2305; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]2306; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2307; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[34:41], v[0:15], v[16:31], v[34:41]2308; GISEL-NEXT: s_clause 0x12309; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off2310; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162311; GISEL-NEXT: s_endpgm2312bb:2313 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.bf8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>, i1 false, i1 false)2314 store <8 x float> %res, ptr addrspace(1) %out2315 ret void2316}2317 2318define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {2319; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_fp8:2320; GFX1250: ; %bb.0: ; %bb2321; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12322; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[34:41], v[0:15], v[16:31], 1.02323; GFX1250-NEXT: s_clause 0x12324; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162325; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off2326; GFX1250-NEXT: s_endpgm2327;2328; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_fp8:2329; GISEL: ; %bb.0: ; %bb2330; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12331; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[34:41], v[0:15], v[16:31], 1.02332; GISEL-NEXT: s_clause 0x12333; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off2334; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162335; GISEL-NEXT: s_endpgm2336bb:2337 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.fp8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)2338 store <8 x float> %res, ptr addrspace(1) %out2339 ret void2340}2341 2342define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {2343; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_fp8_non_splat:2344; GFX1250: ; %bb.0: ; %bb2345; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12346; GFX1250-NEXT: v_dual_mov_b32 v34, 1.0 :: v_dual_mov_b32 v36, 2.02347; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)2348; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v37, v342349; GFX1250-NEXT: v_dual_mov_b32 v38, v34 :: v_dual_mov_b32 v39, v342350; GFX1250-NEXT: v_dual_mov_b32 v40, v34 :: v_dual_mov_b32 v41, v342351; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[34:41], v[0:15], v[16:31], v[34:41]2352; GFX1250-NEXT: s_clause 0x12353; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162354; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off2355; GFX1250-NEXT: s_endpgm2356;2357; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_fp8_non_splat:2358; GISEL: ; %bb.0: ; %bb2359; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12360; GISEL-NEXT: s_mov_b32 s0, 1.02361; GISEL-NEXT: s_mov_b32 s2, 2.02362; GISEL-NEXT: s_mov_b32 s6, s02363; GISEL-NEXT: s_mov_b32 s7, s02364; GISEL-NEXT: s_mov_b32 s1, s02365; GISEL-NEXT: s_mov_b32 s3, s02366; GISEL-NEXT: s_mov_b32 s4, s02367; GISEL-NEXT: s_mov_b32 s5, s02368; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[6:7]2369; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[4:5]2370; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]2371; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]2372; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2373; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[34:41], v[0:15], v[16:31], v[34:41]2374; GISEL-NEXT: s_clause 0x12375; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off2376; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162377; GISEL-NEXT: s_endpgm2378bb:2379 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.fp8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)2380 store <8 x float> %res, ptr addrspace(1) %out2381 ret void2382}2383 2384define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {2385; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_fp8_non_inlineable:2386; GFX1250: ; %bb.0: ; %bb2387; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12388; GFX1250-NEXT: v_mov_b32_e32 v34, 0x404000002389; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)2390; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v36, v342391; GFX1250-NEXT: v_dual_mov_b32 v37, v34 :: v_dual_mov_b32 v38, v342392; GFX1250-NEXT: v_dual_mov_b32 v39, v34 :: v_dual_mov_b32 v40, v342393; GFX1250-NEXT: v_mov_b32_e32 v41, v342394; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[34:41], v[0:15], v[16:31], v[34:41]2395; GFX1250-NEXT: s_clause 0x12396; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162397; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off2398; GFX1250-NEXT: s_endpgm2399;2400; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_fp8_non_inlineable:2401; GISEL: ; %bb.0: ; %bb2402; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12403; GISEL-NEXT: s_mov_b32 s0, 0x404000002404; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2405; GISEL-NEXT: s_mov_b32 s6, s02406; GISEL-NEXT: s_mov_b32 s7, s02407; GISEL-NEXT: s_mov_b32 s1, s02408; GISEL-NEXT: s_mov_b32 s2, s02409; GISEL-NEXT: s_mov_b32 s3, s02410; GISEL-NEXT: s_mov_b32 s4, s02411; GISEL-NEXT: s_mov_b32 s5, s02412; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[6:7]2413; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[4:5]2414; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]2415; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]2416; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2417; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[34:41], v[0:15], v[16:31], v[34:41]2418; GISEL-NEXT: s_clause 0x12419; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off2420; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162421; GISEL-NEXT: s_endpgm2422bb:2423 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.fp8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>, i1 false, i1 false)2424 store <8 x float> %res, ptr addrspace(1) %out2425 ret void2426}2427 2428define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {2429; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_bf8:2430; GFX1250: ; %bb.0: ; %bb2431; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12432; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[34:41], v[0:15], v[16:31], 1.02433; GFX1250-NEXT: s_clause 0x12434; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162435; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off2436; GFX1250-NEXT: s_endpgm2437;2438; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_bf8:2439; GISEL: ; %bb.0: ; %bb2440; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12441; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[34:41], v[0:15], v[16:31], 1.02442; GISEL-NEXT: s_clause 0x12443; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off2444; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162445; GISEL-NEXT: s_endpgm2446bb:2447 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.bf8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)2448 store <8 x float> %res, ptr addrspace(1) %out2449 ret void2450}2451 2452define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {2453; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_bf8_non_splat:2454; GFX1250: ; %bb.0: ; %bb2455; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12456; GFX1250-NEXT: v_dual_mov_b32 v34, 1.0 :: v_dual_mov_b32 v36, 2.02457; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)2458; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v37, v342459; GFX1250-NEXT: v_dual_mov_b32 v38, v34 :: v_dual_mov_b32 v39, v342460; GFX1250-NEXT: v_dual_mov_b32 v40, v34 :: v_dual_mov_b32 v41, v342461; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[34:41], v[0:15], v[16:31], v[34:41]2462; GFX1250-NEXT: s_clause 0x12463; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162464; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off2465; GFX1250-NEXT: s_endpgm2466;2467; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_bf8_non_splat:2468; GISEL: ; %bb.0: ; %bb2469; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12470; GISEL-NEXT: s_mov_b32 s0, 1.02471; GISEL-NEXT: s_mov_b32 s2, 2.02472; GISEL-NEXT: s_mov_b32 s6, s02473; GISEL-NEXT: s_mov_b32 s7, s02474; GISEL-NEXT: s_mov_b32 s1, s02475; GISEL-NEXT: s_mov_b32 s3, s02476; GISEL-NEXT: s_mov_b32 s4, s02477; GISEL-NEXT: s_mov_b32 s5, s02478; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[6:7]2479; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[4:5]2480; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]2481; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]2482; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2483; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[34:41], v[0:15], v[16:31], v[34:41]2484; GISEL-NEXT: s_clause 0x12485; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off2486; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162487; GISEL-NEXT: s_endpgm2488bb:2489 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.bf8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x float> <float 1.0, float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i1 false, i1 false)2490 store <8 x float> %res, ptr addrspace(1) %out2491 ret void2492}2493 2494define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {2495; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_bf8_non_inlineable:2496; GFX1250: ; %bb.0: ; %bb2497; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12498; GFX1250-NEXT: v_mov_b32_e32 v34, 0x404000002499; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)2500; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v36, v342501; GFX1250-NEXT: v_dual_mov_b32 v37, v34 :: v_dual_mov_b32 v38, v342502; GFX1250-NEXT: v_dual_mov_b32 v39, v34 :: v_dual_mov_b32 v40, v342503; GFX1250-NEXT: v_mov_b32_e32 v41, v342504; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[34:41], v[0:15], v[16:31], v[34:41]2505; GFX1250-NEXT: s_clause 0x12506; GFX1250-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162507; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off2508; GFX1250-NEXT: s_endpgm2509;2510; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_bf8_non_inlineable:2511; GISEL: ; %bb.0: ; %bb2512; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12513; GISEL-NEXT: s_mov_b32 s0, 0x404000002514; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2515; GISEL-NEXT: s_mov_b32 s6, s02516; GISEL-NEXT: s_mov_b32 s7, s02517; GISEL-NEXT: s_mov_b32 s1, s02518; GISEL-NEXT: s_mov_b32 s2, s02519; GISEL-NEXT: s_mov_b32 s3, s02520; GISEL-NEXT: s_mov_b32 s4, s02521; GISEL-NEXT: s_mov_b32 s5, s02522; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[6:7]2523; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[4:5]2524; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[2:3]2525; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[0:1]2526; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2527; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[34:41], v[0:15], v[16:31], v[34:41]2528; GISEL-NEXT: s_clause 0x12529; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off2530; GISEL-NEXT: global_store_b128 v[32:33], v[38:41], off offset:162531; GISEL-NEXT: s_endpgm2532bb:2533 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.bf8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 0, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>, i1 false, i1 false)2534 store <8 x float> %res, ptr addrspace(1) %out2535 ret void2536}2537 2538define amdgpu_ps void @test_wmma_f32_32x16x128_f4(<16 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {2539; GFX1250-LABEL: test_wmma_f32_32x16x128_f4:2540; GFX1250: ; %bb.0: ; %bb2541; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12542; GFX1250-NEXT: v_wmma_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], 1.02543; GFX1250-NEXT: s_clause 0x32544; GFX1250-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482545; GFX1250-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322546; GFX1250-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162547; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off2548; GFX1250-NEXT: s_endpgm2549;2550; GISEL-LABEL: test_wmma_f32_32x16x128_f4:2551; GISEL: ; %bb.0: ; %bb2552; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12553; GISEL-NEXT: v_wmma_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], 1.02554; GISEL-NEXT: s_clause 0x32555; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off2556; GISEL-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162557; GISEL-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322558; GISEL-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482559; GISEL-NEXT: s_endpgm2560bb:2561 %res = call <16 x float> @llvm.amdgcn.wmma.f32.32x16x128.f4.v16i32.v8i32.v16f32(<16 x i32> %A, <8 x i32> %B, i16 0, <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>)2562 store <16 x float> %res, ptr addrspace(1) %out2563 ret void2564}2565 2566define amdgpu_ps void @test_wmma_f32_32x16x128_f4_non_splat(<16 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {2567; GFX1250-LABEL: test_wmma_f32_32x16x128_f4_non_splat:2568; GFX1250: ; %bb.0: ; %bb2569; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12570; GFX1250-NEXT: v_dual_mov_b32 v26, 1.0 :: v_dual_mov_b32 v28, 2.02571; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)2572; GFX1250-NEXT: v_dual_mov_b32 v27, v26 :: v_dual_mov_b32 v29, v262573; GFX1250-NEXT: v_dual_mov_b32 v30, v26 :: v_dual_mov_b32 v31, v262574; GFX1250-NEXT: v_dual_mov_b32 v32, v26 :: v_dual_mov_b32 v33, v262575; GFX1250-NEXT: v_dual_mov_b32 v34, v26 :: v_dual_mov_b32 v35, v262576; GFX1250-NEXT: v_dual_mov_b32 v36, v28 :: v_dual_mov_b32 v37, v262577; GFX1250-NEXT: v_dual_mov_b32 v38, v26 :: v_dual_mov_b32 v39, v262578; GFX1250-NEXT: v_dual_mov_b32 v40, v26 :: v_dual_mov_b32 v41, v262579; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)2580; GFX1250-NEXT: v_wmma_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], v[26:41]2581; GFX1250-NEXT: s_clause 0x32582; GFX1250-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482583; GFX1250-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322584; GFX1250-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162585; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off2586; GFX1250-NEXT: s_endpgm2587;2588; GISEL-LABEL: test_wmma_f32_32x16x128_f4_non_splat:2589; GISEL: ; %bb.0: ; %bb2590; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12591; GISEL-NEXT: s_mov_b32 s0, 1.02592; GISEL-NEXT: s_mov_b32 s2, 2.02593; GISEL-NEXT: s_mov_b32 s14, s02594; GISEL-NEXT: s_mov_b32 s15, s02595; GISEL-NEXT: s_mov_b32 s1, s02596; GISEL-NEXT: s_mov_b32 s3, s02597; GISEL-NEXT: s_mov_b32 s4, s02598; GISEL-NEXT: s_mov_b32 s5, s02599; GISEL-NEXT: s_mov_b32 s6, s02600; GISEL-NEXT: s_mov_b32 s7, s02601; GISEL-NEXT: s_mov_b32 s8, s02602; GISEL-NEXT: s_mov_b32 s9, s02603; GISEL-NEXT: s_mov_b32 s10, s22604; GISEL-NEXT: s_mov_b32 s11, s02605; GISEL-NEXT: s_mov_b32 s12, s02606; GISEL-NEXT: s_mov_b32 s13, s02607; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[14:15]2608; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[12:13]2609; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[10:11]2610; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[8:9]2611; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[6:7]2612; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[4:5]2613; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[2:3]2614; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[0:1]2615; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2616; GISEL-NEXT: v_wmma_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], v[26:41]2617; GISEL-NEXT: s_clause 0x32618; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off2619; GISEL-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162620; GISEL-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322621; GISEL-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482622; GISEL-NEXT: s_endpgm2623bb:2624 %res = call <16 x float> @llvm.amdgcn.wmma.f32.32x16x128.f4.v16i32.v8i32.v16f32(<16 x i32> %A, <8 x i32> %B, i16 0, <16 x float> <float 1.0, float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>)2625 store <16 x float> %res, ptr addrspace(1) %out2626 ret void2627}2628 2629define amdgpu_ps void @test_wmma_f32_32x16x128_f4_non_inlineable(<16 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {2630; GFX1250-LABEL: test_wmma_f32_32x16x128_f4_non_inlineable:2631; GFX1250: ; %bb.0: ; %bb2632; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12633; GFX1250-NEXT: v_mov_b32_e32 v26, 0x404000002634; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)2635; GFX1250-NEXT: v_dual_mov_b32 v27, v26 :: v_dual_mov_b32 v28, v262636; GFX1250-NEXT: v_dual_mov_b32 v29, v26 :: v_dual_mov_b32 v30, v262637; GFX1250-NEXT: v_dual_mov_b32 v31, v26 :: v_dual_mov_b32 v32, v262638; GFX1250-NEXT: v_dual_mov_b32 v33, v26 :: v_dual_mov_b32 v34, v262639; GFX1250-NEXT: v_dual_mov_b32 v35, v26 :: v_dual_mov_b32 v36, v262640; GFX1250-NEXT: v_dual_mov_b32 v37, v26 :: v_dual_mov_b32 v38, v262641; GFX1250-NEXT: v_dual_mov_b32 v39, v26 :: v_dual_mov_b32 v40, v262642; GFX1250-NEXT: v_mov_b32_e32 v41, v262643; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)2644; GFX1250-NEXT: v_wmma_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], v[26:41]2645; GFX1250-NEXT: s_clause 0x32646; GFX1250-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482647; GFX1250-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322648; GFX1250-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162649; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off2650; GFX1250-NEXT: s_endpgm2651;2652; GISEL-LABEL: test_wmma_f32_32x16x128_f4_non_inlineable:2653; GISEL: ; %bb.0: ; %bb2654; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12655; GISEL-NEXT: s_mov_b32 s0, 0x404000002656; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2657; GISEL-NEXT: s_mov_b32 s14, s02658; GISEL-NEXT: s_mov_b32 s15, s02659; GISEL-NEXT: s_mov_b32 s1, s02660; GISEL-NEXT: s_mov_b32 s2, s02661; GISEL-NEXT: s_mov_b32 s3, s02662; GISEL-NEXT: s_mov_b32 s4, s02663; GISEL-NEXT: s_mov_b32 s5, s02664; GISEL-NEXT: s_mov_b32 s6, s02665; GISEL-NEXT: s_mov_b32 s7, s02666; GISEL-NEXT: s_mov_b32 s8, s02667; GISEL-NEXT: s_mov_b32 s9, s02668; GISEL-NEXT: s_mov_b32 s10, s02669; GISEL-NEXT: s_mov_b32 s11, s02670; GISEL-NEXT: s_mov_b32 s12, s02671; GISEL-NEXT: s_mov_b32 s13, s02672; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[14:15]2673; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[12:13]2674; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[10:11]2675; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[8:9]2676; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[6:7]2677; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[4:5]2678; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[2:3]2679; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[0:1]2680; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2681; GISEL-NEXT: v_wmma_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], v[26:41]2682; GISEL-NEXT: s_clause 0x32683; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off2684; GISEL-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162685; GISEL-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322686; GISEL-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482687; GISEL-NEXT: s_endpgm2688bb:2689 %res = call <16 x float> @llvm.amdgcn.wmma.f32.32x16x128.f4.v16i32.v8i32.v16f32(<16 x i32> %A, <8 x i32> %B, i16 0, <16 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>)2690 store <16 x float> %res, ptr addrspace(1) %out2691 ret void2692}2693 2694define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4(<16 x i32> %A, <8 x i32> %B, i32 inreg %scale_src0, i32 inreg %scale_src1, ptr addrspace(1) %out) {2695; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4:2696; GFX1250: ; %bb.0: ; %bb2697; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12698; GFX1250-NEXT: v_wmma_scale_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], 1.0, s0, s1 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse2699; GFX1250-NEXT: s_clause 0x32700; GFX1250-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482701; GFX1250-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322702; GFX1250-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162703; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off2704; GFX1250-NEXT: s_endpgm2705;2706; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4:2707; GISEL: ; %bb.0: ; %bb2708; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12709; GISEL-NEXT: v_wmma_scale_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], 1.0, s0, s1 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse2710; GISEL-NEXT: s_clause 0x32711; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off2712; GISEL-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162713; GISEL-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322714; GISEL-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482715; GISEL-NEXT: s_endpgm2716bb:2717 %res = call <16 x float> @llvm.amdgcn.wmma.scale.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 0, <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i32 1, i32 0, i32 %scale_src0, i32 1, i32 0, i32 %scale_src1, i1 true, i1 false)2718 store <16 x float> %res, ptr addrspace(1) %out2719 ret void2720}2721 2722define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_non_splat(<16 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {2723; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4_non_splat:2724; GFX1250: ; %bb.0: ; %bb2725; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12726; GFX1250-NEXT: v_dual_mov_b32 v26, 1.0 :: v_dual_mov_b32 v27, 2.02727; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)2728; GFX1250-NEXT: v_dual_mov_b32 v28, v26 :: v_dual_mov_b32 v29, v262729; GFX1250-NEXT: v_dual_mov_b32 v30, v26 :: v_dual_mov_b32 v31, v262730; GFX1250-NEXT: v_dual_mov_b32 v32, v26 :: v_dual_mov_b32 v33, v262731; GFX1250-NEXT: v_dual_mov_b32 v34, v26 :: v_dual_mov_b32 v35, v262732; GFX1250-NEXT: v_dual_mov_b32 v36, v26 :: v_dual_mov_b32 v37, v262733; GFX1250-NEXT: v_dual_mov_b32 v38, v26 :: v_dual_mov_b32 v39, v262734; GFX1250-NEXT: v_dual_mov_b32 v40, v26 :: v_dual_mov_b32 v41, v262735; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)2736; GFX1250-NEXT: v_wmma_scale_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], v[26:41], 1, 2 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW12737; GFX1250-NEXT: s_clause 0x32738; GFX1250-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482739; GFX1250-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322740; GFX1250-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162741; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off2742; GFX1250-NEXT: s_endpgm2743;2744; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4_non_splat:2745; GISEL: ; %bb.0: ; %bb2746; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12747; GISEL-NEXT: s_mov_b32 s0, 1.02748; GISEL-NEXT: s_mov_b32 s1, 2.02749; GISEL-NEXT: s_mov_b32 s14, s02750; GISEL-NEXT: s_mov_b32 s15, s02751; GISEL-NEXT: s_mov_b32 s2, s02752; GISEL-NEXT: s_mov_b32 s3, s02753; GISEL-NEXT: s_mov_b32 s4, s02754; GISEL-NEXT: s_mov_b32 s5, s02755; GISEL-NEXT: s_mov_b32 s6, s02756; GISEL-NEXT: s_mov_b32 s7, s02757; GISEL-NEXT: s_mov_b32 s8, s02758; GISEL-NEXT: s_mov_b32 s9, s02759; GISEL-NEXT: s_mov_b32 s10, s02760; GISEL-NEXT: s_mov_b32 s11, s02761; GISEL-NEXT: s_mov_b32 s12, s02762; GISEL-NEXT: s_mov_b32 s13, s02763; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[14:15]2764; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[12:13]2765; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[10:11]2766; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[8:9]2767; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[6:7]2768; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[4:5]2769; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[2:3]2770; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[0:1]2771; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2772; GISEL-NEXT: v_wmma_scale_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], v[26:41], 1, 2 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW12773; GISEL-NEXT: s_clause 0x32774; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off2775; GISEL-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162776; GISEL-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322777; GISEL-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482778; GISEL-NEXT: s_endpgm2779bb:2780 %res = call <16 x float> @llvm.amdgcn.wmma.scale.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 0, <16 x float> <float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i32 1, i32 0, i32 1, i32 1, i32 0, i32 2, i1 false, i1 false)2781 store <16 x float> %res, ptr addrspace(1) %out2782 ret void2783}2784 2785define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_non_inlineable(<16 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {2786; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4_non_inlineable:2787; GFX1250: ; %bb.0: ; %bb2788; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12789; GFX1250-NEXT: v_mov_b32_e32 v26, 0x404000002790; GFX1250-NEXT: v_mov_b32_e32 v43, 0x642791; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)2792; GFX1250-NEXT: v_dual_mov_b32 v42, 0x65 :: v_dual_mov_b32 v41, v262793; GFX1250-NEXT: v_dual_mov_b32 v27, v26 :: v_dual_mov_b32 v28, v262794; GFX1250-NEXT: v_dual_mov_b32 v29, v26 :: v_dual_mov_b32 v30, v262795; GFX1250-NEXT: v_dual_mov_b32 v31, v26 :: v_dual_mov_b32 v32, v262796; GFX1250-NEXT: v_dual_mov_b32 v33, v26 :: v_dual_mov_b32 v34, v262797; GFX1250-NEXT: v_dual_mov_b32 v35, v26 :: v_dual_mov_b32 v36, v262798; GFX1250-NEXT: v_dual_mov_b32 v37, v26 :: v_dual_mov_b32 v38, v262799; GFX1250-NEXT: v_dual_mov_b32 v39, v26 :: v_dual_mov_b32 v40, v262800; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)2801; GFX1250-NEXT: v_wmma_scale_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], v[26:41], v43, v42 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse2802; GFX1250-NEXT: s_clause 0x32803; GFX1250-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482804; GFX1250-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322805; GFX1250-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162806; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off2807; GFX1250-NEXT: s_endpgm2808;2809; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4_non_inlineable:2810; GISEL: ; %bb.0: ; %bb2811; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12812; GISEL-NEXT: s_mov_b32 s0, 0x404000002813; GISEL-NEXT: v_mov_b32_e32 v42, 0x642814; GISEL-NEXT: s_mov_b32 s14, s02815; GISEL-NEXT: s_mov_b32 s15, s02816; GISEL-NEXT: s_mov_b32 s1, s02817; GISEL-NEXT: s_mov_b32 s2, s02818; GISEL-NEXT: s_mov_b32 s3, s02819; GISEL-NEXT: s_mov_b32 s4, s02820; GISEL-NEXT: s_mov_b32 s5, s02821; GISEL-NEXT: s_mov_b32 s6, s02822; GISEL-NEXT: s_mov_b32 s7, s02823; GISEL-NEXT: s_mov_b32 s8, s02824; GISEL-NEXT: s_mov_b32 s9, s02825; GISEL-NEXT: s_mov_b32 s10, s02826; GISEL-NEXT: s_mov_b32 s11, s02827; GISEL-NEXT: s_mov_b32 s12, s02828; GISEL-NEXT: s_mov_b32 s13, s02829; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[14:15]2830; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[12:13]2831; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[10:11]2832; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[8:9]2833; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[6:7]2834; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[4:5]2835; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[2:3]2836; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[0:1]2837; GISEL-NEXT: v_mov_b32_e32 v43, 0x652838; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2839; GISEL-NEXT: v_wmma_scale_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], v[26:41], v42, v43 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse2840; GISEL-NEXT: s_clause 0x32841; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off2842; GISEL-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162843; GISEL-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322844; GISEL-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482845; GISEL-NEXT: s_endpgm2846bb:2847 %res = call <16 x float> @llvm.amdgcn.wmma.scale.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 0, <16 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>, i32 1, i32 0, i32 100, i32 1, i32 0, i32 101, i1 true, i1 false)2848 store <16 x float> %res, ptr addrspace(1) %out2849 ret void2850}2851 2852define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4(<16 x i32> %A, <8 x i32> %B, i64 inreg %scale_src0, i64 inreg %scale_src1, ptr addrspace(1) %out) {2853; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4:2854; GFX1250: ; %bb.0: ; %bb2855; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12856; GFX1250-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], 1.0, s[0:1], s[2:3] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse2857; GFX1250-NEXT: s_clause 0x32858; GFX1250-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482859; GFX1250-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322860; GFX1250-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162861; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off2862; GFX1250-NEXT: s_endpgm2863;2864; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4:2865; GISEL: ; %bb.0: ; %bb2866; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12867; GISEL-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], 1.0, s[0:1], s[2:3] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse2868; GISEL-NEXT: s_clause 0x32869; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off2870; GISEL-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162871; GISEL-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322872; GISEL-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482873; GISEL-NEXT: s_endpgm2874bb:2875 %res = call <16 x float> @llvm.amdgcn.wmma.scale16.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 0, <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i32 1, i32 0, i64 %scale_src0, i32 1, i32 0, i64 %scale_src1, i1 true, i1 false)2876 store <16 x float> %res, ptr addrspace(1) %out2877 ret void2878}2879 2880define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_non_splat(<16 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {2881; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4_non_splat:2882; GFX1250: ; %bb.0: ; %bb2883; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12884; GFX1250-NEXT: v_dual_mov_b32 v26, 1.0 :: v_dual_mov_b32 v27, 2.02885; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)2886; GFX1250-NEXT: v_dual_mov_b32 v28, v26 :: v_dual_mov_b32 v29, v262887; GFX1250-NEXT: v_dual_mov_b32 v30, v26 :: v_dual_mov_b32 v31, v262888; GFX1250-NEXT: v_dual_mov_b32 v32, v26 :: v_dual_mov_b32 v33, v262889; GFX1250-NEXT: v_dual_mov_b32 v34, v26 :: v_dual_mov_b32 v35, v262890; GFX1250-NEXT: v_dual_mov_b32 v36, v26 :: v_dual_mov_b32 v37, v262891; GFX1250-NEXT: v_dual_mov_b32 v38, v26 :: v_dual_mov_b32 v39, v262892; GFX1250-NEXT: v_dual_mov_b32 v40, v26 :: v_dual_mov_b32 v41, v262893; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)2894; GFX1250-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], v[26:41], 1, 2 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW12895; GFX1250-NEXT: s_clause 0x32896; GFX1250-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482897; GFX1250-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322898; GFX1250-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162899; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off2900; GFX1250-NEXT: s_endpgm2901;2902; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4_non_splat:2903; GISEL: ; %bb.0: ; %bb2904; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12905; GISEL-NEXT: s_mov_b32 s0, 1.02906; GISEL-NEXT: s_mov_b32 s1, 2.02907; GISEL-NEXT: s_mov_b32 s14, s02908; GISEL-NEXT: s_mov_b32 s15, s02909; GISEL-NEXT: s_mov_b32 s2, s02910; GISEL-NEXT: s_mov_b32 s3, s02911; GISEL-NEXT: s_mov_b32 s4, s02912; GISEL-NEXT: s_mov_b32 s5, s02913; GISEL-NEXT: s_mov_b32 s6, s02914; GISEL-NEXT: s_mov_b32 s7, s02915; GISEL-NEXT: s_mov_b32 s8, s02916; GISEL-NEXT: s_mov_b32 s9, s02917; GISEL-NEXT: s_mov_b32 s10, s02918; GISEL-NEXT: s_mov_b32 s11, s02919; GISEL-NEXT: s_mov_b32 s12, s02920; GISEL-NEXT: s_mov_b32 s13, s02921; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[14:15]2922; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[12:13]2923; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[10:11]2924; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[8:9]2925; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[6:7]2926; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[4:5]2927; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[2:3]2928; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[0:1]2929; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2930; GISEL-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], v[26:41], 1, 2 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW12931; GISEL-NEXT: s_clause 0x32932; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off2933; GISEL-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162934; GISEL-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322935; GISEL-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482936; GISEL-NEXT: s_endpgm2937bb:2938 %res = call <16 x float> @llvm.amdgcn.wmma.scale16.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 0, <16 x float> <float 1.0, float 2.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, i32 1, i32 0, i64 1, i32 1, i32 0, i64 2, i1 false, i1 false)2939 store <16 x float> %res, ptr addrspace(1) %out2940 ret void2941}2942 2943define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_non_inlineable(<16 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {2944; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4_non_inlineable:2945; GFX1250: ; %bb.0: ; %bb2946; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12947; GFX1250-NEXT: v_mov_b32_e32 v26, 0x404000002948; GFX1250-NEXT: v_mov_b64_e32 v[42:43], 0x652949; GFX1250-NEXT: v_mov_b64_e32 v[44:45], 0x642950; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3)2951; GFX1250-NEXT: v_dual_mov_b32 v27, v26 :: v_dual_mov_b32 v28, v262952; GFX1250-NEXT: v_dual_mov_b32 v29, v26 :: v_dual_mov_b32 v30, v262953; GFX1250-NEXT: v_dual_mov_b32 v31, v26 :: v_dual_mov_b32 v32, v262954; GFX1250-NEXT: v_dual_mov_b32 v33, v26 :: v_dual_mov_b32 v34, v262955; GFX1250-NEXT: v_dual_mov_b32 v35, v26 :: v_dual_mov_b32 v36, v262956; GFX1250-NEXT: v_dual_mov_b32 v37, v26 :: v_dual_mov_b32 v38, v262957; GFX1250-NEXT: v_dual_mov_b32 v39, v26 :: v_dual_mov_b32 v40, v262958; GFX1250-NEXT: v_mov_b32_e32 v41, v262959; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)2960; GFX1250-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], v[26:41], v[44:45], v[42:43] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse2961; GFX1250-NEXT: s_clause 0x32962; GFX1250-NEXT: global_store_b128 v[24:25], v[38:41], off offset:482963; GFX1250-NEXT: global_store_b128 v[24:25], v[34:37], off offset:322964; GFX1250-NEXT: global_store_b128 v[24:25], v[30:33], off offset:162965; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off2966; GFX1250-NEXT: s_endpgm2967;2968; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4_non_inlineable:2969; GISEL: ; %bb.0: ; %bb2970; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12971; GISEL-NEXT: s_mov_b32 s0, 0x404000002972; GISEL-NEXT: v_mov_b64_e32 v[42:43], 0x642973; GISEL-NEXT: s_mov_b32 s14, s02974; GISEL-NEXT: s_mov_b32 s15, s02975; GISEL-NEXT: s_mov_b32 s1, s02976; GISEL-NEXT: s_mov_b32 s2, s02977; GISEL-NEXT: s_mov_b32 s3, s02978; GISEL-NEXT: s_mov_b32 s4, s02979; GISEL-NEXT: s_mov_b32 s5, s02980; GISEL-NEXT: s_mov_b32 s6, s02981; GISEL-NEXT: s_mov_b32 s7, s02982; GISEL-NEXT: s_mov_b32 s8, s02983; GISEL-NEXT: s_mov_b32 s9, s02984; GISEL-NEXT: s_mov_b32 s10, s02985; GISEL-NEXT: s_mov_b32 s11, s02986; GISEL-NEXT: s_mov_b32 s12, s02987; GISEL-NEXT: s_mov_b32 s13, s02988; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[14:15]2989; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[12:13]2990; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[10:11]2991; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[8:9]2992; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[6:7]2993; GISEL-NEXT: v_mov_b64_e32 v[30:31], s[4:5]2994; GISEL-NEXT: v_mov_b64_e32 v[28:29], s[2:3]2995; GISEL-NEXT: v_mov_b64_e32 v[26:27], s[0:1]2996; GISEL-NEXT: v_mov_b64_e32 v[44:45], 0x652997; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2998; GISEL-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], v[26:41], v[42:43], v[44:45] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse2999; GISEL-NEXT: s_clause 0x33000; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off3001; GISEL-NEXT: global_store_b128 v[24:25], v[30:33], off offset:163002; GISEL-NEXT: global_store_b128 v[24:25], v[34:37], off offset:323003; GISEL-NEXT: global_store_b128 v[24:25], v[38:41], off offset:483004; GISEL-NEXT: s_endpgm3005bb:3006 %res = call <16 x float> @llvm.amdgcn.wmma.scale16.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 0, <16 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>, i32 1, i32 0, i64 100, i32 1, i32 0, i64 101, i1 true, i1 false)3007 store <16 x float> %res, ptr addrspace(1) %out3008 ret void3009}3010 3011declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x4.f32.v8f32.v2f32(i1, <2 x float>, i1, <2 x float>, i16, <8 x float>, i1, i1)3012declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.bf16.v8f32.v16bf16(i1, <16 x bfloat>, i1, <16 x bfloat>, i16, <8 x float>, i1, i1)3013declare <8 x bfloat> @llvm.amdgcn.wmma.bf16.16x16x32.bf16.v8bf16.v16bf16(i1, <16 x bfloat>, i1, <16 x bfloat>, i16, <8 x bfloat>, i1, i1)3014declare <8 x bfloat> @llvm.amdgcn.wmma.bf16f32.16x16x32.bf16.v8bf16.v16bf16(i1, <16 x bfloat>, i1, <16 x bfloat>, i16, <8 x float>, i1, i1)3015declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.fp8.v8f32.v8i32(<8 x i32>, <8 x i32>, i16, <8 x float>, i1, i1)3016declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.bf8.v8f32.v8i32(<8 x i32>, <8 x i32>, i16, <8 x float>, i1, i1)3017declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.fp8.v8f32.v8i32(<8 x i32>, <8 x i32>, i16, <8 x float>, i1, i1)3018declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.bf8.v8f32.v8i32(<8 x i32>, <8 x i32>, i16, <8 x float>, i1, i1)3019declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.fp8.v8f16.v8i32(<8 x i32>, <8 x i32>, i16, <8 x half>, i1, i1)3020declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.bf8.v8f16.v8i32(<8 x i32>, <8 x i32>, i16, <8 x half>, i1, i1)3021declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.fp8.v8f16.v8i32(<8 x i32>, <8 x i32>, i16, <8 x half>, i1, i1)3022declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.bf8.v8f16.v8i32(<8 x i32>, <8 x i32>, i16, <8 x half>, i1, i1)3023declare <8 x i32> @llvm.amdgcn.wmma.i32.16x16x64.iu8.v8i32.v8i32(i1 immarg, <8 x i32>, i1 immarg, <8 x i32>, <8 x i32>, i1, i1)3024declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1, <16 x half>, i1, <16 x half>, i16, <8 x float>, i1, i1)3025declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x32.f16.v8f16.v16f16(i1, <16 x half>, i1, <16 x half>, i16, <8 x half>, i1, i1)3026declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32, <16 x i32>, i32, <16 x i32>, i16, <8 x float>)3027declare <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32, <16 x i32>, i32, <16 x i32>, i16, <8 x float>, i32, i32, i32, i32, i32, i32, i1, i1)3028declare <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32, <16 x i32>, i32, <16 x i32>, i16, <8 x float>, i32, i32, i64, i32, i32, i64, i1, i1)3029declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.fp8.v8f16.v16i32(<16 x i32>, <16 x i32>, i16, <8 x half>, i1, i1)3030declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.bf8.v8f16.v16i32(<16 x i32>, <16 x i32>, i16, <8 x half>, i1, i1)3031declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.fp8.v8f16.v16i32(<16 x i32>, <16 x i32>, i16, <8 x half>, i1, i1)3032declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.bf8.v8f16.v16i32(<16 x i32>, <16 x i32>, i16, <8 x half>, i1, i1)3033declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.fp8.v8f32.v16i32(<16 x i32>, <16 x i32>, i16, <8 x float>, i1, i1)3034declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.bf8.v8f32.v16i32(<16 x i32>, <16 x i32>, i16, <8 x float>, i1, i1)3035declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.fp8.v8f32.v16i32(<16 x i32>, <16 x i32>, i16, <8 x float>, i1, i1)3036declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.bf8.v8f32.v16i32(<16 x i32>, <16 x i32>, i16, <8 x float>, i1, i1)3037declare <16 x float> @llvm.amdgcn.wmma.f32.32x16x128.f4.v16i32.v8i32.v16f32(<16 x i32>, <8 x i32>, i16, <16 x float>)3038declare <16 x float> @llvm.amdgcn.wmma.scale.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32>, <8 x i32>, i16, <16 x float>, i32, i32, i32, i32, i32, i32, i1, i1)3039declare <16 x float> @llvm.amdgcn.wmma.scale16.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32>, <8 x i32>, i16, <16 x float>, i32, i32, i64, i32, i32, i64, i1, i1)3040