2573 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 42; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s --check-prefix=GFX12503; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -global-isel -global-isel-abort=2 < %s | FileCheck %s --check-prefix=GISEL4 5define amdgpu_ps void @test_wmma_f32_16x16x4_f32_negA(<2 x float> %A, <2 x float> %B, <8 x float> %C, ptr addrspace(1) %out) {6; GFX1250-LABEL: test_wmma_f32_16x16x4_f32_negA:7; GFX1250: ; %bb.0: ; %bb8; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19; GFX1250-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] neg_lo:[1,0,0]10; GFX1250-NEXT: s_clause 0x111; GFX1250-NEXT: global_store_b128 v[12:13], v[8:11], off offset:1612; GFX1250-NEXT: global_store_b128 v[12:13], v[4:7], off13; GFX1250-NEXT: s_endpgm14;15; GISEL-LABEL: test_wmma_f32_16x16x4_f32_negA:16; GISEL: ; %bb.0: ; %bb17; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 118; GISEL-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] neg_lo:[1,0,0]19; GISEL-NEXT: s_clause 0x120; GISEL-NEXT: global_store_b128 v[12:13], v[4:7], off21; GISEL-NEXT: global_store_b128 v[12:13], v[8:11], off offset:1622; GISEL-NEXT: s_endpgm23bb:24 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x4.f32.v8f32.v2f32(i1 1, <2 x float> %A, i1 0, <2 x float> %B, i16 0, <8 x float> %C, i1 false, i1 false)25 store <8 x float> %res, ptr addrspace(1) %out26 ret void27}28 29define amdgpu_ps void @test_wmma_f32_16x16x4_f32_negB(<2 x float> %A, <2 x float> %B, <8 x float> %C, ptr addrspace(1) %out) {30; GFX1250-LABEL: test_wmma_f32_16x16x4_f32_negB:31; GFX1250: ; %bb.0: ; %bb32; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 133; GFX1250-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] neg_lo:[0,1,0]34; GFX1250-NEXT: s_clause 0x135; GFX1250-NEXT: global_store_b128 v[12:13], v[8:11], off offset:1636; GFX1250-NEXT: global_store_b128 v[12:13], v[4:7], off37; GFX1250-NEXT: s_endpgm38;39; GISEL-LABEL: test_wmma_f32_16x16x4_f32_negB:40; GISEL: ; %bb.0: ; %bb41; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 142; GISEL-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] neg_lo:[0,1,0]43; GISEL-NEXT: s_clause 0x144; GISEL-NEXT: global_store_b128 v[12:13], v[4:7], off45; GISEL-NEXT: global_store_b128 v[12:13], v[8:11], off offset:1646; GISEL-NEXT: s_endpgm47bb:48 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x4.f32.v8f32.v2f32(i1 0, <2 x float> %A, i1 1, <2 x float> %B, i16 0, <8 x float> %C, i1 false, i1 false)49 store <8 x float> %res, ptr addrspace(1) %out50 ret void51}52 53define amdgpu_ps void @test_wmma_f32_16x16x4_f32_negC(<2 x float> %A, <2 x float> %B, <8 x float> %C, ptr addrspace(1) %out) {54; GFX1250-LABEL: test_wmma_f32_16x16x4_f32_negC:55; GFX1250: ; %bb.0: ; %bb56; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 157; GFX1250-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] neg_lo:[0,0,1]58; GFX1250-NEXT: s_clause 0x159; GFX1250-NEXT: global_store_b128 v[12:13], v[8:11], off offset:1660; GFX1250-NEXT: global_store_b128 v[12:13], v[4:7], off61; GFX1250-NEXT: s_endpgm62;63; GISEL-LABEL: test_wmma_f32_16x16x4_f32_negC:64; GISEL: ; %bb.0: ; %bb65; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 166; GISEL-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] neg_lo:[0,0,1]67; GISEL-NEXT: s_clause 0x168; GISEL-NEXT: global_store_b128 v[12:13], v[4:7], off69; GISEL-NEXT: global_store_b128 v[12:13], v[8:11], off offset:1670; GISEL-NEXT: s_endpgm71bb:72 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x4.f32.v8f32.v2f32(i1 0, <2 x float> %A, i1 0, <2 x float> %B, i16 1, <8 x float> %C, i1 false, i1 false)73 store <8 x float> %res, ptr addrspace(1) %out74 ret void75}76 77define amdgpu_ps void @test_wmma_f32_16x16x4_f32_neg_absC(<2 x float> %A, <2 x float> %B, <8 x float> %C, ptr addrspace(1) %out) {78; GFX1250-LABEL: test_wmma_f32_16x16x4_f32_neg_absC:79; GFX1250: ; %bb.0: ; %bb80; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 181; GFX1250-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] neg_lo:[0,0,1] neg_hi:[0,0,1]82; GFX1250-NEXT: s_clause 0x183; GFX1250-NEXT: global_store_b128 v[12:13], v[8:11], off offset:1684; GFX1250-NEXT: global_store_b128 v[12:13], v[4:7], off85; GFX1250-NEXT: s_endpgm86;87; GISEL-LABEL: test_wmma_f32_16x16x4_f32_neg_absC:88; GISEL: ; %bb.0: ; %bb89; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 190; GISEL-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] neg_lo:[0,0,1] neg_hi:[0,0,1]91; GISEL-NEXT: s_clause 0x192; GISEL-NEXT: global_store_b128 v[12:13], v[4:7], off93; GISEL-NEXT: global_store_b128 v[12:13], v[8:11], off offset:1694; GISEL-NEXT: s_endpgm95bb:96 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x4.f32.v8f32.v2f32(i1 0, <2 x float> %A, i1 0, <2 x float> %B, i16 3, <8 x float> %C, i1 false, i1 false)97 store <8 x float> %res, ptr addrspace(1) %out98 ret void99}100 101define amdgpu_ps void @test_wmma_f32_16x16x4_f32_ignoreC(<2 x float> %A, <2 x float> %B, <8 x float> %C, ptr addrspace(1) %out) {102; GFX1250-LABEL: test_wmma_f32_16x16x4_f32_ignoreC:103; GFX1250: ; %bb.0: ; %bb104; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1105; GFX1250-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11]106; GFX1250-NEXT: s_clause 0x1107; GFX1250-NEXT: global_store_b128 v[12:13], v[8:11], off offset:16108; GFX1250-NEXT: global_store_b128 v[12:13], v[4:7], off109; GFX1250-NEXT: s_endpgm110;111; GISEL-LABEL: test_wmma_f32_16x16x4_f32_ignoreC:112; GISEL: ; %bb.0: ; %bb113; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1114; GISEL-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11]115; GISEL-NEXT: s_clause 0x1116; GISEL-NEXT: global_store_b128 v[12:13], v[4:7], off117; GISEL-NEXT: global_store_b128 v[12:13], v[8:11], off offset:16118; GISEL-NEXT: s_endpgm119bb:120 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x4.f32.v8f32.v2f32(i1 0, <2 x float> %A, i1 0, <2 x float> %B, i16 4, <8 x float> %C, i1 false, i1 false)121 store <8 x float> %res, ptr addrspace(1) %out122 ret void123}124 125define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_negA(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {126; GFX1250-LABEL: test_wmma_f32_16x16x32_bf16_negA:127; GFX1250: ; %bb.0: ; %bb128; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1129; GFX1250-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[1,0,0] neg_hi:[1,0,0]130; GFX1250-NEXT: s_clause 0x1131; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16132; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off133; GFX1250-NEXT: s_endpgm134;135; GISEL-LABEL: test_wmma_f32_16x16x32_bf16_negA:136; GISEL: ; %bb.0: ; %bb137; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1138; GISEL-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[1,0,0] neg_hi:[1,0,0]139; GISEL-NEXT: s_clause 0x1140; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16141; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off142; GISEL-NEXT: s_endpgm143bb:144 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.bf16.v8f32.v16bf16(i1 1, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 0, <8 x float> %C, i1 false, i1 false)145 store <8 x float> %res, ptr addrspace(1) %out146 ret void147}148 149define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_negB(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {150; GFX1250-LABEL: test_wmma_f32_16x16x32_bf16_negB:151; GFX1250: ; %bb.0: ; %bb152; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1153; GFX1250-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,1,0] neg_hi:[0,1,0]154; GFX1250-NEXT: s_clause 0x1155; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16156; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off157; GFX1250-NEXT: s_endpgm158;159; GISEL-LABEL: test_wmma_f32_16x16x32_bf16_negB:160; GISEL: ; %bb.0: ; %bb161; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1162; GISEL-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,1,0] neg_hi:[0,1,0]163; GISEL-NEXT: s_clause 0x1164; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16165; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off166; GISEL-NEXT: s_endpgm167bb:168 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.bf16.v8f32.v16bf16(i1 0, <16 x bfloat> %A, i1 1, <16 x bfloat> %B, i16 0, <8 x float> %C, i1 false, i1 false)169 store <8 x float> %res, ptr addrspace(1) %out170 ret void171}172 173define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_negC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {174; GFX1250-LABEL: test_wmma_f32_16x16x32_bf16_negC:175; GFX1250: ; %bb.0: ; %bb176; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1177; GFX1250-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]178; GFX1250-NEXT: s_clause 0x1179; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16180; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off181; GFX1250-NEXT: s_endpgm182;183; GISEL-LABEL: test_wmma_f32_16x16x32_bf16_negC:184; GISEL: ; %bb.0: ; %bb185; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1186; GISEL-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]187; GISEL-NEXT: s_clause 0x1188; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16189; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off190; GISEL-NEXT: s_endpgm191bb:192 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.bf16.v8f32.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 1, <8 x float> %C, i1 false, i1 false)193 store <8 x float> %res, ptr addrspace(1) %out194 ret void195}196 197define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_neg_absC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {198; GFX1250-LABEL: test_wmma_f32_16x16x32_bf16_neg_absC:199; GFX1250: ; %bb.0: ; %bb200; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1201; GFX1250-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]202; GFX1250-NEXT: s_clause 0x1203; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16204; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off205; GFX1250-NEXT: s_endpgm206;207; GISEL-LABEL: test_wmma_f32_16x16x32_bf16_neg_absC:208; GISEL: ; %bb.0: ; %bb209; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1210; GISEL-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]211; GISEL-NEXT: s_clause 0x1212; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16213; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off214; GISEL-NEXT: s_endpgm215bb:216 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.bf16.v8f32.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 3, <8 x float> %C, i1 false, i1 false)217 store <8 x float> %res, ptr addrspace(1) %out218 ret void219}220 221define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_ignoreC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {222; GFX1250-LABEL: test_wmma_f32_16x16x32_bf16_ignoreC:223; GFX1250: ; %bb.0: ; %bb224; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1225; GFX1250-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23]226; GFX1250-NEXT: s_clause 0x1227; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16228; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off229; GFX1250-NEXT: s_endpgm230;231; GISEL-LABEL: test_wmma_f32_16x16x32_bf16_ignoreC:232; GISEL: ; %bb.0: ; %bb233; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1234; GISEL-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23]235; GISEL-NEXT: s_clause 0x1236; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16237; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off238; GISEL-NEXT: s_endpgm239bb:240 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.bf16.v8f32.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 4, <8 x float> %C, i1 false, i1 false)241 store <8 x float> %res, ptr addrspace(1) %out242 ret void243}244 245define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_negA(<16 x bfloat> %A, <16 x bfloat> %B, <8 x bfloat> %C, ptr addrspace(1) %out) {246; GFX1250-LABEL: test_wmma_bf16_16x16x32_bf16_negA:247; GFX1250: ; %bb.0: ; %bb248; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1249; GFX1250-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[1,0,0] neg_hi:[1,0,0]250; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off251; GFX1250-NEXT: s_endpgm252;253; GISEL-LABEL: test_wmma_bf16_16x16x32_bf16_negA:254; GISEL: ; %bb.0: ; %bb255; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1256; GISEL-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[1,0,0] neg_hi:[1,0,0]257; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off258; GISEL-NEXT: s_endpgm259bb:260 %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16.16x16x32.bf16.v8bf16.v16bf16(i1 1, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 0, <8 x bfloat> %C, i1 false, i1 false)261 store <8 x bfloat> %res, ptr addrspace(1) %out262 ret void263}264 265define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_negB(<16 x bfloat> %A, <16 x bfloat> %B, <8 x bfloat> %C, ptr addrspace(1) %out) {266; GFX1250-LABEL: test_wmma_bf16_16x16x32_bf16_negB:267; GFX1250: ; %bb.0: ; %bb268; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1269; GFX1250-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,1,0] neg_hi:[0,1,0]270; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off271; GFX1250-NEXT: s_endpgm272;273; GISEL-LABEL: test_wmma_bf16_16x16x32_bf16_negB:274; GISEL: ; %bb.0: ; %bb275; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1276; GISEL-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,1,0] neg_hi:[0,1,0]277; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off278; GISEL-NEXT: s_endpgm279bb:280 %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16.16x16x32.bf16.v8bf16.v16bf16(i1 0, <16 x bfloat> %A, i1 1, <16 x bfloat> %B, i16 0, <8 x bfloat> %C, i1 false, i1 false)281 store <8 x bfloat> %res, ptr addrspace(1) %out282 ret void283}284 285define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_negC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x bfloat> %C, ptr addrspace(1) %out) {286; GFX1250-LABEL: test_wmma_bf16_16x16x32_bf16_negC:287; GFX1250: ; %bb.0: ; %bb288; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1289; GFX1250-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]290; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off291; GFX1250-NEXT: s_endpgm292;293; GISEL-LABEL: test_wmma_bf16_16x16x32_bf16_negC:294; GISEL: ; %bb.0: ; %bb295; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1296; GISEL-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]297; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off298; GISEL-NEXT: s_endpgm299bb:300 %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16.16x16x32.bf16.v8bf16.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 1, <8 x bfloat> %C, i1 false, i1 false)301 store <8 x bfloat> %res, ptr addrspace(1) %out302 ret void303}304 305define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_neg_absC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x bfloat> %C, ptr addrspace(1) %out) {306; GFX1250-LABEL: test_wmma_bf16_16x16x32_bf16_neg_absC:307; GFX1250: ; %bb.0: ; %bb308; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1309; GFX1250-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]310; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off311; GFX1250-NEXT: s_endpgm312;313; GISEL-LABEL: test_wmma_bf16_16x16x32_bf16_neg_absC:314; GISEL: ; %bb.0: ; %bb315; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1316; GISEL-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]317; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off318; GISEL-NEXT: s_endpgm319bb:320 %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16.16x16x32.bf16.v8bf16.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 3, <8 x bfloat> %C, i1 false, i1 false)321 store <8 x bfloat> %res, ptr addrspace(1) %out322 ret void323}324 325define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_ignoreC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x bfloat> %C, ptr addrspace(1) %out) {326; GFX1250-LABEL: test_wmma_bf16_16x16x32_bf16_ignoreC:327; GFX1250: ; %bb.0: ; %bb328; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1329; GFX1250-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19]330; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off331; GFX1250-NEXT: s_endpgm332;333; GISEL-LABEL: test_wmma_bf16_16x16x32_bf16_ignoreC:334; GISEL: ; %bb.0: ; %bb335; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1336; GISEL-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19]337; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off338; GISEL-NEXT: s_endpgm339bb:340 %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16.16x16x32.bf16.v8bf16.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 4, <8 x bfloat> %C, i1 false, i1 false)341 store <8 x bfloat> %res, ptr addrspace(1) %out342 ret void343}344 345define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_negA(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {346; GFX1250-LABEL: test_wmma_bf16f32_16x16x32_bf16_negA:347; GFX1250: ; %bb.0: ; %bb348; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1349; GFX1250-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] neg_lo:[1,0,0] neg_hi:[1,0,0]350; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off351; GFX1250-NEXT: s_endpgm352;353; GISEL-LABEL: test_wmma_bf16f32_16x16x32_bf16_negA:354; GISEL: ; %bb.0: ; %bb355; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1356; GISEL-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] neg_lo:[1,0,0] neg_hi:[1,0,0]357; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off358; GISEL-NEXT: s_endpgm359bb:360 %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16f32.16x16x32.bf16.v8bf16.v16bf16(i1 1, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 0, <8 x float> %C, i1 false, i1 false)361 store <8 x bfloat> %res, ptr addrspace(1) %out362 ret void363}364 365define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_negB(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {366; GFX1250-LABEL: test_wmma_bf16f32_16x16x32_bf16_negB:367; GFX1250: ; %bb.0: ; %bb368; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1369; GFX1250-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] neg_lo:[0,1,0] neg_hi:[0,1,0]370; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off371; GFX1250-NEXT: s_endpgm372;373; GISEL-LABEL: test_wmma_bf16f32_16x16x32_bf16_negB:374; GISEL: ; %bb.0: ; %bb375; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1376; GISEL-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] neg_lo:[0,1,0] neg_hi:[0,1,0]377; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off378; GISEL-NEXT: s_endpgm379bb:380 %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16f32.16x16x32.bf16.v8bf16.v16bf16(i1 0, <16 x bfloat> %A, i1 1, <16 x bfloat> %B, i16 0, <8 x float> %C, i1 false, i1 false)381 store <8 x bfloat> %res, ptr addrspace(1) %out382 ret void383}384 385define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_negC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {386; GFX1250-LABEL: test_wmma_bf16f32_16x16x32_bf16_negC:387; GFX1250: ; %bb.0: ; %bb388; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1389; GFX1250-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]390; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off391; GFX1250-NEXT: s_endpgm392;393; GISEL-LABEL: test_wmma_bf16f32_16x16x32_bf16_negC:394; GISEL: ; %bb.0: ; %bb395; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1396; GISEL-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]397; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off398; GISEL-NEXT: s_endpgm399bb:400 %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16f32.16x16x32.bf16.v8bf16.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 1, <8 x float> %C, i1 false, i1 false)401 store <8 x bfloat> %res, ptr addrspace(1) %out402 ret void403}404 405define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_neg_absC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {406; GFX1250-LABEL: test_wmma_bf16f32_16x16x32_bf16_neg_absC:407; GFX1250: ; %bb.0: ; %bb408; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1409; GFX1250-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]410; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off411; GFX1250-NEXT: s_endpgm412;413; GISEL-LABEL: test_wmma_bf16f32_16x16x32_bf16_neg_absC:414; GISEL: ; %bb.0: ; %bb415; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1416; GISEL-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]417; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off418; GISEL-NEXT: s_endpgm419bb:420 %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16f32.16x16x32.bf16.v8bf16.v16bf16(i1 0, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 3, <8 x float> %C, i1 false, i1 false)421 store <8 x bfloat> %res, ptr addrspace(1) %out422 ret void423}424 425define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_ignoreC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {426; GFX1250-LABEL: test_wmma_bf16f32_16x16x32_bf16_ignoreC:427; GFX1250: ; %bb.0: ; %bb428; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1429; GFX1250-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] neg_lo:[1,0,0] neg_hi:[1,0,0]430; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off431; GFX1250-NEXT: s_endpgm432;433; GISEL-LABEL: test_wmma_bf16f32_16x16x32_bf16_ignoreC:434; GISEL: ; %bb.0: ; %bb435; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1436; GISEL-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] neg_lo:[1,0,0] neg_hi:[1,0,0]437; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off438; GISEL-NEXT: s_endpgm439bb:440 %res = call <8 x bfloat> @llvm.amdgcn.wmma.bf16f32.16x16x32.bf16.v8bf16.v16bf16(i1 1, <16 x bfloat> %A, i1 0, <16 x bfloat> %B, i16 4, <8 x float> %C, i1 false, i1 false)441 store <8 x bfloat> %res, ptr addrspace(1) %out442 ret void443}444 445define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8_negC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {446; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_fp8_negC:447; GFX1250: ; %bb.0: ; %bb448; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1449; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]450; GFX1250-NEXT: s_clause 0x1451; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16452; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off453; GFX1250-NEXT: s_endpgm454;455; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_fp8_negC:456; GISEL: ; %bb.0: ; %bb457; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1458; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]459; GISEL-NEXT: s_clause 0x1460; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off461; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16462; GISEL-NEXT: s_endpgm463bb:464 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.fp8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 1, <8 x float> %C, i1 false, i1 false)465 store <8 x float> %res, ptr addrspace(1) %out466 ret void467}468 469define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8_neg_absC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {470; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_fp8_neg_absC:471; GFX1250: ; %bb.0: ; %bb472; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1473; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]474; GFX1250-NEXT: s_clause 0x1475; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16476; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off477; GFX1250-NEXT: s_endpgm478;479; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_fp8_neg_absC:480; GISEL: ; %bb.0: ; %bb481; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1482; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]483; GISEL-NEXT: s_clause 0x1484; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off485; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16486; GISEL-NEXT: s_endpgm487bb:488 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.fp8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 3, <8 x float> %C, i1 false, i1 false)489 store <8 x float> %res, ptr addrspace(1) %out490 ret void491}492 493define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8_ignoreC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {494; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_fp8_ignoreC:495; GFX1250: ; %bb.0: ; %bb496; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1497; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[16:23], v[0:7], v[8:15], v[16:23]498; GFX1250-NEXT: s_clause 0x1499; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16500; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off501; GFX1250-NEXT: s_endpgm502;503; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_fp8_ignoreC:504; GISEL: ; %bb.0: ; %bb505; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1506; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[16:23], v[0:7], v[8:15], v[16:23]507; GISEL-NEXT: s_clause 0x1508; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off509; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16510; GISEL-NEXT: s_endpgm511bb:512 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.fp8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 4, <8 x float> %C, i1 false, i1 false)513 store <8 x float> %res, ptr addrspace(1) %out514 ret void515}516 517define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8_negC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {518; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_bf8_negC:519; GFX1250: ; %bb.0: ; %bb520; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1521; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]522; GFX1250-NEXT: s_clause 0x1523; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16524; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off525; GFX1250-NEXT: s_endpgm526;527; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_bf8_negC:528; GISEL: ; %bb.0: ; %bb529; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1530; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]531; GISEL-NEXT: s_clause 0x1532; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off533; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16534; GISEL-NEXT: s_endpgm535bb:536 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.bf8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 1, <8 x float> %C, i1 false, i1 false)537 store <8 x float> %res, ptr addrspace(1) %out538 ret void539}540 541define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8_neg_absC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {542; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_bf8_neg_absC:543; GFX1250: ; %bb.0: ; %bb544; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1545; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]546; GFX1250-NEXT: s_clause 0x1547; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16548; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off549; GFX1250-NEXT: s_endpgm550;551; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_bf8_neg_absC:552; GISEL: ; %bb.0: ; %bb553; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1554; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]555; GISEL-NEXT: s_clause 0x1556; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off557; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16558; GISEL-NEXT: s_endpgm559bb:560 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.bf8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 3, <8 x float> %C, i1 false, i1 false)561 store <8 x float> %res, ptr addrspace(1) %out562 ret void563}564 565define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8_ignoreC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {566; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_bf8_ignoreC:567; GFX1250: ; %bb.0: ; %bb568; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1569; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[16:23], v[0:7], v[8:15], v[16:23]570; GFX1250-NEXT: s_clause 0x1571; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16572; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off573; GFX1250-NEXT: s_endpgm574;575; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_bf8_ignoreC:576; GISEL: ; %bb.0: ; %bb577; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1578; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[16:23], v[0:7], v[8:15], v[16:23]579; GISEL-NEXT: s_clause 0x1580; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off581; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16582; GISEL-NEXT: s_endpgm583bb:584 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.bf8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 4, <8 x float> %C, i1 false, i1 false)585 store <8 x float> %res, ptr addrspace(1) %out586 ret void587}588 589define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8_negC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {590; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_fp8_negC:591; GFX1250: ; %bb.0: ; %bb592; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1593; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]594; GFX1250-NEXT: s_clause 0x1595; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16596; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off597; GFX1250-NEXT: s_endpgm598;599; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_fp8_negC:600; GISEL: ; %bb.0: ; %bb601; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1602; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]603; GISEL-NEXT: s_clause 0x1604; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off605; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16606; GISEL-NEXT: s_endpgm607bb:608 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.fp8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 1, <8 x float> %C, i1 false, i1 false)609 store <8 x float> %res, ptr addrspace(1) %out610 ret void611}612 613define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8_neg_absC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {614; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_fp8_neg_absC:615; GFX1250: ; %bb.0: ; %bb616; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1617; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]618; GFX1250-NEXT: s_clause 0x1619; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16620; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off621; GFX1250-NEXT: s_endpgm622;623; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_fp8_neg_absC:624; GISEL: ; %bb.0: ; %bb625; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1626; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]627; GISEL-NEXT: s_clause 0x1628; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off629; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16630; GISEL-NEXT: s_endpgm631bb:632 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.fp8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 3, <8 x float> %C, i1 false, i1 false)633 store <8 x float> %res, ptr addrspace(1) %out634 ret void635}636 637define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8_ignoreC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {638; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_fp8_ignoreC:639; GFX1250: ; %bb.0: ; %bb640; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1641; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[16:23], v[0:7], v[8:15], v[16:23]642; GFX1250-NEXT: s_clause 0x1643; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16644; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off645; GFX1250-NEXT: s_endpgm646;647; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_fp8_ignoreC:648; GISEL: ; %bb.0: ; %bb649; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1650; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[16:23], v[0:7], v[8:15], v[16:23]651; GISEL-NEXT: s_clause 0x1652; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off653; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16654; GISEL-NEXT: s_endpgm655bb:656 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.fp8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 4, <8 x float> %C, i1 false, i1 false)657 store <8 x float> %res, ptr addrspace(1) %out658 ret void659}660 661define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8_negC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {662; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_bf8_negC:663; GFX1250: ; %bb.0: ; %bb664; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1665; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]666; GFX1250-NEXT: s_clause 0x1667; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16668; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off669; GFX1250-NEXT: s_endpgm670;671; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_bf8_negC:672; GISEL: ; %bb.0: ; %bb673; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1674; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]675; GISEL-NEXT: s_clause 0x1676; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off677; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16678; GISEL-NEXT: s_endpgm679bb:680 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.bf8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 1, <8 x float> %C, i1 false, i1 false)681 store <8 x float> %res, ptr addrspace(1) %out682 ret void683}684 685define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8_neg_absC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {686; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_bf8_neg_absC:687; GFX1250: ; %bb.0: ; %bb688; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1689; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]690; GFX1250-NEXT: s_clause 0x1691; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16692; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off693; GFX1250-NEXT: s_endpgm694;695; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_bf8_neg_absC:696; GISEL: ; %bb.0: ; %bb697; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1698; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]699; GISEL-NEXT: s_clause 0x1700; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off701; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16702; GISEL-NEXT: s_endpgm703bb:704 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.bf8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 3, <8 x float> %C, i1 false, i1 false)705 store <8 x float> %res, ptr addrspace(1) %out706 ret void707}708 709define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8_ignoreC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {710; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_bf8_ignoreC:711; GFX1250: ; %bb.0: ; %bb712; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1713; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[16:23], v[0:7], v[8:15], v[16:23]714; GFX1250-NEXT: s_clause 0x1715; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16716; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off717; GFX1250-NEXT: s_endpgm718;719; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_bf8_ignoreC:720; GISEL: ; %bb.0: ; %bb721; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1722; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[16:23], v[0:7], v[8:15], v[16:23]723; GISEL-NEXT: s_clause 0x1724; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off725; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16726; GISEL-NEXT: s_endpgm727bb:728 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.bf8.v8f32.v8i32(<8 x i32> %A, <8 x i32> %B, i16 4, <8 x float> %C, i1 false, i1 false)729 store <8 x float> %res, ptr addrspace(1) %out730 ret void731}732 733define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8_negC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {734; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_fp8_negC:735; GFX1250: ; %bb.0: ; %bb736; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1737; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]738; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off739; GFX1250-NEXT: s_endpgm740;741; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_fp8_negC:742; GISEL: ; %bb.0: ; %bb743; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1744; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]745; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off746; GISEL-NEXT: s_endpgm747bb:748 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.fp8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 1, <8 x half> %C, i1 false, i1 false)749 store <8 x half> %res, ptr addrspace(1) %out750 ret void751}752 753define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8_neg_absC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {754; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_fp8_neg_absC:755; GFX1250: ; %bb.0: ; %bb756; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1757; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]758; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off759; GFX1250-NEXT: s_endpgm760;761; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_fp8_neg_absC:762; GISEL: ; %bb.0: ; %bb763; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1764; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]765; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off766; GISEL-NEXT: s_endpgm767bb:768 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.fp8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 3, <8 x half> %C, i1 false, i1 false)769 store <8 x half> %res, ptr addrspace(1) %out770 ret void771}772 773define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8_ignoreC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {774; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_fp8_ignoreC:775; GFX1250: ; %bb.0: ; %bb776; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1777; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[16:19], v[0:7], v[8:15], v[16:19]778; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off779; GFX1250-NEXT: s_endpgm780;781; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_fp8_ignoreC:782; GISEL: ; %bb.0: ; %bb783; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1784; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[16:19], v[0:7], v[8:15], v[16:19]785; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off786; GISEL-NEXT: s_endpgm787bb:788 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.fp8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 4, <8 x half> %C, i1 false, i1 false)789 store <8 x half> %res, ptr addrspace(1) %out790 ret void791}792 793define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8_negC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {794; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_bf8_negC:795; GFX1250: ; %bb.0: ; %bb796; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1797; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]798; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off799; GFX1250-NEXT: s_endpgm800;801; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_bf8_negC:802; GISEL: ; %bb.0: ; %bb803; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1804; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]805; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off806; GISEL-NEXT: s_endpgm807bb:808 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.bf8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 1, <8 x half> %C, i1 false, i1 false)809 store <8 x half> %res, ptr addrspace(1) %out810 ret void811}812 813define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8_neg_absC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {814; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_bf8_neg_absC:815; GFX1250: ; %bb.0: ; %bb816; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1817; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]818; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off819; GFX1250-NEXT: s_endpgm820;821; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_bf8_neg_absC:822; GISEL: ; %bb.0: ; %bb823; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1824; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]825; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off826; GISEL-NEXT: s_endpgm827bb:828 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.bf8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 3, <8 x half> %C, i1 false, i1 false)829 store <8 x half> %res, ptr addrspace(1) %out830 ret void831}832 833define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8_ignoreC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {834; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_bf8_ignoreC:835; GFX1250: ; %bb.0: ; %bb836; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1837; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[16:19], v[0:7], v[8:15], v[16:19]838; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off839; GFX1250-NEXT: s_endpgm840;841; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_bf8_ignoreC:842; GISEL: ; %bb.0: ; %bb843; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1844; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[16:19], v[0:7], v[8:15], v[16:19]845; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off846; GISEL-NEXT: s_endpgm847bb:848 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.bf8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 4, <8 x half> %C, i1 false, i1 false)849 store <8 x half> %res, ptr addrspace(1) %out850 ret void851}852 853define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8_negC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {854; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_fp8_negC:855; GFX1250: ; %bb.0: ; %bb856; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1857; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]858; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off859; GFX1250-NEXT: s_endpgm860;861; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_fp8_negC:862; GISEL: ; %bb.0: ; %bb863; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1864; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]865; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off866; GISEL-NEXT: s_endpgm867bb:868 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.fp8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 1, <8 x half> %C, i1 false, i1 false)869 store <8 x half> %res, ptr addrspace(1) %out870 ret void871}872 873define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8_neg_absC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {874; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_fp8_neg_absC:875; GFX1250: ; %bb.0: ; %bb876; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1877; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]878; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off879; GFX1250-NEXT: s_endpgm880;881; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_fp8_neg_absC:882; GISEL: ; %bb.0: ; %bb883; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1884; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]885; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off886; GISEL-NEXT: s_endpgm887bb:888 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.fp8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 3, <8 x half> %C, i1 false, i1 false)889 store <8 x half> %res, ptr addrspace(1) %out890 ret void891}892 893define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8_ignoreC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {894; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_fp8_ignoreC:895; GFX1250: ; %bb.0: ; %bb896; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1897; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[16:19], v[0:7], v[8:15], v[16:19]898; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off899; GFX1250-NEXT: s_endpgm900;901; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_fp8_ignoreC:902; GISEL: ; %bb.0: ; %bb903; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1904; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[16:19], v[0:7], v[8:15], v[16:19]905; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off906; GISEL-NEXT: s_endpgm907bb:908 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.fp8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 4, <8 x half> %C, i1 false, i1 false)909 store <8 x half> %res, ptr addrspace(1) %out910 ret void911}912 913define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8_negC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {914; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_bf8_negC:915; GFX1250: ; %bb.0: ; %bb916; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1917; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]918; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off919; GFX1250-NEXT: s_endpgm920;921; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_bf8_negC:922; GISEL: ; %bb.0: ; %bb923; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1924; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]925; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off926; GISEL-NEXT: s_endpgm927bb:928 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.bf8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 1, <8 x half> %C, i1 false, i1 false)929 store <8 x half> %res, ptr addrspace(1) %out930 ret void931}932 933define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8_neg_absC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {934; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_bf8_neg_absC:935; GFX1250: ; %bb.0: ; %bb936; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1937; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]938; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off939; GFX1250-NEXT: s_endpgm940;941; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_bf8_neg_absC:942; GISEL: ; %bb.0: ; %bb943; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1944; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]945; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off946; GISEL-NEXT: s_endpgm947bb:948 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.bf8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 3, <8 x half> %C, i1 false, i1 false)949 store <8 x half> %res, ptr addrspace(1) %out950 ret void951}952 953define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8_ignoreC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {954; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_bf8_ignoreC:955; GFX1250: ; %bb.0: ; %bb956; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1957; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[16:19], v[0:7], v[8:15], v[16:19]958; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off959; GFX1250-NEXT: s_endpgm960;961; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_bf8_ignoreC:962; GISEL: ; %bb.0: ; %bb963; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1964; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[16:19], v[0:7], v[8:15], v[16:19]965; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off966; GISEL-NEXT: s_endpgm967bb:968 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.bf8.v8f16.v8i32(<8 x i32> %A, <8 x i32> %B, i16 4, <8 x half> %C, i1 false, i1 false)969 store <8 x half> %res, ptr addrspace(1) %out970 ret void971}972 973define amdgpu_ps void @test_wmma_i32_16x16x64_iu8_signedA(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out) {974; GFX1250-LABEL: test_wmma_i32_16x16x64_iu8_signedA:975; GFX1250: ; %bb.0: ; %bb976; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1977; GFX1250-NEXT: v_wmma_i32_16x16x64_iu8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[1,0,0]978; GFX1250-NEXT: s_clause 0x1979; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16980; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off981; GFX1250-NEXT: s_endpgm982;983; GISEL-LABEL: test_wmma_i32_16x16x64_iu8_signedA:984; GISEL: ; %bb.0: ; %bb985; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1986; GISEL-NEXT: v_wmma_i32_16x16x64_iu8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[1,0,0]987; GISEL-NEXT: s_clause 0x1988; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off989; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:16990; GISEL-NEXT: s_endpgm991bb:992 %res = call <8 x i32> @llvm.amdgcn.wmma.i32.16x16x64.iu8.v8i32.v8i32(i1 1, <8 x i32> %A, i1 0, <8 x i32> %B, <8 x i32> %C, i1 false, i1 false)993 store <8 x i32> %res, ptr addrspace(1) %out994 ret void995}996 997define amdgpu_ps void @test_wmma_i32_16x16x64_iu8_signedB(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out) {998; GFX1250-LABEL: test_wmma_i32_16x16x64_iu8_signedB:999; GFX1250: ; %bb.0: ; %bb1000; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11001; GFX1250-NEXT: v_wmma_i32_16x16x64_iu8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,1,0]1002; GFX1250-NEXT: s_clause 0x11003; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:161004; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off1005; GFX1250-NEXT: s_endpgm1006;1007; GISEL-LABEL: test_wmma_i32_16x16x64_iu8_signedB:1008; GISEL: ; %bb.0: ; %bb1009; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11010; GISEL-NEXT: v_wmma_i32_16x16x64_iu8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,1,0]1011; GISEL-NEXT: s_clause 0x11012; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off1013; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:161014; GISEL-NEXT: s_endpgm1015bb:1016 %res = call <8 x i32> @llvm.amdgcn.wmma.i32.16x16x64.iu8.v8i32.v8i32(i1 0, <8 x i32> %A, i1 1, <8 x i32> %B, <8 x i32> %C, i1 false, i1 false)1017 store <8 x i32> %res, ptr addrspace(1) %out1018 ret void1019}1020 1021define amdgpu_ps void @test_wmma_f32_16x16x32_f16_negA(<16 x half> %A, <16 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {1022; GFX1250-LABEL: test_wmma_f32_16x16x32_f16_negA:1023; GFX1250: ; %bb.0: ; %bb1024; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11025; GFX1250-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[1,0,0] neg_hi:[1,0,0]1026; GFX1250-NEXT: s_clause 0x11027; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:161028; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off1029; GFX1250-NEXT: s_endpgm1030;1031; GISEL-LABEL: test_wmma_f32_16x16x32_f16_negA:1032; GISEL: ; %bb.0: ; %bb1033; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11034; GISEL-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[1,0,0] neg_hi:[1,0,0]1035; GISEL-NEXT: s_clause 0x11036; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off1037; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:161038; GISEL-NEXT: s_endpgm1039bb:1040 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 1, <16 x half> %A, i1 0, <16 x half> %B, i16 0, <8 x float> %C, i1 false, i1 false)1041 store <8 x float> %res, ptr addrspace(1) %out1042 ret void1043}1044 1045define amdgpu_ps void @test_wmma_f32_16x16x32_f16_negB(<16 x half> %A, <16 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {1046; GFX1250-LABEL: test_wmma_f32_16x16x32_f16_negB:1047; GFX1250: ; %bb.0: ; %bb1048; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11049; GFX1250-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,1,0] neg_hi:[0,1,0]1050; GFX1250-NEXT: s_clause 0x11051; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:161052; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off1053; GFX1250-NEXT: s_endpgm1054;1055; GISEL-LABEL: test_wmma_f32_16x16x32_f16_negB:1056; GISEL: ; %bb.0: ; %bb1057; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11058; GISEL-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,1,0] neg_hi:[0,1,0]1059; GISEL-NEXT: s_clause 0x11060; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off1061; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:161062; GISEL-NEXT: s_endpgm1063bb:1064 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 0, <16 x half> %A, i1 1, <16 x half> %B, i16 0, <8 x float> %C, i1 false, i1 false)1065 store <8 x float> %res, ptr addrspace(1) %out1066 ret void1067}1068 1069define amdgpu_ps void @test_wmma_f32_16x16x32_f16_negC(<16 x half> %A, <16 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {1070; GFX1250-LABEL: test_wmma_f32_16x16x32_f16_negC:1071; GFX1250: ; %bb.0: ; %bb1072; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11073; GFX1250-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]1074; GFX1250-NEXT: s_clause 0x11075; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:161076; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off1077; GFX1250-NEXT: s_endpgm1078;1079; GISEL-LABEL: test_wmma_f32_16x16x32_f16_negC:1080; GISEL: ; %bb.0: ; %bb1081; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11082; GISEL-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]1083; GISEL-NEXT: s_clause 0x11084; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off1085; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:161086; GISEL-NEXT: s_endpgm1087bb:1088 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 0, <16 x half> %A, i1 0, <16 x half> %B, i16 1, <8 x float> %C, i1 false, i1 false)1089 store <8 x float> %res, ptr addrspace(1) %out1090 ret void1091}1092 1093define amdgpu_ps void @test_wmma_f32_16x16x32_f16_neg_absC(<16 x half> %A, <16 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {1094; GFX1250-LABEL: test_wmma_f32_16x16x32_f16_neg_absC:1095; GFX1250: ; %bb.0: ; %bb1096; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11097; GFX1250-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]1098; GFX1250-NEXT: s_clause 0x11099; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:161100; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off1101; GFX1250-NEXT: s_endpgm1102;1103; GISEL-LABEL: test_wmma_f32_16x16x32_f16_neg_absC:1104; GISEL: ; %bb.0: ; %bb1105; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11106; GISEL-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]1107; GISEL-NEXT: s_clause 0x11108; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off1109; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:161110; GISEL-NEXT: s_endpgm1111bb:1112 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 0, <16 x half> %A, i1 0, <16 x half> %B, i16 3, <8 x float> %C, i1 false, i1 false)1113 store <8 x float> %res, ptr addrspace(1) %out1114 ret void1115}1116 1117define amdgpu_ps void @test_wmma_f32_16x16x32_f16_ignoreC(<16 x half> %A, <16 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {1118; GFX1250-LABEL: test_wmma_f32_16x16x32_f16_ignoreC:1119; GFX1250: ; %bb.0: ; %bb1120; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11121; GFX1250-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23]1122; GFX1250-NEXT: s_clause 0x11123; GFX1250-NEXT: global_store_b128 v[24:25], v[20:23], off offset:161124; GFX1250-NEXT: global_store_b128 v[24:25], v[16:19], off1125; GFX1250-NEXT: s_endpgm1126;1127; GISEL-LABEL: test_wmma_f32_16x16x32_f16_ignoreC:1128; GISEL: ; %bb.0: ; %bb1129; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11130; GISEL-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23]1131; GISEL-NEXT: s_clause 0x11132; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off1133; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:161134; GISEL-NEXT: s_endpgm1135bb:1136 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 0, <16 x half> %A, i1 0, <16 x half> %B, i16 4, <8 x float> %C, i1 false, i1 false)1137 store <8 x float> %res, ptr addrspace(1) %out1138 ret void1139}1140 1141define amdgpu_ps void @test_wmma_f16_16x16x32_f16_negA(<16 x half> %A, <16 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {1142; GFX1250-LABEL: test_wmma_f16_16x16x32_f16_negA:1143; GFX1250: ; %bb.0: ; %bb1144; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11145; GFX1250-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[1,0,0] neg_hi:[1,0,0]1146; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off1147; GFX1250-NEXT: s_endpgm1148;1149; GISEL-LABEL: test_wmma_f16_16x16x32_f16_negA:1150; GISEL: ; %bb.0: ; %bb1151; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11152; GISEL-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[1,0,0] neg_hi:[1,0,0]1153; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off1154; GISEL-NEXT: s_endpgm1155bb:1156 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x32.f16.v8f16.v16f16(i1 1, <16 x half> %A, i1 0, <16 x half> %B, i16 0, <8 x half> %C, i1 false, i1 false)1157 store <8 x half> %res, ptr addrspace(1) %out1158 ret void1159}1160 1161define amdgpu_ps void @test_wmma_f16_16x16x32_f16_negB(<16 x half> %A, <16 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {1162; GFX1250-LABEL: test_wmma_f16_16x16x32_f16_negB:1163; GFX1250: ; %bb.0: ; %bb1164; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11165; GFX1250-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,1,0] neg_hi:[0,1,0]1166; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off1167; GFX1250-NEXT: s_endpgm1168;1169; GISEL-LABEL: test_wmma_f16_16x16x32_f16_negB:1170; GISEL: ; %bb.0: ; %bb1171; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11172; GISEL-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,1,0] neg_hi:[0,1,0]1173; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off1174; GISEL-NEXT: s_endpgm1175bb:1176 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x32.f16.v8f16.v16f16(i1 0, <16 x half> %A, i1 1, <16 x half> %B, i16 0, <8 x half> %C, i1 false, i1 false)1177 store <8 x half> %res, ptr addrspace(1) %out1178 ret void1179}1180 1181define amdgpu_ps void @test_wmma_f16_16x16x32_f16_negC(<16 x half> %A, <16 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {1182; GFX1250-LABEL: test_wmma_f16_16x16x32_f16_negC:1183; GFX1250: ; %bb.0: ; %bb1184; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11185; GFX1250-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]1186; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off1187; GFX1250-NEXT: s_endpgm1188;1189; GISEL-LABEL: test_wmma_f16_16x16x32_f16_negC:1190; GISEL: ; %bb.0: ; %bb1191; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11192; GISEL-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]1193; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off1194; GISEL-NEXT: s_endpgm1195bb:1196 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x32.f16.v8f16.v16f16(i1 0, <16 x half> %A, i1 0, <16 x half> %B, i16 1, <8 x half> %C, i1 false, i1 false)1197 store <8 x half> %res, ptr addrspace(1) %out1198 ret void1199}1200 1201define amdgpu_ps void @test_wmma_f16_16x16x32_f16_neg_absC(<16 x half> %A, <16 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {1202; GFX1250-LABEL: test_wmma_f16_16x16x32_f16_neg_absC:1203; GFX1250: ; %bb.0: ; %bb1204; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11205; GFX1250-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]1206; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off1207; GFX1250-NEXT: s_endpgm1208;1209; GISEL-LABEL: test_wmma_f16_16x16x32_f16_neg_absC:1210; GISEL: ; %bb.0: ; %bb1211; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11212; GISEL-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]1213; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off1214; GISEL-NEXT: s_endpgm1215bb:1216 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x32.f16.v8f16.v16f16(i1 0, <16 x half> %A, i1 0, <16 x half> %B, i16 3, <8 x half> %C, i1 false, i1 false)1217 store <8 x half> %res, ptr addrspace(1) %out1218 ret void1219}1220 1221define amdgpu_ps void @test_wmma_f16_16x16x32_f16_ignoreC(<16 x half> %A, <16 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {1222; GFX1250-LABEL: test_wmma_f16_16x16x32_f16_ignoreC:1223; GFX1250: ; %bb.0: ; %bb1224; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11225; GFX1250-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19]1226; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off1227; GFX1250-NEXT: s_endpgm1228;1229; GISEL-LABEL: test_wmma_f16_16x16x32_f16_ignoreC:1230; GISEL: ; %bb.0: ; %bb1231; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11232; GISEL-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19]1233; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off1234; GISEL-NEXT: s_endpgm1235bb:1236 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x32.f16.v8f16.v16f16(i1 0, <16 x half> %A, i1 0, <16 x half> %B, i16 4, <8 x half> %C, i1 false, i1 false)1237 store <8 x half> %res, ptr addrspace(1) %out1238 ret void1239}1240 1241define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_negC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1242; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_negC:1243; GFX1250: ; %bb.0: ; %bb1244; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11245; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]1246; GFX1250-NEXT: s_clause 0x11247; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161248; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1249; GFX1250-NEXT: s_endpgm1250;1251; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_negC:1252; GISEL: ; %bb.0: ; %bb1253; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11254; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]1255; GISEL-NEXT: s_clause 0x11256; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1257; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161258; GISEL-NEXT: s_endpgm1259bb:1260 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 1, <8 x float> %C)1261 store <8 x float> %res, ptr addrspace(1) %out1262 ret void1263}1264 1265define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1266; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_neg_absC:1267; GFX1250: ; %bb.0: ; %bb1268; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11269; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]1270; GFX1250-NEXT: s_clause 0x11271; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161272; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1273; GFX1250-NEXT: s_endpgm1274;1275; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_neg_absC:1276; GISEL: ; %bb.0: ; %bb1277; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11278; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]1279; GISEL-NEXT: s_clause 0x11280; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1281; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161282; GISEL-NEXT: s_endpgm1283bb:1284 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 3, <8 x float> %C)1285 store <8 x float> %res, ptr addrspace(1) %out1286 ret void1287}1288 1289define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1290; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_ignoreC:1291; GFX1250: ; %bb.0: ; %bb1292; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11293; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39]1294; GFX1250-NEXT: s_clause 0x11295; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161296; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1297; GFX1250-NEXT: s_endpgm1298;1299; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_ignoreC:1300; GISEL: ; %bb.0: ; %bb1301; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11302; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39]1303; GISEL-NEXT: s_clause 0x11304; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1305; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161306; GISEL-NEXT: s_endpgm1307bb:1308 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 4, <8 x float> %C)1309 store <8 x float> %res, ptr addrspace(1) %out1310 ret void1311}1312 1313define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_negC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1314; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_negC:1315; GFX1250: ; %bb.0: ; %bb1316; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11317; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1]1318; GFX1250-NEXT: s_clause 0x11319; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161320; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1321; GFX1250-NEXT: s_endpgm1322;1323; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_negC:1324; GISEL: ; %bb.0: ; %bb1325; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11326; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1]1327; GISEL-NEXT: s_clause 0x11328; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1329; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161330; GISEL-NEXT: s_endpgm1331bb:1332 %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 1, <8 x float> %C, i32 1, i32 0, i32 2, i32 1, i32 0, i32 4, i1 false, i1 false)1333 store <8 x float> %res, ptr addrspace(1) %out1334 ret void1335}1336 1337define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1338; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_neg_absC:1339; GFX1250: ; %bb.0: ; %bb1340; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11341; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1] neg_hi:[0,0,1]1342; GFX1250-NEXT: s_clause 0x11343; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161344; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1345; GFX1250-NEXT: s_endpgm1346;1347; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_neg_absC:1348; GISEL: ; %bb.0: ; %bb1349; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11350; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1] neg_hi:[0,0,1]1351; GISEL-NEXT: s_clause 0x11352; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1353; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161354; GISEL-NEXT: s_endpgm1355bb:1356 %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 3, <8 x float> %C, i32 1, i32 0, i32 2, i32 1, i32 0, i32 4, i1 false, i1 false)1357 store <8 x float> %res, ptr addrspace(1) %out1358 ret void1359}1360 1361define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1362; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_ignoreC:1363; GFX1250: ; %bb.0: ; %bb1364; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11365; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW11366; GFX1250-NEXT: s_clause 0x11367; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161368; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1369; GFX1250-NEXT: s_endpgm1370;1371; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_ignoreC:1372; GISEL: ; %bb.0: ; %bb1373; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11374; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW11375; GISEL-NEXT: s_clause 0x11376; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1377; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161378; GISEL-NEXT: s_endpgm1379bb:1380 %res = call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 4, <8 x float> %C, i32 1, i32 0, i32 2, i32 1, i32 0, i32 4, i1 false, i1 false)1381 store <8 x float> %res, ptr addrspace(1) %out1382 ret void1383}1384 1385define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_negC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1386; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_negC:1387; GFX1250: ; %bb.0: ; %bb1388; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11389; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1]1390; GFX1250-NEXT: s_clause 0x11391; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161392; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1393; GFX1250-NEXT: s_endpgm1394;1395; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_negC:1396; GISEL: ; %bb.0: ; %bb1397; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11398; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1]1399; GISEL-NEXT: s_clause 0x11400; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1401; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161402; GISEL-NEXT: s_endpgm1403bb:1404 %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 1, <8 x float> %C, i32 1, i32 0, i64 2, i32 1, i32 0, i64 4, i1 false, i1 false)1405 store <8 x float> %res, ptr addrspace(1) %out1406 ret void1407}1408 1409define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1410; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_neg_absC:1411; GFX1250: ; %bb.0: ; %bb1412; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11413; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1] neg_hi:[0,0,1]1414; GFX1250-NEXT: s_clause 0x11415; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161416; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1417; GFX1250-NEXT: s_endpgm1418;1419; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_neg_absC:1420; GISEL: ; %bb.0: ; %bb1421; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11422; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1] neg_hi:[0,0,1]1423; GISEL-NEXT: s_clause 0x11424; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1425; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161426; GISEL-NEXT: s_endpgm1427bb:1428 %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 3, <8 x float> %C, i32 1, i32 0, i64 2, i32 1, i32 0, i64 4, i1 false, i1 false)1429 store <8 x float> %res, ptr addrspace(1) %out1430 ret void1431}1432 1433define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1434; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_ignoreC:1435; GFX1250: ; %bb.0: ; %bb1436; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11437; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW11438; GFX1250-NEXT: s_clause 0x11439; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161440; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1441; GFX1250-NEXT: s_endpgm1442;1443; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_ignoreC:1444; GISEL: ; %bb.0: ; %bb1445; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11446; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW11447; GISEL-NEXT: s_clause 0x11448; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1449; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161450; GISEL-NEXT: s_endpgm1451bb:1452 %res = call <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 4, <8 x float> %C, i32 1, i32 0, i64 2, i32 1, i32 0, i64 4, i1 false, i1 false)1453 store <8 x float> %res, ptr addrspace(1) %out1454 ret void1455}1456 1457define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8_negC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {1458; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_fp8_negC:1459; GFX1250: ; %bb.0: ; %bb1460; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11461; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1]1462; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off1463; GFX1250-NEXT: s_endpgm1464;1465; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_fp8_negC:1466; GISEL: ; %bb.0: ; %bb1467; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11468; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1]1469; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off1470; GISEL-NEXT: s_endpgm1471bb:1472 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.fp8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 1, <8 x half> %C, i1 false, i1 false)1473 store <8 x half> %res, ptr addrspace(1) %out1474 ret void1475}1476 1477define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {1478; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_fp8_neg_absC:1479; GFX1250: ; %bb.0: ; %bb1480; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11481; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1] neg_hi:[0,0,1]1482; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off1483; GFX1250-NEXT: s_endpgm1484;1485; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_fp8_neg_absC:1486; GISEL: ; %bb.0: ; %bb1487; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11488; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1] neg_hi:[0,0,1]1489; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off1490; GISEL-NEXT: s_endpgm1491bb:1492 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.fp8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 3, <8 x half> %C, i1 false, i1 false)1493 store <8 x half> %res, ptr addrspace(1) %out1494 ret void1495}1496 1497define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {1498; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_fp8_ignoreC:1499; GFX1250: ; %bb.0: ; %bb1500; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11501; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[32:35], v[0:15], v[16:31], v[32:35]1502; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off1503; GFX1250-NEXT: s_endpgm1504;1505; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_fp8_ignoreC:1506; GISEL: ; %bb.0: ; %bb1507; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11508; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[32:35], v[0:15], v[16:31], v[32:35]1509; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off1510; GISEL-NEXT: s_endpgm1511bb:1512 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.fp8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 4, <8 x half> %C, i1 false, i1 false)1513 store <8 x half> %res, ptr addrspace(1) %out1514 ret void1515}1516 1517define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8_negC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {1518; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_bf8_negC:1519; GFX1250: ; %bb.0: ; %bb1520; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11521; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1]1522; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off1523; GFX1250-NEXT: s_endpgm1524;1525; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_bf8_negC:1526; GISEL: ; %bb.0: ; %bb1527; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11528; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1]1529; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off1530; GISEL-NEXT: s_endpgm1531bb:1532 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.bf8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 1, <8 x half> %C, i1 false, i1 false)1533 store <8 x half> %res, ptr addrspace(1) %out1534 ret void1535}1536 1537define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {1538; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_bf8_neg_absC:1539; GFX1250: ; %bb.0: ; %bb1540; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11541; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1] neg_hi:[0,0,1]1542; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off1543; GFX1250-NEXT: s_endpgm1544;1545; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_bf8_neg_absC:1546; GISEL: ; %bb.0: ; %bb1547; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11548; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1] neg_hi:[0,0,1]1549; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off1550; GISEL-NEXT: s_endpgm1551bb:1552 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.bf8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 3, <8 x half> %C, i1 false, i1 false)1553 store <8 x half> %res, ptr addrspace(1) %out1554 ret void1555}1556 1557define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {1558; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_bf8_ignoreC:1559; GFX1250: ; %bb.0: ; %bb1560; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11561; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[32:35], v[0:15], v[16:31], v[32:35]1562; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off1563; GFX1250-NEXT: s_endpgm1564;1565; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_bf8_ignoreC:1566; GISEL: ; %bb.0: ; %bb1567; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11568; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[32:35], v[0:15], v[16:31], v[32:35]1569; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off1570; GISEL-NEXT: s_endpgm1571bb:1572 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.bf8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 4, <8 x half> %C, i1 false, i1 false)1573 store <8 x half> %res, ptr addrspace(1) %out1574 ret void1575}1576 1577define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8_negC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {1578; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_fp8_negC:1579; GFX1250: ; %bb.0: ; %bb1580; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11581; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1]1582; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off1583; GFX1250-NEXT: s_endpgm1584;1585; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_fp8_negC:1586; GISEL: ; %bb.0: ; %bb1587; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11588; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1]1589; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off1590; GISEL-NEXT: s_endpgm1591bb:1592 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.fp8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 1, <8 x half> %C, i1 false, i1 false)1593 store <8 x half> %res, ptr addrspace(1) %out1594 ret void1595}1596 1597define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {1598; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_fp8_neg_absC:1599; GFX1250: ; %bb.0: ; %bb1600; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11601; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1] neg_hi:[0,0,1]1602; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off1603; GFX1250-NEXT: s_endpgm1604;1605; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_fp8_neg_absC:1606; GISEL: ; %bb.0: ; %bb1607; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11608; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1] neg_hi:[0,0,1]1609; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off1610; GISEL-NEXT: s_endpgm1611bb:1612 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.fp8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 3, <8 x half> %C, i1 false, i1 false)1613 store <8 x half> %res, ptr addrspace(1) %out1614 ret void1615}1616 1617define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {1618; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_fp8_ignoreC:1619; GFX1250: ; %bb.0: ; %bb1620; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11621; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[32:35], v[0:15], v[16:31], v[32:35]1622; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off1623; GFX1250-NEXT: s_endpgm1624;1625; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_fp8_ignoreC:1626; GISEL: ; %bb.0: ; %bb1627; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11628; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[32:35], v[0:15], v[16:31], v[32:35]1629; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off1630; GISEL-NEXT: s_endpgm1631bb:1632 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.fp8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 4, <8 x half> %C, i1 false, i1 false)1633 store <8 x half> %res, ptr addrspace(1) %out1634 ret void1635}1636 1637define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8_negC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {1638; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_bf8_negC:1639; GFX1250: ; %bb.0: ; %bb1640; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11641; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1]1642; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off1643; GFX1250-NEXT: s_endpgm1644;1645; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_bf8_negC:1646; GISEL: ; %bb.0: ; %bb1647; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11648; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1]1649; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off1650; GISEL-NEXT: s_endpgm1651bb:1652 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.bf8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 1, <8 x half> %C, i1 false, i1 false)1653 store <8 x half> %res, ptr addrspace(1) %out1654 ret void1655}1656 1657define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {1658; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_bf8_neg_absC:1659; GFX1250: ; %bb.0: ; %bb1660; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11661; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1] neg_hi:[0,0,1]1662; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off1663; GFX1250-NEXT: s_endpgm1664;1665; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_bf8_neg_absC:1666; GISEL: ; %bb.0: ; %bb1667; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11668; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1] neg_hi:[0,0,1]1669; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off1670; GISEL-NEXT: s_endpgm1671bb:1672 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.bf8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 3, <8 x half> %C, i1 false, i1 false)1673 store <8 x half> %res, ptr addrspace(1) %out1674 ret void1675}1676 1677define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {1678; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_bf8_ignoreC:1679; GFX1250: ; %bb.0: ; %bb1680; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11681; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[32:35], v[0:15], v[16:31], v[32:35]1682; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off1683; GFX1250-NEXT: s_endpgm1684;1685; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_bf8_ignoreC:1686; GISEL: ; %bb.0: ; %bb1687; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11688; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[32:35], v[0:15], v[16:31], v[32:35]1689; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off1690; GISEL-NEXT: s_endpgm1691bb:1692 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.bf8.v8f16.v16i32(<16 x i32> %A, <16 x i32> %B, i16 4, <8 x half> %C, i1 false, i1 false)1693 store <8 x half> %res, ptr addrspace(1) %out1694 ret void1695}1696 1697define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8_negC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1698; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_fp8_negC:1699; GFX1250: ; %bb.0: ; %bb1700; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11701; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]1702; GFX1250-NEXT: s_clause 0x11703; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161704; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1705; GFX1250-NEXT: s_endpgm1706;1707; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_fp8_negC:1708; GISEL: ; %bb.0: ; %bb1709; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11710; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]1711; GISEL-NEXT: s_clause 0x11712; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1713; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161714; GISEL-NEXT: s_endpgm1715bb:1716 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.fp8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 1, <8 x float> %C, i1 false, i1 false)1717 store <8 x float> %res, ptr addrspace(1) %out1718 ret void1719}1720 1721define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1722; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_fp8_neg_absC:1723; GFX1250: ; %bb.0: ; %bb1724; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11725; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]1726; GFX1250-NEXT: s_clause 0x11727; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161728; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1729; GFX1250-NEXT: s_endpgm1730;1731; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_fp8_neg_absC:1732; GISEL: ; %bb.0: ; %bb1733; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11734; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]1735; GISEL-NEXT: s_clause 0x11736; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1737; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161738; GISEL-NEXT: s_endpgm1739bb:1740 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.fp8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 3, <8 x float> %C, i1 false, i1 false)1741 store <8 x float> %res, ptr addrspace(1) %out1742 ret void1743}1744 1745define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1746; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_fp8_ignoreC:1747; GFX1250: ; %bb.0: ; %bb1748; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11749; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[32:39], v[0:15], v[16:31], v[32:39]1750; GFX1250-NEXT: s_clause 0x11751; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161752; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1753; GFX1250-NEXT: s_endpgm1754;1755; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_fp8_ignoreC:1756; GISEL: ; %bb.0: ; %bb1757; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11758; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[32:39], v[0:15], v[16:31], v[32:39]1759; GISEL-NEXT: s_clause 0x11760; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1761; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161762; GISEL-NEXT: s_endpgm1763bb:1764 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.fp8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 4, <8 x float> %C, i1 false, i1 false)1765 store <8 x float> %res, ptr addrspace(1) %out1766 ret void1767}1768 1769define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8_negC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1770; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_bf8_negC:1771; GFX1250: ; %bb.0: ; %bb1772; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11773; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]1774; GFX1250-NEXT: s_clause 0x11775; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161776; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1777; GFX1250-NEXT: s_endpgm1778;1779; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_bf8_negC:1780; GISEL: ; %bb.0: ; %bb1781; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11782; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]1783; GISEL-NEXT: s_clause 0x11784; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1785; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161786; GISEL-NEXT: s_endpgm1787bb:1788 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.bf8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 1, <8 x float> %C, i1 false, i1 false)1789 store <8 x float> %res, ptr addrspace(1) %out1790 ret void1791}1792 1793define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1794; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_bf8_neg_absC:1795; GFX1250: ; %bb.0: ; %bb1796; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11797; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]1798; GFX1250-NEXT: s_clause 0x11799; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161800; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1801; GFX1250-NEXT: s_endpgm1802;1803; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_bf8_neg_absC:1804; GISEL: ; %bb.0: ; %bb1805; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11806; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]1807; GISEL-NEXT: s_clause 0x11808; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1809; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161810; GISEL-NEXT: s_endpgm1811bb:1812 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.bf8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 3, <8 x float> %C, i1 false, i1 false)1813 store <8 x float> %res, ptr addrspace(1) %out1814 ret void1815}1816 1817define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1818; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_bf8_ignoreC:1819; GFX1250: ; %bb.0: ; %bb1820; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11821; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[32:39], v[0:15], v[16:31], v[32:39]1822; GFX1250-NEXT: s_clause 0x11823; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161824; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1825; GFX1250-NEXT: s_endpgm1826;1827; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_bf8_ignoreC:1828; GISEL: ; %bb.0: ; %bb1829; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11830; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[32:39], v[0:15], v[16:31], v[32:39]1831; GISEL-NEXT: s_clause 0x11832; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1833; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161834; GISEL-NEXT: s_endpgm1835bb:1836 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.bf8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 4, <8 x float> %C, i1 false, i1 false)1837 store <8 x float> %res, ptr addrspace(1) %out1838 ret void1839}1840 1841define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8_negC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1842; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_fp8_negC:1843; GFX1250: ; %bb.0: ; %bb1844; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11845; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]1846; GFX1250-NEXT: s_clause 0x11847; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161848; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1849; GFX1250-NEXT: s_endpgm1850;1851; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_fp8_negC:1852; GISEL: ; %bb.0: ; %bb1853; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11854; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]1855; GISEL-NEXT: s_clause 0x11856; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1857; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161858; GISEL-NEXT: s_endpgm1859bb:1860 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.fp8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 1, <8 x float> %C, i1 false, i1 false)1861 store <8 x float> %res, ptr addrspace(1) %out1862 ret void1863}1864 1865define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1866; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_fp8_neg_absC:1867; GFX1250: ; %bb.0: ; %bb1868; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11869; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]1870; GFX1250-NEXT: s_clause 0x11871; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161872; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1873; GFX1250-NEXT: s_endpgm1874;1875; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_fp8_neg_absC:1876; GISEL: ; %bb.0: ; %bb1877; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11878; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]1879; GISEL-NEXT: s_clause 0x11880; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1881; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161882; GISEL-NEXT: s_endpgm1883bb:1884 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.fp8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 3, <8 x float> %C, i1 false, i1 false)1885 store <8 x float> %res, ptr addrspace(1) %out1886 ret void1887}1888 1889define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1890; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_fp8_ignoreC:1891; GFX1250: ; %bb.0: ; %bb1892; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11893; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[32:39], v[0:15], v[16:31], v[32:39]1894; GFX1250-NEXT: s_clause 0x11895; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161896; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1897; GFX1250-NEXT: s_endpgm1898;1899; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_fp8_ignoreC:1900; GISEL: ; %bb.0: ; %bb1901; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11902; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[32:39], v[0:15], v[16:31], v[32:39]1903; GISEL-NEXT: s_clause 0x11904; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1905; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161906; GISEL-NEXT: s_endpgm1907bb:1908 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.fp8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 4, <8 x float> %C, i1 false, i1 false)1909 store <8 x float> %res, ptr addrspace(1) %out1910 ret void1911}1912 1913define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8_negC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1914; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_bf8_negC:1915; GFX1250: ; %bb.0: ; %bb1916; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11917; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]1918; GFX1250-NEXT: s_clause 0x11919; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161920; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1921; GFX1250-NEXT: s_endpgm1922;1923; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_bf8_negC:1924; GISEL: ; %bb.0: ; %bb1925; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11926; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]1927; GISEL-NEXT: s_clause 0x11928; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1929; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161930; GISEL-NEXT: s_endpgm1931bb:1932 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.bf8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 1, <8 x float> %C, i1 false, i1 false)1933 store <8 x float> %res, ptr addrspace(1) %out1934 ret void1935}1936 1937define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1938; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_bf8_neg_absC:1939; GFX1250: ; %bb.0: ; %bb1940; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11941; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]1942; GFX1250-NEXT: s_clause 0x11943; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161944; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1945; GFX1250-NEXT: s_endpgm1946;1947; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_bf8_neg_absC:1948; GISEL: ; %bb.0: ; %bb1949; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11950; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]1951; GISEL-NEXT: s_clause 0x11952; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1953; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161954; GISEL-NEXT: s_endpgm1955bb:1956 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.bf8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 3, <8 x float> %C, i1 false, i1 false)1957 store <8 x float> %res, ptr addrspace(1) %out1958 ret void1959}1960 1961define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {1962; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_bf8_ignoreC:1963; GFX1250: ; %bb.0: ; %bb1964; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11965; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[32:39], v[0:15], v[16:31], v[32:39]1966; GFX1250-NEXT: s_clause 0x11967; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161968; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off1969; GFX1250-NEXT: s_endpgm1970;1971; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_bf8_ignoreC:1972; GISEL: ; %bb.0: ; %bb1973; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11974; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[32:39], v[0:15], v[16:31], v[32:39]1975; GISEL-NEXT: s_clause 0x11976; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off1977; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:161978; GISEL-NEXT: s_endpgm1979bb:1980 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.bf8.v8f32.v16i32(<16 x i32> %A, <16 x i32> %B, i16 4, <8 x float> %C, i1 false, i1 false)1981 store <8 x float> %res, ptr addrspace(1) %out1982 ret void1983}1984 1985define amdgpu_ps void @test_wmma_f32_32x16x128_f4_negC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {1986; GFX1250-LABEL: test_wmma_f32_32x16x128_f4_negC:1987; GFX1250: ; %bb.0: ; %bb1988; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11989; GFX1250-NEXT: v_wmma_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39] neg_lo:[0,0,1]1990; GFX1250-NEXT: s_clause 0x31991; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:481992; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off offset:321993; GFX1250-NEXT: global_store_b128 v[40:41], v[28:31], off offset:161994; GFX1250-NEXT: global_store_b128 v[40:41], v[24:27], off1995; GFX1250-NEXT: s_endpgm1996;1997; GISEL-LABEL: test_wmma_f32_32x16x128_f4_negC:1998; GISEL: ; %bb.0: ; %bb1999; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12000; GISEL-NEXT: v_wmma_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39] neg_lo:[0,0,1]2001; GISEL-NEXT: s_clause 0x32002; GISEL-NEXT: global_store_b128 v[40:41], v[24:27], off2003; GISEL-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162004; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322005; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482006; GISEL-NEXT: s_endpgm2007bb:2008 %res = call <16 x float> @llvm.amdgcn.wmma.f32.32x16x128.f4.v16i32.v8i32.v16f32(<16 x i32> %A, <8 x i32> %B, i16 1, <16 x float> %C)2009 store <16 x float> %res, ptr addrspace(1) %out2010 ret void2011}2012 2013define amdgpu_ps void @test_wmma_f32_32x16x128_f4_neg_absC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {2014; GFX1250-LABEL: test_wmma_f32_32x16x128_f4_neg_absC:2015; GFX1250: ; %bb.0: ; %bb2016; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12017; GFX1250-NEXT: v_wmma_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39] neg_lo:[0,0,1] neg_hi:[0,0,1]2018; GFX1250-NEXT: s_clause 0x32019; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482020; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322021; GFX1250-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162022; GFX1250-NEXT: global_store_b128 v[40:41], v[24:27], off2023; GFX1250-NEXT: s_endpgm2024;2025; GISEL-LABEL: test_wmma_f32_32x16x128_f4_neg_absC:2026; GISEL: ; %bb.0: ; %bb2027; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12028; GISEL-NEXT: v_wmma_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39] neg_lo:[0,0,1] neg_hi:[0,0,1]2029; GISEL-NEXT: s_clause 0x32030; GISEL-NEXT: global_store_b128 v[40:41], v[24:27], off2031; GISEL-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162032; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322033; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482034; GISEL-NEXT: s_endpgm2035bb:2036 %res = call <16 x float> @llvm.amdgcn.wmma.f32.32x16x128.f4.v16i32.v8i32.v16f32(<16 x i32> %A, <8 x i32> %B, i16 3, <16 x float> %C)2037 store <16 x float> %res, ptr addrspace(1) %out2038 ret void2039}2040 2041define amdgpu_ps void @test_wmma_f32_32x16x128_f4_ignoreC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {2042; GFX1250-LABEL: test_wmma_f32_32x16x128_f4_ignoreC:2043; GFX1250: ; %bb.0: ; %bb2044; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12045; GFX1250-NEXT: v_wmma_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39]2046; GFX1250-NEXT: s_clause 0x32047; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482048; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322049; GFX1250-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162050; GFX1250-NEXT: global_store_b128 v[40:41], v[24:27], off2051; GFX1250-NEXT: s_endpgm2052;2053; GISEL-LABEL: test_wmma_f32_32x16x128_f4_ignoreC:2054; GISEL: ; %bb.0: ; %bb2055; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12056; GISEL-NEXT: v_wmma_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39]2057; GISEL-NEXT: s_clause 0x32058; GISEL-NEXT: global_store_b128 v[40:41], v[24:27], off2059; GISEL-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162060; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322061; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482062; GISEL-NEXT: s_endpgm2063bb:2064 %res = call <16 x float> @llvm.amdgcn.wmma.f32.32x16x128.f4.v16i32.v8i32.v16f32(<16 x i32> %A, <8 x i32> %B, i16 4, <16 x float> %C)2065 store <16 x float> %res, ptr addrspace(1) %out2066 ret void2067}2068 2069define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_negC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {2070; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4_negC:2071; GFX1250: ; %bb.0: ; %bb2072; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12073; GFX1250-NEXT: v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1]2074; GFX1250-NEXT: s_clause 0x32075; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482076; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322077; GFX1250-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162078; GFX1250-NEXT: global_store_b128 v[40:41], v[24:27], off2079; GFX1250-NEXT: s_endpgm2080;2081; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4_negC:2082; GISEL: ; %bb.0: ; %bb2083; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12084; GISEL-NEXT: v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1]2085; GISEL-NEXT: s_clause 0x32086; GISEL-NEXT: global_store_b128 v[40:41], v[24:27], off2087; GISEL-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162088; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322089; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482090; GISEL-NEXT: s_endpgm2091bb:2092 %res = call <16 x float> @llvm.amdgcn.wmma.scale.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 1, <16 x float> %C, i32 1, i32 0, i32 2, i32 1, i32 0, i32 4, i1 false, i1 false)2093 store <16 x float> %res, ptr addrspace(1) %out2094 ret void2095}2096 2097define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_neg_absC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {2098; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4_neg_absC:2099; GFX1250: ; %bb.0: ; %bb2100; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12101; GFX1250-NEXT: v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1] neg_hi:[0,0,1]2102; GFX1250-NEXT: s_clause 0x32103; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482104; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322105; GFX1250-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162106; GFX1250-NEXT: global_store_b128 v[40:41], v[24:27], off2107; GFX1250-NEXT: s_endpgm2108;2109; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4_neg_absC:2110; GISEL: ; %bb.0: ; %bb2111; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12112; GISEL-NEXT: v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1] neg_hi:[0,0,1]2113; GISEL-NEXT: s_clause 0x32114; GISEL-NEXT: global_store_b128 v[40:41], v[24:27], off2115; GISEL-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162116; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322117; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482118; GISEL-NEXT: s_endpgm2119bb:2120 %res = call <16 x float> @llvm.amdgcn.wmma.scale.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 3, <16 x float> %C, i32 1, i32 0, i32 2, i32 1, i32 0, i32 4, i1 false, i1 false)2121 store <16 x float> %res, ptr addrspace(1) %out2122 ret void2123}2124 2125define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_ignoreC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {2126; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4_ignoreC:2127; GFX1250: ; %bb.0: ; %bb2128; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12129; GFX1250-NEXT: v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW12130; GFX1250-NEXT: s_clause 0x32131; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482132; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322133; GFX1250-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162134; GFX1250-NEXT: global_store_b128 v[40:41], v[24:27], off2135; GFX1250-NEXT: s_endpgm2136;2137; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4_ignoreC:2138; GISEL: ; %bb.0: ; %bb2139; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12140; GISEL-NEXT: v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW12141; GISEL-NEXT: s_clause 0x32142; GISEL-NEXT: global_store_b128 v[40:41], v[24:27], off2143; GISEL-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162144; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322145; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482146; GISEL-NEXT: s_endpgm2147bb:2148 %res = call <16 x float> @llvm.amdgcn.wmma.scale.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 4, <16 x float> %C, i32 1, i32 0, i32 2, i32 1, i32 0, i32 4, i1 false, i1 false)2149 store <16 x float> %res, ptr addrspace(1) %out2150 ret void2151}2152 2153define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_negC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {2154; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4_negC:2155; GFX1250: ; %bb.0: ; %bb2156; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12157; GFX1250-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1]2158; GFX1250-NEXT: s_clause 0x32159; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482160; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322161; GFX1250-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162162; GFX1250-NEXT: global_store_b128 v[40:41], v[24:27], off2163; GFX1250-NEXT: s_endpgm2164;2165; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4_negC:2166; GISEL: ; %bb.0: ; %bb2167; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12168; GISEL-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1]2169; GISEL-NEXT: s_clause 0x32170; GISEL-NEXT: global_store_b128 v[40:41], v[24:27], off2171; GISEL-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162172; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322173; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482174; GISEL-NEXT: s_endpgm2175bb:2176 %res = call <16 x float> @llvm.amdgcn.wmma.scale16.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 1, <16 x float> %C, i32 1, i32 0, i64 2, i32 1, i32 0, i64 4, i1 false, i1 false)2177 store <16 x float> %res, ptr addrspace(1) %out2178 ret void2179}2180 2181define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_neg_absC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {2182; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4_neg_absC:2183; GFX1250: ; %bb.0: ; %bb2184; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12185; GFX1250-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1] neg_hi:[0,0,1]2186; GFX1250-NEXT: s_clause 0x32187; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482188; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322189; GFX1250-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162190; GFX1250-NEXT: global_store_b128 v[40:41], v[24:27], off2191; GFX1250-NEXT: s_endpgm2192;2193; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4_neg_absC:2194; GISEL: ; %bb.0: ; %bb2195; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12196; GISEL-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1] neg_hi:[0,0,1]2197; GISEL-NEXT: s_clause 0x32198; GISEL-NEXT: global_store_b128 v[40:41], v[24:27], off2199; GISEL-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162200; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322201; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482202; GISEL-NEXT: s_endpgm2203bb:2204 %res = call <16 x float> @llvm.amdgcn.wmma.scale16.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 3, <16 x float> %C, i32 1, i32 0, i64 2, i32 1, i32 0, i64 4, i1 false, i1 false)2205 store <16 x float> %res, ptr addrspace(1) %out2206 ret void2207}2208 2209define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_ignoreC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {2210; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4_ignoreC:2211; GFX1250: ; %bb.0: ; %bb2212; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12213; GFX1250-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW12214; GFX1250-NEXT: s_clause 0x32215; GFX1250-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482216; GFX1250-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322217; GFX1250-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162218; GFX1250-NEXT: global_store_b128 v[40:41], v[24:27], off2219; GFX1250-NEXT: s_endpgm2220;2221; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4_ignoreC:2222; GISEL: ; %bb.0: ; %bb2223; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12224; GISEL-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW12225; GISEL-NEXT: s_clause 0x32226; GISEL-NEXT: global_store_b128 v[40:41], v[24:27], off2227; GISEL-NEXT: global_store_b128 v[40:41], v[28:31], off offset:162228; GISEL-NEXT: global_store_b128 v[40:41], v[32:35], off offset:322229; GISEL-NEXT: global_store_b128 v[40:41], v[36:39], off offset:482230; GISEL-NEXT: s_endpgm2231bb:2232 %res = call <16 x float> @llvm.amdgcn.wmma.scale16.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32> %A, <8 x i32> %B, i16 4, <16 x float> %C, i32 1, i32 0, i64 2, i32 1, i32 0, i64 4, i1 false, i1 false)2233 store <16 x float> %res, ptr addrspace(1) %out2234 ret void2235}2236 2237define amdgpu_ps void @test_swmmac_f32_16x16x64_bf16_negA(<16 x bfloat> %A, <32 x bfloat> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {2238; GFX1250-LABEL: test_swmmac_f32_16x16x64_bf16_negA:2239; GFX1250: ; %bb.0: ; %bb2240; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12241; GFX1250-NEXT: v_swmmac_f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[1,0,0] neg_hi:[1,0,0]2242; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v332243; GFX1250-NEXT: s_clause 0x12244; GFX1250-NEXT: global_store_b128 v[34:35], v[28:31], off offset:162245; GFX1250-NEXT: global_store_b128 v[34:35], v[24:27], off2246; GFX1250-NEXT: s_endpgm2247;2248; GISEL-LABEL: test_swmmac_f32_16x16x64_bf16_negA:2249; GISEL: ; %bb.0: ; %bb2250; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12251; GISEL-NEXT: v_swmmac_f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[1,0,0] neg_hi:[1,0,0]2252; GISEL-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v332253; GISEL-NEXT: s_clause 0x12254; GISEL-NEXT: global_store_b128 v[34:35], v[28:31], off offset:162255; GISEL-NEXT: global_store_b128 v[34:35], v[24:27], off2256; GISEL-NEXT: s_endpgm2257bb:2258 %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x64.bf16.v8f32.v16bf16.v32bf16.i16(i1 1, <16 x bfloat> %A, i1 0, <32 x bfloat> %B, <8 x float> %C, i16 %Index, i1 false, i1 false)2259 store <8 x float> %res, ptr addrspace(1) %out2260 ret void2261}2262 2263define amdgpu_ps void @test_swmmac_f32_16x16x64_bf16_negB(<16 x bfloat> %A, <32 x bfloat> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {2264; GFX1250-LABEL: test_swmmac_f32_16x16x64_bf16_negB:2265; GFX1250: ; %bb.0: ; %bb2266; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12267; GFX1250-NEXT: v_swmmac_f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[0,1,0] neg_hi:[0,1,0]2268; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v332269; GFX1250-NEXT: s_clause 0x12270; GFX1250-NEXT: global_store_b128 v[34:35], v[28:31], off offset:162271; GFX1250-NEXT: global_store_b128 v[34:35], v[24:27], off2272; GFX1250-NEXT: s_endpgm2273;2274; GISEL-LABEL: test_swmmac_f32_16x16x64_bf16_negB:2275; GISEL: ; %bb.0: ; %bb2276; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12277; GISEL-NEXT: v_swmmac_f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[0,1,0] neg_hi:[0,1,0]2278; GISEL-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v332279; GISEL-NEXT: s_clause 0x12280; GISEL-NEXT: global_store_b128 v[34:35], v[28:31], off offset:162281; GISEL-NEXT: global_store_b128 v[34:35], v[24:27], off2282; GISEL-NEXT: s_endpgm2283bb:2284 %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x64.bf16.v8f32.v16bf16.v32bf16.i16(i1 0, <16 x bfloat> %A, i1 1, <32 x bfloat> %B, <8 x float> %C, i16 %Index, i1 false, i1 false)2285 store <8 x float> %res, ptr addrspace(1) %out2286 ret void2287}2288 2289define amdgpu_ps void @test_swmmac_bf16_16x16x64_bf16_negA(<16 x bfloat> %A, <32 x bfloat> %B, <8 x bfloat> %C, i16 %Index, ptr addrspace(1) %out) {2290; GFX1250-LABEL: test_swmmac_bf16_16x16x64_bf16_negA:2291; GFX1250: ; %bb.0: ; %bb2292; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12293; GFX1250-NEXT: v_swmmac_bf16_16x16x64_bf16 v[24:27], v[0:7], v[8:23], v28 neg_lo:[1,0,0] neg_hi:[1,0,0]2294; GFX1250-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v292295; GFX1250-NEXT: global_store_b128 v[30:31], v[24:27], off2296; GFX1250-NEXT: s_endpgm2297;2298; GISEL-LABEL: test_swmmac_bf16_16x16x64_bf16_negA:2299; GISEL: ; %bb.0: ; %bb2300; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12301; GISEL-NEXT: v_swmmac_bf16_16x16x64_bf16 v[24:27], v[0:7], v[8:23], v28 neg_lo:[1,0,0] neg_hi:[1,0,0]2302; GISEL-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v292303; GISEL-NEXT: global_store_b128 v[30:31], v[24:27], off2304; GISEL-NEXT: s_endpgm2305bb:2306 %res = call <8 x bfloat> @llvm.amdgcn.swmmac.bf16.16x16x64.bf16.v8bf16.v16bf16.v32bf16.i16(i1 1, <16 x bfloat> %A, i1 0, <32 x bfloat> %B, <8 x bfloat> %C, i16 %Index, i1 false, i1 false)2307 store <8 x bfloat> %res, ptr addrspace(1) %out2308 ret void2309}2310 2311define amdgpu_ps void @test_swmmac_bf16_16x16x64_bf16_negB(<16 x bfloat> %A, <32 x bfloat> %B, <8 x bfloat> %C, i16 %Index, ptr addrspace(1) %out) {2312; GFX1250-LABEL: test_swmmac_bf16_16x16x64_bf16_negB:2313; GFX1250: ; %bb.0: ; %bb2314; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12315; GFX1250-NEXT: v_swmmac_bf16_16x16x64_bf16 v[24:27], v[0:7], v[8:23], v28 neg_lo:[0,1,0] neg_hi:[0,1,0]2316; GFX1250-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v292317; GFX1250-NEXT: global_store_b128 v[30:31], v[24:27], off2318; GFX1250-NEXT: s_endpgm2319;2320; GISEL-LABEL: test_swmmac_bf16_16x16x64_bf16_negB:2321; GISEL: ; %bb.0: ; %bb2322; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12323; GISEL-NEXT: v_swmmac_bf16_16x16x64_bf16 v[24:27], v[0:7], v[8:23], v28 neg_lo:[0,1,0] neg_hi:[0,1,0]2324; GISEL-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v292325; GISEL-NEXT: global_store_b128 v[30:31], v[24:27], off2326; GISEL-NEXT: s_endpgm2327bb:2328 %res = call <8 x bfloat> @llvm.amdgcn.swmmac.bf16.16x16x64.bf16.v8bf16.v16bf16.v32bf16.i16(i1 0, <16 x bfloat> %A, i1 1, <32 x bfloat> %B, <8 x bfloat> %C, i16 %Index, i1 false, i1 false)2329 store <8 x bfloat> %res, ptr addrspace(1) %out2330 ret void2331}2332 2333define amdgpu_ps void @test_swmmac_bf16f32_16x16x64_bf16_negA(<16 x bfloat> %A, <32 x bfloat> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {2334; GFX1250-LABEL: test_swmmac_bf16f32_16x16x64_bf16_negA:2335; GFX1250: ; %bb.0: ; %bb2336; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12337; GFX1250-NEXT: v_swmmac_bf16f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[1,0,0] neg_hi:[1,0,0]2338; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v332339; GFX1250-NEXT: s_clause 0x12340; GFX1250-NEXT: global_store_b128 v[34:35], v[28:31], off offset:162341; GFX1250-NEXT: global_store_b128 v[34:35], v[24:27], off2342; GFX1250-NEXT: s_endpgm2343;2344; GISEL-LABEL: test_swmmac_bf16f32_16x16x64_bf16_negA:2345; GISEL: ; %bb.0: ; %bb2346; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12347; GISEL-NEXT: v_swmmac_bf16f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[1,0,0] neg_hi:[1,0,0]2348; GISEL-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v332349; GISEL-NEXT: s_clause 0x12350; GISEL-NEXT: global_store_b128 v[34:35], v[28:31], off offset:162351; GISEL-NEXT: global_store_b128 v[34:35], v[24:27], off2352; GISEL-NEXT: s_endpgm2353bb:2354 %res = call <8 x float> @llvm.amdgcn.swmmac.bf16f32.16x16x64.bf16.v8f32.v16bf16.v32bf16.i16(i1 1, <16 x bfloat> %A, i1 0, <32 x bfloat> %B, <8 x float> %C, i16 %Index, i1 false, i1 false)2355 store <8 x float> %res, ptr addrspace(1) %out2356 ret void2357}2358 2359define amdgpu_ps void @test_swmmac_bf16f32_16x16x64_bf16_negB(<16 x bfloat> %A, <32 x bfloat> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {2360; GFX1250-LABEL: test_swmmac_bf16f32_16x16x64_bf16_negB:2361; GFX1250: ; %bb.0: ; %bb2362; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12363; GFX1250-NEXT: v_swmmac_bf16f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[0,1,0] neg_hi:[0,1,0]2364; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v332365; GFX1250-NEXT: s_clause 0x12366; GFX1250-NEXT: global_store_b128 v[34:35], v[28:31], off offset:162367; GFX1250-NEXT: global_store_b128 v[34:35], v[24:27], off2368; GFX1250-NEXT: s_endpgm2369;2370; GISEL-LABEL: test_swmmac_bf16f32_16x16x64_bf16_negB:2371; GISEL: ; %bb.0: ; %bb2372; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12373; GISEL-NEXT: v_swmmac_bf16f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[0,1,0] neg_hi:[0,1,0]2374; GISEL-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v332375; GISEL-NEXT: s_clause 0x12376; GISEL-NEXT: global_store_b128 v[34:35], v[28:31], off offset:162377; GISEL-NEXT: global_store_b128 v[34:35], v[24:27], off2378; GISEL-NEXT: s_endpgm2379bb:2380 %res = call <8 x float> @llvm.amdgcn.swmmac.bf16f32.16x16x64.bf16.v8f32.v16bf16.v32bf16.i16(i1 0, <16 x bfloat> %A, i1 1, <32 x bfloat> %B, <8 x float> %C, i16 %Index, i1 false, i1 false)2381 store <8 x float> %res, ptr addrspace(1) %out2382 ret void2383}2384 2385define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8_signedA(<8 x i32> %A, <16 x i32> %B, <8 x i32> %C, i64 %Index, ptr addrspace(1) %out) {2386; GFX1250-LABEL: test_swmmac_i32_16x16x128_iu8_signedA:2387; GFX1250: ; %bb.0: ; %bb2388; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12389; GFX1250-NEXT: v_swmmac_i32_16x16x128_iu8 v[24:31], v[0:7], v[8:23], v[32:33] neg_lo:[1,0,0]2390; GFX1250-NEXT: s_clause 0x12391; GFX1250-NEXT: global_store_b128 v[34:35], v[28:31], off offset:162392; GFX1250-NEXT: global_store_b128 v[34:35], v[24:27], off2393; GFX1250-NEXT: s_endpgm2394;2395; GISEL-LABEL: test_swmmac_i32_16x16x128_iu8_signedA:2396; GISEL: ; %bb.0: ; %bb2397; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12398; GISEL-NEXT: v_swmmac_i32_16x16x128_iu8 v[24:31], v[0:7], v[8:23], v[32:33] neg_lo:[1,0,0]2399; GISEL-NEXT: s_clause 0x12400; GISEL-NEXT: global_store_b128 v[34:35], v[24:27], off2401; GISEL-NEXT: global_store_b128 v[34:35], v[28:31], off offset:162402; GISEL-NEXT: s_endpgm2403bb:2404 %res = call <8 x i32> @llvm.amdgcn.swmmac.i32.16x16x128.iu8.v8i32.v8i32.v16i32.i64(i1 1, <8 x i32> %A, i1 0, <16 x i32> %B, <8 x i32> %C, i64 %Index, i1 false, i1 false)2405 store <8 x i32> %res, ptr addrspace(1) %out2406 ret void2407}2408 2409define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8_signedB(<8 x i32> %A, <16 x i32> %B, <8 x i32> %C, i64 %Index, ptr addrspace(1) %out) {2410; GFX1250-LABEL: test_swmmac_i32_16x16x128_iu8_signedB:2411; GFX1250: ; %bb.0: ; %bb2412; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12413; GFX1250-NEXT: v_swmmac_i32_16x16x128_iu8 v[24:31], v[0:7], v[8:23], v[32:33] neg_lo:[0,1,0]2414; GFX1250-NEXT: s_clause 0x12415; GFX1250-NEXT: global_store_b128 v[34:35], v[28:31], off offset:162416; GFX1250-NEXT: global_store_b128 v[34:35], v[24:27], off2417; GFX1250-NEXT: s_endpgm2418;2419; GISEL-LABEL: test_swmmac_i32_16x16x128_iu8_signedB:2420; GISEL: ; %bb.0: ; %bb2421; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12422; GISEL-NEXT: v_swmmac_i32_16x16x128_iu8 v[24:31], v[0:7], v[8:23], v[32:33] neg_lo:[0,1,0]2423; GISEL-NEXT: s_clause 0x12424; GISEL-NEXT: global_store_b128 v[34:35], v[24:27], off2425; GISEL-NEXT: global_store_b128 v[34:35], v[28:31], off offset:162426; GISEL-NEXT: s_endpgm2427bb:2428 %res = call <8 x i32> @llvm.amdgcn.swmmac.i32.16x16x128.iu8.v8i32.v8i32.v16i32.i64(i1 0, <8 x i32> %A, i1 1, <16 x i32> %B, <8 x i32> %C, i64 %Index, i1 false, i1 false)2429 store <8 x i32> %res, ptr addrspace(1) %out2430 ret void2431}2432 2433define amdgpu_ps void @test_swmmac_f32_16x16x64_f16_negA(<16 x half> %A, <32 x half> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {2434; GFX1250-LABEL: test_swmmac_f32_16x16x64_f16_negA:2435; GFX1250: ; %bb.0: ; %bb2436; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12437; GFX1250-NEXT: v_swmmac_f32_16x16x64_f16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[1,0,0] neg_hi:[1,0,0]2438; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v332439; GFX1250-NEXT: s_clause 0x12440; GFX1250-NEXT: global_store_b128 v[34:35], v[28:31], off offset:162441; GFX1250-NEXT: global_store_b128 v[34:35], v[24:27], off2442; GFX1250-NEXT: s_endpgm2443;2444; GISEL-LABEL: test_swmmac_f32_16x16x64_f16_negA:2445; GISEL: ; %bb.0: ; %bb2446; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12447; GISEL-NEXT: v_swmmac_f32_16x16x64_f16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[1,0,0] neg_hi:[1,0,0]2448; GISEL-NEXT: v_dual_mov_b32 v36, v33 :: v_dual_mov_b32 v37, v342449; GISEL-NEXT: s_clause 0x12450; GISEL-NEXT: global_store_b128 v[36:37], v[24:27], off2451; GISEL-NEXT: global_store_b128 v[36:37], v[28:31], off offset:162452; GISEL-NEXT: s_endpgm2453bb:2454 %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x64.f16.v8f32.v16f16.v32f16.i16(i1 1, <16 x half> %A, i1 0, <32 x half> %B, <8 x float> %C, i16 %Index, i1 false, i1 false)2455 store <8 x float> %res, ptr addrspace(1) %out2456 ret void2457}2458 2459define amdgpu_ps void @test_swmmac_f32_16x16x64_f16_negB(<16 x half> %A, <32 x half> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {2460; GFX1250-LABEL: test_swmmac_f32_16x16x64_f16_negB:2461; GFX1250: ; %bb.0: ; %bb2462; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12463; GFX1250-NEXT: v_swmmac_f32_16x16x64_f16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[0,1,0] neg_hi:[0,1,0]2464; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v332465; GFX1250-NEXT: s_clause 0x12466; GFX1250-NEXT: global_store_b128 v[34:35], v[28:31], off offset:162467; GFX1250-NEXT: global_store_b128 v[34:35], v[24:27], off2468; GFX1250-NEXT: s_endpgm2469;2470; GISEL-LABEL: test_swmmac_f32_16x16x64_f16_negB:2471; GISEL: ; %bb.0: ; %bb2472; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12473; GISEL-NEXT: v_swmmac_f32_16x16x64_f16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[0,1,0] neg_hi:[0,1,0]2474; GISEL-NEXT: v_dual_mov_b32 v36, v33 :: v_dual_mov_b32 v37, v342475; GISEL-NEXT: s_clause 0x12476; GISEL-NEXT: global_store_b128 v[36:37], v[24:27], off2477; GISEL-NEXT: global_store_b128 v[36:37], v[28:31], off offset:162478; GISEL-NEXT: s_endpgm2479bb:2480 %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x64.f16.v8f32.v16f16.v32f16.i16(i1 0, <16 x half> %A, i1 1, <32 x half> %B, <8 x float> %C, i16 %Index, i1 false, i1 false)2481 store <8 x float> %res, ptr addrspace(1) %out2482 ret void2483}2484 2485define amdgpu_ps void @test_swmmac_f16_16x16x64_f16_negA(<16 x half> %A, <32 x half> %B, <8 x half> %C, i16 %Index, ptr addrspace(1) %out) {2486; GFX1250-LABEL: test_swmmac_f16_16x16x64_f16_negA:2487; GFX1250: ; %bb.0: ; %bb2488; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12489; GFX1250-NEXT: v_swmmac_f16_16x16x64_f16 v[24:27], v[0:7], v[8:23], v28 neg_lo:[1,0,0] neg_hi:[1,0,0]2490; GFX1250-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v292491; GFX1250-NEXT: global_store_b128 v[30:31], v[24:27], off2492; GFX1250-NEXT: s_endpgm2493;2494; GISEL-LABEL: test_swmmac_f16_16x16x64_f16_negA:2495; GISEL: ; %bb.0: ; %bb2496; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12497; GISEL-NEXT: v_swmmac_f16_16x16x64_f16 v[24:27], v[0:7], v[8:23], v28 neg_lo:[1,0,0] neg_hi:[1,0,0]2498; GISEL-NEXT: v_dual_mov_b32 v32, v29 :: v_dual_mov_b32 v33, v302499; GISEL-NEXT: global_store_b128 v[32:33], v[24:27], off2500; GISEL-NEXT: s_endpgm2501bb:2502 %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x64.f16.v8f16.v16f16.v32f16.i16(i1 1, <16 x half> %A, i1 0, <32 x half> %B, <8 x half> %C, i16 %Index, i1 false, i1 false)2503 store <8 x half> %res, ptr addrspace(1) %out2504 ret void2505}2506 2507define amdgpu_ps void @test_swmmac_f16_16x16x64_f16_negB(<16 x half> %A, <32 x half> %B, <8 x half> %C, i16 %Index, ptr addrspace(1) %out) {2508; GFX1250-LABEL: test_swmmac_f16_16x16x64_f16_negB:2509; GFX1250: ; %bb.0: ; %bb2510; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12511; GFX1250-NEXT: v_swmmac_f16_16x16x64_f16 v[24:27], v[0:7], v[8:23], v28 neg_lo:[0,1,0] neg_hi:[0,1,0]2512; GFX1250-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v292513; GFX1250-NEXT: global_store_b128 v[30:31], v[24:27], off2514; GFX1250-NEXT: s_endpgm2515;2516; GISEL-LABEL: test_swmmac_f16_16x16x64_f16_negB:2517; GISEL: ; %bb.0: ; %bb2518; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12519; GISEL-NEXT: v_swmmac_f16_16x16x64_f16 v[24:27], v[0:7], v[8:23], v28 neg_lo:[0,1,0] neg_hi:[0,1,0]2520; GISEL-NEXT: v_dual_mov_b32 v32, v29 :: v_dual_mov_b32 v33, v302521; GISEL-NEXT: global_store_b128 v[32:33], v[24:27], off2522; GISEL-NEXT: s_endpgm2523bb:2524 %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x64.f16.v8f16.v16f16.v32f16.i16(i1 0, <16 x half> %A, i1 1, <32 x half> %B, <8 x half> %C, i16 %Index, i1 false, i1 false)2525 store <8 x half> %res, ptr addrspace(1) %out2526 ret void2527}2528 2529declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x4.f32.v8f32.v2f32(i1, <2 x float>, i1, <2 x float>, i16, <8 x float>, i1, i1)2530declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.bf16.v8f32.v16bf16(i1, <16 x bfloat>, i1, <16 x bfloat>, i16, <8 x float>, i1, i1)2531declare <8 x bfloat> @llvm.amdgcn.wmma.bf16.16x16x32.bf16.v8bf16.v16bf16(i1, <16 x bfloat>, i1, <16 x bfloat>, i16, <8 x bfloat>, i1, i1)2532declare <8 x bfloat> @llvm.amdgcn.wmma.bf16f32.16x16x32.bf16.v8bf16.v16bf16(i1, <16 x bfloat>, i1, <16 x bfloat>, i16, <8 x float>, i1, i1)2533declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.fp8.v8f32.v8i32(<8 x i32>, <8 x i32>, i16, <8 x float>, i1, i1)2534declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.fp8.bf8.v8f32.v8i32(<8 x i32>, <8 x i32>, i16, <8 x float>, i1, i1)2535declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.fp8.v8f32.v8i32(<8 x i32>, <8 x i32>, i16, <8 x float>, i1, i1)2536declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.bf8.v8f32.v8i32(<8 x i32>, <8 x i32>, i16, <8 x float>, i1, i1)2537declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.fp8.v8f16.v8i32(<8 x i32>, <8 x i32>, i16, <8 x half>, i1, i1)2538declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.fp8.bf8.v8f16.v8i32(<8 x i32>, <8 x i32>, i16, <8 x half>, i1, i1)2539declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.fp8.v8f16.v8i32(<8 x i32>, <8 x i32>, i16, <8 x half>, i1, i1)2540declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x64.bf8.bf8.v8f16.v8i32(<8 x i32>, <8 x i32>, i16, <8 x half>, i1, i1)2541declare <8 x i32> @llvm.amdgcn.wmma.i32.16x16x64.iu8.v8i32.v8i32(i1 immarg, <8 x i32>, i1 immarg, <8 x i32>, <8 x i32>, i1, i1)2542declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1, <16 x half>, i1, <16 x half>, i16, <8 x float>, i1, i1)2543declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x32.f16.v8f16.v16f16(i1, <16 x half>, i1, <16 x half>, i16, <8 x half>, i1, i1)2544declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32, <16 x i32>, i32, <16 x i32>, i16, <8 x float>)2545declare <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32, <16 x i32>, i32, <16 x i32>, i16, <8 x float>, i32, i32, i32, i32, i32, i32, i1, i1)2546declare <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32, <16 x i32>, i32, <16 x i32>, i16, <8 x float>, i32, i32, i64, i32, i32, i64, i1, i1)2547declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.fp8.v8f16.v16i32(<16 x i32>, <16 x i32>, i16, <8 x half>, i1, i1)2548declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.fp8.bf8.v8f16.v16i32(<16 x i32>, <16 x i32>, i16, <8 x half>, i1, i1)2549declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.fp8.v8f16.v16i32(<16 x i32>, <16 x i32>, i16, <8 x half>, i1, i1)2550declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x128.bf8.bf8.v8f16.v16i32(<16 x i32>, <16 x i32>, i16, <8 x half>, i1, i1)2551declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.fp8.v8f32.v16i32(<16 x i32>, <16 x i32>, i16, <8 x float>, i1, i1)2552declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.fp8.bf8.v8f32.v16i32(<16 x i32>, <16 x i32>, i16, <8 x float>, i1, i1)2553declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.fp8.v8f32.v16i32(<16 x i32>, <16 x i32>, i16, <8 x float>, i1, i1)2554declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x128.bf8.bf8.v8f32.v16i32(<16 x i32>, <16 x i32>, i16, <8 x float>, i1, i1)2555declare <16 x float> @llvm.amdgcn.wmma.f32.32x16x128.f4.v16i32.v8i32.v16f32(<16 x i32>, <8 x i32>, i16, <16 x float>)2556declare <16 x float> @llvm.amdgcn.wmma.scale.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32>, <8 x i32>, i16, <16 x float>, i32, i32, i32, i32, i32, i32, i1, i1)2557declare <16 x float> @llvm.amdgcn.wmma.scale16.f32.32x16x128.f4.v16f32.v16i32.v8i32(<16 x i32>, <8 x i32>, i16, <16 x float>, i32, i32, i64, i32, i32, i64, i1, i1)2558 2559declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x64.bf16.v8f32.v16bf16.v32bf16.i16(i1, <16 x bfloat>, i1, <32 x bfloat>, <8 x float>, i16, i1, i1)2560declare <8 x bfloat> @llvm.amdgcn.swmmac.bf16.16x16x64.bf16.v8bf16.v16bf16.v32bf16.i16(i1, <16 x bfloat>, i1, <32 x bfloat>, <8 x bfloat>, i16, i1, i1)2561declare <8 x float> @llvm.amdgcn.swmmac.bf16f32.16x16x64.bf16.v8f32.v16bf16.v32bf16.i16(i1, <16 x bfloat>, i1, <32 x bfloat>, <8 x float>, i16, i1, i1)2562declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.fp8.fp8.v8f32.v8i32.v16i32.i64(<8 x i32>, <16 x i32>, <8 x float>, i64, i1, i1)2563declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.fp8.bf8.v8f32.v8i32.v16i32.i64(<8 x i32>, <16 x i32>, <8 x float>, i64, i1, i1)2564declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.bf8.fp8.v8f32.v8i32.v16i32.i64(<8 x i32>, <16 x i32>, <8 x float>, i64, i1, i1)2565declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.bf8.bf8.v8f32.v8i32.v16i32.i64(<8 x i32>, <16 x i32>, <8 x float>, i64, i1, i1)2566declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.fp8.fp8.v8f16.v8i32.v16i32.i64(<8 x i32>, <16 x i32>, <8 x half>, i64, i1, i1)2567declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.fp8.bf8.v8f16.v8i32.v16i32.i64(<8 x i32>, <16 x i32>, <8 x half>, i64, i1, i1)2568declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.bf8.fp8.v8f16.v8i32.v16i32.i64(<8 x i32>, <16 x i32>, <8 x half>, i64, i1, i1)2569declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.bf8.bf8.v8f16.v8i32.v16i32.i64(<8 x i32>, <16 x i32>, <8 x half>, i64, i1, i1)2570declare <8 x i32> @llvm.amdgcn.swmmac.i32.16x16x128.iu8.v8i32.v8i32.v16i32.i64(i1 immarg, <8 x i32>, i1 immarg, <16 x i32>, <8 x i32>, i64, i1, i1)2571declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x64.f16.v8f32.v16f16.v32f16.i16(i1, <16 x half>, i1, <32 x half>, <8 x float>, i16, i1, i1)2572declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x64.f16.v8f16.v16f16.v32f16.i16(i1, <16 x half>, i1, <32 x half>, <8 x half>, i16, i1, i1)2573