476 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck %s --check-prefix=GFX123 4define amdgpu_ps void @test_wmma_f32_16x16x16_f16_imm(<8 x half> %A, <8 x half> %B, ptr addrspace(1) %out) {5; GFX12-LABEL: test_wmma_f32_16x16x16_f16_imm:6; GFX12: ; %bb.0: ; %bb7; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[10:17], v[0:3], v[4:7], 1.08; GFX12-NEXT: s_clause 0x19; GFX12-NEXT: global_store_b128 v[8:9], v[10:13], off10; GFX12-NEXT: global_store_b128 v[8:9], v[14:17], off offset:1611; GFX12-NEXT: s_endpgm12bb:13 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v8f16.v8f16.v8f32(<8 x half> %A, <8 x half> %B, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>)14 store <8 x float> %res, ptr addrspace(1) %out15 ret void16}17 18define amdgpu_ps void @test_wmma_f32_16x16x16_f16_imm_non_inlineable(<8 x half> %A, <8 x half> %B, ptr addrspace(1) %out) {19; GFX12-LABEL: test_wmma_f32_16x16x16_f16_imm_non_inlineable:20; GFX12: ; %bb.0: ; %bb21; GFX12-NEXT: s_mov_b32 s0, 0x4040000022; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)23; GFX12-NEXT: s_mov_b32 s7, s024; GFX12-NEXT: s_mov_b32 s1, s025; GFX12-NEXT: s_mov_b32 s2, s026; GFX12-NEXT: s_mov_b32 s3, s027; GFX12-NEXT: s_mov_b32 s4, s028; GFX12-NEXT: s_mov_b32 s5, s029; GFX12-NEXT: s_mov_b32 s6, s030; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)31; GFX12-NEXT: v_dual_mov_b32 v17, s7 :: v_dual_mov_b32 v16, s632; GFX12-NEXT: v_dual_mov_b32 v15, s5 :: v_dual_mov_b32 v14, s433; GFX12-NEXT: v_dual_mov_b32 v13, s3 :: v_dual_mov_b32 v12, s234; GFX12-NEXT: v_dual_mov_b32 v11, s1 :: v_dual_mov_b32 v10, s035; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[10:17], v[0:3], v[4:7], v[10:17]36; GFX12-NEXT: s_clause 0x137; GFX12-NEXT: global_store_b128 v[8:9], v[10:13], off38; GFX12-NEXT: global_store_b128 v[8:9], v[14:17], off offset:1639; GFX12-NEXT: s_endpgm40bb:41 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v8f16.v8f16.v8f32(<8 x half> %A, <8 x half> %B, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>)42 store <8 x float> %res, ptr addrspace(1) %out43 ret void44}45 46define amdgpu_ps void @test_wmma_f32_16x16x16_bf16_imm(<8 x i16> %A, <8 x i16> %B, ptr addrspace(1) %out) {47; GFX12-LABEL: test_wmma_f32_16x16x16_bf16_imm:48; GFX12: ; %bb.0: ; %bb49; GFX12-NEXT: v_wmma_f32_16x16x16_bf16 v[10:17], v[0:3], v[4:7], 1.050; GFX12-NEXT: s_clause 0x151; GFX12-NEXT: global_store_b128 v[8:9], v[10:13], off52; GFX12-NEXT: global_store_b128 v[8:9], v[14:17], off offset:1653; GFX12-NEXT: s_endpgm54bb:55 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf16.v8f32.v8i16.v8i16.v8f32(<8 x i16> %A, <8 x i16> %B, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>)56 store <8 x float> %res, ptr addrspace(1) %out57 ret void58}59 60define amdgpu_ps void @test_wmma_f32_16x16x16_bf16_imm_non_inlineable(<8 x i16> %A, <8 x i16> %B, ptr addrspace(1) %out) {61; GFX12-LABEL: test_wmma_f32_16x16x16_bf16_imm_non_inlineable:62; GFX12: ; %bb.0: ; %bb63; GFX12-NEXT: s_mov_b32 s0, 0x4040000064; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)65; GFX12-NEXT: s_mov_b32 s7, s066; GFX12-NEXT: s_mov_b32 s1, s067; GFX12-NEXT: s_mov_b32 s2, s068; GFX12-NEXT: s_mov_b32 s3, s069; GFX12-NEXT: s_mov_b32 s4, s070; GFX12-NEXT: s_mov_b32 s5, s071; GFX12-NEXT: s_mov_b32 s6, s072; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)73; GFX12-NEXT: v_dual_mov_b32 v17, s7 :: v_dual_mov_b32 v16, s674; GFX12-NEXT: v_dual_mov_b32 v15, s5 :: v_dual_mov_b32 v14, s475; GFX12-NEXT: v_dual_mov_b32 v13, s3 :: v_dual_mov_b32 v12, s276; GFX12-NEXT: v_dual_mov_b32 v11, s1 :: v_dual_mov_b32 v10, s077; GFX12-NEXT: v_wmma_f32_16x16x16_bf16 v[10:17], v[0:3], v[4:7], v[10:17]78; GFX12-NEXT: s_clause 0x179; GFX12-NEXT: global_store_b128 v[8:9], v[10:13], off80; GFX12-NEXT: global_store_b128 v[8:9], v[14:17], off offset:1681; GFX12-NEXT: s_endpgm82bb:83 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf16.v8f32.v8i16.v8i16.v8f32(<8 x i16> %A, <8 x i16> %B, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>)84 store <8 x float> %res, ptr addrspace(1) %out85 ret void86}87 88define amdgpu_ps void @test_wmma_f16_16x16x16_f16_imm(<8 x half> %A, <8 x half> %B, ptr addrspace(1) %out) {89; GFX12-LABEL: test_wmma_f16_16x16x16_f16_imm:90; GFX12: ; %bb.0: ; %bb91; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[10:13], v[0:3], v[4:7], 1.092; GFX12-NEXT: global_store_b128 v[8:9], v[10:13], off93; GFX12-NEXT: s_endpgm94bb:95 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16.v8f16.v8f16(<8 x half> %A, <8 x half> %B, <8 x half> <half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 0)96 store <8 x half> %res, ptr addrspace(1) %out97 ret void98}99 100define amdgpu_ps void @test_wmma_f16_16x16x16_f16_imm_non_inlineable(<8 x half> %A, <8 x half> %B, ptr addrspace(1) %out) {101; GFX12-LABEL: test_wmma_f16_16x16x16_f16_imm_non_inlineable:102; GFX12: ; %bb.0: ; %bb103; GFX12-NEXT: s_mov_b32 s0, 0x42004200104; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)105; GFX12-NEXT: s_mov_b32 s3, s0106; GFX12-NEXT: s_mov_b32 s1, s0107; GFX12-NEXT: s_mov_b32 s2, s0108; GFX12-NEXT: v_dual_mov_b32 v13, s3 :: v_dual_mov_b32 v12, s2109; GFX12-NEXT: v_dual_mov_b32 v11, s1 :: v_dual_mov_b32 v10, s0110; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)111; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[10:13], v[0:3], v[4:7], v[10:13]112; GFX12-NEXT: global_store_b128 v[8:9], v[10:13], off113; GFX12-NEXT: s_endpgm114bb:115 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16.v8f16.v8f16(<8 x half> %A, <8 x half> %B, <8 x half> <half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0, half 3.0>, i1 0)116 store <8 x half> %res, ptr addrspace(1) %out117 ret void118}119 120define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_imm(<8 x i16> %A, <8 x i16> %B, ptr addrspace(1) %out) {121; GFX12-LABEL: test_wmma_bf16_16x16x16_bf16_imm:122; GFX12: ; %bb.0: ; %bb123; GFX12-NEXT: s_mov_b32 s0, 0x3f803f80124; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)125; GFX12-NEXT: s_mov_b32 s3, s0126; GFX12-NEXT: s_mov_b32 s1, s0127; GFX12-NEXT: s_mov_b32 s2, s0128; GFX12-NEXT: v_dual_mov_b32 v13, s3 :: v_dual_mov_b32 v12, s2129; GFX12-NEXT: v_dual_mov_b32 v11, s1 :: v_dual_mov_b32 v10, s0130; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)131; GFX12-NEXT: v_wmma_bf16_16x16x16_bf16 v[10:13], v[0:3], v[4:7], v[10:13]132; GFX12-NEXT: global_store_b128 v[8:9], v[10:13], off133; GFX12-NEXT: s_endpgm134bb:135 %res = call <8 x i16> @llvm.amdgcn.wmma.bf16.16x16x16.bf16.v8i16.v8i16.v8i16.v8i16(<8 x i16> %A, <8 x i16> %B, <8 x i16> <i16 16256, i16 16256, i16 16256, i16 16256, i16 16256, i16 16256, i16 16256, i16 16256>, i1 0)136 store <8 x i16> %res, ptr addrspace(1) %out137 ret void138}139 140define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_imm_non_inlineable(<8 x i16> %A, <8 x i16> %B, ptr addrspace(1) %out) {141; GFX12-LABEL: test_wmma_bf16_16x16x16_bf16_imm_non_inlineable:142; GFX12: ; %bb.0: ; %bb143; GFX12-NEXT: s_mov_b32 s0, 0x3fc03fc0144; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)145; GFX12-NEXT: s_mov_b32 s3, s0146; GFX12-NEXT: s_mov_b32 s1, s0147; GFX12-NEXT: s_mov_b32 s2, s0148; GFX12-NEXT: v_dual_mov_b32 v13, s3 :: v_dual_mov_b32 v12, s2149; GFX12-NEXT: v_dual_mov_b32 v11, s1 :: v_dual_mov_b32 v10, s0150; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)151; GFX12-NEXT: v_wmma_bf16_16x16x16_bf16 v[10:13], v[0:3], v[4:7], v[10:13]152; GFX12-NEXT: global_store_b128 v[8:9], v[10:13], off153; GFX12-NEXT: s_endpgm154bb:155 %res = call <8 x i16> @llvm.amdgcn.wmma.bf16.16x16x16.bf16.v8i16.v8i16.v8i16.v8i16(<8 x i16> %A, <8 x i16> %B, <8 x i16> <i16 16320, i16 16320, i16 16320, i16 16320, i16 16320, i16 16320, i16 16320, i16 16320>, i1 0)156 store <8 x i16> %res, ptr addrspace(1) %out157 ret void158}159 160define amdgpu_ps void @test_wmma_i32_16x16x16_iu8_imm(<2 x i32> %A, <2 x i32> %B, ptr addrspace(1) %out) {161; GFX12-LABEL: test_wmma_i32_16x16x16_iu8_imm:162; GFX12: ; %bb.0: ; %bb163; GFX12-NEXT: v_wmma_i32_16x16x16_iu8 v[6:13], v[0:1], v[2:3], 1164; GFX12-NEXT: s_clause 0x1165; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off166; GFX12-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16167; GFX12-NEXT: s_endpgm168bb:169 %res = call <8 x i32> @llvm.amdgcn.wmma.i32.16x16x16.iu8.v8i32.v2i32.v2i32.v8i32(i1 0, <2 x i32> %A, i1 0, <2 x i32> %B, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, i1 0)170 store <8 x i32> %res, ptr addrspace(1) %out171 ret void172}173 174define amdgpu_ps void @test_wmma_i32_16x16x16_iu8_imm_non_inlineable(<2 x i32> %A, <2 x i32> %B, ptr addrspace(1) %out) {175; GFX12-LABEL: test_wmma_i32_16x16x16_iu8_imm_non_inlineable:176; GFX12: ; %bb.0: ; %bb177; GFX12-NEXT: s_movk_i32 s0, 0x80178; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)179; GFX12-NEXT: s_mov_b32 s7, s0180; GFX12-NEXT: s_mov_b32 s1, s0181; GFX12-NEXT: s_mov_b32 s2, s0182; GFX12-NEXT: s_mov_b32 s3, s0183; GFX12-NEXT: s_mov_b32 s4, s0184; GFX12-NEXT: s_mov_b32 s5, s0185; GFX12-NEXT: s_mov_b32 s6, s0186; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)187; GFX12-NEXT: v_dual_mov_b32 v13, s7 :: v_dual_mov_b32 v12, s6188; GFX12-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v10, s4189; GFX12-NEXT: v_dual_mov_b32 v9, s3 :: v_dual_mov_b32 v8, s2190; GFX12-NEXT: v_dual_mov_b32 v7, s1 :: v_dual_mov_b32 v6, s0191; GFX12-NEXT: v_wmma_i32_16x16x16_iu8 v[6:13], v[0:1], v[2:3], v[6:13]192; GFX12-NEXT: s_clause 0x1193; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off194; GFX12-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16195; GFX12-NEXT: s_endpgm196bb:197 %res = call <8 x i32> @llvm.amdgcn.wmma.i32.16x16x16.iu8.v8i32.v2i32.v2i32.v8i32(i1 0, <2 x i32> %A, i1 0, <2 x i32> %B, <8 x i32> <i32 128, i32 128, i32 128, i32 128, i32 128, i32 128, i32 128, i32 128>, i1 0)198 store <8 x i32> %res, ptr addrspace(1) %out199 ret void200}201 202define amdgpu_ps void @test_wmma_i32_16x16x16_iu4_imm(i32 %A, i32 %B, ptr addrspace(1) %out) {203; GFX12-LABEL: test_wmma_i32_16x16x16_iu4_imm:204; GFX12: ; %bb.0: ; %bb205; GFX12-NEXT: v_wmma_i32_16x16x16_iu4 v[4:11], v0, v1, 1206; GFX12-NEXT: s_clause 0x1207; GFX12-NEXT: global_store_b128 v[2:3], v[4:7], off208; GFX12-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16209; GFX12-NEXT: s_endpgm210bb:211 %res = call <8 x i32> @llvm.amdgcn.wmma.i32.16x16x16.iu4.v8i32.i32.i32.v8i32(i1 0, i32 %A, i1 0, i32 %B, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, i1 0)212 store <8 x i32> %res, ptr addrspace(1) %out213 ret void214}215 216define amdgpu_ps void @test_wmma_i32_16x16x16_iu4_imm_non_inlineable(i32 %A, i32 %B, ptr addrspace(1) %out) {217; GFX12-LABEL: test_wmma_i32_16x16x16_iu4_imm_non_inlineable:218; GFX12: ; %bb.0: ; %bb219; GFX12-NEXT: s_movk_i32 s0, 0x80220; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)221; GFX12-NEXT: s_mov_b32 s7, s0222; GFX12-NEXT: s_mov_b32 s1, s0223; GFX12-NEXT: s_mov_b32 s2, s0224; GFX12-NEXT: s_mov_b32 s3, s0225; GFX12-NEXT: s_mov_b32 s4, s0226; GFX12-NEXT: s_mov_b32 s5, s0227; GFX12-NEXT: s_mov_b32 s6, s0228; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)229; GFX12-NEXT: v_dual_mov_b32 v11, s7 :: v_dual_mov_b32 v10, s6230; GFX12-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v8, s4231; GFX12-NEXT: v_dual_mov_b32 v7, s3 :: v_dual_mov_b32 v6, s2232; GFX12-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0233; GFX12-NEXT: v_wmma_i32_16x16x16_iu4 v[4:11], v0, v1, v[4:11]234; GFX12-NEXT: s_clause 0x1235; GFX12-NEXT: global_store_b128 v[2:3], v[4:7], off236; GFX12-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16237; GFX12-NEXT: s_endpgm238bb:239 %res = call <8 x i32> @llvm.amdgcn.wmma.i32.16x16x16.iu4.v8i32.i32.i32.v8i32(i1 0, i32 %A, i1 0, i32 %B, <8 x i32> <i32 128, i32 128, i32 128, i32 128, i32 128, i32 128, i32 128, i32 128>, i1 0)240 store <8 x i32> %res, ptr addrspace(1) %out241 ret void242}243 244define amdgpu_ps void @test_wmma_f32_16x16x16_fp8_fp8_imm(<2 x i32> %A, <2 x i32> %B, ptr addrspace(1) %out) {245; GFX12-LABEL: test_wmma_f32_16x16x16_fp8_fp8_imm:246; GFX12: ; %bb.0: ; %bb247; GFX12-NEXT: v_wmma_f32_16x16x16_fp8_fp8 v[6:13], v[0:1], v[2:3], 1.0248; GFX12-NEXT: s_clause 0x1249; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off250; GFX12-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16251; GFX12-NEXT: s_endpgm252bb:253 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.fp8.v8f32.v2i32.v2i32.v8f32(<2 x i32> %A, <2 x i32> %B, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>)254 store <8 x float> %res, ptr addrspace(1) %out255 ret void256}257 258define amdgpu_ps void @test_wmma_f32_16x16x16_fp8_fp8_imm_non_inlineable(<2 x i32> %A, <2 x i32> %B, ptr addrspace(1) %out) {259; GFX12-LABEL: test_wmma_f32_16x16x16_fp8_fp8_imm_non_inlineable:260; GFX12: ; %bb.0: ; %bb261; GFX12-NEXT: s_mov_b32 s0, 0x40400000262; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)263; GFX12-NEXT: s_mov_b32 s7, s0264; GFX12-NEXT: s_mov_b32 s1, s0265; GFX12-NEXT: s_mov_b32 s2, s0266; GFX12-NEXT: s_mov_b32 s3, s0267; GFX12-NEXT: s_mov_b32 s4, s0268; GFX12-NEXT: s_mov_b32 s5, s0269; GFX12-NEXT: s_mov_b32 s6, s0270; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)271; GFX12-NEXT: v_dual_mov_b32 v13, s7 :: v_dual_mov_b32 v12, s6272; GFX12-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v10, s4273; GFX12-NEXT: v_dual_mov_b32 v9, s3 :: v_dual_mov_b32 v8, s2274; GFX12-NEXT: v_dual_mov_b32 v7, s1 :: v_dual_mov_b32 v6, s0275; GFX12-NEXT: v_wmma_f32_16x16x16_fp8_fp8 v[6:13], v[0:1], v[2:3], v[6:13]276; GFX12-NEXT: s_clause 0x1277; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off278; GFX12-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16279; GFX12-NEXT: s_endpgm280bb:281 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.fp8.v8f32.v2i32.v2i32.v8f32(<2 x i32> %A, <2 x i32> %B, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>)282 store <8 x float> %res, ptr addrspace(1) %out283 ret void284}285 286define amdgpu_ps void @test_wmma_f32_16x16x16_bf8_fp8_imm(<2 x i32> %A, <2 x i32> %B, ptr addrspace(1) %out) {287; GFX12-LABEL: test_wmma_f32_16x16x16_bf8_fp8_imm:288; GFX12: ; %bb.0: ; %bb289; GFX12-NEXT: v_wmma_f32_16x16x16_bf8_fp8 v[6:13], v[0:1], v[2:3], 1.0290; GFX12-NEXT: s_clause 0x1291; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off292; GFX12-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16293; GFX12-NEXT: s_endpgm294bb:295 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.fp8.v8f32.v2i32.v2i32.v8f32(<2 x i32> %A, <2 x i32> %B, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>)296 store <8 x float> %res, ptr addrspace(1) %out297 ret void298}299 300define amdgpu_ps void @test_wmma_f32_16x16x16_bf8_fp8_imm_non_inlineable(<2 x i32> %A, <2 x i32> %B, ptr addrspace(1) %out) {301; GFX12-LABEL: test_wmma_f32_16x16x16_bf8_fp8_imm_non_inlineable:302; GFX12: ; %bb.0: ; %bb303; GFX12-NEXT: s_mov_b32 s0, 0x40400000304; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)305; GFX12-NEXT: s_mov_b32 s7, s0306; GFX12-NEXT: s_mov_b32 s1, s0307; GFX12-NEXT: s_mov_b32 s2, s0308; GFX12-NEXT: s_mov_b32 s3, s0309; GFX12-NEXT: s_mov_b32 s4, s0310; GFX12-NEXT: s_mov_b32 s5, s0311; GFX12-NEXT: s_mov_b32 s6, s0312; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)313; GFX12-NEXT: v_dual_mov_b32 v13, s7 :: v_dual_mov_b32 v12, s6314; GFX12-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v10, s4315; GFX12-NEXT: v_dual_mov_b32 v9, s3 :: v_dual_mov_b32 v8, s2316; GFX12-NEXT: v_dual_mov_b32 v7, s1 :: v_dual_mov_b32 v6, s0317; GFX12-NEXT: v_wmma_f32_16x16x16_bf8_fp8 v[6:13], v[0:1], v[2:3], v[6:13]318; GFX12-NEXT: s_clause 0x1319; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off320; GFX12-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16321; GFX12-NEXT: s_endpgm322bb:323 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.fp8.v8f32.v2i32.v2i32.v8f32(<2 x i32> %A, <2 x i32> %B, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>)324 store <8 x float> %res, ptr addrspace(1) %out325 ret void326}327 328define amdgpu_ps void @test_wmma_f32_16x16x16_fp8_bf8_imm(<2 x i32> %A, <2 x i32> %B, ptr addrspace(1) %out) {329; GFX12-LABEL: test_wmma_f32_16x16x16_fp8_bf8_imm:330; GFX12: ; %bb.0: ; %bb331; GFX12-NEXT: v_wmma_f32_16x16x16_fp8_bf8 v[6:13], v[0:1], v[2:3], 1.0332; GFX12-NEXT: s_clause 0x1333; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off334; GFX12-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16335; GFX12-NEXT: s_endpgm336bb:337 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.bf8.v8f32.v2i32.v2i32.v8f32(<2 x i32> %A, <2 x i32> %B, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>)338 store <8 x float> %res, ptr addrspace(1) %out339 ret void340}341 342define amdgpu_ps void @test_wmma_f32_16x16x16_fp8_bf8_imm_non_inlineable(<2 x i32> %A, <2 x i32> %B, ptr addrspace(1) %out) {343; GFX12-LABEL: test_wmma_f32_16x16x16_fp8_bf8_imm_non_inlineable:344; GFX12: ; %bb.0: ; %bb345; GFX12-NEXT: s_mov_b32 s0, 0x40400000346; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)347; GFX12-NEXT: s_mov_b32 s7, s0348; GFX12-NEXT: s_mov_b32 s1, s0349; GFX12-NEXT: s_mov_b32 s2, s0350; GFX12-NEXT: s_mov_b32 s3, s0351; GFX12-NEXT: s_mov_b32 s4, s0352; GFX12-NEXT: s_mov_b32 s5, s0353; GFX12-NEXT: s_mov_b32 s6, s0354; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)355; GFX12-NEXT: v_dual_mov_b32 v13, s7 :: v_dual_mov_b32 v12, s6356; GFX12-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v10, s4357; GFX12-NEXT: v_dual_mov_b32 v9, s3 :: v_dual_mov_b32 v8, s2358; GFX12-NEXT: v_dual_mov_b32 v7, s1 :: v_dual_mov_b32 v6, s0359; GFX12-NEXT: v_wmma_f32_16x16x16_fp8_bf8 v[6:13], v[0:1], v[2:3], v[6:13]360; GFX12-NEXT: s_clause 0x1361; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off362; GFX12-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16363; GFX12-NEXT: s_endpgm364bb:365 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.bf8.v8f32.v2i32.v2i32.v8f32(<2 x i32> %A, <2 x i32> %B, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>)366 store <8 x float> %res, ptr addrspace(1) %out367 ret void368}369 370define amdgpu_ps void @test_wmma_f32_16x16x16_bf8_bf8_imm(<2 x i32> %A, <2 x i32> %B, ptr addrspace(1) %out) {371; GFX12-LABEL: test_wmma_f32_16x16x16_bf8_bf8_imm:372; GFX12: ; %bb.0: ; %bb373; GFX12-NEXT: v_wmma_f32_16x16x16_bf8_bf8 v[6:13], v[0:1], v[2:3], 1.0374; GFX12-NEXT: s_clause 0x1375; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off376; GFX12-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16377; GFX12-NEXT: s_endpgm378bb:379 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.bf8.v8f32.v2i32.v2i32.v8f32(<2 x i32> %A, <2 x i32> %B, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>)380 store <8 x float> %res, ptr addrspace(1) %out381 ret void382}383 384define amdgpu_ps void @test_wmma_f32_16x16x16_bf8_bf8_imm_non_inlineable(<2 x i32> %A, <2 x i32> %B, ptr addrspace(1) %out) {385; GFX12-LABEL: test_wmma_f32_16x16x16_bf8_bf8_imm_non_inlineable:386; GFX12: ; %bb.0: ; %bb387; GFX12-NEXT: s_mov_b32 s0, 0x40400000388; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)389; GFX12-NEXT: s_mov_b32 s7, s0390; GFX12-NEXT: s_mov_b32 s1, s0391; GFX12-NEXT: s_mov_b32 s2, s0392; GFX12-NEXT: s_mov_b32 s3, s0393; GFX12-NEXT: s_mov_b32 s4, s0394; GFX12-NEXT: s_mov_b32 s5, s0395; GFX12-NEXT: s_mov_b32 s6, s0396; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)397; GFX12-NEXT: v_dual_mov_b32 v13, s7 :: v_dual_mov_b32 v12, s6398; GFX12-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v10, s4399; GFX12-NEXT: v_dual_mov_b32 v9, s3 :: v_dual_mov_b32 v8, s2400; GFX12-NEXT: v_dual_mov_b32 v7, s1 :: v_dual_mov_b32 v6, s0401; GFX12-NEXT: v_wmma_f32_16x16x16_bf8_bf8 v[6:13], v[0:1], v[2:3], v[6:13]402; GFX12-NEXT: s_clause 0x1403; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off404; GFX12-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16405; GFX12-NEXT: s_endpgm406bb:407 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.bf8.v8f32.v2i32.v2i32.v8f32(<2 x i32> %A, <2 x i32> %B, <8 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>)408 store <8 x float> %res, ptr addrspace(1) %out409 ret void410}411 412define amdgpu_ps void @test_wmma_i32_16x16x32_iu4_imm(<2 x i32> %A, <2 x i32> %B, ptr addrspace(1) %out) {413; GFX12-LABEL: test_wmma_i32_16x16x32_iu4_imm:414; GFX12: ; %bb.0: ; %bb415; GFX12-NEXT: v_wmma_i32_16x16x32_iu4 v[6:13], v[0:1], v[2:3], 1416; GFX12-NEXT: s_clause 0x1417; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off418; GFX12-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16419; GFX12-NEXT: s_endpgm420bb:421 %res = call <8 x i32> @llvm.amdgcn.wmma.i32.16x16x32.iu4.v8i32.v2i32.v2i32.v8i32(i1 0, <2 x i32> %A, i1 0, <2 x i32> %B, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, i1 0)422 store <8 x i32> %res, ptr addrspace(1) %out423 ret void424}425 426define amdgpu_ps void @test_wmma_i32_16x16x32_iu4_imm_non_inlineable(<2 x i32> %A, <2 x i32> %B, ptr addrspace(1) %out) {427; GFX12-LABEL: test_wmma_i32_16x16x32_iu4_imm_non_inlineable:428; GFX12: ; %bb.0: ; %bb429; GFX12-NEXT: s_movk_i32 s0, 0x80430; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)431; GFX12-NEXT: s_mov_b32 s7, s0432; GFX12-NEXT: s_mov_b32 s1, s0433; GFX12-NEXT: s_mov_b32 s2, s0434; GFX12-NEXT: s_mov_b32 s3, s0435; GFX12-NEXT: s_mov_b32 s4, s0436; GFX12-NEXT: s_mov_b32 s5, s0437; GFX12-NEXT: s_mov_b32 s6, s0438; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)439; GFX12-NEXT: v_dual_mov_b32 v13, s7 :: v_dual_mov_b32 v12, s6440; GFX12-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v10, s4441; GFX12-NEXT: v_dual_mov_b32 v9, s3 :: v_dual_mov_b32 v8, s2442; GFX12-NEXT: v_dual_mov_b32 v7, s1 :: v_dual_mov_b32 v6, s0443; GFX12-NEXT: v_wmma_i32_16x16x32_iu4 v[6:13], v[0:1], v[2:3], v[6:13]444; GFX12-NEXT: s_clause 0x1445; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off446; GFX12-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16447; GFX12-NEXT: s_endpgm448bb:449 %res = call <8 x i32> @llvm.amdgcn.wmma.i32.16x16x32.iu4.v8i32.v2i32.v2i32.v8i32(i1 0, <2 x i32> %A, i1 0, <2 x i32> %B, <8 x i32> <i32 128, i32 128, i32 128, i32 128, i32 128, i32 128, i32 128, i32 128>, i1 0)450 store <8 x i32> %res, ptr addrspace(1) %out451 ret void452}453 454declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v8f16.v8f16.v8f32(<8 x half>, <8 x half>, <8 x float>)455declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf16.v8f32.v8i16.v8i16.v8f32(<8 x i16>, <8 x i16>, <8 x float>)456declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16.v8f16.v8f16(<8 x half>, <8 x half>, <8 x half>, i1 immarg)457declare <8 x i16> @llvm.amdgcn.wmma.bf16.16x16x16.bf16.v8i16.v8i16.v8i16.v8i16(<8 x i16>, <8 x i16>, <8 x i16>, i1 immarg)458declare <8 x i32> @llvm.amdgcn.wmma.i32.16x16x16.iu8.v8i32.v2i32.v2i32.v8i32(i1 immarg, <2 x i32>, i1 immarg, <2 x i32>, <8 x i32>, i1 immarg)459declare <8 x i32> @llvm.amdgcn.wmma.i32.16x16x16.iu4.v8i32.i32.i32.v8i32(i1 immarg, i32, i1 immarg, i32, <8 x i32>, i1 immarg)460declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.fp8.v8f32.v2i32.v2i32.v8f32(<2 x i32>, <2 x i32>, <8 x float>)461declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.bf8.v8f32.v2i32.v2i32.v8f32(<2 x i32>, <2 x i32>, <8 x float>)462declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.fp8.v8f32.v2i32.v2i32.v8f32(<2 x i32>, <2 x i32>, <8 x float>)463declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.bf8.v8f32.v2i32.v2i32.v8f32(<2 x i32>, <2 x i32>, <8 x float>)464declare <8 x i32> @llvm.amdgcn.wmma.i32.16x16x32.iu4.v8i32.v2i32.v2i32.v8i32(i1 immarg, <2 x i32>, i1 immarg, <2 x i32>, <8 x i32>, i1 immarg)465declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16.v8f32.v8f16.v16f16.v8f32.i16(<8 x half>, <16 x half>, <8 x float>, i16)466declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf16.v8f32.v8i16.v16i16.v8f32.i16(<8 x i16>, <16 x i16>, <8 x float>, i16)467declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x32.f16.v8f16.v8f16.v16f16.v8f16.i16(<8 x half>, <16 x half>, <8 x half>, i16)468declare <8 x i16> @llvm.amdgcn.swmmac.bf16.16x16x32.bf16.v8i16.v8i16.v16i16.v8i16.i16(<8 x i16>, <16 x i16>, <8 x i16>, i16)469declare <8 x i32> @llvm.amdgcn.swmmac.i32.16x16x32.iu8.v8i32.v2i32.v4i32.v8i32.i16(i1 immarg, <2 x i32>, i1 immarg, <4 x i32>, <8 x i32>, i16 %Index, i1 immarg)470declare <8 x i32> @llvm.amdgcn.swmmac.i32.16x16x32.iu4.v8i32.i32.v2i32.v8i32.i16(i1 immarg, i32, i1 immarg, <2 x i32>, <8 x i32>, i16 %Index, i1 immarg)471declare <8 x i32> @llvm.amdgcn.swmmac.i32.16x16x64.iu4.v8i32.v2i32.v4i32.v8i32.i32(i1 immarg, <2 x i32>, i1 immarg, <4 x i32>, <8 x i32>, i32 %Index, i1 immarg)472declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.fp8.v8f32.v2i32.v4i32.v8f32.i16(<2 x i32>, <4 x i32>, <8 x float>, i16)473declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.bf8.v8f32.v2i32.v4i32.v8f32.i16(<2 x i32>, <4 x i32>, <8 x float>, i16)474declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.fp8.v8f32.v2i32.v4i32.v8f32.i16(<2 x i32>, <4 x i32>, <8 x float>, i16)475declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.bf8.v8f32.v2i32.v4i32.v8f32.i16(<2 x i32>, <4 x i32>, <8 x float>, i16)476