444 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck %s --check-prefix=GFX123 4define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negA(<8 x half> %A, <8 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {5; GFX12-LABEL: test_wmma_f32_16x16x16_f16_negA:6; GFX12: ; %bb.0: ; %bb7; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[8:15], v[0:3], v[4:7], v[8:15] neg_lo:[1,0,0] neg_hi:[1,0,0]8; GFX12-NEXT: s_clause 0x19; GFX12-NEXT: global_store_b128 v[16:17], v[12:15], off offset:1610; GFX12-NEXT: global_store_b128 v[16:17], v[8:11], off11; GFX12-NEXT: s_endpgm12bb:13 %fneg.A = fneg <8 x half> %A14 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v8f16(<8 x half> %fneg.A, <8 x half> %B, <8 x float> %C)15 store <8 x float> %res, ptr addrspace(1) %out16 ret void17}18 19define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negB(<8 x half> %A, <8 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {20; GFX12-LABEL: test_wmma_f32_16x16x16_f16_negB:21; GFX12: ; %bb.0: ; %bb22; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[8:15], v[0:3], v[4:7], v[8:15] neg_lo:[0,1,0] neg_hi:[0,1,0]23; GFX12-NEXT: s_clause 0x124; GFX12-NEXT: global_store_b128 v[16:17], v[12:15], off offset:1625; GFX12-NEXT: global_store_b128 v[16:17], v[8:11], off26; GFX12-NEXT: s_endpgm27bb:28 %fneg.B = fneg <8 x half> %B29 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v8f16(<8 x half> %A, <8 x half> %fneg.B, <8 x float> %C)30 store <8 x float> %res, ptr addrspace(1) %out31 ret void32}33 34define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negC(<8 x half> %A, <8 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {35; GFX12-LABEL: test_wmma_f32_16x16x16_f16_negC:36; GFX12: ; %bb.0: ; %bb37; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[8:15], v[0:3], v[4:7], v[8:15] neg_lo:[0,0,1]38; GFX12-NEXT: s_clause 0x139; GFX12-NEXT: global_store_b128 v[16:17], v[12:15], off offset:1640; GFX12-NEXT: global_store_b128 v[16:17], v[8:11], off41; GFX12-NEXT: s_endpgm42bb:43 %fneg.C = fneg <8 x float> %C44 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v8f16(<8 x half> %A, <8 x half> %B, <8 x float> %fneg.C)45 store <8 x float> %res, ptr addrspace(1) %out46 ret void47}48 49define amdgpu_ps void @test_wmma_f32_16x16x16_f16_absC(<8 x half> %A, <8 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {50; GFX12-LABEL: test_wmma_f32_16x16x16_f16_absC:51; GFX12: ; %bb.0: ; %bb52; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[8:15], v[0:3], v[4:7], v[8:15] neg_hi:[0,0,1]53; GFX12-NEXT: s_clause 0x154; GFX12-NEXT: global_store_b128 v[16:17], v[12:15], off offset:1655; GFX12-NEXT: global_store_b128 v[16:17], v[8:11], off56; GFX12-NEXT: s_endpgm57bb:58 %fabs.C = call <8 x float> @llvm.fabs.v8f32(<8 x float> %C)59 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v8f16(<8 x half> %A, <8 x half> %B, <8 x float> %fabs.C)60 store <8 x float> %res, ptr addrspace(1) %out61 ret void62}63 64define amdgpu_ps void @test_wmma_f32_16x16x16_bf16_negC(<8 x i16> %A, <8 x i16> %B, <8 x float> %C, ptr addrspace(1) %out) {65; GFX12-LABEL: test_wmma_f32_16x16x16_bf16_negC:66; GFX12: ; %bb.0: ; %bb67; GFX12-NEXT: v_wmma_f32_16x16x16_bf16 v[8:15], v[0:3], v[4:7], v[8:15] neg_lo:[0,0,1]68; GFX12-NEXT: s_clause 0x169; GFX12-NEXT: global_store_b128 v[16:17], v[12:15], off offset:1670; GFX12-NEXT: global_store_b128 v[16:17], v[8:11], off71; GFX12-NEXT: s_endpgm72bb:73 %fneg.C = fneg <8 x float> %C74 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf16.v8f32.v8i16(<8 x i16> %A, <8 x i16> %B, <8 x float> %fneg.C)75 store <8 x float> %res, ptr addrspace(1) %out76 ret void77}78 79define amdgpu_ps void @test_wmma_f32_16x16x16_bf16_absC(<8 x i16> %A, <8 x i16> %B, <8 x float> %C, ptr addrspace(1) %out) {80; GFX12-LABEL: test_wmma_f32_16x16x16_bf16_absC:81; GFX12: ; %bb.0: ; %bb82; GFX12-NEXT: v_wmma_f32_16x16x16_bf16 v[8:15], v[0:3], v[4:7], v[8:15] neg_hi:[0,0,1]83; GFX12-NEXT: s_clause 0x184; GFX12-NEXT: global_store_b128 v[16:17], v[12:15], off offset:1685; GFX12-NEXT: global_store_b128 v[16:17], v[8:11], off86; GFX12-NEXT: s_endpgm87bb:88 %fabs.C = call <8 x float> @llvm.fabs.v8f32(<8 x float> %C)89 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf16.v8f32.v8i16(<8 x i16> %A, <8 x i16> %B, <8 x float> %fabs.C)90 store <8 x float> %res, ptr addrspace(1) %out91 ret void92}93 94define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negA(<8 x half> %A, <8 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {95; GFX12-LABEL: test_wmma_f16_16x16x16_f16_negA:96; GFX12: ; %bb.0: ; %bb97; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[8:11], v[0:3], v[4:7], v[8:11] neg_lo:[1,0,0] neg_hi:[1,0,0]98; GFX12-NEXT: global_store_b128 v[12:13], v[8:11], off99; GFX12-NEXT: s_endpgm100bb:101 %fneg.A = fneg <8 x half> %A102 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<8 x half> %fneg.A, <8 x half> %B, <8 x half> %C, i1 0)103 store <8 x half> %res, ptr addrspace(1) %out104 ret void105}106 107define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negB(<8 x half> %A, <8 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {108; GFX12-LABEL: test_wmma_f16_16x16x16_f16_negB:109; GFX12: ; %bb.0: ; %bb110; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[8:11], v[0:3], v[4:7], v[8:11] neg_lo:[0,1,0] neg_hi:[0,1,0]111; GFX12-NEXT: global_store_b128 v[12:13], v[8:11], off112; GFX12-NEXT: s_endpgm113bb:114 %fneg.B = fneg <8 x half> %B115 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<8 x half> %A, <8 x half> %fneg.B, <8 x half> %C, i1 0)116 store <8 x half> %res, ptr addrspace(1) %out117 ret void118}119 120define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negC(<8 x half> %A, <8 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {121; GFX12-LABEL: test_wmma_f16_16x16x16_f16_negC:122; GFX12: ; %bb.0: ; %bb123; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[8:11], v[0:3], v[4:7], v[8:11] neg_lo:[0,0,1]124; GFX12-NEXT: global_store_b128 v[12:13], v[8:11], off125; GFX12-NEXT: s_endpgm126bb:127 %fneg.C = fneg <8 x half> %C128 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<8 x half> %A, <8 x half> %B, <8 x half> %fneg.C, i1 0)129 store <8 x half> %res, ptr addrspace(1) %out130 ret void131}132 133define amdgpu_ps void @test_wmma_f16_16x16x16_f16_absC(<8 x half> %A, <8 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {134; GFX12-LABEL: test_wmma_f16_16x16x16_f16_absC:135; GFX12: ; %bb.0: ; %bb136; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[8:11], v[0:3], v[4:7], v[8:11] neg_hi:[0,0,1]137; GFX12-NEXT: global_store_b128 v[12:13], v[8:11], off138; GFX12-NEXT: s_endpgm139bb:140 %fabs.C = call <8 x half> @llvm.fabs.v8f16(<8 x half> %C)141 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<8 x half> %A, <8 x half> %B, <8 x half> %fabs.C, i1 0)142 store <8 x half> %res, ptr addrspace(1) %out143 ret void144}145 146define amdgpu_ps void @test_wmma_f32_16x16x16_fp8_fp8_negC(<2 x i32> %A, <2 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {147; GFX12-LABEL: test_wmma_f32_16x16x16_fp8_fp8_negC:148; GFX12: ; %bb.0: ; %bb149; GFX12-NEXT: v_wmma_f32_16x16x16_fp8_fp8 v[4:11], v[0:1], v[2:3], v[4:11] neg_lo:[0,0,1]150; GFX12-NEXT: s_clause 0x1151; GFX12-NEXT: global_store_b128 v[12:13], v[8:11], off offset:16152; GFX12-NEXT: global_store_b128 v[12:13], v[4:7], off153; GFX12-NEXT: s_endpgm154bb:155 %fneg.C = fneg <8 x float> %C156 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.fp8.v8f32.v2i32(<2 x i32> %A, <2 x i32> %B, <8 x float> %fneg.C)157 store <8 x float> %res, ptr addrspace(1) %out158 ret void159}160 161define amdgpu_ps void @test_wmma_f32_16x16x16_fp8_fp8_absC(<2 x i32> %A, <2 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {162; GFX12-LABEL: test_wmma_f32_16x16x16_fp8_fp8_absC:163; GFX12: ; %bb.0: ; %bb164; GFX12-NEXT: v_wmma_f32_16x16x16_fp8_fp8 v[4:11], v[0:1], v[2:3], v[4:11] neg_hi:[0,0,1]165; GFX12-NEXT: s_clause 0x1166; GFX12-NEXT: global_store_b128 v[12:13], v[8:11], off offset:16167; GFX12-NEXT: global_store_b128 v[12:13], v[4:7], off168; GFX12-NEXT: s_endpgm169bb:170 %fabs.C = call <8 x float> @llvm.fabs.v8f32(<8 x float> %C)171 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.fp8.v8f32.v2i32(<2 x i32> %A, <2 x i32> %B, <8 x float> %fabs.C)172 store <8 x float> %res, ptr addrspace(1) %out173 ret void174}175 176define amdgpu_ps void @test_wmma_f32_16x16x16_bf8_fp8_negC(<2 x i32> %A, <2 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {177; GFX12-LABEL: test_wmma_f32_16x16x16_bf8_fp8_negC:178; GFX12: ; %bb.0: ; %bb179; GFX12-NEXT: v_wmma_f32_16x16x16_bf8_fp8 v[4:11], v[0:1], v[2:3], v[4:11] neg_lo:[0,0,1]180; GFX12-NEXT: s_clause 0x1181; GFX12-NEXT: global_store_b128 v[12:13], v[8:11], off offset:16182; GFX12-NEXT: global_store_b128 v[12:13], v[4:7], off183; GFX12-NEXT: s_endpgm184bb:185 %fneg.C = fneg <8 x float> %C186 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.fp8.v8f32.v2i32(<2 x i32> %A, <2 x i32> %B, <8 x float> %fneg.C)187 store <8 x float> %res, ptr addrspace(1) %out188 ret void189}190 191define amdgpu_ps void @test_wmma_f32_16x16x16_bf8_fp8_absC(<2 x i32> %A, <2 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {192; GFX12-LABEL: test_wmma_f32_16x16x16_bf8_fp8_absC:193; GFX12: ; %bb.0: ; %bb194; GFX12-NEXT: v_wmma_f32_16x16x16_bf8_fp8 v[4:11], v[0:1], v[2:3], v[4:11] neg_hi:[0,0,1]195; GFX12-NEXT: s_clause 0x1196; GFX12-NEXT: global_store_b128 v[12:13], v[8:11], off offset:16197; GFX12-NEXT: global_store_b128 v[12:13], v[4:7], off198; GFX12-NEXT: s_endpgm199bb:200 %fabs.C = call <8 x float> @llvm.fabs.v8f32(<8 x float> %C)201 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.fp8.v8f32.v2i32(<2 x i32> %A, <2 x i32> %B, <8 x float> %fabs.C)202 store <8 x float> %res, ptr addrspace(1) %out203 ret void204}205 206define amdgpu_ps void @test_wmma_f32_16x16x16_fp8_bf8_negC(<2 x i32> %A, <2 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {207; GFX12-LABEL: test_wmma_f32_16x16x16_fp8_bf8_negC:208; GFX12: ; %bb.0: ; %bb209; GFX12-NEXT: v_wmma_f32_16x16x16_fp8_bf8 v[4:11], v[0:1], v[2:3], v[4:11] neg_lo:[0,0,1]210; GFX12-NEXT: s_clause 0x1211; GFX12-NEXT: global_store_b128 v[12:13], v[8:11], off offset:16212; GFX12-NEXT: global_store_b128 v[12:13], v[4:7], off213; GFX12-NEXT: s_endpgm214bb:215 %fneg.C = fneg <8 x float> %C216 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.bf8.v8f32.v2i32(<2 x i32> %A, <2 x i32> %B, <8 x float> %fneg.C)217 store <8 x float> %res, ptr addrspace(1) %out218 ret void219}220 221define amdgpu_ps void @test_wmma_f32_16x16x16_fp8_bf8_absC(<2 x i32> %A, <2 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {222; GFX12-LABEL: test_wmma_f32_16x16x16_fp8_bf8_absC:223; GFX12: ; %bb.0: ; %bb224; GFX12-NEXT: v_wmma_f32_16x16x16_fp8_bf8 v[4:11], v[0:1], v[2:3], v[4:11] neg_hi:[0,0,1]225; GFX12-NEXT: s_clause 0x1226; GFX12-NEXT: global_store_b128 v[12:13], v[8:11], off offset:16227; GFX12-NEXT: global_store_b128 v[12:13], v[4:7], off228; GFX12-NEXT: s_endpgm229bb:230 %fabs.C = call <8 x float> @llvm.fabs.v8f32(<8 x float> %C)231 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.bf8.v8f32.v2i32(<2 x i32> %A, <2 x i32> %B, <8 x float> %fabs.C)232 store <8 x float> %res, ptr addrspace(1) %out233 ret void234}235 236define amdgpu_ps void @test_wmma_f32_16x16x16_bf8_bf8_negC(<2 x i32> %A, <2 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {237; GFX12-LABEL: test_wmma_f32_16x16x16_bf8_bf8_negC:238; GFX12: ; %bb.0: ; %bb239; GFX12-NEXT: v_wmma_f32_16x16x16_bf8_bf8 v[4:11], v[0:1], v[2:3], v[4:11] neg_lo:[0,0,1]240; GFX12-NEXT: s_clause 0x1241; GFX12-NEXT: global_store_b128 v[12:13], v[8:11], off offset:16242; GFX12-NEXT: global_store_b128 v[12:13], v[4:7], off243; GFX12-NEXT: s_endpgm244bb:245 %fneg.C = fneg <8 x float> %C246 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.bf8.v8f32.v2i32(<2 x i32> %A, <2 x i32> %B, <8 x float> %fneg.C)247 store <8 x float> %res, ptr addrspace(1) %out248 ret void249}250 251define amdgpu_ps void @test_wmma_f32_16x16x16_bf8_bf8_absC(<2 x i32> %A, <2 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {252; GFX12-LABEL: test_wmma_f32_16x16x16_bf8_bf8_absC:253; GFX12: ; %bb.0: ; %bb254; GFX12-NEXT: v_wmma_f32_16x16x16_bf8_bf8 v[4:11], v[0:1], v[2:3], v[4:11] neg_hi:[0,0,1]255; GFX12-NEXT: s_clause 0x1256; GFX12-NEXT: global_store_b128 v[12:13], v[8:11], off offset:16257; GFX12-NEXT: global_store_b128 v[12:13], v[4:7], off258; GFX12-NEXT: s_endpgm259bb:260 %fabs.C = call <8 x float> @llvm.fabs.v8f32(<8 x float> %C)261 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.bf8.v8f32.v2i32(<2 x i32> %A, <2 x i32> %B, <8 x float> %fabs.C)262 store <8 x float> %res, ptr addrspace(1) %out263 ret void264}265 266define amdgpu_ps void @test_swmmac_f32_16x16x32_f16_negA(<8 x half> %A, <16 x half> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {267; GFX12-LABEL: test_swmmac_f32_16x16x32_f16_negA:268; GFX12: ; %bb.0: ; %bb269; GFX12-NEXT: v_swmmac_f32_16x16x32_f16 v[12:19], v[0:3], v[4:11], v20 neg_lo:[1,0,0] neg_hi:[1,0,0]270; GFX12-NEXT: s_clause 0x1271; GFX12-NEXT: global_store_b128 v[21:22], v[16:19], off offset:16272; GFX12-NEXT: global_store_b128 v[21:22], v[12:15], off273; GFX12-NEXT: s_endpgm274bb:275 %fneg.A = fneg <8 x half> %A276 %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16.v8f32.v8f16.v16f16.i16(<8 x half> %fneg.A, <16 x half> %B, <8 x float> %C, i16 %Index)277 store <8 x float> %res, ptr addrspace(1) %out278 ret void279}280 281define amdgpu_ps void @test_swmmac_f32_16x16x32_f16_negB(<8 x half> %A, <16 x half> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {282; GFX12-LABEL: test_swmmac_f32_16x16x32_f16_negB:283; GFX12: ; %bb.0: ; %bb284; GFX12-NEXT: v_swmmac_f32_16x16x32_f16 v[12:19], v[0:3], v[4:11], v20 neg_lo:[0,1,0] neg_hi:[0,1,0]285; GFX12-NEXT: s_clause 0x1286; GFX12-NEXT: global_store_b128 v[21:22], v[16:19], off offset:16287; GFX12-NEXT: global_store_b128 v[21:22], v[12:15], off288; GFX12-NEXT: s_endpgm289bb:290 %fneg.B = fneg <16 x half> %B291 %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16.v8f32.v8f16.v16f16.i16(<8 x half> %A, <16 x half> %fneg.B, <8 x float> %C, i16 %Index)292 store <8 x float> %res, ptr addrspace(1) %out293 ret void294}295 296define amdgpu_ps void @test_swmmac_f16_16x16x32_f16_negA(<8 x half> %A, <16 x half> %B, <8 x half> %C, i16 %Index, ptr addrspace(1) %out) {297; GFX12-LABEL: test_swmmac_f16_16x16x32_f16_negA:298; GFX12: ; %bb.0: ; %bb299; GFX12-NEXT: v_swmmac_f16_16x16x32_f16 v[12:15], v[0:3], v[4:11], v16 neg_lo:[1,0,0] neg_hi:[1,0,0]300; GFX12-NEXT: global_store_b128 v[17:18], v[12:15], off301; GFX12-NEXT: s_endpgm302bb:303 %fneg.A = fneg <8 x half> %A304 %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x32.f16.v8f16.v8f16.v16f16.i16(<8 x half> %fneg.A, <16 x half> %B, <8 x half> %C, i16 %Index)305 store <8 x half> %res, ptr addrspace(1) %out306 ret void307}308 309define amdgpu_ps void @test_swmmac_f16_16x16x32_f16_negB(<8 x half> %A, <16 x half> %B, <8 x half> %C, i16 %Index, ptr addrspace(1) %out) {310; GFX12-LABEL: test_swmmac_f16_16x16x32_f16_negB:311; GFX12: ; %bb.0: ; %bb312; GFX12-NEXT: v_swmmac_f16_16x16x32_f16 v[12:15], v[0:3], v[4:11], v16 neg_lo:[0,1,0] neg_hi:[0,1,0]313; GFX12-NEXT: global_store_b128 v[17:18], v[12:15], off314; GFX12-NEXT: s_endpgm315bb:316 %fneg.B = fneg <16 x half> %B317 %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x32.f16.v8f16.v8f16.v16f16.i16(<8 x half> %A, <16 x half> %fneg.B, <8 x half> %C, i16 %Index)318 store <8 x half> %res, ptr addrspace(1) %out319 ret void320}321 322; both neg and abs patterns (wmma matrix C f32 or f16 )323 324define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negabsC(<8 x half> %A, <8 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {325; GFX12-LABEL: test_wmma_f32_16x16x16_f16_negabsC:326; GFX12: ; %bb.0: ; %bb327; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[8:15], v[0:3], v[4:7], v[8:15] neg_lo:[0,0,1] neg_hi:[0,0,1]328; GFX12-NEXT: s_clause 0x1329; GFX12-NEXT: global_store_b128 v[16:17], v[12:15], off offset:16330; GFX12-NEXT: global_store_b128 v[16:17], v[8:11], off331; GFX12-NEXT: s_endpgm332bb:333 %fabs.C = call <8 x float> @llvm.fabs.v8f32(<8 x float> %C)334 %fneg.fabs.C = fneg <8 x float> %fabs.C335 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v8f16(<8 x half> %A, <8 x half> %B, <8 x float> %fneg.fabs.C)336 store <8 x float> %res, ptr addrspace(1) %out337 ret void338}339 340define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negabsC(<8 x half> %A, <8 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {341; GFX12-LABEL: test_wmma_f16_16x16x16_f16_negabsC:342; GFX12: ; %bb.0: ; %bb343; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[8:11], v[0:3], v[4:7], v[8:11] neg_lo:[0,0,1] neg_hi:[0,0,1]344; GFX12-NEXT: global_store_b128 v[12:13], v[8:11], off345; GFX12-NEXT: s_endpgm346bb:347 %fabs.C = call <8 x half> @llvm.fabs.v8f16(<8 x half> %C)348 %fneg.fabs.C = fneg <8 x half> %fabs.C349 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<8 x half> %A, <8 x half> %B, <8 x half> %fneg.fabs.C, i1 0)350 store <8 x half> %res, ptr addrspace(1) %out351 ret void352}353 354define amdgpu_ps void @test_wmma_f32_16x16x16_f16_neg_partial_fabsA(<8 x half> %A, <8 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {355; GFX12-LABEL: test_wmma_f32_16x16x16_f16_neg_partial_fabsA:356; GFX12: ; %bb.0: ; %bb357; GFX12-NEXT: v_and_b32_e32 v11, 0x7fffffff, v11358; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)359; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[8:15], v[0:3], v[4:7], v[8:15] neg_lo:[0,0,1]360; GFX12-NEXT: s_clause 0x1361; GFX12-NEXT: global_store_b128 v[16:17], v[12:15], off offset:16362; GFX12-NEXT: global_store_b128 v[16:17], v[8:11], off363; GFX12-NEXT: s_endpgm364bb:365 %el3 = extractelement <8 x float> %C, i32 3366 %el3.fabs = call float @llvm.fabs.f32(float %el3)367 %partial.fabs.C = insertelement <8 x float> %C, float %el3.fabs, i32 3368 %fneg.partial.fabs.C = fneg <8 x float> %partial.fabs.C369 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v8f16(<8 x half> %A, <8 x half> %B, <8 x float> %fneg.partial.fabs.C)370 store <8 x float> %res, ptr addrspace(1) %out371 ret void372}373 374; A or B matrix modifier and constant in C375 376define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negA_constantC(<8 x half> %A, <8 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {377; GFX12-LABEL: test_wmma_f32_16x16x16_f16_negA_constantC:378; GFX12: ; %bb.0: ; %bb379; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[10:17], v[0:3], v[4:7], 1.0 neg_lo:[1,0,0] neg_hi:[1,0,0]380; GFX12-NEXT: s_clause 0x1381; GFX12-NEXT: global_store_b128 v[8:9], v[14:17], off offset:16382; GFX12-NEXT: global_store_b128 v[8:9], v[10:13], off383; GFX12-NEXT: s_endpgm384bb:385 %fneg.A = fneg <8 x half> %A386 %res = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v8f16(<8 x half> %fneg.A, <8 x half> %B, <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>)387 store <8 x float> %res, ptr addrspace(1) %out388 ret void389}390 391define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negB_constantC(<8 x half> %A, <8 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {392; GFX12-LABEL: test_wmma_f16_16x16x16_f16_negB_constantC:393; GFX12: ; %bb.0: ; %bb394; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[10:13], v[0:3], v[4:7], 1.0 neg_lo:[0,1,0] neg_hi:[0,1,0]395; GFX12-NEXT: global_store_b128 v[8:9], v[10:13], off396; GFX12-NEXT: s_endpgm397bb:398 %fneg.B = fneg <8 x half> %B399 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<8 x half> %A, <8 x half> %fneg.B, <8 x half> <half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0, half 1.0>, i1 0)400 store <8 x half> %res, ptr addrspace(1) %out401 ret void402}403 404; pack f16 elements with v_perm_b32 since they don't come from same b32405 406define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negC_pack(<8 x half> %A, <8 x half> %B, ptr %Caddr, ptr addrspace(1) %out) {407; GFX12-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:408; GFX12: ; %bb.0: ; %bb409; GFX12-NEXT: s_clause 0x1410; GFX12-NEXT: flat_load_b128 v[12:15], v[8:9] offset:16411; GFX12-NEXT: flat_load_b128 v[16:19], v[8:9]412; GFX12-NEXT: s_wait_loadcnt_dscnt 0x101413; GFX12-NEXT: v_perm_b32 v15, v15, v14, 0x5040100414; GFX12-NEXT: v_perm_b32 v14, v13, v12, 0x5040100415; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0416; GFX12-NEXT: v_perm_b32 v13, v19, v18, 0x5040100417; GFX12-NEXT: v_perm_b32 v12, v17, v16, 0x5040100418; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)419; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[12:15], v[0:3], v[4:7], v[12:15] neg_lo:[0,0,1]420; GFX12-NEXT: global_store_b128 v[10:11], v[12:15], off421; GFX12-NEXT: s_endpgm422bb:423 %C = load <16 x half>, ptr %Caddr424 %C_shuffle = shufflevector <16 x half> %C, <16 x half> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>425 %fneg.C_shuffle = fneg <8 x half> %C_shuffle426 %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<8 x half> %A, <8 x half> %B, <8 x half> %fneg.C_shuffle , i1 0)427 store <8 x half> %res, ptr addrspace(1) %out428 ret void429}430 431declare <8 x half> @llvm.fabs.v8f16(<8 x half>)432declare <8 x float> @llvm.fabs.v8f32(<8 x float>)433declare float @llvm.fabs.f32(float)434 435declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v8f16(<8 x half>, <8 x half>, <8 x float>)436declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf16.v8f32.v8i16(<8 x i16>, <8 x i16>, <8 x float>)437declare <8 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<8 x half>, <8 x half>, <8 x half>, i1 immarg)438declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.fp8.v8f32.v2i32(<2 x i32>, <2 x i32>, <8 x float>)439declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.bf8.v8f32.v2i32(<2 x i32>, <2 x i32>, <8 x float>)440declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.fp8.v8f32.v2i32(<2 x i32>, <2 x i32>, <8 x float>)441declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.bf8.v8f32.v2i32(<2 x i32>, <2 x i32>, <8 x float>)442declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16.v8f32.v8f16.v16f16.i16(<8 x half>, <16 x half>, <8 x float>, i16)443declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x32.f16.v8f16.v8f16.v16f16.i16(<8 x half>, <16 x half>, <8 x half>, i16)444