414 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -mattr=+wavefrontsize64 < %s | FileCheck %s --check-prefixes=GFX12,GFX12-TRUE163; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -mattr=+wavefrontsize64 < %s | FileCheck %s --check-prefixes=GFX12,GFX12-FAKE164 5define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negA(<4 x half> %A, <4 x half> %B, <4 x float> %C, ptr addrspace(1) %out) {6; GFX12-LABEL: test_wmma_f32_16x16x16_f16_negA:7; GFX12: ; %bb.0: ; %bb8; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[4:7], v[0:1], v[2:3], v[4:7] neg_lo:[1,0,0] neg_hi:[1,0,0]9; GFX12-NEXT: global_store_b128 v[8:9], v[4:7], off10; GFX12-NEXT: s_endpgm11bb:12 %fneg.A = fneg <4 x half> %A13 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v4f32.v4f16(<4 x half> %fneg.A, <4 x half> %B, <4 x float> %C)14 store <4 x float> %res, ptr addrspace(1) %out15 ret void16}17 18define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negB(<4 x half> %A, <4 x half> %B, <4 x float> %C, ptr addrspace(1) %out) {19; GFX12-LABEL: test_wmma_f32_16x16x16_f16_negB:20; GFX12: ; %bb.0: ; %bb21; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[4:7], v[0:1], v[2:3], v[4:7] neg_lo:[0,1,0] neg_hi:[0,1,0]22; GFX12-NEXT: global_store_b128 v[8:9], v[4:7], off23; GFX12-NEXT: s_endpgm24bb:25 %fneg.B = fneg <4 x half> %B26 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v4f32.v4f16(<4 x half> %A, <4 x half> %fneg.B, <4 x float> %C)27 store <4 x float> %res, ptr addrspace(1) %out28 ret void29}30 31define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negC(<4 x half> %A, <4 x half> %B, <4 x float> %C, ptr addrspace(1) %out) {32; GFX12-LABEL: test_wmma_f32_16x16x16_f16_negC:33; GFX12: ; %bb.0: ; %bb34; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[4:7], v[0:1], v[2:3], v[4:7] neg_lo:[0,0,1]35; GFX12-NEXT: global_store_b128 v[8:9], v[4:7], off36; GFX12-NEXT: s_endpgm37bb:38 %fneg.C = fneg <4 x float> %C39 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v4f32.v4f16(<4 x half> %A, <4 x half> %B, <4 x float> %fneg.C)40 store <4 x float> %res, ptr addrspace(1) %out41 ret void42}43 44define amdgpu_ps void @test_wmma_f32_16x16x16_f16_absC(<4 x half> %A, <4 x half> %B, <4 x float> %C, ptr addrspace(1) %out) {45; GFX12-LABEL: test_wmma_f32_16x16x16_f16_absC:46; GFX12: ; %bb.0: ; %bb47; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[4:7], v[0:1], v[2:3], v[4:7] neg_hi:[0,0,1]48; GFX12-NEXT: global_store_b128 v[8:9], v[4:7], off49; GFX12-NEXT: s_endpgm50bb:51 %fabs.C = call <4 x float> @llvm.fabs.v4f32(<4 x float> %C)52 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v4f32.v4f16(<4 x half> %A, <4 x half> %B, <4 x float> %fabs.C)53 store <4 x float> %res, ptr addrspace(1) %out54 ret void55}56 57define amdgpu_ps void @test_wmma_f32_16x16x16_bf16_negC(<4 x i16> %A, <4 x i16> %B, <4 x float> %C, ptr addrspace(1) %out) {58; GFX12-LABEL: test_wmma_f32_16x16x16_bf16_negC:59; GFX12: ; %bb.0: ; %bb60; GFX12-NEXT: v_wmma_f32_16x16x16_bf16 v[4:7], v[0:1], v[2:3], v[4:7] neg_lo:[0,0,1]61; GFX12-NEXT: global_store_b128 v[8:9], v[4:7], off62; GFX12-NEXT: s_endpgm63bb:64 %fneg.C = fneg <4 x float> %C65 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf16.v4f32.v4i16(<4 x i16> %A, <4 x i16> %B, <4 x float> %fneg.C)66 store <4 x float> %res, ptr addrspace(1) %out67 ret void68}69 70define amdgpu_ps void @test_wmma_f32_16x16x16_bf16_absC(<4 x i16> %A, <4 x i16> %B, <4 x float> %C, ptr addrspace(1) %out) {71; GFX12-LABEL: test_wmma_f32_16x16x16_bf16_absC:72; GFX12: ; %bb.0: ; %bb73; GFX12-NEXT: v_wmma_f32_16x16x16_bf16 v[4:7], v[0:1], v[2:3], v[4:7] neg_hi:[0,0,1]74; GFX12-NEXT: global_store_b128 v[8:9], v[4:7], off75; GFX12-NEXT: s_endpgm76bb:77 %fabs.C = call <4 x float> @llvm.fabs.v4f32(<4 x float> %C)78 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf16.v4f32.v4i16(<4 x i16> %A, <4 x i16> %B, <4 x float> %fabs.C)79 store <4 x float> %res, ptr addrspace(1) %out80 ret void81}82 83define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negA(<4 x half> %A, <4 x half> %B, <4 x half> %C, ptr addrspace(1) %out) {84; GFX12-LABEL: test_wmma_f16_16x16x16_f16_negA:85; GFX12: ; %bb.0: ; %bb86; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[4:5], v[0:1], v[2:3], v[4:5] neg_lo:[1,0,0] neg_hi:[1,0,0]87; GFX12-NEXT: global_store_b64 v[6:7], v[4:5], off88; GFX12-NEXT: s_endpgm89bb:90 %fneg.A = fneg <4 x half> %A91 %res = call <4 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<4 x half> %fneg.A, <4 x half> %B, <4 x half> %C, i1 0)92 store <4 x half> %res, ptr addrspace(1) %out93 ret void94}95 96define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negB(<4 x half> %A, <4 x half> %B, <4 x half> %C, ptr addrspace(1) %out) {97; GFX12-LABEL: test_wmma_f16_16x16x16_f16_negB:98; GFX12: ; %bb.0: ; %bb99; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[4:5], v[0:1], v[2:3], v[4:5] neg_lo:[0,1,0] neg_hi:[0,1,0]100; GFX12-NEXT: global_store_b64 v[6:7], v[4:5], off101; GFX12-NEXT: s_endpgm102bb:103 %fneg.B = fneg <4 x half> %B104 %res = call <4 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<4 x half> %A, <4 x half> %fneg.B, <4 x half> %C, i1 0)105 store <4 x half> %res, ptr addrspace(1) %out106 ret void107}108 109define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negC(<4 x half> %A, <4 x half> %B, <4 x half> %C, ptr addrspace(1) %out) {110; GFX12-LABEL: test_wmma_f16_16x16x16_f16_negC:111; GFX12: ; %bb.0: ; %bb112; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[4:5], v[0:1], v[2:3], v[4:5] neg_lo:[0,0,1]113; GFX12-NEXT: global_store_b64 v[6:7], v[4:5], off114; GFX12-NEXT: s_endpgm115bb:116 %fneg.C = fneg <4 x half> %C117 %res = call <4 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<4 x half> %A, <4 x half> %B, <4 x half> %fneg.C, i1 0)118 store <4 x half> %res, ptr addrspace(1) %out119 ret void120}121 122define amdgpu_ps void @test_wmma_f16_16x16x16_f16_absC(<4 x half> %A, <4 x half> %B, <4 x half> %C, ptr addrspace(1) %out) {123; GFX12-LABEL: test_wmma_f16_16x16x16_f16_absC:124; GFX12: ; %bb.0: ; %bb125; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[4:5], v[0:1], v[2:3], v[4:5] neg_hi:[0,0,1]126; GFX12-NEXT: global_store_b64 v[6:7], v[4:5], off127; GFX12-NEXT: s_endpgm128bb:129 %fabs.C = call <4 x half> @llvm.fabs.v4f16(<4 x half> %C)130 %res = call <4 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<4 x half> %A, <4 x half> %B, <4 x half> %fabs.C, i1 0)131 store <4 x half> %res, ptr addrspace(1) %out132 ret void133}134 135define amdgpu_ps void @test_wmma_f32_16x16x16_fp8_fp8_negC(i32 %A, i32 %B, <4 x float> %C, ptr addrspace(1) %out) {136; GFX12-LABEL: test_wmma_f32_16x16x16_fp8_fp8_negC:137; GFX12: ; %bb.0: ; %bb138; GFX12-NEXT: v_wmma_f32_16x16x16_fp8_fp8 v[2:5], v0, v1, v[2:5] neg_lo:[0,0,1]139; GFX12-NEXT: global_store_b128 v[6:7], v[2:5], off140; GFX12-NEXT: s_endpgm141bb:142 %fneg.C = fneg <4 x float> %C143 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.fp8.v4f32.i32(i32 %A, i32 %B, <4 x float> %fneg.C)144 store <4 x float> %res, ptr addrspace(1) %out145 ret void146}147 148define amdgpu_ps void @test_wmma_f32_16x16x16_fp8_fp8_absC(i32 %A, i32 %B, <4 x float> %C, ptr addrspace(1) %out) {149; GFX12-LABEL: test_wmma_f32_16x16x16_fp8_fp8_absC:150; GFX12: ; %bb.0: ; %bb151; GFX12-NEXT: v_wmma_f32_16x16x16_fp8_fp8 v[2:5], v0, v1, v[2:5] neg_hi:[0,0,1]152; GFX12-NEXT: global_store_b128 v[6:7], v[2:5], off153; GFX12-NEXT: s_endpgm154bb:155 %fabs.C = call <4 x float> @llvm.fabs.v4f32(<4 x float> %C)156 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.fp8.v4f32.i32(i32 %A, i32 %B, <4 x float> %fabs.C)157 store <4 x float> %res, ptr addrspace(1) %out158 ret void159}160 161define amdgpu_ps void @test_wmma_f32_16x16x16_bf8_fp8_negC(i32 %A, i32 %B, <4 x float> %C, ptr addrspace(1) %out) {162; GFX12-LABEL: test_wmma_f32_16x16x16_bf8_fp8_negC:163; GFX12: ; %bb.0: ; %bb164; GFX12-NEXT: v_wmma_f32_16x16x16_bf8_fp8 v[2:5], v0, v1, v[2:5] neg_lo:[0,0,1]165; GFX12-NEXT: global_store_b128 v[6:7], v[2:5], off166; GFX12-NEXT: s_endpgm167bb:168 %fneg.C = fneg <4 x float> %C169 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.fp8.v4f32.i32(i32 %A, i32 %B, <4 x float> %fneg.C)170 store <4 x float> %res, ptr addrspace(1) %out171 ret void172}173 174define amdgpu_ps void @test_wmma_f32_16x16x16_bf8_fp8_absC(i32 %A, i32 %B, <4 x float> %C, ptr addrspace(1) %out) {175; GFX12-LABEL: test_wmma_f32_16x16x16_bf8_fp8_absC:176; GFX12: ; %bb.0: ; %bb177; GFX12-NEXT: v_wmma_f32_16x16x16_bf8_fp8 v[2:5], v0, v1, v[2:5] neg_hi:[0,0,1]178; GFX12-NEXT: global_store_b128 v[6:7], v[2:5], off179; GFX12-NEXT: s_endpgm180bb:181 %fabs.C = call <4 x float> @llvm.fabs.v4f32(<4 x float> %C)182 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.fp8.v4f32.i32(i32 %A, i32 %B, <4 x float> %fabs.C)183 store <4 x float> %res, ptr addrspace(1) %out184 ret void185}186 187define amdgpu_ps void @test_wmma_f32_16x16x16_fp8_bf8_negC(i32 %A, i32 %B, <4 x float> %C, ptr addrspace(1) %out) {188; GFX12-LABEL: test_wmma_f32_16x16x16_fp8_bf8_negC:189; GFX12: ; %bb.0: ; %bb190; GFX12-NEXT: v_wmma_f32_16x16x16_fp8_bf8 v[2:5], v0, v1, v[2:5] neg_lo:[0,0,1]191; GFX12-NEXT: global_store_b128 v[6:7], v[2:5], off192; GFX12-NEXT: s_endpgm193bb:194 %fneg.C = fneg <4 x float> %C195 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.bf8.v4f32.i32(i32 %A, i32 %B, <4 x float> %fneg.C)196 store <4 x float> %res, ptr addrspace(1) %out197 ret void198}199 200define amdgpu_ps void @test_wmma_f32_16x16x16_fp8_bf8_absC(i32 %A, i32 %B, <4 x float> %C, ptr addrspace(1) %out) {201; GFX12-LABEL: test_wmma_f32_16x16x16_fp8_bf8_absC:202; GFX12: ; %bb.0: ; %bb203; GFX12-NEXT: v_wmma_f32_16x16x16_fp8_bf8 v[2:5], v0, v1, v[2:5] neg_hi:[0,0,1]204; GFX12-NEXT: global_store_b128 v[6:7], v[2:5], off205; GFX12-NEXT: s_endpgm206bb:207 %fabs.C = call <4 x float> @llvm.fabs.v4f32(<4 x float> %C)208 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.bf8.v4f32.i32(i32 %A, i32 %B, <4 x float> %fabs.C)209 store <4 x float> %res, ptr addrspace(1) %out210 ret void211}212 213define amdgpu_ps void @test_wmma_f32_16x16x16_bf8_bf8_negC(i32 %A, i32 %B, <4 x float> %C, ptr addrspace(1) %out) {214; GFX12-LABEL: test_wmma_f32_16x16x16_bf8_bf8_negC:215; GFX12: ; %bb.0: ; %bb216; GFX12-NEXT: v_wmma_f32_16x16x16_bf8_bf8 v[2:5], v0, v1, v[2:5] neg_lo:[0,0,1]217; GFX12-NEXT: global_store_b128 v[6:7], v[2:5], off218; GFX12-NEXT: s_endpgm219bb:220 %fneg.C = fneg <4 x float> %C221 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.bf8.v4f32.i32(i32 %A, i32 %B, <4 x float> %fneg.C)222 store <4 x float> %res, ptr addrspace(1) %out223 ret void224}225 226define amdgpu_ps void @test_wmma_f32_16x16x16_bf8_bf8_absC(i32 %A, i32 %B, <4 x float> %C, ptr addrspace(1) %out) {227; GFX12-LABEL: test_wmma_f32_16x16x16_bf8_bf8_absC:228; GFX12: ; %bb.0: ; %bb229; GFX12-NEXT: v_wmma_f32_16x16x16_bf8_bf8 v[2:5], v0, v1, v[2:5] neg_hi:[0,0,1]230; GFX12-NEXT: global_store_b128 v[6:7], v[2:5], off231; GFX12-NEXT: s_endpgm232bb:233 %fabs.C = call <4 x float> @llvm.fabs.v4f32(<4 x float> %C)234 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.bf8.v4f32.i32(i32 %A, i32 %B, <4 x float> %fabs.C)235 store <4 x float> %res, ptr addrspace(1) %out236 ret void237}238 239define amdgpu_ps void @test_swmmac_f32_16x16x32_f16_negA(<4 x half> %A, <8 x half> %B, <4 x float> %C, i16 %Index, ptr addrspace(1) %out) {240; GFX12-LABEL: test_swmmac_f32_16x16x32_f16_negA:241; GFX12: ; %bb.0: ; %bb242; GFX12-NEXT: v_swmmac_f32_16x16x32_f16 v[6:9], v[0:1], v[2:5], v10 neg_lo:[1,0,0] neg_hi:[1,0,0]243; GFX12-NEXT: global_store_b128 v[11:12], v[6:9], off244; GFX12-NEXT: s_endpgm245bb:246 %fneg.A = fneg <4 x half> %A247 %res = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16.v4f32.v4f16.v8f16.i16(<4 x half> %fneg.A, <8 x half> %B, <4 x float> %C, i16 %Index)248 store <4 x float> %res, ptr addrspace(1) %out249 ret void250}251 252define amdgpu_ps void @test_swmmac_f32_16x16x32_f16_negB(<4 x half> %A, <8 x half> %B, <4 x float> %C, i16 %Index, ptr addrspace(1) %out) {253; GFX12-LABEL: test_swmmac_f32_16x16x32_f16_negB:254; GFX12: ; %bb.0: ; %bb255; GFX12-NEXT: v_swmmac_f32_16x16x32_f16 v[6:9], v[0:1], v[2:5], v10 neg_lo:[0,1,0] neg_hi:[0,1,0]256; GFX12-NEXT: global_store_b128 v[11:12], v[6:9], off257; GFX12-NEXT: s_endpgm258bb:259 %fneg.B = fneg <8 x half> %B260 %res = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16.v4f32.v4f16.v8f16.i16(<4 x half> %A, <8 x half> %fneg.B, <4 x float> %C, i16 %Index)261 store <4 x float> %res, ptr addrspace(1) %out262 ret void263}264 265define amdgpu_ps void @test_swmmac_f16_16x16x32_f16_negA(<4 x half> %A, <8 x half> %B, <4 x half> %C, i16 %Index, ptr addrspace(1) %out) {266; GFX12-LABEL: test_swmmac_f16_16x16x32_f16_negA:267; GFX12: ; %bb.0: ; %bb268; GFX12-NEXT: v_swmmac_f16_16x16x32_f16 v[6:7], v[0:1], v[2:5], v8 neg_lo:[1,0,0] neg_hi:[1,0,0]269; GFX12-NEXT: global_store_b64 v[9:10], v[6:7], off270; GFX12-NEXT: s_endpgm271bb:272 %fneg.A = fneg <4 x half> %A273 %res = call <4 x half> @llvm.amdgcn.swmmac.f16.16x16x32.f16.v4f16.v4f16.v8f16.i16(<4 x half> %fneg.A, <8 x half> %B, <4 x half> %C, i16 %Index)274 store <4 x half> %res, ptr addrspace(1) %out275 ret void276}277 278define amdgpu_ps void @test_swmmac_f16_16x16x32_f16_negB(<4 x half> %A, <8 x half> %B, <4 x half> %C, i16 %Index, ptr addrspace(1) %out) {279; GFX12-LABEL: test_swmmac_f16_16x16x32_f16_negB:280; GFX12: ; %bb.0: ; %bb281; GFX12-NEXT: v_swmmac_f16_16x16x32_f16 v[6:7], v[0:1], v[2:5], v8 neg_lo:[0,1,0] neg_hi:[0,1,0]282; GFX12-NEXT: global_store_b64 v[9:10], v[6:7], off283; GFX12-NEXT: s_endpgm284bb:285 %fneg.B = fneg <8 x half> %B286 %res = call <4 x half> @llvm.amdgcn.swmmac.f16.16x16x32.f16.v4f16.v4f16.v8f16.i16(<4 x half> %A, <8 x half> %fneg.B, <4 x half> %C, i16 %Index)287 store <4 x half> %res, ptr addrspace(1) %out288 ret void289}290 291; both neg and abs patterns (wmma matrix C f32 or f16 )292 293define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negabsC(<4 x half> %A, <4 x half> %B, <4 x float> %C, ptr addrspace(1) %out) {294; GFX12-LABEL: test_wmma_f32_16x16x16_f16_negabsC:295; GFX12: ; %bb.0: ; %bb296; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[4:7], v[0:1], v[2:3], v[4:7] neg_lo:[0,0,1] neg_hi:[0,0,1]297; GFX12-NEXT: global_store_b128 v[8:9], v[4:7], off298; GFX12-NEXT: s_endpgm299bb:300 %fabs.C = call <4 x float> @llvm.fabs.v4f32(<4 x float> %C)301 %fneg.fabs.C = fneg <4 x float> %fabs.C302 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v4f32.v4f16(<4 x half> %A, <4 x half> %B, <4 x float> %fneg.fabs.C)303 store <4 x float> %res, ptr addrspace(1) %out304 ret void305}306 307define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negabsC(<4 x half> %A, <4 x half> %B, <4 x half> %C, ptr addrspace(1) %out) {308; GFX12-LABEL: test_wmma_f16_16x16x16_f16_negabsC:309; GFX12: ; %bb.0: ; %bb310; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[4:5], v[0:1], v[2:3], v[4:5] neg_lo:[0,0,1] neg_hi:[0,0,1]311; GFX12-NEXT: global_store_b64 v[6:7], v[4:5], off312; GFX12-NEXT: s_endpgm313bb:314 %fabs.C = call <4 x half> @llvm.fabs.v4f16(<4 x half> %C)315 %fneg.fabs.C = fneg <4 x half> %fabs.C316 %res = call <4 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<4 x half> %A, <4 x half> %B, <4 x half> %fneg.fabs.C, i1 0)317 store <4 x half> %res, ptr addrspace(1) %out318 ret void319}320 321define amdgpu_ps void @test_wmma_f32_16x16x16_f16_neg_partial_fabsA(<4 x half> %A, <4 x half> %B, <4 x float> %C, ptr addrspace(1) %out) {322; GFX12-LABEL: test_wmma_f32_16x16x16_f16_neg_partial_fabsA:323; GFX12: ; %bb.0: ; %bb324; GFX12-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7325; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)326; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[4:7], v[0:1], v[2:3], v[4:7] neg_lo:[0,0,1]327; GFX12-NEXT: global_store_b128 v[8:9], v[4:7], off328; GFX12-NEXT: s_endpgm329bb:330 %el3 = extractelement <4 x float> %C, i32 3331 %el3.fabs = call float @llvm.fabs.f32(float %el3)332 %partial.fabs.C = insertelement <4 x float> %C, float %el3.fabs, i32 3333 %fneg.partial.fabs.C = fneg <4 x float> %partial.fabs.C334 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v4f32.v4f16(<4 x half> %A, <4 x half> %B, <4 x float> %fneg.partial.fabs.C)335 store <4 x float> %res, ptr addrspace(1) %out336 ret void337}338 339; A or B matrix modifier and constant in C340 341define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negA_constantC(<4 x half> %A, <4 x half> %B, <4 x float> %C, ptr addrspace(1) %out) {342; GFX12-LABEL: test_wmma_f32_16x16x16_f16_negA_constantC:343; GFX12: ; %bb.0: ; %bb344; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[6:9], v[0:1], v[2:3], 1.0 neg_lo:[1,0,0] neg_hi:[1,0,0]345; GFX12-NEXT: global_store_b128 v[4:5], v[6:9], off346; GFX12-NEXT: s_endpgm347bb:348 %fneg.A = fneg <4 x half> %A349 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v4f32.v4f16(<4 x half> %fneg.A, <4 x half> %B, <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>)350 store <4 x float> %res, ptr addrspace(1) %out351 ret void352}353 354define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negB_constantC(<4 x half> %A, <4 x half> %B, <4 x half> %C, ptr addrspace(1) %out) {355; GFX12-LABEL: test_wmma_f16_16x16x16_f16_negB_constantC:356; GFX12: ; %bb.0: ; %bb357; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[6:7], v[0:1], v[2:3], 1.0 neg_lo:[0,1,0] neg_hi:[0,1,0]358; GFX12-NEXT: global_store_b64 v[4:5], v[6:7], off359; GFX12-NEXT: s_endpgm360bb:361 %fneg.B = fneg <4 x half> %B362 %res = call <4 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<4 x half> %A, <4 x half> %fneg.B, <4 x half> <half 1.0, half 1.0, half 1.0, half 1.0>, i1 0)363 store <4 x half> %res, ptr addrspace(1) %out364 ret void365}366 367; pack f16 elements with v_perm_b32 since they don't come from same b32368 369define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negC_pack(<4 x half> %A, <4 x half> %B, ptr %Caddr, ptr addrspace(1) %out) {370; GFX12-TRUE16-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:371; GFX12-TRUE16: ; %bb.0: ; %bb372; GFX12-TRUE16-NEXT: flat_load_b128 v[8:11], v[4:5]373; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0374; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.h, v11.l375; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.h, v9.l376; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)377; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v10378; GFX12-TRUE16-NEXT: v_wmma_f16_16x16x16_f16 v[8:9], v[0:1], v[2:3], v[8:9] neg_lo:[0,0,1]379; GFX12-TRUE16-NEXT: global_store_b64 v[6:7], v[8:9], off380; GFX12-TRUE16-NEXT: s_endpgm381;382; GFX12-FAKE16-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:383; GFX12-FAKE16: ; %bb.0: ; %bb384; GFX12-FAKE16-NEXT: flat_load_b128 v[8:11], v[4:5]385; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0386; GFX12-FAKE16-NEXT: v_perm_b32 v5, v11, v10, 0x5040100387; GFX12-FAKE16-NEXT: v_perm_b32 v4, v9, v8, 0x5040100388; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)389; GFX12-FAKE16-NEXT: v_wmma_f16_16x16x16_f16 v[4:5], v[0:1], v[2:3], v[4:5] neg_lo:[0,0,1]390; GFX12-FAKE16-NEXT: global_store_b64 v[6:7], v[4:5], off391; GFX12-FAKE16-NEXT: s_endpgm392bb:393 %C = load <8 x half>, ptr %Caddr394 %C_shuffle = shufflevector <8 x half> %C, <8 x half> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>395 %fneg.C_shuffle = fneg <4 x half> %C_shuffle396 %res = call <4 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<4 x half> %A, <4 x half> %B, <4 x half> %fneg.C_shuffle , i1 0)397 store <4 x half> %res, ptr addrspace(1) %out398 ret void399}400 401declare <4 x half> @llvm.fabs.v4f16(<4 x half>)402declare <4 x float> @llvm.fabs.v4f32(<4 x float>)403declare float @llvm.fabs.f32(float)404 405declare <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v4f32.v4f16(<4 x half>, <4 x half>, <4 x float>)406declare <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf16.v4f32.v4i16(<4 x i16>, <4 x i16>, <4 x float>)407declare <4 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<4 x half>, <4 x half>, <4 x half>, i1 immarg)408declare <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.fp8.v4f32.i32(i32, i32, <4 x float>)409declare <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.bf8.v4f32.i32(i32, i32, <4 x float>)410declare <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.fp8.v4f32.i32(i32, i32, <4 x float>)411declare <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.bf8.v4f32.i32(i32, i32, <4 x float>)412declare <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16.v4f32.v4f16.v8f16.i16(<4 x half>, <8 x half>, <4 x float>, i16)413declare <4 x half> @llvm.amdgcn.swmmac.f16.16x16x32.f16.v4f16.v4f16.v8f16.i16(<4 x half>, <8 x half>, <4 x half>, i16)414