290 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 < %s | FileCheck %s --check-prefix=GFX123 4define amdgpu_ps void @test_wmma_f32_16x16x16_f16(<4 x half> %A, <4 x half> %B, <4 x float> %C, ptr addrspace(1) %out) {5; GFX12-LABEL: test_wmma_f32_16x16x16_f16:6; GFX12: ; %bb.0: ; %bb7; GFX12-NEXT: v_wmma_f32_16x16x16_f16 v[4:7], v[0:1], v[2:3], v[4:7]8; GFX12-NEXT: global_store_b128 v[8:9], v[4:7], off9; GFX12-NEXT: s_endpgm10bb:11 %res = call <4 x float>@llvm.amdgcn.wmma.f32.16x16x16.f16.v4f32.v4f16(<4 x half> %A, <4 x half> %B, <4 x float> %C)12 store <4 x float> %res, ptr addrspace(1) %out13 ret void14}15 16define amdgpu_ps void @test_wmma_f32_16x16x16_bf16(<4 x i16> %A, <4 x i16> %B, <4 x float> %C, ptr addrspace(1) %out) {17; GFX12-LABEL: test_wmma_f32_16x16x16_bf16:18; GFX12: ; %bb.0: ; %bb19; GFX12-NEXT: v_wmma_f32_16x16x16_bf16 v[4:7], v[0:1], v[2:3], v[4:7]20; GFX12-NEXT: global_store_b128 v[8:9], v[4:7], off21; GFX12-NEXT: s_endpgm22bb:23 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf16.v4f32.v4i16(<4 x i16> %A, <4 x i16> %B, <4 x float> %C)24 store <4 x float> %res, ptr addrspace(1) %out25 ret void26}27 28define amdgpu_ps void @test_wmma_f16_16x16x16_f16(<4 x half> %A, <4 x half> %B, <4 x half> %C, ptr addrspace(1) %out) {29; GFX12-LABEL: test_wmma_f16_16x16x16_f16:30; GFX12: ; %bb.0: ; %bb31; GFX12-NEXT: v_wmma_f16_16x16x16_f16 v[4:5], v[0:1], v[2:3], v[4:5]32; GFX12-NEXT: global_store_b64 v[6:7], v[4:5], off33; GFX12-NEXT: s_endpgm34bb:35 %res = call <4 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v4f16.v4f16(<4 x half> %A, <4 x half> %B, <4 x half> %C, i1 0)36 store <4 x half> %res, ptr addrspace(1) %out37 ret void38}39 40define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16(<4 x i16> %A, <4 x i16> %B, <4 x i16> %C, ptr addrspace(1) %out) {41; GFX12-LABEL: test_wmma_bf16_16x16x16_bf16:42; GFX12: ; %bb.0: ; %bb43; GFX12-NEXT: v_wmma_bf16_16x16x16_bf16 v[4:5], v[0:1], v[2:3], v[4:5]44; GFX12-NEXT: global_store_b64 v[6:7], v[4:5], off45; GFX12-NEXT: s_endpgm46bb:47 %res = call <4 x i16> @llvm.amdgcn.wmma.bf16.16x16x16.bf16.v4i16.v4i16(<4 x i16> %A, <4 x i16> %B, <4 x i16> %C, i1 0)48 store <4 x i16> %res, ptr addrspace(1) %out49 ret void50}51 52define amdgpu_ps void @test_wmma_i32_16x16x16_iu8(i32 %A, i32 %B, <4 x i32> %C, ptr addrspace(1) %out) {53; GFX12-LABEL: test_wmma_i32_16x16x16_iu8:54; GFX12: ; %bb.0: ; %bb55; GFX12-NEXT: v_wmma_i32_16x16x16_iu8 v[2:5], v0, v1, v[2:5]56; GFX12-NEXT: global_store_b128 v[6:7], v[2:5], off57; GFX12-NEXT: s_endpgm58bb:59 %res = call <4 x i32> @llvm.amdgcn.wmma.i32.16x16x16.iu8.v4i32.i32(i1 0, i32 %A, i1 0, i32 %B, <4 x i32> %C, i1 0)60 store <4 x i32> %res, ptr addrspace(1) %out61 ret void62}63 64define amdgpu_ps void @test_wmma_i32_16x16x16_iu4(i32 %A, i32 %B, <4 x i32> %C, ptr addrspace(1) %out) {65; GFX12-LABEL: test_wmma_i32_16x16x16_iu4:66; GFX12: ; %bb.0: ; %bb67; GFX12-NEXT: v_wmma_i32_16x16x16_iu4 v[2:5], v0, v1, v[2:5]68; GFX12-NEXT: global_store_b128 v[6:7], v[2:5], off69; GFX12-NEXT: s_endpgm70bb:71 %res = call <4 x i32> @llvm.amdgcn.wmma.i32.16x16x16.iu4.v4i32.i32(i1 0, i32 %A, i1 0, i32 %B, <4 x i32> %C, i1 0)72 store <4 x i32> %res, ptr addrspace(1) %out73 ret void74}75 76define amdgpu_ps void @test_wmma_f32_16x16x16_fp8_fp8(i32 %A, i32 %B, <4 x float> %C, ptr addrspace(1) %out) {77; GFX12-LABEL: test_wmma_f32_16x16x16_fp8_fp8:78; GFX12: ; %bb.0: ; %bb79; GFX12-NEXT: v_wmma_f32_16x16x16_fp8_fp8 v[2:5], v0, v1, v[2:5]80; GFX12-NEXT: global_store_b128 v[6:7], v[2:5], off81; GFX12-NEXT: s_endpgm82bb:83 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.fp8.v4f32.i32(i32 %A, i32 %B, <4 x float> %C)84 store <4 x float> %res, ptr addrspace(1) %out85 ret void86}87 88define amdgpu_ps void @test_wmma_f32_16x16x16_bf8_fp8(i32 %A, i32 %B, <4 x float> %C, ptr addrspace(1) %out) {89; GFX12-LABEL: test_wmma_f32_16x16x16_bf8_fp8:90; GFX12: ; %bb.0: ; %bb91; GFX12-NEXT: v_wmma_f32_16x16x16_bf8_fp8 v[2:5], v0, v1, v[2:5]92; GFX12-NEXT: global_store_b128 v[6:7], v[2:5], off93; GFX12-NEXT: s_endpgm94bb:95 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.fp8.v4f32.i32(i32 %A, i32 %B, <4 x float> %C)96 store <4 x float> %res, ptr addrspace(1) %out97 ret void98}99 100define amdgpu_ps void @test_wmma_f32_16x16x16_fp8_bf8(i32 %A, i32 %B, <4 x float> %C, ptr addrspace(1) %out) {101; GFX12-LABEL: test_wmma_f32_16x16x16_fp8_bf8:102; GFX12: ; %bb.0: ; %bb103; GFX12-NEXT: v_wmma_f32_16x16x16_fp8_bf8 v[2:5], v0, v1, v[2:5]104; GFX12-NEXT: global_store_b128 v[6:7], v[2:5], off105; GFX12-NEXT: s_endpgm106bb:107 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.bf8.v4f32.i32(i32 %A, i32 %B, <4 x float> %C)108 store <4 x float> %res, ptr addrspace(1) %out109 ret void110}111 112define amdgpu_ps void @test_wmma_f32_16x16x16_bf8_bf8(i32 %A, i32 %B, <4 x float> %C, ptr addrspace(1) %out) {113; GFX12-LABEL: test_wmma_f32_16x16x16_bf8_bf8:114; GFX12: ; %bb.0: ; %bb115; GFX12-NEXT: v_wmma_f32_16x16x16_bf8_bf8 v[2:5], v0, v1, v[2:5]116; GFX12-NEXT: global_store_b128 v[6:7], v[2:5], off117; GFX12-NEXT: s_endpgm118bb:119 %res = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.bf8.v4f32.i32(i32 %A, i32 %B, <4 x float> %C)120 store <4 x float> %res, ptr addrspace(1) %out121 ret void122}123 124define amdgpu_ps void @test_wmma_i32_16x16x32_iu4(i32 %A, i32 %B, <4 x i32> %C, ptr addrspace(1) %out) {125; GFX12-LABEL: test_wmma_i32_16x16x32_iu4:126; GFX12: ; %bb.0: ; %bb127; GFX12-NEXT: v_wmma_i32_16x16x32_iu4 v[2:5], v0, v1, v[2:5]128; GFX12-NEXT: global_store_b128 v[6:7], v[2:5], off129; GFX12-NEXT: s_endpgm130bb:131 %res = call <4 x i32> @llvm.amdgcn.wmma.i32.16x16x32.iu4.v4i32.i32(i1 0, i32 %A, i1 0, i32 %B, <4 x i32> %C, i1 0)132 store <4 x i32> %res, ptr addrspace(1) %out133 ret void134}135 136define amdgpu_ps void @test_swmmac_f32_16x16x32_f16(<4 x half> %A, <8 x half> %B, <4 x float> %C, i8 %Index, ptr addrspace(1) %out) {137; GFX12-LABEL: test_swmmac_f32_16x16x32_f16:138; GFX12: ; %bb.0: ; %bb139; GFX12-NEXT: v_swmmac_f32_16x16x32_f16 v[6:9], v[0:1], v[2:5], v10140; GFX12-NEXT: global_store_b128 v[11:12], v[6:9], off141; GFX12-NEXT: s_endpgm142bb:143 %res = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16.v4f32.v4f16.v8f16.i8(<4 x half> %A, <8 x half> %B, <4 x float> %C, i8 %Index)144 store <4 x float> %res, ptr addrspace(1) %out145 ret void146}147 148define amdgpu_ps void @test_swmmac_f32_16x16x32_bf16(<4 x i16> %A, <8 x i16> %B, <4 x float> %C, i8 %Index, ptr addrspace(1) %out) {149; GFX12-LABEL: test_swmmac_f32_16x16x32_bf16:150; GFX12: ; %bb.0: ; %bb151; GFX12-NEXT: v_swmmac_f32_16x16x32_bf16 v[6:9], v[0:1], v[2:5], v10152; GFX12-NEXT: global_store_b128 v[11:12], v[6:9], off153; GFX12-NEXT: s_endpgm154bb:155 %res = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf16.v4f32.v4i16.v8i16.i8(<4 x i16> %A, <8 x i16> %B, <4 x float> %C, i8 %Index)156 store <4 x float> %res, ptr addrspace(1) %out157 ret void158}159 160define amdgpu_ps void @test_swmmac_f16_16x16x32_f16(<4 x half> %A, <8 x half> %B, <4 x half> %C, i8 %Index, ptr addrspace(1) %out) {161; GFX12-LABEL: test_swmmac_f16_16x16x32_f16:162; GFX12: ; %bb.0: ; %bb163; GFX12-NEXT: v_swmmac_f16_16x16x32_f16 v[6:7], v[0:1], v[2:5], v8164; GFX12-NEXT: global_store_b64 v[9:10], v[6:7], off165; GFX12-NEXT: s_endpgm166bb:167 %res = call <4 x half> @llvm.amdgcn.swmmac.f16.16x16x32.f16.v4f16.v4f16.v8f16.i8(<4 x half> %A, <8 x half> %B, <4 x half> %C, i8 %Index)168 store <4 x half> %res, ptr addrspace(1) %out169 ret void170}171 172define amdgpu_ps void @test_swmmac_bf16_16x16x32_bf16(<4 x i16> %A, <8 x i16> %B, <4 x i16> %C, i8 %Index, ptr addrspace(1) %out) {173; GFX12-LABEL: test_swmmac_bf16_16x16x32_bf16:174; GFX12: ; %bb.0: ; %bb175; GFX12-NEXT: v_swmmac_bf16_16x16x32_bf16 v[6:7], v[0:1], v[2:5], v8176; GFX12-NEXT: global_store_b64 v[9:10], v[6:7], off177; GFX12-NEXT: s_endpgm178bb:179 %res = call <4 x i16> @llvm.amdgcn.swmmac.bf16.16x16x32.bf16.v4i16.v4i16.v8i16.i8(<4 x i16> %A, <8 x i16> %B, <4 x i16> %C, i8 %Index)180 store <4 x i16> %res, ptr addrspace(1) %out181 ret void182}183 184define amdgpu_ps void @test_swmmac_i32_16x16x32_iu8(i32 %A, <2 x i32> %B, <4 x i32> %C, i8 %Index, ptr addrspace(1) %out) {185; GFX12-LABEL: test_swmmac_i32_16x16x32_iu8:186; GFX12: ; %bb.0: ; %bb187; GFX12-NEXT: v_swmmac_i32_16x16x32_iu8 v[3:6], v0, v[1:2], v7188; GFX12-NEXT: global_store_b128 v[8:9], v[3:6], off189; GFX12-NEXT: s_endpgm190bb:191 %res = call <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x32.iu8.v4i32.i32.v2i32.i8(i1 0, i32 %A, i1 0, <2 x i32> %B, <4 x i32> %C, i8 %Index, i1 0)192 store <4 x i32> %res, ptr addrspace(1) %out193 ret void194}195 196define amdgpu_ps void @test_swmmac_i32_16x16x32_iu4(i32 %A, i32 %B, <4 x i32> %C, i16 %Index, ptr addrspace(1) %out) {197; GFX12-LABEL: test_swmmac_i32_16x16x32_iu4:198; GFX12: ; %bb.0: ; %bb199; GFX12-NEXT: v_swmmac_i32_16x16x32_iu4 v[2:5], v0, v1, v6200; GFX12-NEXT: global_store_b128 v[7:8], v[2:5], off201; GFX12-NEXT: s_endpgm202bb:203 %res = call <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x32.iu4.v4i32.i32.i32.i16(i1 0, i32 %A, i1 0, i32 %B, <4 x i32> %C, i16 %Index, i1 0)204 store <4 x i32> %res, ptr addrspace(1) %out205 ret void206}207 208define amdgpu_ps void @test_swmmac_i32_16x16x64_iu4(i32 %A, <2 x i32> %B, <4 x i32> %C, i16 %Index, ptr addrspace(1) %out) {209; GFX12-LABEL: test_swmmac_i32_16x16x64_iu4:210; GFX12: ; %bb.0: ; %bb211; GFX12-NEXT: v_swmmac_i32_16x16x64_iu4 v[3:6], v0, v[1:2], v7212; GFX12-NEXT: global_store_b128 v[8:9], v[3:6], off213; GFX12-NEXT: s_endpgm214bb:215 %res = call <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x64.iu4.v4i32.i32.v2i32.i16(i1 0, i32 %A, i1 0, <2 x i32> %B, <4 x i32> %C, i16 %Index, i1 0)216 store <4 x i32> %res, ptr addrspace(1) %out217 ret void218}219 220define amdgpu_ps void @test_swmmac_f32_16x16x32_fp8_fp8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index, ptr addrspace(1) %out) {221; GFX12-LABEL: test_swmmac_f32_16x16x32_fp8_fp8:222; GFX12: ; %bb.0: ; %bb223; GFX12-NEXT: v_swmmac_f32_16x16x32_fp8_fp8 v[3:6], v0, v[1:2], v7224; GFX12-NEXT: global_store_b128 v[8:9], v[3:6], off225; GFX12-NEXT: s_endpgm226bb:227 %res = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.fp8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index)228 store <4 x float> %res, ptr addrspace(1) %out229 ret void230}231 232define amdgpu_ps void @test_swmmac_f32_16x16x32_fp8_bf8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index, ptr addrspace(1) %out) {233; GFX12-LABEL: test_swmmac_f32_16x16x32_fp8_bf8:234; GFX12: ; %bb.0: ; %bb235; GFX12-NEXT: v_swmmac_f32_16x16x32_fp8_bf8 v[3:6], v0, v[1:2], v7236; GFX12-NEXT: global_store_b128 v[8:9], v[3:6], off237; GFX12-NEXT: s_endpgm238bb:239 %res = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.bf8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index)240 store <4 x float> %res, ptr addrspace(1) %out241 ret void242}243 244define amdgpu_ps void @test_swmmac_f32_16x16x32_bf8_fp8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index, ptr addrspace(1) %out) {245; GFX12-LABEL: test_swmmac_f32_16x16x32_bf8_fp8:246; GFX12: ; %bb.0: ; %bb247; GFX12-NEXT: v_swmmac_f32_16x16x32_bf8_fp8 v[3:6], v0, v[1:2], v7248; GFX12-NEXT: global_store_b128 v[8:9], v[3:6], off249; GFX12-NEXT: s_endpgm250bb:251 %res = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.fp8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index)252 store <4 x float> %res, ptr addrspace(1) %out253 ret void254}255 256define amdgpu_ps void @test_swmmac_f32_16x16x32_bf8_bf8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index, ptr addrspace(1) %out) {257; GFX12-LABEL: test_swmmac_f32_16x16x32_bf8_bf8:258; GFX12: ; %bb.0: ; %bb259; GFX12-NEXT: v_swmmac_f32_16x16x32_bf8_bf8 v[3:6], v0, v[1:2], v7260; GFX12-NEXT: global_store_b128 v[8:9], v[3:6], off261; GFX12-NEXT: s_endpgm262bb:263 %res = call <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.bf8.v4f32.i32.v2i32.i8(i32 %A, <2 x i32> %B, <4 x float> %C, i8 %Index)264 store <4 x float> %res, ptr addrspace(1) %out265 ret void266}267 268declare <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v4f32.v4f16(<4 x half>, <4 x half>, <4 x float>)269declare <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf16.v4f32.v4i16(<4 x i16>, <4 x i16>, <4 x float>)270declare <4 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v4f16.v4f16(<4 x half>, <4 x half>, <4 x half>, i1 immarg)271declare <4 x i16> @llvm.amdgcn.wmma.bf16.16x16x16.bf16.v4i16.v4i16(<4 x i16>, <4 x i16>, <4 x i16>, i1 immarg)272declare <4 x i32> @llvm.amdgcn.wmma.i32.16x16x16.iu8.v4i32.i32(i1 immarg, i32, i1 immarg, i32, <4 x i32>, i1 immarg)273declare <4 x i32> @llvm.amdgcn.wmma.i32.16x16x16.iu4.v4i32.i32(i1 immarg, i32, i1 immarg, i32, <4 x i32>, i1 immarg)274declare <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.fp8.v4f32.i32(i32, i32, <4 x float>)275declare <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.bf8.v4f32.i32(i32, i32, <4 x float>)276declare <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.fp8.v4f32.i32.(i32, i32, <4 x float>)277declare <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.bf8.bf8.v4f32.i32(i32, i32, <4 x float>)278declare <4 x i32> @llvm.amdgcn.wmma.i32.16x16x32.iu4.v4i32.i32(i1 immarg, i32, i1 immarg, i32, <4 x i32>, i1 immarg)279declare <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.f16.v4f32.v4f16.v8f16.i8(<4 x half>, <8 x half>, <4 x float>, i8)280declare <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf16.v4f32.v4i16.v8i16.i8(<4 x i16>, <8 x i16>, <4 x float>, i8)281declare <4 x half> @llvm.amdgcn.swmmac.f16.16x16x32.f16.v4f16.v4f16.v8f16.i8(<4 x half>, <8 x half>, <4 x half>, i8)282declare <4 x i16> @llvm.amdgcn.swmmac.bf16.16x16x32.bf16.v4i16.v4i16.v8i16.i8(<4 x i16>, <8 x i16>, <4 x i16>, i8)283declare <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x32.iu8.v4i32.i32.v2i32.i8(i1 immarg, i32, i1 immarg, <2 x i32>, <4 x i32>, i8 %Index, i1 immarg)284declare <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x32.iu4.v4i32.i32.i32.i16(i1 immarg, i32, i1 immarg, i32, <4 x i32>, i16 %Index, i1 immarg)285declare <4 x i32> @llvm.amdgcn.swmmac.i32.16x16x64.iu4.v4i32.i32.v2i32.i16(i1 immarg, i32, i1 immarg, <2 x i32>, <4 x i32>, i16 %Index, i1 immarg)286declare <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.fp8.v4f32.i32.v2i32.i8(i32, <2 x i32>, <4 x float>, i8)287declare <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.bf8.v4f32.i32.v2i32.i8(i32, <2 x i32>, <4 x float>, i8)288declare <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.fp8.v4f32.i32.v2i32.i8(i32, <2 x i32>, <4 x float>, i8)289declare <4 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.bf8.v4f32.i32.v2i32.i8(i32, <2 x i32>, <4 x float>, i8)290